fix(dataabc): drop NaN values in key_to_dict/key_to_lists (#1211)

The dropna filter compared values against float("nan") using ==, which is
always False (NaN != NaN). As a result NaN values were never dropped when
dropna=True, letting them leak into key_to_series/key_to_array and downstream
resampling.

Use pd.isna() to detect NaN, matching the rest of the module. Add regression
tests that fail before and pass after the fix.

Co-authored-by: Cornelius Mund <cornim@users.noreply.github.com>
Co-authored-by: Normann <github@koldrack.com>

src/akkudoktoreos/core/dataabc.py

@@ -1043,7 +1043,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
             if (
                 record.date_time is None
                 or (dropna and getattr(record, key, None) is None)
-                or (dropna and getattr(record, key, None) == float("nan"))
+                or (dropna and pd.isna(getattr(record, key, None)))
             ):
                 continue
             record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)
@@ -1122,7 +1122,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
             if (
                 record.date_time is None
                 or (getattr(record, key, None) is None)  # key is not in record
-                or (dropna and getattr(record, key, None) == float("nan"))
+                or (dropna and pd.isna(getattr(record, key, None)))
             ):
                 continue
             record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)

tests/test_dataabcsequence.py

@@ -975,6 +975,44 @@ class TestDataSequence:
         assert dates == [to_datetime(datetime(2023, 11, 5)), to_datetime(datetime(2023, 11, 6))]
         assert values == [0.8, 0.9]
 
+    async def test_key_to_dict_dropna_removes_nan(self, sequence):
+        """`dropna=True` (default) must drop records whose value is NaN, not just None."""
+        record1 = self.create_test_record(datetime(2023, 11, 5), 0.8)
+        record2 = self.create_test_record(datetime(2023, 11, 6), float("nan"))
+        record3 = self.create_test_record(datetime(2023, 11, 7), 0.9)
+        await sequence.insert_by_datetime(record1)
+        await sequence.insert_by_datetime(record2)
+        await sequence.insert_by_datetime(record3)
+
+        # Default dropna=True must drop the NaN record.
+        data_dict = await sequence.key_to_dict("data_value")
+        assert to_datetime(datetime(2023, 11, 6), as_string=True) not in data_dict
+        assert data_dict[to_datetime(datetime(2023, 11, 5), as_string=True)] == 0.8
+        assert data_dict[to_datetime(datetime(2023, 11, 7), as_string=True)] == 0.9
+
+        # With dropna=False the NaN record must be kept.
+        data_dict_keep = await sequence.key_to_dict("data_value", dropna=False)
+        assert pd.isna(data_dict_keep[to_datetime(datetime(2023, 11, 6), as_string=True)])
+
+    async def test_key_to_lists_dropna_removes_nan(self, sequence):
+        """`dropna=True` (default) must drop records whose value is NaN, not just None."""
+        record1 = self.create_test_record(datetime(2023, 11, 5), 0.8)
+        record2 = self.create_test_record(datetime(2023, 11, 6), float("nan"))
+        record3 = self.create_test_record(datetime(2023, 11, 7), 0.9)
+        await sequence.insert_by_datetime(record1)
+        await sequence.insert_by_datetime(record2)
+        await sequence.insert_by_datetime(record3)
+
+        # Default dropna=True must drop the NaN record.
+        dates, values = await sequence.key_to_lists("data_value")
+        assert dates == [to_datetime(datetime(2023, 11, 5)), to_datetime(datetime(2023, 11, 7))]
+        assert values == [0.8, 0.9]
+
+        # With dropna=False the NaN record must be kept.
+        dates_keep, values_keep = await sequence.key_to_lists("data_value", dropna=False)
+        assert len(values_keep) == 3
+        assert pd.isna(values_keep[1])
+
     async def test_to_dataframe_full_data(self, sequence):
         """Test conversion of all records to a DataFrame without filtering."""
         record1 = self.create_test_record("2024-01-01T12:00:00Z", 10)