From b7610e8274055834042d1f033b436a40935209c1 Mon Sep 17 00:00:00 2001 From: Normann Date: Sun, 11 Oct 2026 08:39:20 +0200 Subject: [PATCH] fix: detect recent Energy-Charts cadence for feed-in coverage (#1362) * fix: use recent original feed-in cadence for publication coverage * test: cover mixed Energy-Charts feed-in source cadence --- .../prediction/feedintariffenergycharts.py | 61 ++++++-- tests/test_feedintariffenergycharts.py | 146 ++++++++++++++++++ 2 files changed, 194 insertions(+), 13 deletions(-) diff --git a/src/akkudoktoreos/prediction/feedintariffenergycharts.py b/src/akkudoktoreos/prediction/feedintariffenergycharts.py index c4f5d258..c0343999 100644 --- a/src/akkudoktoreos/prediction/feedintariffenergycharts.py +++ b/src/akkudoktoreos/prediction/feedintariffenergycharts.py @@ -55,6 +55,31 @@ class FeedInTariffEnergyCharts(FeedInTariffProvider): """Return the unique identifier for the Energy-Charts feed-in tariff provider.""" return "FeedInTariffEnergyCharts" + def _coverage_resolution_seconds(self, source_series: pd.Series) -> int: + """Infer the recent source cadence without changing prediction resolution. + + The caller excludes predicted timestamps. Four matching spacings among the + latest five identify a short cadence transition, allowing one exceptional + gap. Otherwise use the median over the last 24 hours; sparse data uses + the shared resolution helper's hourly fallback. + """ + if source_series.empty: + return 3600 + + recent_series = source_series.sort_index() + recent_series = recent_series[ + recent_series.index >= recent_series.index[-1] - pd.Timedelta(hours=24) + ] + index = pd.DatetimeIndex(recent_series.index).drop_duplicates() + deltas = index.to_series().diff().dropna().dt.total_seconds().tail(5) + counts = deltas.value_counts() + if not counts.empty and counts.iloc[0] >= 4: + resolution = float(counts.index[0]) + if resolution > 0 and 3600 % resolution == 0: + return int(resolution) + + return self._resolution_seconds(recent_series) + def _has_complete_published_horizon( self, *, now: pd.Timestamp, resolution_seconds: int ) -> bool: @@ -186,12 +211,18 @@ class FeedInTariffEnergyCharts(FeedInTariffProvider): # Determine if update is needed and what start date is really necessary needs_update = False + raw_series: Optional[pd.Series] = None if self.highest_orig_datetime: - raw_history = await self.key_to_raw_series( + source_end = to_datetime(self.highest_orig_datetime).add(seconds=1) + raw_series = await self.key_to_raw_series( key="feed_in_tariff_raw_wh", - start_datetime=start_datetime, - end_datetime=gross_start_datetime, + end_datetime=max(gross_start_datetime, source_end), ) + raw_history = raw_series[ + (raw_series.index >= pd.Timestamp(start_datetime)) + & (raw_series.index < pd.Timestamp(gross_start_datetime)) + ] + raw_series = raw_series[raw_series.index <= pd.Timestamp(self.highest_orig_datetime)] if raw_history.empty: # We need the default start date (35 days in past) @@ -209,12 +240,14 @@ class FeedInTariffEnergyCharts(FeedInTariffProvider): elif force_update: # Use default start date in case of forced update needs_update = True - elif not self._has_complete_published_horizon( - now=now, resolution_seconds=resolution_seconds - ): - # We have enough history, but not every expected source interval. - start_datetime = gross_start_datetime - needs_update = True + else: + source_resolution_seconds = self._coverage_resolution_seconds(raw_series) + if not self._has_complete_published_horizon( + now=now, resolution_seconds=source_resolution_seconds + ): + # We have enough history, but not every expected source interval. + start_datetime = gross_start_datetime + needs_update = True else: needs_update = True @@ -244,6 +277,7 @@ class FeedInTariffEnergyCharts(FeedInTariffProvider): raise ValueError("No Energy-Charts feed-in tariff data available") self.highest_orig_datetime = to_datetime(series_data.index.max()) await self.key_from_series("feed_in_tariff_raw_wh", series_data) + raw_series = None # Reload the newly fetched source before forecasting. # Newly fetched data widens the window that needs its gross # (fee-inclusive) values recomputed. gross_start_datetime = to_datetime(series_data.index.min()) @@ -268,10 +302,11 @@ class FeedInTariffEnergyCharts(FeedInTariffProvider): logger.error(error_msg) raise ValueError(error_msg) - raw_series = await self.key_to_raw_series( - key="feed_in_tariff_raw_wh", - end_datetime=to_datetime(self.highest_orig_datetime).add(seconds=1), - ) + if raw_series is None: + raw_series = await self.key_to_raw_series( + key="feed_in_tariff_raw_wh", + end_datetime=to_datetime(self.highest_orig_datetime).add(seconds=1), + ) resolution_seconds = self._resolution_seconds(raw_series) slots_per_hour = 3600 // resolution_seconds diff --git a/tests/test_feedintariffenergycharts.py b/tests/test_feedintariffenergycharts.py index dbeb4888..3b966124 100644 --- a/tests/test_feedintariffenergycharts.py +++ b/tests/test_feedintariffenergycharts.py @@ -1,6 +1,7 @@ # ruff: noqa: S101 import json +from collections.abc import Callable from pathlib import Path from unittest.mock import Mock, patch @@ -107,6 +108,151 @@ class TestFeedInTariffEnergyCharts: is expected ) + @pytest.mark.parametrize( + ("old_minutes", "recent_minutes", "expected_seconds"), + [ + (60, [15, 15, 15, 15], 900), + (15, [60, 60, 60, 60], 3600), + (60, [15, 15, 60, 15, 15], 900), + (15, [60, 60, 120, 60, 60], 3600), + (60, [15, 15, 15], 3600), + (60, [15, 60, 15, 60, 15], 3600), + ], + ) + def test_coverage_resolution_handles_transitions_and_gaps( + self, provider, old_minutes, recent_minutes, expected_seconds + ): + """A short consistent run overrides old history; ambiguous runs use its median.""" + transition = pd.Timestamp("2026-01-15 18:00", tz="Europe/Berlin") + old_index = pd.date_range( + start=transition - pd.Timedelta(days=1), end=transition, freq=f"{old_minutes}min" + ) + recent_index = transition + pd.to_timedelta(np.cumsum(recent_minutes), unit="min") + source = pd.Series(0.0001, index=old_index.append(recent_index)) + assert provider._coverage_resolution_seconds(source) == expected_seconds + + @pytest.mark.asyncio + @pytest.mark.parametrize("host_timezone", ["UTC", "Europe/Berlin"]) + @pytest.mark.parametrize("history_interval_minutes", [15, 60]) + @pytest.mark.parametrize("recent_source_points", [None, 5]) + @pytest.mark.parametrize( + ("now", "last_price", "interval_minutes", "needs_update"), + [ + ("2026-01-15 13:59:59", "2026-01-15 23:00", 15, True), + ("2026-01-15 13:59:59", "2026-01-15 23:45", 15, False), + ("2026-01-15 13:59:59", "2026-01-15 23:00", 60, False), + ("2026-01-15 14:00:00", "2026-01-16 23:00", 15, True), + ("2026-01-15 14:00:00", "2026-01-16 23:45", 15, False), + ("2026-01-15 14:00:00", "2026-01-16 23:00", 60, False), + ("2026-03-28 14:00:00", "2026-03-29 23:45", 15, False), + ("2026-03-28 14:00:00", "2026-03-29 23:30", 15, True), + ("2026-10-24 14:00:00", "2026-10-25 23:45", 15, False), + ("2026-10-24 14:00:00", "2026-10-25 23:30", 15, True), + ], + ) + async def test_update_data_uses_source_cadence_for_coverage( + self, + provider, + set_other_timezone: Callable[[str], str], + host_timezone, + history_interval_minutes, + recent_source_points, + now, + last_price, + interval_minutes, + needs_update, + ): + """Refresh incomplete original prices, independent of host zone or predicted tail.""" + set_other_timezone(host_timezone) + fixed_now = pd.Timestamp(now, tz="Europe/Berlin") + start = to_datetime(fixed_now, in_timezone="Europe/Berlin").start_of("day") + last_original = to_datetime( + pd.Timestamp(last_price, tz="Europe/Berlin"), in_timezone="Europe/Berlin" + ) + get_ems().set_start_datetime(start) + source_start = ( + start + if recent_source_points is None + else last_original.subtract(minutes=(recent_source_points - 1) * interval_minutes) + ) + source_index = pd.date_range( + start=source_start, end=last_original, freq=f"{interval_minutes}min" + ) + history_index = pd.date_range( + start=start.subtract(days=35), + end=source_index[0], + freq=f"{history_interval_minutes}min", + inclusive="left", + ) + await provider.key_from_series( + "feed_in_tariff_raw_wh", pd.Series(0.0001, index=history_index.append(source_index)) + ) + provider.highest_orig_datetime = last_original + + # The predicted tail uses the same key and must not influence source coverage. + predicted_minutes = 60 if interval_minutes == 15 else 15 + predicted_index = pd.date_range( + start=last_original.add(minutes=predicted_minutes), + periods=120, + freq=f"{predicted_minutes}min", + ) + await provider.key_from_series( + "feed_in_tariff_raw_wh", pd.Series(0.00005, index=predicted_index) + ) + + published_end = start.add(days=1 if fixed_now.hour < 14 else 2) + response_index = pd.date_range( + start=start, end=published_end, freq=f"{interval_minutes}min", inclusive="left" + ) + response = EnergyChartsElecPrice( + license_info="", + unix_seconds=[int(timestamp.timestamp()) for timestamp in response_index], + price=[200.0] * len(response_index), + unit="EUR/MWh", + deprecated=False, + ) + + def predict(history: np.ndarray, hours: int, slots_per_hour: int = 1) -> np.ndarray: + return np.full(hours, 0.00005) + + original_read = FeedInTariffEnergyCharts.key_to_raw_series + source_reads = [] + + async def read_source(self, *args, **kwargs): + if "start_datetime" not in kwargs: + source_reads.append(kwargs) + return await original_read(self, *args, **kwargs) + + with ( + patch("akkudoktoreos.prediction.feedintariffenergycharts.pd", wraps=pd) as pandas, + patch.object(provider, "_request_forecast", return_value=response) as request, + patch.object(provider, "_predict", side_effect=predict), + patch.object(FeedInTariffEnergyCharts, "key_to_raw_series", read_source), + ): + pandas.Timestamp.now.return_value = fixed_now + await provider._update_data(force_update=False) + + assert len(source_reads) == (2 if needs_update else 1) + if needs_update: + request.assert_called_once_with( + start_date=start.in_timezone(host_timezone).format("YYYY-MM-DD"), + force_update=False, + ) + assert pd.Timestamp(provider.highest_orig_datetime) == response_index[-1] + fetched = await provider.key_to_raw_series( + key="feed_in_tariff_raw_wh", + start_datetime=last_original, + end_datetime=published_end, + ) + expected_index = response_index[response_index >= pd.Timestamp(last_original)].tz_convert( + "UTC" + ) + assert fetched.index.equals(expected_index) + np.testing.assert_allclose(fetched.to_numpy(), 0.0002) + else: + request.assert_not_called() + assert provider.highest_orig_datetime == last_original + @patch("requests.get") def test_request_forecast_uses_feedintariff_bidding_zone(