From 983a23f0af614fe4b71c35a878783cd20ffb57ad Mon Sep 17 00:00:00 2001 From: Andreas Date: Wed, 9 Sep 2026 12:57:39 +0200 Subject: [PATCH] fix(elecprice): survive a day-ahead source that has not published yet Every morning before the day-ahead auction is published, /v1/prediction/update answered 400 and no prediction was produced at all. The provider asks for prices starting at the run day, because an existing history sets past_days to 0. The optimization horizon always reaches past the last published price, so an update is always considered necessary - and SMARD publishes the next day around midday. Between midnight and publication the requested window therefore contains nothing, and ElecPriceSMARD raised "SMARD response contains no usable day-ahead prices", which failed the whole prediction update rather than only that provider. ElecPriceEnergyCharts and its SMARD subclass now keep their existing history and let the ETS/median branch extrapolate the remaining slots, the same fallback FeedInTariffEnergyCharts already had. A cold start without any history stays fatal. ElecPriceSMARD also separates the two cases it used to conflate: a period the source has not published yet now reports the latest value it does have, and only a response without a single price still reads as unusable. Fixes the cache noise this produced as well. cache_in_file claimed its cache entry before calling the wrapped function, so a raising function left an empty file behind and every later call within the TTL logged "Read failed: Ran out of input" before refetching. The entry is now created only after the call returns. --- CHANGELOG.md | 13 ++++++ src/akkudoktoreos/core/cache.py | 6 ++- .../prediction/elecpriceenergycharts.py | 44 ++++++++++++------ .../prediction/elecpricesmard.py | 20 ++++++-- tests/test_cache.py | 31 +++++++++++++ tests/test_elecpriceenergycharts.py | 46 +++++++++++++++++-- tests/test_elecpricesmard.py | 43 +++++++++++++++++ 7 files changed, 183 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c3abb0d..8f1804de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -231,6 +231,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). historical data exists, the existing history is kept and the remaining slots are extrapolated via ETS instead of failing. A genuine cold start (no data at all) still fails. +- A day-ahead price source that has not published the next day yet no longer fails the whole + prediction update. `ElecPriceEnergyCharts` and its `ElecPriceSMARD` subclass ask for prices + starting at the run day, while the optimization horizon always reaches past the last published + price, so every morning before the auction is published the request came back empty and the + provider raised - answering `/v1/prediction/update` with 400 until the source caught up. The + provider now keeps its existing history and extrapolates the remaining slots via ETS, the same + way `FeedInTariffEnergyCharts` already did. A cold start with no history at all still fails. +- `ElecPriceSMARD` now distinguishes a lagging publication from a broken response. A window the + source cannot serve yet reports the latest value it does have, instead of claiming the response + contained no usable prices. +- `cache_in_file` no longer leaves an empty cache entry behind when the wrapped function raises. + The entry was claimed before the call, so every later call within the TTL first failed to read + it ("Ran out of input") before refetching. The entry is now created only after the call returns. - The deprecated `/gesamtlast` endpoint no longer forces a full provider refresh on every call. Forcing bypassed the provider caches and hammered external APIs, so a single flaky provider could 404 the whole load prediction. It now defaults to a cache-aware update and diff --git a/src/akkudoktoreos/core/cache.py b/src/akkudoktoreos/core/cache.py index 0dc30789..8976c9a7 100644 --- a/src/akkudoktoreos/core/cache.py +++ b/src/akkudoktoreos/core/cache.py @@ -1016,6 +1016,11 @@ def cache_in_file( force_update = True if force_update or cache_file is None: # Otherwise, call the function and save its result to the cache + # Run first and only then claim a cache entry. Creating the entry + # up front left an empty file behind whenever the function raised, + # and every later call within the TTL then failed to read it + # ("Ran out of input") before refetching anyway. + result = func(*args, **kwargs) logger.debug("Created cache file for function: " + func.__name__) cache_file = CacheFileStore().create( key, @@ -1026,7 +1031,6 @@ def cache_in_file( until_date=until_date, with_ttl=with_ttl, ) - result = func(*args, **kwargs) try: # Assure we have an empty file cache_file.truncate(0) diff --git a/src/akkudoktoreos/prediction/elecpriceenergycharts.py b/src/akkudoktoreos/prediction/elecpriceenergycharts.py index c847cbd7..a5db55e7 100644 --- a/src/akkudoktoreos/prediction/elecpriceenergycharts.py +++ b/src/akkudoktoreos/prediction/elecpriceenergycharts.py @@ -166,9 +166,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider): highest_orig_datetime = orig_datetime # Convert EUR/MWh to EUR/Wh and add the configured retail price components. - price_wh = self._price_with_charges( - price_eur_per_mwh / 1_000_000, orig_datetime - ) + price_wh = self._price_with_charges(price_eur_per_mwh / 1_000_000, orig_datetime) # Store in series series_data.at[orig_datetime] = price_wh @@ -279,8 +277,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider): if needs_update: logger.info( - "Update {} is needed, last in history: {}, " - "force_update={}, history_refresh={}", + "Update {} is needed, last in history: {}, " "force_update={}, history_refresh={}", self.provider_id(), self.highest_orig_datetime, bool(force_update), @@ -290,15 +287,36 @@ class ElecPriceEnergyCharts(ElecPriceProvider): start_date = to_datetime( self.ems_start_datetime - to_duration(f"{past_days} days"), as_string="YYYY-MM-DD" ) - # Get Energy-Charts electricity price data - energy_charts_data = self._request_forecast( - start_date=start_date, force_update=force_update - ) # type: ignore + try: + # Get Energy-Charts electricity price data + energy_charts_data = self._request_forecast( + start_date=start_date, force_update=force_update + ) # type: ignore - # Parse and store data - series_data = self._parse_data(energy_charts_data) - self.highest_orig_datetime = series_data.index.max() - self.key_from_series("elecprice_marketprice_wh", series_data) + # Parse and store data + series_data = self._parse_data(energy_charts_data) + if series_data.empty: + raise ValueError("No electricity price data available") + self.highest_orig_datetime = series_data.index.max() + self.key_from_series("elecprice_marketprice_wh", series_data) + except Exception as exc: + if self.highest_orig_datetime is None: + # Cold start: there is no history to fall back to, so a failed + # fetch is fatal. + raise + # The horizon reaches past the last published price on every run, + # so an upstream that has not published the next day yet is the + # normal case, not an outage - and neither is a transient API + # failure a reason to fail the whole prediction update. Keep the + # history and let the ETS/median branch below extrapolate the + # remaining slots. + logger.warning( + "{} update failed ({}); keeping existing history until {} and " + "extrapolating the remaining slots.", + self.provider_id(), + exc, + self.highest_orig_datetime, + ) else: logger.info( "No update {} is needed, last in history: {}", diff --git a/src/akkudoktoreos/prediction/elecpricesmard.py b/src/akkudoktoreos/prediction/elecpricesmard.py index b7913be1..3c193d38 100644 --- a/src/akkudoktoreos/prediction/elecpricesmard.py +++ b/src/akkudoktoreos/prediction/elecpricesmard.py @@ -154,6 +154,9 @@ class ElecPriceSMARD(ElecPriceEnergyCharts): values_by_timestamp: dict[int, float] = {} latest_created = 0 + latest_published_ms: Optional[int] = None + start_ms = int(start_datetime.timestamp() * 1000) + end_ms = int(end_datetime.timestamp() * 1000) for chunk_timestamp in chunk_timestamps: chunk_url = ( f"{SMARD_BASE_URL}/{filter_id}/{region}/" @@ -164,12 +167,23 @@ class ElecPriceSMARD(ElecPriceEnergyCharts): for timestamp_ms, price_eur_mwh in chunk.series: if price_eur_mwh is None: continue - if int(start_datetime.timestamp() * 1000) <= timestamp_ms <= int( - end_datetime.timestamp() * 1000 - ): + if latest_published_ms is None or timestamp_ms > latest_published_ms: + latest_published_ms = timestamp_ms + if start_ms <= timestamp_ms <= end_ms: values_by_timestamp[timestamp_ms] = price_eur_mwh if not values_by_timestamp: + # SMARD answered correctly; it simply has not published the requested + # period yet. Say so, because the caller keeps its history and + # extrapolates in that case instead of treating it as a broken API. + if latest_published_ms is not None: + latest_published = to_datetime( + latest_published_ms / 1000, in_timezone=self.config.general.timezone + ) + raise ValueError( + f"SMARD has not published day-ahead prices for the requested period yet " + f"(from {start_datetime}); latest published value is {latest_published}" + ) raise ValueError("SMARD response contains no usable day-ahead prices") ordered_values = sorted(values_by_timestamp.items()) diff --git a/tests/test_cache.py b/tests/test_cache.py index 76835864..57e7b09f 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -560,6 +560,37 @@ class TestCacheFileDecorators: cache_file.seek(0) # Move to the start of the file assert cache_file.read() == "Some expensive computation result" + def test_cache_in_file_decorator_discards_the_entry_when_the_call_raises( + self, cache_file_store + ): + """A failing call must not leave an empty cache file behind. + + The entry is created before the wrapped function runs, so a raising + function used to leave an empty file that every later call within the TTL + failed to unpickle ("Ran out of input") before falling back to a refetch. + """ + cache_file_store.clear(clear_all=True) + assert len(cache_file_store._store) == 0 + + calls = [] + + @cache_in_file(mode="w+") + def failing_function(until_date=None): + calls.append(1) + raise ValueError("upstream has nothing to offer yet") + + until = datetime.now() + timedelta(days=1) + with pytest.raises(ValueError, match="upstream has nothing"): + failing_function(until_date=until) + + assert len(cache_file_store._store) == 0 + + # The next call runs the function again and reports the same failure + # rather than a confusing unpickling error from an empty file. + with pytest.raises(ValueError, match="upstream has nothing"): + failing_function(until_date=until) + assert len(calls) == 2 + def test_cache_in_file_decorator_uses_cache(self, cache_file_store): """Test that the cache_in_file decorator reuses cached file on subsequent calls.""" # Clear store to assure it is empty diff --git a/tests/test_elecpriceenergycharts.py b/tests/test_elecpriceenergycharts.py index fcfe1d29..a97e1de1 100644 --- a/tests/test_elecpriceenergycharts.py +++ b/tests/test_elecpriceenergycharts.py @@ -267,9 +267,8 @@ def test_market_price_charge_round_trip(provider): ) -@patch("requests.get") -def test_update_data_with_incomplete_forecast(mock_get, provider): - """Test `_update_data` with incomplete or missing forecast data.""" +def _mock_empty_forecast(mock_get) -> None: + """Let the API answer correctly but without any price rows.""" incomplete_data: dict = { "license_info": "", "unix_seconds": [], @@ -281,11 +280,52 @@ def test_update_data_with_incomplete_forecast(mock_get, provider): mock_response.status_code = 200 mock_response.content = json.dumps(incomplete_data) mock_get.return_value = mock_response + + +@patch("requests.get") +def test_update_data_with_incomplete_forecast_is_fatal_on_cold_start(mock_get, provider): + """Without any history there is nothing to fall back to.""" + _mock_empty_forecast(mock_get) + provider.highest_orig_datetime = None logger.info("The following errors are intentional and part of the test.") with pytest.raises(ValueError): provider._update_data(force_update=True) +@patch("requests.get") +def test_update_data_with_incomplete_forecast_keeps_existing_history( + mock_get, provider, sample_energycharts_json, cache_store +): + """An upstream without new prices must not fail the whole prediction update. + + The horizon always reaches past the last published price, so a day-ahead + source that has not published the next day yet is the normal case. The + provider keeps its history and extrapolates the remaining slots. + """ + # Establish a history first. + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_energycharts_json) + mock_get.return_value = mock_response + cache_store.clear(clear_all=True) + get_ems().set_start_datetime(to_datetime("2024-12-11 00:00:00", in_timezone="Europe/Berlin")) + provider.highest_orig_datetime = None + provider.update_data(force_enable=True, force_update=True) + + before = provider.highest_orig_datetime + assert before is not None + records_before = len(provider) + + # The next refresh finds nothing new upstream. + _mock_empty_forecast(mock_get) + cache_store.clear(clear_all=True) + logger.info("The following errors are intentional and part of the test.") + provider._update_data(force_update=True) + + assert provider.highest_orig_datetime == before + assert len(provider) == records_before + + @pytest.mark.parametrize( "status_code, exception", [(400, requests.exceptions.HTTPError), (500, requests.exceptions.HTTPError), (200, None)], diff --git a/tests/test_elecpricesmard.py b/tests/test_elecpricesmard.py index e4c57bbd..ca577524 100644 --- a/tests/test_elecpricesmard.py +++ b/tests/test_elecpricesmard.py @@ -75,3 +75,46 @@ def test_chunk_selection_includes_preceding_overlapping_chunk(provider): def test_smard_provider_is_enabled(provider): assert provider.enabled() + + +@patch("akkudoktoreos.prediction.elecpricesmard.requests.get") +def test_unpublished_period_names_the_latest_published_value(mock_get, provider): + """A day-ahead source that lags is not a broken response. + + SMARD publishes the next day around midday. Until then a run whose horizon + already reaches into that day asks for a window SMARD cannot serve yet. The + error has to say that, because the caller keeps its history and extrapolates + in that case rather than treating the API as broken. + """ + chunk_start = 1785103200000 + # The chunk holds prices, but all of them end before the requested window. + mock_get.side_effect = [ + _response({"timestamps": [chunk_start]}), + _response( + { + "meta_data": {"version": 1, "created": 1785500527370}, + "series": [[chunk_start, 86.04], [chunk_start + 900000, 84.5]], + } + ), + ] + + with pytest.raises(ValueError, match="has not published day-ahead prices"): + provider._request_forecast(start_date="2026-07-28", force_update=True) + + +@patch("akkudoktoreos.prediction.elecpricesmard.requests.get") +def test_empty_series_still_reports_an_unusable_response(mock_get, provider): + """A chunk without a single price is a different problem and says so.""" + chunk_start = 1785103200000 + mock_get.side_effect = [ + _response({"timestamps": [chunk_start]}), + _response( + { + "meta_data": {"version": 1, "created": 1785500527370}, + "series": [[chunk_start, None]], + } + ), + ] + + with pytest.raises(ValueError, match="no usable day-ahead prices"): + provider._request_forecast(start_date="2026-07-27", force_update=True)