fix(pvforecast): survive a transient Open-Meteo outage

A single 503 from Open-Meteo answered /v1/prediction/update with 400 and left
every provider after PVForecastAkkudoktorLocal unrun: the provider raised on the
first failed request, and PredictionContainer.update_data re-raises whatever an
enabled provider raises. Open-Meteo returns 503 while it rotates its model runs
and 429 when the free tier is briefly saturated; both clear within seconds.

Retryable responses (429, 500, 502, 503, 504) and connection errors are now
retried three times with a growing pause, matching what the SMARD provider
already does. If the fetch still fails and the stored forecast reaches past the
run start, that forecast is kept for one more run rather than failing the update
- a forecast one run old beats no forecast at all. A cold start with nothing
stored still fails, because then there really is no PV forecast.

CHANGELOG.md

@@ -241,6 +241,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - `ElecPriceSMARD` now distinguishes a lagging publication from a broken response. A window the
   source cannot serve yet reports the latest value it does have, instead of claiming the response
   contained no usable prices.
+- A weather-API outage no longer takes the whole prediction update with it.
+  `PVForecastAkkudoktorLocal` raised on the first failed Open-Meteo request, and
+  `PredictionContainer.update_data` re-raises whatever an enabled provider raises, so every
+  provider after it was skipped and `/v1/prediction/update` answered 400 - over a 503 that
+  Open-Meteo clears within seconds while rotating its model runs. Retryable responses (429 and
+  5xx) and connection errors are now retried three times with a growing pause, and if the fetch
+  still fails while a stored forecast reaches past the run start, that forecast is kept for one
+  more run instead of failing the update. A cold start with no stored forecast still fails.
 - `cache_in_file` no longer leaves an empty cache entry behind when the wrapped function raises.
   The entry was claimed before the call, so every later call within the TTL first failed to read
   it ("Ran out of input") before refetching. The entry is now created only after the call returns.

src/akkudoktoreos/prediction/pvforecastakkudoktorlocal.py

@@ -30,6 +30,7 @@ Note also that ``direct_radiation`` in the Open-Meteo API is beam irradiance on
 """
 
 import math
+import time
 from typing import Any, Optional
 
 import numpy as np
@@ -379,13 +380,26 @@ class PVForecastAkkudoktorLocal(PVForecastProvider):
             "models": ",".join(settings.weather_models),
         }
 
+        response = None
+        for attempt in range(1, 4):
             try:
-            response = requests.get(OPENMETEO_URL, params=params, timeout=30)
+                response = requests.get(OPENMETEO_URL, params=params, timeout=(5, 30))
                 logger.debug(f"Requesting Open-Meteo forecast: {response.url}")
                 response.raise_for_status()
+                break
             except requests.RequestException as e:
+                response = None
+                status = getattr(e.response, "status_code", None)
+                # Open-Meteo answers 503 while it rotates its model runs and 429
+                # when the free tier is briefly saturated. Both clear in seconds.
+                retryable = status is None or status in (429, 500, 502, 503, 504)
+                if not retryable or attempt == 3:
                     logger.error(f"Failed to fetch weather for local pvforecast: {e}")
                     raise RuntimeError("Failed to fetch weather from Open-Meteo API") from e
+                logger.warning(
+                    "Open-Meteo request attempt {}/3 failed for local pvforecast: {}", attempt, e
+                )
+                time.sleep(2 * attempt)
 
         data = response.json()
         if block not in data:
@@ -1099,6 +1113,13 @@ class PVForecastAkkudoktorLocal(PVForecastProvider):
 
     # ------------------------------------------------------------------ update
 
+    def _holds_usable_forecast(self) -> bool:
+        """Whether the stored forecast still reaches into the optimization horizon."""
+        latest = self.max_datetime
+        if latest is None:
+            return False
+        return compare_datetimes(latest, self.ems_start_datetime).gt
+
     def _update_data(self, force_update: Optional[bool] = False) -> None:
         """Compute the PV forecast and store it as PVForecastDataRecord entries."""
         if not self.enabled():
@@ -1110,7 +1131,24 @@ class PVForecastAkkudoktorLocal(PVForecastProvider):
             logger.error(f"Configuration error: {error_msg}")
             raise ValueError(error_msg)
 
+        try:
             data = self._request_forecast(force_update=force_update)  # type: ignore[call-arg]
+        except Exception as exc:
+            if not self._holds_usable_forecast():
+                # Nothing stored that still covers the horizon - the caller has
+                # to know there is no PV forecast at all.
+                raise
+            # A momentary weather-API outage must not fail the whole prediction
+            # update and take every provider after this one down with it. The
+            # forecast from the previous run still covers the horizon; it ages
+            # by one run, which beats having none.
+            logger.warning(
+                "PVForecastAkkudoktorLocal update failed ({}); keeping the forecast from the "
+                "previous run until {}.",
+                exc,
+                self.max_datetime,
+            )
+            return
         frame = self._forecast_frame(data)
         if frame.empty:
             logger.warning("Open-Meteo returned no weather rows for local pvforecast.")

tests/test_pvforecastakkudoktorlocal.py

@@ -1,14 +1,15 @@
 """Tests for the native (pvlib) PV forecast provider."""
 
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import numpy as np
 import pandas as pd
 import pendulum
 import pvlib
 import pytest
+import requests
 
-from akkudoktoreos.core.coreabc import get_measurement
+from akkudoktoreos.core.coreabc import get_ems, get_measurement
 from akkudoktoreos.prediction.pvforecastakkudoktorlocal import (
     PVForecastAkkudoktorLocal,
     PVForecastAkkudoktorLocalCommonSettings,
@@ -491,3 +492,78 @@ def test_forecast_frame_applies_the_calibration(pvforecast_instance):
     calibrated = pvforecast_instance._forecast_frame(synthetic_openmeteo())
     ratio = calibrated["ac_power"].sum() / raw["ac_power"].sum()
     assert ratio == pytest.approx(0.8, abs=0.03)
+
+
+def test_transient_weather_outage_is_retried(pvforecast_instance):
+    """Open-Meteo answers 503 while rotating model runs; that clears in seconds."""
+    error = requests.exceptions.HTTPError("503 Server Error")
+    error.response = Mock(status_code=503)
+    good = Mock()
+    good.url = "https://api.open-meteo.com/v1/forecast"
+    good.raise_for_status.return_value = None
+    good.json.return_value = synthetic_openmeteo()
+
+    failing = Mock()
+    failing.url = good.url
+    failing.raise_for_status.side_effect = error
+
+    with (
+        patch(
+            "akkudoktoreos.prediction.pvforecastakkudoktorlocal.requests.get",
+            side_effect=[failing, good],
+        ) as request,
+        patch("akkudoktoreos.prediction.pvforecastakkudoktorlocal.time.sleep"),
+    ):
+        data = pvforecast_instance._request_forecast(force_update=True)
+
+    assert request.call_count == 2
+    assert "minutely_15" in data
+
+
+def test_weather_outage_keeps_the_previous_forecast(pvforecast_instance):
+    """One dead weather API must not take the whole prediction update down.
+
+    `PredictionContainer.update_data` re-raises whatever an enabled provider
+    raises, so every provider after this one would be skipped and the endpoint
+    would answer 400. A forecast that is one run old still covers the horizon.
+    """
+    # The stored forecast has to reach past the run start for the fallback to be
+    # worth anything, so put the run inside the synthetic window.
+    get_ems().set_start_datetime(WINDOW_START.add(days=1))
+    with patch.object(
+        pvforecast_instance,
+        "_request_forecast",
+        return_value=synthetic_openmeteo(),
+    ):
+        pvforecast_instance._update_data(force_update=True)
+    stored = pvforecast_instance.max_datetime
+    assert stored is not None
+    records_before = len(pvforecast_instance)
+
+    with patch.object(
+        pvforecast_instance,
+        "_request_forecast",
+        side_effect=RuntimeError("Failed to fetch weather from Open-Meteo API"),
+    ):
+        pvforecast_instance._update_data(force_update=True)
+
+    assert pvforecast_instance.max_datetime == stored
+    assert len(pvforecast_instance) == records_before
+
+
+def test_weather_outage_without_any_forecast_still_fails(pvforecast_instance):
+    """A cold start has nothing to fall back to, so the caller must hear about it.
+
+    The stored forecast lives in the backing store, not in `records`, so an empty
+    store is what a cold start actually looks like.
+    """
+    pvforecast_instance.records = []
+    with patch.object(
+        type(pvforecast_instance), "db_timestamp_range", return_value=(None, None)
+    ), patch.object(
+        pvforecast_instance,
+        "_request_forecast",
+        side_effect=RuntimeError("Failed to fetch weather from Open-Meteo API"),
+    ):
+        with pytest.raises(RuntimeError, match="Failed to fetch weather"):
+            pvforecast_instance._update_data(force_update=True)