diff --git a/docs/akkudoktoreos/prediction.md b/docs/akkudoktoreos/prediction.md index ed94720f..693388f5 100644 --- a/docs/akkudoktoreos/prediction.md +++ b/docs/akkudoktoreos/prediction.md @@ -1014,7 +1014,17 @@ The prediction keys for the PV forecast data are: The `PVForecastForecastSolar` provider retrieves PV power forecasts from the free [Forecast.Solar](https://forecast.solar) API. No API key is required for the public endpoint; an optional key raises the rate limit. The location is taken from `general.latitude`/`longitude` and -the system geometry from the configured `planes` (one request per plane, summed per timestamp). +the system geometry from the configured `planes` (one request per plane). + +The provider integrates Forecast.Solar's instantaneous power samples into 15-minute +interval-average W before summing the planes. Sunrise and sunset retain their actual +timestamps during integration; explicit zero boundaries establish nighttime zeros within +each forecast day. Missing daylight samples, invalid values, and unavailable days remain +missing. GENETIC converts the averages to Wh for its configured 15-minute or hourly slots. +Refreshing the provider replaces older samples and removes unavailable forecast tails. +On the first refresh after upgrading, legacy forecasts with non-aligned timestamps are +cleared, including history, so they cannot distort cadence detection. Later refreshes +retain the normalized history. ```python { diff --git a/src/akkudoktoreos/optimization/genetic/configrequest.py b/src/akkudoktoreos/optimization/genetic/configrequest.py index 93175cf3..d0fba862 100644 --- a/src/akkudoktoreos/optimization/genetic/configrequest.py +++ b/src/akkudoktoreos/optimization/genetic/configrequest.py @@ -224,8 +224,15 @@ class ConfigOptimizationRequest( ("prices", prices), ("feed-in tariff", tariffs), ): - if not all(math.isfinite(value) for value in values[first:last]): - raise ValueError(f"Missing or invalid {key} within the control horizon.") + missing = [index for index in range(first, last) if not math.isfinite(values[index])] + if missing: + left = origin.add(seconds=missing[0] * settings.interval_sec) + right = left.add(seconds=settings.interval_sec) + raise ValueError( + f"Missing or invalid {key} within the control horizon. " + f"First invalid interval: [{left.isoformat()}, {right.isoformat()}); " + f"{len(missing)} invalid interval(s)." + ) if any(value < 0 for values in (pv, load) for value in values[first:last]): raise ValueError("PV and load energy must be nonnegative.") previous = ems.genetic_solution() diff --git a/src/akkudoktoreos/prediction/pvforecastforecastsolar.py b/src/akkudoktoreos/prediction/pvforecastforecastsolar.py index de22b6f5..3c558425 100644 --- a/src/akkudoktoreos/prediction/pvforecastforecastsolar.py +++ b/src/akkudoktoreos/prediction/pvforecastforecastsolar.py @@ -6,10 +6,10 @@ request covers a single plane: GET https://api.forecast.solar[/{api_key}]/estimate/{lat}/{lon}/{dec}/{az}/{kwp} -``result.watts`` is the instantaneous AC power per timestamp — exactly what the -optimizer consumes as ``pvforecast_ac_power``. EOS plants with several roof -planes issue one request per plane and the instantaneous powers are summed per -timestamp. +``result.watts`` contains instantaneous AC power, including non-aligned sunrise +and sunset samples. Integrate each plane's piecewise-linear power curve into +15-minute interval averages before summing planes. This is the interval-start +power representation consumed by EOS; conversion to slot Wh happens downstream. Note on conventions: - Forecast.Solar azimuth is -180=N, -90=E, 0=S, 90=W, whereas EOS @@ -19,6 +19,7 @@ Note on conventions: to resolve them to absolute instants before EOS resamples them. """ +import math import re from typing import Any, Optional @@ -30,11 +31,12 @@ from pydantic import Field from akkudoktoreos.config.configabc import SettingsBaseModel from akkudoktoreos.core.cache import cache_in_file from akkudoktoreos.prediction.pvforecastabc import PVForecastProvider -from akkudoktoreos.utils.datetimeutil import to_datetime +from akkudoktoreos.utils.datetimeutil import DateTime, to_datetime FORECAST_SOLAR_BASE = "https://api.forecast.solar" _TZ_SUFFIX = re.compile(r"([zZ]|[+-]\d\d:?\d\d)$") +_INTERVAL_SECONDS = 900 class PVForecastForecastSolarCommonSettings(SettingsBaseModel): @@ -65,7 +67,7 @@ class PVForecastForecastSolar(PVForecastProvider): settings = self.config.pvforecast.forecastsolar return settings.api_key if settings is not None else None - def _to_utc_datetime(self, local_ts: Any, iana_tz: Optional[str]) -> Any: + def _to_utc_datetime(self, local_ts: Any, iana_tz: Optional[str]) -> DateTime: """Resolve a Forecast.Solar wall-clock timestamp to a timezone-aware datetime.""" s = str(local_ts).strip() if _TZ_SUFFIX.search(s): @@ -76,6 +78,63 @@ class PVForecastForecastSolar(PVForecastProvider): raise ValueError(f"Expected a datetime, got {local_ts!r}") return to_datetime(dt.isoformat()) + def _interval_power(self, watts: dict[str, Any], timezone: Optional[str]) -> dict[int, float]: + """Integrate instantaneous samples without bridging missing daylight data. + + Forecast.Solar supplies at most hourly daylight samples. A larger gap, + invalid power, or an unobserved day remains missing. Only an explicit + zero at the first/last sample of a day establishes the adjacent night, + up to that local day's midnight. Never extend into an unforecast day. + UTC seconds preserve energy and both occurrences of a DST fall-back hour. + """ + tz = timezone or str(self.config.general.timezone) + days: dict[DateTime, dict[int, float]] = {} + for ts, power in watts.items(): + try: + date = self._to_utc_datetime(ts, timezone).in_timezone(tz) + except (ValueError, TypeError) as exc: + logger.warning(f"Forecast.Solar: skipping unparseable timestamp {ts!r}: {exc}") + continue + try: + value = float(power) + except (ValueError, TypeError): + value = math.nan + if not math.isfinite(value) or value < 0: + value = math.nan + days.setdefault(date.start_of("day"), {})[int(date.timestamp())] = value + + averages: dict[int, float] = {} + for day, samples in sorted(days.items()): + start = int(day.timestamp()) + end = int(day.add(days=1).timestamp()) + points = sorted(samples.items()) + segments: list[tuple[int, int, float, float]] = [] + if len(points) >= 2: + if points[0][1] == 0: + segments.append((start, points[0][0], 0.0, 0.0)) + for (left, p_left), (right, p_right) in zip(points, points[1:]): + if right - left <= 3600 and math.isfinite(p_left) and math.isfinite(p_right): + segments.append((left, right, p_left, p_right)) + if points[-1][1] == 0: + segments.append((points[-1][0], end, 0.0, 0.0)) + + for slot in range(start, end, _INTERVAL_SECONDS): + energy = 0.0 + covered = 0 + for left, right, p_left, p_right in segments: + a = max(slot, left) + b = min(slot + _INTERVAL_SECONDS, right) + if b <= a: + continue + slope = (p_right - p_left) / (right - left) + # Exact trapezoidal integral of this linear segment, in W s. + energy += (p_left + slope * ((a + b) / 2 - left)) * (b - a) + covered += b - a + averages[slot] = ( + energy / _INTERVAL_SECONDS if covered == _INTERVAL_SECONDS else math.nan + ) + return averages + def _plane_url(self, plane: Any) -> str: """Build the single-plane estimate URL for the given plane configuration.""" latitude = self.config.general.latitude @@ -105,8 +164,7 @@ class PVForecastForecastSolar(PVForecastProvider): if not planes: raise ValueError("PVForecastForecastSolar needs at least one pvforecast.planes entry") - summed: dict[str, float] = {} - timezone: Optional[str] = None + plane_power: list[dict[int, float]] = [] for plane in planes: url = self._plane_url(plane) logger.debug(f"Requesting Forecast.Solar estimate: {url}") @@ -117,17 +175,22 @@ class PVForecastForecastSolar(PVForecastProvider): logger.error(f"Failed to fetch pvforecast from Forecast.Solar: {e}") raise RuntimeError("Failed to fetch pvforecast from Forecast.Solar API") from e data = response.json() - if timezone is None: - timezone = (data.get("message", {}).get("info", {}) or {}).get("timezone") + timezone = (data.get("message", {}).get("info", {}) or {}).get("timezone") watts = (data.get("result", {}) or {}).get("watts", {}) or {} - for ts, power in watts.items(): - try: - summed[ts] = summed.get(ts, 0.0) + float(power) - except (TypeError, ValueError): - continue + plane_power.append(self._interval_power(watts, timezone)) + + # Align before adding: planes can have different sunrise/sunset timestamps. + # Missing coverage on any plane must not become a valid partial plant forecast. + stamps = sorted({stamp for power in plane_power for stamp in power}) + summed = { + to_datetime(stamp, in_timezone="UTC").isoformat(): sum( + power.get(stamp, math.nan) for power in plane_power + ) + for stamp in stamps + } self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) - return {"timezone": timezone, "watts": summed} + return {"timezone": "UTC", "watts": summed} async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update forecast data in the PVForecastDataRecord format.""" @@ -139,6 +202,16 @@ class PVForecastForecastSolar(PVForecastProvider): timezone = body.get("timezone") watts = body.get("watts", {}) + if watts: + # Replace the refreshed forecast, including old irregular timestamps + # and stale tails. Cadence inference also sees historical rows, so + # invalidate legacy non-aligned forecasts once on upgrade. Subsequent + # refreshes preserve the normalized forecast history. + stored = await self.key_to_raw_series("pvforecast_ac_power", dropna=False) + legacy = any(stamp.timestamp() % _INTERVAL_SECONDS for stamp in stored.index) + start = min(self._to_utc_datetime(ts, timezone) for ts in watts) + await self.delete_by_datetime(start_datetime=None if legacy else start) + count = 0 for ts, power_w in sorted(watts.items()): try: @@ -146,7 +219,7 @@ class PVForecastForecastSolar(PVForecastProvider): except Exception as e: # noqa: BLE001 - skip unparseable rows logger.warning(f"Forecast.Solar: skipping unparseable timestamp {ts!r}: {e}") continue - value = round(float(power_w), 1) + value = float(power_w) await self.update_value( date, {"pvforecast_ac_power": value, "pvforecast_dc_power": value}, diff --git a/tests/test_geneticconfigoptimizerequest.py b/tests/test_geneticconfigoptimizerequest.py index befd8bc7..a37385f1 100644 --- a/tests/test_geneticconfigoptimizerequest.py +++ b/tests/test_geneticconfigoptimizerequest.py @@ -1,7 +1,7 @@ """Configuration-owned requests and the real automatic GENETIC path.""" from types import SimpleNamespace -from unittest.mock import AsyncMock, Mock +from unittest.mock import AsyncMock, Mock, patch import httpx import numpy as np @@ -21,6 +21,7 @@ from akkudoktoreos.optimization.optimization import ( OptimizationAlgorithm, OptimizationCommonSettings, ) +from akkudoktoreos.prediction.pvforecastforecastsolar import PVForecastForecastSolar from akkudoktoreos.utils.datetimeutil import to_datetime @@ -90,6 +91,80 @@ def configured_request(config_eos, monkeypatch): return config_eos, ems, measurement, data +@pytest.mark.asyncio +@pytest.mark.parametrize("interval", [900, 3600]) +async def test_forecastsolar_resolves_without_runtime_override( + configured_request, monkeypatch, interval +): + config, ems, _, data = configured_request + config.merge_settings_from_dict( + { + "general": {"latitude": 52.5, "longitude": 13.4, "timezone": "Europe/Berlin"}, + "prediction": {"hours": 48}, + "optimization": {"genetic": {"interval_sec": interval, "horizon_hours": 24}}, + "pvforecast": { + "provider": "PVForecastForecastSolar", + "planes": [ + {"surface_tilt": 45, "surface_azimuth": 217, "peakpower": 8.05}, + ], + }, + } + ) + ems.start_datetime = to_datetime("2026-11-02T10:00:00+01:00", in_timezone="Europe/Berlin") + ems.observation_datetime = ems.start_datetime + data["forecasts"] = {} + pv = PVForecastForecastSolar(config=config.load) + watts = { + f"2026-11-{day} {time}": power + for day in ("02", "03") + for time, power in { + "07:16:18": 0, + "08:00:00": 1000, + "09:00:00": 1000, + "10:00:00": 1000, + "11:00:00": 1000, + "12:00:00": 1000, + "13:00:00": 1000, + "14:00:00": 1000, + "15:00:00": 1000, + "16:00:00": 1000, + "16:55:54": 0, + }.items() + } + + async def read(key, **kwargs): + if key == "pvforecast_ac_power": + return await pv.key_to_raw_series(key, **kwargs) + value = { + "loadforecast_power_w": 2000, + "elecprice_marketprice_wh": 0.0003, + "feed_in_tariff_wh": 0.00008, + }[key] + return pd.Series( + [value] * 72, index=pd.date_range("2026-11-02T00:00:00+01:00", periods=72, freq="h") + ) + + async def update(): + # Isolate this synthetic response from the provider's on-disk cache. + await pv._update_data(force_update=True) + + prediction = SimpleNamespace(update_data=AsyncMock(side_effect=update), key_to_raw_series=read) + monkeypatch.setattr(ConfigOptimizationRequest, "prediction", prediction) + response = Mock() + response.json.return_value = { + "result": {"watts": watts}, + "message": {"info": {"timezone": "Europe/Berlin"}}, + } + with patch("requests.get", return_value=response): + parameters = await ConfigOptimizationRequest.model_validate(data).resolve() + prediction.update_data.assert_awaited_once() + energy = parameters.ems.pv_forecast_wh + assert energy[10 * 3600 // interval] == pytest.approx(1000 * interval / 3600) + assert energy[17 * 3600 // interval : 24 * 3600 // interval] == [0] * (7 * 3600 // interval) + assert np.isfinite(energy[10 * 3600 // interval : 34 * 3600 // interval]).all() + await pv.delete_by_datetime() + + @pytest.mark.asyncio async def test_hardware_costs_and_state_are_resolved_without_config_changes(configured_request): config, _, _, data = configured_request @@ -208,8 +283,10 @@ async def test_control_gaps_fail_but_shorter_tail_is_accepted(configured_request parameters = await ConfigOptimizationRequest.model_validate(data).resolve() assert len(parameters.ems.total_load) == 50 data["forecasts"]["electricity_price_per_wh"][42] = np.nan - with pytest.raises(ValueError, match="control horizon"): + with pytest.raises(ValueError, match="control horizon") as error: await ConfigOptimizationRequest.model_validate(data).resolve() + assert "2026-09-12T10:30:00+00:00, 2026-09-12T10:45:00+00:00" in str(error.value) + assert "1 invalid interval(s)" in str(error.value) @pytest.mark.asyncio diff --git a/tests/test_pvforecastforecastsolar.py b/tests/test_pvforecastforecastsolar.py index 643b39d2..ae228ac8 100644 --- a/tests/test_pvforecastforecastsolar.py +++ b/tests/test_pvforecastforecastsolar.py @@ -1,10 +1,14 @@ +import math from unittest.mock import call, patch +import numpy as np import pendulum import pytest import requests +from akkudoktoreos.optimization.genetic.forecast import bounded_forecast_array from akkudoktoreos.prediction.pvforecastforecastsolar import PVForecastForecastSolar +from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration def _config(config_eos, planes=None, api_key=None): @@ -55,10 +59,14 @@ async def test_update_data_resolves_tz_and_sets_power(pvforecast_instance): "watts": {"2025-01-01 12:00:00": 1200.0, "2025-01-01 13:00:00": 1500.0}, } with patch.object(pvforecast_instance, "_request_forecast", return_value=body), \ - patch.object(PVForecastForecastSolar, "update_value") as mock_update: + patch.object(PVForecastForecastSolar, "update_value") as mock_update, \ + patch.object(PVForecastForecastSolar, "delete_by_datetime") as mock_delete: await pvforecast_instance._update_data() assert mock_update.call_count == 2 + mock_delete.assert_awaited_once_with( + start_datetime=to_datetime("2025-01-01T12:00:00+01:00") + ) expected = [ call( pendulum.datetime(2025, 1, 1, 12, 0, tz="Europe/Berlin"), @@ -105,13 +113,13 @@ def test_request_forecast_sums_planes(config_eos): config=config_eos.load, start_datetime=pendulum.datetime(2025, 1, 1, tz="UTC") ) responses = [ - _http({"2025-01-01 12:00:00": 1000.0}), - _http({"2025-01-01 12:00:00": 800.0}), + _http({"2025-01-01 12:00:00": 1000.0, "2025-01-01 13:00:00": 1000.0}), + _http({"2025-01-01 12:00:00": 800.0, "2025-01-01 13:00:00": 800.0}), ] with patch("requests.get", side_effect=responses) as mock_get: body = pv._request_forecast(force_update=True) # type: ignore assert mock_get.call_count == 2 - assert body["watts"]["2025-01-01 12:00:00"] == 1800.0 + assert body["watts"]["2025-01-01T11:00:00+00:00"] == 1800.0 @pytest.mark.asyncio @@ -129,3 +137,161 @@ def test_request_forecast_raises_on_http_error(pvforecast_instance): with pytest.raises(RuntimeError) as exc_info: pvforecast_instance._request_forecast(force_update=True) assert "Failed to fetch pvforecast from Forecast.Solar" in str(exc_info.value) + + +def _day_watts(day="2026-11-02"): + """The irregular sunrise/sunset samples reported in #1485.""" + return { + f"{day} {time}": power + for time, power in { + "07:16:18": 0.0, + "08:00:00": 84.0, + "09:00:00": 292.0, + "10:00:00": 709.0, + "11:00:00": 1225.0, + "12:00:00": 1296.0, + "13:00:00": 1289.0, + "14:00:00": 1894.0, + "15:00:00": 2509.0, + "16:00:00": 1777.0, + "16:55:54": 0.0, + }.items() + } + + +def test_instantaneous_power_is_integrated_and_nights_are_zero(pvforecast_instance): + averages = pvforecast_instance._interval_power(_day_watts(), "Europe/Berlin") + assert len(averages) == 96 + assert all(math.isfinite(value) for value in averages.values()) + at = lambda time: int(to_datetime(f"2026-11-02T{time}+01:00").timestamp()) + # A linear ramp 84 -> 292 W over an hour has 110 W average in its first quarter. + assert averages[at("08:00:00")] == pytest.approx(110.0) + # Integrating into Wh must conserve the entire daylight curve's energy. + points = [ + (int(to_datetime(ts, in_timezone="Europe/Berlin").timestamp()), p) + for ts, p in _day_watts().items() + ] + expected_wh = sum((b - a) * (pa + pb) / 7200 for (a, pa), (b, pb) in zip(points, points[1:])) + assert sum(averages.values()) / 4 == pytest.approx(expected_wh) + assert all( + value == 0 + for stamp, value in averages.items() + if stamp < at("07:00:00") or stamp >= at("17:00:00") + ) + + +@pytest.mark.parametrize("change", ["missing", "invalid", "negative", "infinite"]) +def test_bad_daylight_coverage_remains_missing(pvforecast_instance, change): + watts = _day_watts() + key = "2026-11-02 12:00:00" + if change == "missing": + del watts[key] + else: + watts[key] = {"invalid": "bad", "negative": -1, "infinite": math.inf}[change] + averages = pvforecast_instance._interval_power(watts, "Europe/Berlin") + at = int(to_datetime("2026-11-02T11:00:00+01:00").timestamp()) + assert all(math.isnan(averages[at + i * 900]) for i in range(8)) + assert averages[at + 8 * 900] == pytest.approx((1289 * 7 + 1894) / 8) + + +def test_partial_days_and_missing_days_are_not_invented(pvforecast_instance): + watts = {"2026-11-02 12:00:00": 1000, "2026-11-02 13:00:00": 1000, "2026-11-04 12:00:00": 1000} + averages = pvforecast_instance._interval_power(watts, "Europe/Berlin") + assert len(averages) == 192 + assert sum(math.isfinite(value) for value in averages.values()) == 4 + assert max(averages) < to_datetime("2026-11-05T00:00:00+01:00").timestamp() + + +@pytest.mark.parametrize("day,hours", [("2026-03-29", 23), ("2026-10-25", 25)]) +def test_dst_days_have_elapsed_time_slots(pvforecast_instance, day, hours): + averages = pvforecast_instance._interval_power(_day_watts(day), "Europe/Berlin") + assert len(averages) == hours * 4 + assert all(math.isfinite(value) for value in averages.values()) + assert np.diff(sorted(averages)).tolist() == [900] * (hours * 4 - 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("interval", [900, 3600]) +async def test_provider_to_genetic_and_prediction_list(pvforecast_instance, interval): + """Exercise real persistence/resampling, including cleanup of pre-fix samples.""" + pv = pvforecast_instance + start = to_datetime("2026-11-02T00:00:00+01:00", in_timezone="Europe/Berlin") + # Upgrade/refresh must remove the irregular sunset and an old forecast tail. + await pv.update_value( + start.subtract(days=1).add(hours=16, minutes=55, seconds=54), "pvforecast_ac_power", 0.0 + ) + await pv.update_value(start.add(hours=16, minutes=55, seconds=54), "pvforecast_ac_power", 0.0) + await pv.update_value(start.add(days=3), "pvforecast_ac_power", 123.0) + watts = _day_watts() | _day_watts("2026-11-03") + with patch("requests.get", return_value=_http(watts)): + await pv._update_data(force_update=True) + raw = await pv.key_to_raw_series("pvforecast_ac_power", dropna=False) + assert len(raw) == 192 + assert np.diff(raw.index.as_unit("ns").asi8 / 1e9).tolist() == [900] * 191 + values = await bounded_forecast_array( + pv, + key="pvforecast_ac_power", + start_datetime=start, + end_datetime=start.add(days=2), + interval=to_duration(interval), + ) + assert np.isfinite(values).all() + listed = await pv.key_to_array( + "pvforecast_ac_power", + start_datetime=start, + end_datetime=start.add(days=2), + interval=to_duration(interval), + ) + assert listed == pytest.approx(values) + assert values[17 * 3600 // interval : 24 * 3600 // interval].tolist() == [0.0] * ( + 7 * 3600 // interval + ) + # Preserve the missing tail instead of extrapolating zeros into a third day. + tail = await bounded_forecast_array( + pv, + key="pvforecast_ac_power", + start_datetime=start.add(days=2), + end_datetime=start.add(days=2, hours=1), + interval=to_duration(interval), + ) + assert np.isnan(tail).all() + # Once migrated, a normal refresh retains already normalized history. + with patch("requests.get", return_value=_http(_day_watts("2026-11-03"))): + await pv._update_data(force_update=True) + raw = await pv.key_to_raw_series("pvforecast_ac_power", dropna=False) + assert len(raw) == 192 + await pv.delete_by_datetime() + + +def test_planes_are_integrated_before_summing(config_eos): + _config( + config_eos, + planes=[ + {"surface_tilt": 30, "surface_azimuth": 90, "peakpower": 3}, + {"surface_tilt": 30, "surface_azimuth": 270, "peakpower": 3}, + ], + ) + pv = PVForecastForecastSolar(config=config_eos.load) + first = { + "2026-11-02 07:10:00": 0, + "2026-11-02 08:00:00": 1000, + "2026-11-02 09:00:00": 1000, + "2026-11-02 09:45:00": 0, + } + second = { + "2026-11-02 07:25:00": 0, + "2026-11-02 08:00:00": 500, + "2026-11-02 09:00:00": 500, + "2026-11-02 09:55:00": 0, + } + expected = pv._interval_power(first, "Europe/Berlin") + other = pv._interval_power(second, "Europe/Berlin") + with patch("requests.get", side_effect=[_http(first), _http(second)]): + body = pv._request_forecast(force_update=True) # type: ignore[call-arg] + for ts, value in body["watts"].items(): + stamp = int(to_datetime(ts).timestamp()) + assert value == pytest.approx(expected[stamp] + other[stamp]) + # An absent plane may not silently produce the other plane's forecast. + with patch("requests.get", side_effect=[_http(first), _http({})]): + body = pv._request_forecast(force_update=True) # type: ignore[call-arg] + assert all(math.isnan(value) for value in body["watts"].values())