Mirrors/EOS
1
0
Fork 0
mirror of https://github.com/Akkudoktor-EOS/EOS.git synced 2026-11-09 15:24:01 +01:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(pvforecast): add adaptive per-plane PV learning and diagnostics (#1422)

Browse Source 
PV arrays with different orientations and curtailment need separate plant calibration and weather correction. This opt-in provider publishes normal EOS AC/DC forecast keys at 15-minute resolution.

- Preserve daily per-plane energy/shape calibration and validated short-term local/trend selection. Curtailment-sensitive arrays require a fresh release flag or an explicitly configured learning guard.
-  Add switchable empirical solar-angle response relative to separate eligible reference arrays, removing their common weather error. This learns angular residuals, not physical angles or a measured geometric horizon.
-  Archive actually received weather model inputs, first local receipt and frozen per-member PV candidates. Returned past weather is excluded; unknown source issuance remains explicit. Learn model weights and forecast-regime bias separately at 2–6, 6–24 and 24–48 hours.
- New long-horizon candidates remain in shadow until their genuinely issued outputs improve equal-day MAE by the configured margin, win on at least 60% of validation days and pass a matched-support energy-bias guard. Insufficient evidence retains calibration. Daily yields still require complete local days, including DST.
- Expose fitted parameters, independent training/validation day counts, errors, selected methods and persistence in the existing status API, with additional normal prediction keys for Grafana. Retain physical DC through angular/weather corrections an  enforce individual inverter AC limits.

Co-authored-by: Normann <github@koldrack.com>
Co-authored-by: Christin <chrissi@von-perbandt.de>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: YMT <yn276364@dal.ca>
Co-authored-by: sarankumar1325 <120080287+sarankumar1325@users.noreply.github.com>
This commit is contained in:
Andreas
2026-11-03 15:31:32 +01:00
committed by GitHub
co-authored by Normann Christin Claude Opus 5.5 YMT sarankumar1325
parent 298d46a527
commit f804d48072
24 changed files with 6063 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/_generated/configexample.md
+70
View File
@@ -328,6 +328,56 @@
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5
},
"adaptive": {
"backend": "local",
"resolution_minutes": 15,
"forecast_days": null,
"past_days": null,
"weather_models": [
"best_match",
"icon_d2"
],
"transposition_model": "perez",
"albedo": 0.25,
"inverter_efficiency": 0.96,
"temperature_coefficient": -0.36,
"apply_iam": true,
"shift_to_interval_start": true,
"calibration_enabled": false,
"calibration_days": 30,
"calibration_reference_days": 30,
"calibration_outage_filter_enabled": true,
"calibration_outage_threshold": 0.55,
"calibration_min_healthy_days": 3,
"calibration_azimuth_bin_degrees": 45,
"calibration_prior_kwh": 5.0,
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5,
"regional_weather_model": "icon_d2",
"regional_horizon_hours": 6,
"learning_days": 21,
"min_training_days": 5,
"max_ac_power_w": 0,
"nominal_total_peakpower_kw": null,
"diagnostics_days": 7,
"archive_retention_days": 60,
"nowcast_enabled": true,
"trend_enabled": true,
"solar_response_enabled": true,
"weather_response_enabled": true,
"correction_mode": "auto",
"validation_days": 7,
"min_validation_days": 3,
"min_validation_samples": 32,
"min_validation_improvement": 0.02,
"training_delay_minutes": 15,
"total_measurement_key": null,
"total_curtailment_sensitive": false,
"total_curtailment_key": null,
"total_learning_guard_key": null,
"total_learning_guard_max": null,
"total_learning_guard_max_gap_seconds": 600
},
"pvforecastimport": {
"import_file_path": null,
"import_json": null
@@ -360,6 +410,16 @@
},
"planes": [
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 10.0,
"surface_azimuth": 180.0,
"userhorizon": [
@@ -382,6 +442,16 @@
"strings_per_inverter": 2
},
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 20.0,
"surface_azimuth": 90.0,
"userhorizon": [
docs/_generated/configpvforecast.md
+282
View File
@@ -7,6 +7,7 @@
| Name | Environment Variable | Type | Read-Only | Default | Description |
| ---- | -------------------- | ---- | --------- | ------- | ----------- |
| adaptive | `EOS_PVFORECAST__ADAPTIVE` | `PVForecastAdaptiveAkkudoktorCommonSettings` | `rw` | `required` | Adaptive Akkudoktor: configure string measurements and independent reference flags on the planes. |
| akkudoktor | `EOS_PVFORECAST__AKKUDOKTOR` | `PVForecastAkkudoktorLocalCommonSettings` | `rw` | `required` | Akkudoktor forecast backend and local calibration settings |
| forecastsolar | `EOS_PVFORECAST__FORECASTSOLAR` | `PVForecastForecastSolarCommonSettings` | `rw` | `required` | ForecastSolar provider settings |
| homeassistant | `EOS_PVFORECAST__HOMEASSISTANT` | `PVForecastHomeAssistantCommonSettings` | `rw` | `required` | Home Assistant provider settings |
@@ -61,6 +62,56 @@
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5
},
"adaptive": {
"backend": "local",
"resolution_minutes": 15,
"forecast_days": null,
"past_days": null,
"weather_models": [
"best_match",
"icon_d2"
],
"transposition_model": "perez",
"albedo": 0.25,
"inverter_efficiency": 0.96,
"temperature_coefficient": -0.36,
"apply_iam": true,
"shift_to_interval_start": true,
"calibration_enabled": false,
"calibration_days": 30,
"calibration_reference_days": 30,
"calibration_outage_filter_enabled": true,
"calibration_outage_threshold": 0.55,
"calibration_min_healthy_days": 3,
"calibration_azimuth_bin_degrees": 45,
"calibration_prior_kwh": 5.0,
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5,
"regional_weather_model": "icon_d2",
"regional_horizon_hours": 6,
"learning_days": 21,
"min_training_days": 5,
"max_ac_power_w": 0,
"nominal_total_peakpower_kw": null,
"diagnostics_days": 7,
"archive_retention_days": 60,
"nowcast_enabled": true,
"trend_enabled": true,
"solar_response_enabled": true,
"weather_response_enabled": true,
"correction_mode": "auto",
"validation_days": 7,
"min_validation_days": 3,
"min_validation_samples": 32,
"min_validation_improvement": 0.02,
"training_delay_minutes": 15,
"total_measurement_key": null,
"total_curtailment_sensitive": false,
"total_curtailment_key": null,
"total_learning_guard_key": null,
"total_learning_guard_max": null,
"total_learning_guard_max_gap_seconds": 600
},
"pvforecastimport": {
"import_file_path": null,
"import_json": null
@@ -93,6 +144,16 @@
},
"planes": [
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 10.0,
"surface_azimuth": 180.0,
"userhorizon": [
@@ -115,6 +176,16 @@
"strings_per_inverter": 2
},
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 20.0,
"surface_azimuth": 90.0,
"userhorizon": [
@@ -177,6 +248,56 @@
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5
},
"adaptive": {
"backend": "local",
"resolution_minutes": 15,
"forecast_days": null,
"past_days": null,
"weather_models": [
"best_match",
"icon_d2"
],
"transposition_model": "perez",
"albedo": 0.25,
"inverter_efficiency": 0.96,
"temperature_coefficient": -0.36,
"apply_iam": true,
"shift_to_interval_start": true,
"calibration_enabled": false,
"calibration_days": 30,
"calibration_reference_days": 30,
"calibration_outage_filter_enabled": true,
"calibration_outage_threshold": 0.55,
"calibration_min_healthy_days": 3,
"calibration_azimuth_bin_degrees": 45,
"calibration_prior_kwh": 5.0,
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5,
"regional_weather_model": "icon_d2",
"regional_horizon_hours": 6,
"learning_days": 21,
"min_training_days": 5,
"max_ac_power_w": 0,
"nominal_total_peakpower_kw": null,
"diagnostics_days": 7,
"archive_retention_days": 60,
"nowcast_enabled": true,
"trend_enabled": true,
"solar_response_enabled": true,
"weather_response_enabled": true,
"correction_mode": "auto",
"validation_days": 7,
"min_validation_days": 3,
"min_validation_samples": 32,
"min_validation_improvement": 0.02,
"training_delay_minutes": 15,
"total_measurement_key": null,
"total_curtailment_sensitive": false,
"total_curtailment_key": null,
"total_learning_guard_key": null,
"total_learning_guard_max": null,
"total_learning_guard_max_gap_seconds": 600
},
"pvforecastimport": {
"import_file_path": null,
"import_json": null
@@ -209,6 +330,16 @@
},
"planes": [
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 10.0,
"surface_azimuth": 180.0,
"userhorizon": [
@@ -231,6 +362,16 @@
"strings_per_inverter": 2
},
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 20.0,
"surface_azimuth": 90.0,
"userhorizon": [
@@ -256,6 +397,7 @@
"max_planes": 1,
"providers": [
"PVForecastAkkudoktor",
"PVForecastAdaptiveAkkudoktor",
"PVForecastForecastSolar",
"PVForecastHomeAssistant",
"PVForecastImport",
@@ -459,14 +601,24 @@
| Name | Type | Read-Only | Default | Description |
| ---- | ---- | --------- | ------- | ----------- |
| albedo | `Optional[float]` | `rw` | `0.2` | Proportion of the light hitting the ground that it reflects back. |
| correction_group | `Optional[str]` | `rw` | `None` | Optional group of planes sharing short-term validation. Unset independent references share one group; other planes are separate. |
| curtailment_key | `Optional[str]` | `rw` | `None` | Optional measured flag: 0 explicitly unrestricted, 1 curtailed. Missing or stale flags never authorize learning a curtailment-sensitive plane. |
| curtailment_sensitive | `bool` | `rw` | `False` | Battery-full curtailment possible; exclude from unrestricted weather learning. |
| inverter_model | `Optional[str]` | `rw` | `None` | Model of the inverter of this plane. |
| inverter_paco | `Optional[int]` | `rw` | `None` | AC power rating of the inverter [W]. |
| learning_guard_key | `Optional[str]` | `rw` | `None` | Optional site-specific positive learning guard, e.g. battery SoC, as an alternative to a binary curtailment flag. |
| learning_guard_max | `Optional[float]` | `rw` | `None` | Learn only while every supported guard value is below this bound. This is an explicit site assumption, not automatic curtailment detection. |
| learning_guard_max_gap_seconds | `int` | `rw` | `600` | None |
| loss | `Optional[float]` | `rw` | `14.0` | Sum of PV system losses in percent |
| measurement_key | `Optional[str]` | `rw` | `None` | Matching EOS power measurement key for adaptive learning. |
| module_model | `Optional[str]` | `rw` | `None` | Model of the PV modules of this plane. |
| modules_per_string | `Optional[int]` | `rw` | `None` | Number of the PV modules of the strings of this plane. |
| mountingplace | `Optional[str]` | `rw` | `building` | Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated. |
| name | `Optional[str]` | `rw` | `None` | Readable PV string name. |
| nowcast_reference | `bool` | `rw` | `False` | Use this independent array as a local weather reference. |
| optimal_surface_tilt | `Optional[bool]` | `rw` | `False` | Calculate the optimum tilt angle. Ignored for two-axis tracking. |
| optimalangles | `Optional[bool]` | `rw` | `False` | Calculate the optimum tilt and azimuth angles. Ignored for two-axis tracking. |
| parameters_estimated | `bool` | `rw` | `False` | Geometry and peak power are effective fitted model estimates, not verified nameplate data. |
| peakpower | `Optional[float]` | `rw` | `None` | Nominal power of PV system in kW. |
| pvtechchoice | `Optional[str]` | `rw` | `crystSi` | PV technology. One of 'crystSi', 'CIS', 'CdTe', 'Unknown'. |
| strings_per_inverter | `Optional[int]` | `rw` | `None` | Number of the strings of the inverter of this plane. |
@@ -487,6 +639,16 @@
"pvforecast": {
"planes": [
{
"name": null,
"measurement_key": null,
"nowcast_reference": false,
"curtailment_sensitive": false,
"parameters_estimated": false,
"correction_group": null,
"curtailment_key": null,
"learning_guard_key": null,
"learning_guard_max": null,
"learning_guard_max_gap_seconds": 600,
"surface_tilt": 10.0,
"surface_azimuth": 180.0,
"userhorizon": [
@@ -654,3 +816,123 @@
}
```
<!-- pyml enable line-length -->
### Automatic defaults; configure measurements and reference flags on each plane
<!-- pyml disable line-length -->
:::{table} pvforecast::adaptive
:widths: 10 10 5 5 30
:align: left
| Name | Type | Read-Only | Default | Description |
| ---- | ---- | --------- | ------- | ----------- |
| albedo | `float` | `rw` | `0.25` | Ground albedo used for planes that do not set their own. |
| apply_iam | `bool` | `rw` | `True` | Apply the ASHRAE incidence-angle modifier to the beam component. |
| archive_retention_days | `int` | `rw` | `60` | None |
| backend | `Literal['remote', 'local']` | `rw` | `local` | None |
| calibration_azimuth_bin_degrees | `int` | `rw` | `45` | Width of the solar-azimuth bins for the correction. 0 fits a single global factor only. |
| calibration_days | `int` | `rw` | `30` | Length of the measurement window used to fit the correction. |
| calibration_enabled | `bool` | `rw` | `False` | None |
| calibration_max_factor | `float` | `rw` | `1.5` | Upper clamp on any fitted correction factor. |
| calibration_min_factor | `float` | `rw` | `0.5` | Lower clamp on any fitted correction factor. |
| calibration_min_healthy_days | `int` | `rw` | `3` | Minimum number of healthy days used for a fit. Older healthy days from the reference window are added when the recent window contains fewer. |
| calibration_outage_filter_enabled | `bool` | `rw` | `True` | Exclude days whose measured production is far below the recent healthy plant level. This prevents inverter, battery and curtailment events from being learned as permanent PV model losses. |
| calibration_outage_threshold | `float` | `rw` | `0.55` | A day is treated as unavailable when its measured/modelled energy ratio is below this fraction of the robust healthy reference ratio. |
| calibration_prior_kwh | `float` | `rw` | `5.0` | Shrinkage strength: a bin needs this much modelled energy before its own factor outweighs the global one. Higher is more conservative. |
| calibration_reference_days | `int` | `rw` | `30` | Lookback used to distinguish healthy production from outages or curtailment. If the calibration window contains too few healthy days, the most recent healthy days from this reference window are used. |
| correction_mode | `Literal['auto', 'calibrated', 'local', 'trend']` | `rw` | `auto` | Choose validated short-term candidates per group/lead, or explicitly fix a correction stage. |
| diagnostics_days | `int` | `rw` | `7` | None |
| forecast_days | `Optional[int]` | `rw` | `None` | Forecast horizon in days (1-16). Leave empty to derive it from `prediction.hours`, which is what keeps the optimizer's tail horizon fed. |
| inverter_efficiency | `float` | `rw` | `0.96` | Nominal inverter efficiency (PVWatts eta_inv_nom). |
| learning_days | `int` | `rw` | `21` | None |
| max_ac_power_w | `float` | `rw` | `0` | None |
| min_training_days | `int` | `rw` | `5` | None |
| min_validation_days | `int` | `rw` | `3` | None |
| min_validation_improvement | `float` | `rw` | `0.02` | None |
| min_validation_samples | `int` | `rw` | `32` | None |
| nominal_total_peakpower_kw | `Optional[float]` | `rw` | `None` | Known installed nameplate total; fitted plane powers may be effective estimates. |
| nowcast_enabled | `bool` | `rw` | `True` | None |
| past_days | `Optional[int]` | `rw` | `None` | Days of past data to request (0-92). Leave empty to derive it from `prediction.historic_hours`. |
| regional_horizon_hours | `float` | `rw` | `6` | None |
| regional_weather_model | `Optional[str]` | `rw` | `icon_d2` | None |
| resolution_minutes | `int` | `rw` | `15` | Forecast resolution in minutes. 15 requests Open-Meteo's `minutely_15` block (natively resolved over Central Europe and North America, interpolated from hourly elsewhere); 60 requests the `hourly` block. |
| shift_to_interval_start | `bool` | `rw` | `True` | Open-Meteo stamps an interval mean with the interval END. EOS labels an interval by its START, so records are shifted back by one interval. Disable only to compare like-for-like against a provider that does not. |
| solar_response_enabled | `bool` | `rw` | `True` | Learn empirical angular response relative to separate, eligible reference arrays; activate only after issued-candidate validation. |
| temperature_coefficient | `float` | `rw` | `-0.36` | Module power temperature coefficient in %/degC (negative). Matches the `cellCoEff` the akkudoktor.net forecast uses. |
| total_curtailment_key | `Optional[str]` | `rw` | `None` | None |
| total_curtailment_sensitive | `bool` | `rw` | `False` | None |
| total_learning_guard_key | `Optional[str]` | `rw` | `None` | None |
| total_learning_guard_max | `Optional[float]` | `rw` | `None` | None |
| total_learning_guard_max_gap_seconds | `int` | `rw` | `600` | None |
| total_measurement_key | `Optional[str]` | `rw` | `None` | Optional whole-plant power channel when individual plane measurements are unavailable. |
| training_delay_minutes | `int` | `rw` | `15` | None |
| transposition_model | `str` | `rw` | `perez` | pvlib sky-diffuse transposition model: isotropic, klucher, haydavies, reindl, king or perez. |
| trend_enabled | `bool` | `rw` | `True` | None |
| validation_days | `int` | `rw` | `7` | None |
| weather_models | `list[str]` | `rw` | `['best_match']` | Open-Meteo weather models to request. Listing more than one turns the input into a poor-man's ensemble: the members are averaged per variable, which is the cheapest reliable way to cut irradiance forecast error. Costs no extra API calls. |
| weather_response_enabled | `bool` | `rw` | `True` | Archive received model weather and learn horizon/regime corrections; activate only after issued-candidate validation. |
:::
<!-- pyml enable line-length -->
<!-- pyml disable no-emphasis-as-heading -->
**Example Input/Output**
<!-- pyml enable no-emphasis-as-heading -->
<!-- pyml disable line-length -->
```json
{
"pvforecast": {
"adaptive": {
"backend": "local",
"resolution_minutes": 15,
"forecast_days": null,
"past_days": null,
"weather_models": [
"best_match",
"icon_d2"
],
"transposition_model": "perez",
"albedo": 0.25,
"inverter_efficiency": 0.96,
"temperature_coefficient": -0.36,
"apply_iam": true,
"shift_to_interval_start": true,
"calibration_enabled": false,
"calibration_days": 30,
"calibration_reference_days": 30,
"calibration_outage_filter_enabled": true,
"calibration_outage_threshold": 0.55,
"calibration_min_healthy_days": 3,
"calibration_azimuth_bin_degrees": 45,
"calibration_prior_kwh": 5.0,
"calibration_min_factor": 0.5,
"calibration_max_factor": 1.5,
"regional_weather_model": "icon_d2",
"regional_horizon_hours": 6.0,
"learning_days": 21,
"min_training_days": 5,
"max_ac_power_w": 0.0,
"nominal_total_peakpower_kw": null,
"diagnostics_days": 7,
"archive_retention_days": 60,
"nowcast_enabled": true,
"trend_enabled": true,
"solar_response_enabled": true,
"weather_response_enabled": true,
"correction_mode": "auto",
"validation_days": 7,
"min_validation_days": 3,
"min_validation_samples": 32,
"min_validation_improvement": 0.02,
"training_delay_minutes": 15,
"total_measurement_key": null,
"total_curtailment_sensitive": false,
"total_curtailment_key": null,
"total_learning_guard_key": null,
"total_learning_guard_max": null,
"total_learning_guard_max_gap_seconds": 600
}
}
}
```
<!-- pyml enable line-length -->
docs/_generated/openapi.md
+112 -1
View File
@@ -1,6 +1,6 @@
# Akkudoktor-EOS
**Version**: `v0.3.0.dev2610301534325811`
**Version**: `v0.3.0.dev2611011668106370`
<!-- pyml disable line-length -->
**Description**: This project provides a comprehensive solution for simulating and optimizing an energy system based on renewable energy sources. With a focus on photovoltaic (PV) systems, battery storage (batteries), load management (consumer requirements), heat pumps, electric vehicles, and consideration of electricity price data, this system enables forecasting and optimization of energy flow and costs over a specified period.
@@ -1699,6 +1699,117 @@ Args:
---
## GET /v1/prediction/pv-adaptive/comparisons
<!-- pyml disable line-length -->
**Links**: [local](http://localhost:8503/docs#/default/comparisons_v1_prediction_pv-adaptive_comparisons_get), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/comparisons_v1_prediction_pv-adaptive_comparisons_get)
<!-- pyml enable line-length -->
Comparisons
<!-- pyml disable line-length -->
```python
"""
Export genuinely issued forecasts with later measured truth, issuance and lead.
"""
```
<!-- pyml enable line-length -->
**Parameters**:
- `days` (query, optional): No description provided.
- `limit` (query, optional): No description provided.
- `since_ms` (query, optional): No description provided.
**Responses**:
- **200**: Successful Response
- **422**: Validation Error
---
## GET /v1/prediction/pv-adaptive/latest
<!-- pyml disable line-length -->
**Links**: [local](http://localhost:8503/docs#/default/latest_v1_prediction_pv-adaptive_latest_get), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/latest_v1_prediction_pv-adaptive_latest_get)
<!-- pyml enable line-length -->
Latest
<!-- pyml disable line-length -->
```python
"""
Latest complete output curve; validation uses the separate immutable archive.
"""
```
<!-- pyml enable line-length -->
**Responses**:
- **200**: Successful Response
---
## PUT /v1/prediction/pv-adaptive/samples
<!-- pyml disable line-length -->
**Links**: [local](http://localhost:8503/docs#/default/samples_v1_prediction_pv-adaptive_samples_put), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/samples_v1_prediction_pv-adaptive_samples_put)
<!-- pyml enable line-length -->
Samples
<!-- pyml disable line-length -->
```python
"""
Store original power readings in the bounded adaptive archive, without zero filling.
"""
```
<!-- pyml enable line-length -->
**Request Body**:
- `application/json`: {
"items": {
"$ref": "#/components/schemas/MeasurementSample"
},
"type": "array",
"maxItems": 10000,
"title": "Values"
}
**Responses**:
- **200**: Successful Response
- **422**: Validation Error
---
## GET /v1/prediction/pv-adaptive/status
<!-- pyml disable line-length -->
**Links**: [local](http://localhost:8503/docs#/default/status_v1_prediction_pv-adaptive_status_get), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/status_v1_prediction_pv-adaptive_status_get)
<!-- pyml enable line-length -->
Status
<!-- pyml disable line-length -->
```python
"""
Status contains honest cold-start counts and matched per-lead error metrics.
"""
```
<!-- pyml enable line-length -->
**Responses**:
- **200**: Successful Response
---
## GET /v1/prediction/pvforecast/pvlib/inverters
<!-- pyml disable line-length -->
docs/akkudoktoreos/pvakkuadapt.md
+364
View File
@@ -0,0 +1,364 @@
# Adaptive Akkudoktor PV forecast
`PVForecastAdaptiveAkkudoktor` is an opt-in provider that combines local
Open-Meteo/pvlib forecasts with measured production. It publishes the normal
`pvforecast_ac_power` and `pvforecast_dc_power` keys, so the optimizer can use
it without an external PV forecast override.
## What learns
- Each measured plane has an energy gain and a smooth daily correction.
Training uses completed previous days and genuinely issued forecasts.
- Independent, uncurtailed reference planes correct the next two hours. The
correction fades with forecast distance and does not carry over into tomorrow.
- A separate residual/trend regression learns each quarter-hour lead from
archived features and subsequently available measurements. Until five suitable
days and 100 pairs per lead exist, it retains the local candidate.
- The default `correction_mode: "auto"` chooses calibration, local correction or
trend separately for each measurement group and lead. It compares genuinely
issued candidates on completed previous days. Cold starts retain calibration.
Fixed `calibrated`, `local` and `trend` modes are available for explicit control.
- An optional empirical angular response learns repeatable differences between
arrays after dividing out the common error of separate reference arrays.
It uses smooth solar azimuth/elevation features, an identity prior and bounded
factors. It does not identify physical angles or a geometric horizon.
- A separate weather response learns weights for the actually received weather
model members and bias by forecast cloud regime, separately at 2–6, 6–24 and
24–48 hours. Open-Meteo source issuance is unknown; the archive explicitly
records first local receipt and retains only future intervals from that response.
Returned past weather is never treated as a historical forecast.
- New solar/weather candidates run in shadow until their **frozen issued outputs**
beat calibration on at least `min_validation_days` independent completed days
and `min_validation_samples` daylight pairs. Selection uses equal day weighting,
a minimum MAE improvement, wins on at least 60% of days, and an energy-bias
guard on identical observation support. Incomplete days are not reported as
complete daily yields. Models refitted to old labels do not validate themselves.
Training uses `min_training_days`; genuine validation therefore takes additional
days after cold start. The calibrated forecast remains available throughout.
ICON D2 is preferred near term, with a transition to the other configured
weather models between four and six hours. Missing daylight radiation cannot
become zero production. A failed weather update retains a usable old forecast.
The provider does not identify physical module angles from live measurements.
Configure each actual plane separately. Optional initial response profiles in
the learning archive are fixed starting estimates, distinct from live training.
More training data do not guarantee a lower error on every day.
## Configuration
Merge the following sections into the existing EOS configuration, keeping the
site location, devices, tariffs and other settings. Replace the example planes
with your actual plant. Azimuth is clockwise from north; peak power is in kWp.
Use a distinct power channel per measured plane, or one whole-plant power
channel as described below. Explicitly mark independent weather references.
When none are marked, unrestricted planes serve as references. If every plane
may be curtailed and no suitable reference exists, the physical forecast still
works, with local corrections disabled.
```json
{
"database": {"provider": "SQLite"},
"pvforecast": {
"provider": "PVForecastAdaptiveAkkudoktor",
"adaptive": {
"weather_models": ["icon_seamless", "icon_d2"],
"regional_weather_model": "icon_d2",
"learning_days": 21,
"min_training_days": 5,
"archive_retention_days": 60,
"nowcast_enabled": true,
"trend_enabled": true,
"solar_response_enabled": true,
"weather_response_enabled": true,
"correction_mode": "auto"
},
"planes": [
{
"name": "Hybrid inverter",
"peakpower": 5.0,
"surface_tilt": 20,
"surface_azimuth": 180,
"measurement_key": "pv_hybrid_w",
"curtailment_sensitive": true
},
{
"name": "Independent microinverter",
"peakpower": 1.0,
"surface_tilt": 30,
"surface_azimuth": 180,
"measurement_key": "pv_micro_w",
"nowcast_reference": true
}
]
},
"measurement": {
"channels": {
"pv_hybrid_w": {
"quantity": "power",
"unit": "W",
"integration_method": "hold",
"max_gap_seconds": 600
},
"pv_micro_w": {
"quantity": "power",
"unit": "W",
"integration_method": "hold",
"max_gap_seconds": 600
}
}
}
}
```
Mark battery-full-curtailed strings as `curtailment_sensitive`. Without further
evidence they retain the physical calibration and do not learn unrestricted
weather gain. An optional `curtailment_key` names an original measured flag:
`0` means explicitly unrestricted and `1` means curtailed. Supply it through
the sample endpoint, or a configured EOS measurement key. Missing, stale or
estimated flags do not authorize learning. A partial day needs at least eight
unrestricted daylight intervals; at least `min_training_days` suitable days
are still required. A full battery alone is not a universal curtailment flag.
Where a site has a documented relation between a measured value and unrestricted
operation, `learning_guard_key` plus `learning_guard_max` is an alternative:
all supported values must be below the configured bound. Set
`learning_guard_max_gap_seconds` to the signal's actual maximum support gap
(default 600). For example, a battery-SoC guard is a site-specific assumption,
not inverter curtailment detection; it must not be used to ignore other limits.
Status exposes the guard and bound. Prefer a real curtailment flag when available.
Whole-plant mode has corresponding `total_learning_guard_key`,
`total_learning_guard_max` and `total_learning_guard_max_gap_seconds` options.
Independent reference planes share one correction group by default; other
planes are separate. Set `correction_group` on planes to define your own groups.
The weather residual supplies shadow candidates to every group. A group uses
one only when its own unrestricted observations validate it. No string name,
inverter manufacturer, SQL database, Node-RED or Grafana is required.
For a site with only total production measurements, set
`pvforecast.adaptive.total_measurement_key` to that power channel. Keep the
physical planes, but omit their individual `measurement_key` values. Configure
the total channel's unit, integration and gap semantics in `measurement.channels`.
Use `total_curtailment_sensitive` and optionally `total_curtailment_key` when
the whole plant may be limited. Per-plane learning diagnostics are unavailable
in this mode; inverter limits are still applied to the physical planes.
The automatic selector uses a configurable `validation_days` window (default
7), at least `min_validation_days` completed days (3) and
`min_validation_samples` distinct target intervals per lead (32). Each target
gets one vote regardless of update frequency, and days have equal weight.
An extra stage must improve mean absolute error by at least
`min_validation_improvement` (0.02 = 2%). Historical candidates were issued
before their target values existed; fitting today's coefficients on those
labels is never used as retrospective validation. This does not guarantee an
improvement on every future day.
Training waits until `training_delay_minutes` after local midnight (default
15) for the previous day's closing samples. Until then, the persisted earlier
model remains usable. Plane-specific AC limits are reapplied after corrections.
DC retains the original physical DC estimate and its correction; it is not
reconstructed from an already clipped AC value.
An existing installation with configured inverter AC ratings starts a new model
signature for this bounded candidate definition. The archive and raw measurements
are retained. Installations without such ratings retain their physical signature.
Send original measured power, not energy counters, to the ordinary measurement
API or directly to `PUT /v1/prediction/pv-adaptive/samples`. The direct endpoint
accepts at most 10,000 samples per request, assigned plane keys, nonnegative
values and explicit time zones. It rejects future samples. A sample has this
shape; substitute the real acquisition timestamp and value:
```json
[
{
"date_time": "2026-01-01T12:00:00+01:00",
"key": "pv_micro_w",
"value": 350.0
}
]
```
Supply all configured planes regularly, including legitimate night-time zeros.
EOS integrates complete 15-minute intervals with actual temporal coverage.
Missing readings stay missing. Use the `linear` integration method instead of
`hold` only if that matches the sensor's semantics.
## Install this branch on a Linux NUC
### Update an existing source installation
Stop EOS and back up its configuration and data directory. In your existing
EOS checkout, update to the PR branch and synchronize dependencies:
```bash
git fetch origin
git switch feat/adaptive-pv-learning
git pull --ff-only
uv sync --frozen
git rev-parse HEAD
```
Record the printed commit for a reproducible installation. Keep any local
changes safe before switching branches. For a fresh checkout, clone the branch
with the command shown in the Docker section below, then run `uv sync --frozen`.
Merge the PV configuration into the existing `EOS.config.json`, retaining your
devices, tariffs and native data paths. In particular, preserve
`general.data_folder_path`, `general.data_output_subpath` and the environment
variables used by your existing launcher, including `EOS_CONFIG_DIR` and
`EOS_DATA_DIR` where set. Select `PVForecastAdaptiveAkkudoktor` as the provider
and configure the per-plane measurement channels above.
To retain an existing adaptive instance's learning, transfer its configuration
and SQLite archive together. Stop the source instance while copying, or use
SQLite's backup API for a consistent live copy. With the target EOS stopped,
back up any existing target archive and copy the transferred `learning.sqlite3`
to `<data_folder_path>/<data_output_subpath>/adaptive-pv/learning.sqlite3`.
The default output subpath is `output`; the destination must be writable by
the user running EOS. A configuration patch alone does not transfer learning.
Restart EOS with your existing service or launcher. For a manual LAN start
from the checkout, using the same configuration environment as before:
```bash
uv run python -m akkudoktoreos.server.eos --host 0.0.0.0 --eosdash-host 0.0.0.0
```
Retain your existing port options if customized. Check the adaptive `status`
and `latest` endpoints below, including tomorrow's forecast, then redirect
Node-RED's EOS measurement delivery, optimizer requests and diagnostic export
to the NUC's EOS host and port. Remove any external `pv_forecast_wh` override
from optimizer requests so the native provider supplies the PV forecast.
Switch control and export delivery to one EOS instance when activating it.
Check that a valid optimization plan is produced using the adaptive provider.
This provider does not require changing battery or optimizer settings.
### Optional Docker build
The PR branch is source code, not an automatically published DockerHub image.
For a container installation with Docker already installed, build it locally:
```bash
git clone --branch feat/adaptive-pv-learning --single-branch \
https://github.com/Akkudoktor-EOS/EOS.git eos-adaptive-pv
cd eos-adaptive-pv
git rev-parse HEAD
docker build --build-arg BUILD_VERSION=adaptive-pv -t eos:adaptive-pv .
docker volume create eos-adaptive-data
docker run -d --name eos-adaptive-pv --restart unless-stopped \
-p 8505:8503 -p 8506:8504 \
-v eos-adaptive-data:/data eos:adaptive-pv
```
These example host ports allow a separate instance; select available ports on
your NUC. Configure
it through EOS before switching measurement delivery and the optimizer client
to the new instance. A fresh volume begins without your plant configuration.
The container stores configuration at `/data/config/EOS.config.json` and the
learning archive at `/data/output/adaptive-pv/learning.sqlite3`. The volume
must survive container replacement. The forecast needs internet access to
Open-Meteo; local weather correction needs your sensor delivery as well.
To move an existing adaptive instance, back up its configuration and learning
archive together while the source instance is stopped, or use SQLite's backup
API for a consistent live archive copy. Transfer them into the corresponding
paths of the new data volume while the target instance is stopped. Set
`general.data_folder_path` to `/data` and `general.data_output_subpath` to
`output` in the container configuration. Check other machine-specific paths
in the existing configuration before starting the target.
Keep the original instance's configuration and data available until the new
installation has produced a valid forecast and an optimization using it.
## Learning progress and Grafana
The configured EOS database retains samples, issued forecasts and trained models across
restarts. Its default retention is 60 days; the learning window is 21 days.
Changing the plant, weather configuration or response profiles changes the
training signature so incompatible histories are not mixed.
The following HTTP endpoints are available:
| Endpoint | Purpose |
| --- | --- |
| `GET /v1/prediction/pv-adaptive/status` | State, training days/counts, per-plane gain and matched errors |
| `GET /v1/prediction/pv-adaptive/latest` | Latest forecast with raw, calibrated, local and adaptive power |
| `GET /v1/prediction/pv-adaptive/comparisons` | Issued predictions paired with later observations |
| `PUT /v1/prediction/pv-adaptive/samples` | Bounded ingestion of original plane readings |
Status includes `selection` with the chosen method, validation-day/sample
counts and candidate errors per group and lead. `algorithm_version` identifies
the guarded output; `metrics_algorithm_versions` identifies the histories in
the current error window. Model readiness is separate from demonstrated skill.
Metrics cover the complete configured window, independently of the public
comparison export's page limit. `daily_metrics` compares frozen pre-midnight
forecasts with complete measured local days, including 23/25-hour DST days.
Incomplete days carry null energy values rather than invented zeros.
For a compact learning panel, plot matched `mae_w` from status `metrics` over
time for one scope and horizon, with `method` distinguishing the stages. Do
not compare different observation sets or display missing errors as zero.
The API does not require Grafana or MariaDB. Existing Node-RED/Grafana bridges
must point to the new EOS address after migration.
`state` reports `initializing`, `reference_missing`, `learning`, `active` or
`night`. A fresh installation still provides a physical forecast while it
collects genuinely issued training pairs. Historical sensor readings alone
cannot recreate missing issued forecasts.
Check `/v1/prediction/pv-adaptive/latest` for tomorrow as well as today. The
native output is mean power in W per interval; daily kWh is the sum of W times
0.25 divided by 1000 for a complete local day. Keep time stamps when aligning
with the optimizer, including on daylight-saving transitions.
To revert, select the previous PV provider and restore its configuration.
Retain the data directory or container volume with the learned archive.
## EOS database and legacy migration
Durable adaptive records use the existing EOS `Database` frontend and its configured
`SQLite` or `LMDB` backend, under `pvakkuadapt.*` namespaces. No separate persistent
SQLite database is created. An in-memory relational index serves the provider's
bounded analytical queries. Database calls are made through the asynchronous EOS
frontend from background numerical workers; REST handlers do not perform migration
on the web event loop. Pruning removes only adaptive namespaces' expired records.
Set `database.provider` to `SQLite` or `LMDB` before enabling persistent learning.
With `null` or `NoDB`, learning is explicitly volatile and an old private file is
not imported. `/status.storage` exposes the selected backend and persistence state.
Changing the global database provider also affects other EOS persistence; choose it
as part of the installation configuration.
On first persistent startup, an existing
`<data_output_subpath>/adaptive-pv/learning.sqlite3` is opened read-only and copied
through the EOS frontend. Forecast IDs, issue times, first measurement availability,
physical signatures and fitted models are preserved. A completion marker is written
after all migrated records are flushed. Interrupted migration can be retried.
Keep the original file as a rollback backup; subsequent writes use the common EOS
backend. For later migrations between computers, stop EOS and copy the configured
EOS data directory (including `db/`) together with its configuration. Instructions
above mentioning the legacy SQLite archive apply to old installations only.
## Inspect long-horizon learning
`/v1/prediction/pv-adaptive/status` retains its existing fields and additionally
exposes `storage` and `long_response`: fitted angular coefficients, sample/day counts,
reference keys, weather member weights, regime factors, and per-group/horizon
validation counts, MAE, energy bias on matched intervals, and the selected method.
A fitted model and an activated correction are separate states. An unfitted or
unvalidated candidate retains calibration. Angular learning needs a separate
reference array; total-only installations can learn weather response but cannot
separate their own shading from common weather error.
Normal prediction keys additionally include
`pvforecast_adaptive_solar_ac_power`, `pvforecast_adaptive_weather_ac_power`,
`pvforecast_adaptive_long_validation_days`, and `pvforecast_adaptive_long_active`.
The candidate curves are diagnostic; optimization consumes `pvforecast_ac_power`,
which includes only selected corrections. These keys can be queried in Grafana
without enlarging an existing dashboard. Status distinguishes the established
short-term algorithm version from long-response version 1. Hourly immutable
snapshots bound long-horizon storage; repeated polling cannot create extra evidence.
The last received weather inputs and all candidates are retained for the configured
archive retention period. Disabling either response switch prevents its activation.
docs/index.md
+1
View File
@@ -49,6 +49,7 @@ akkudoktoreos/optimpost.md
akkudoktoreos/optimauto.md
akkudoktoreos/resource.md
akkudoktoreos/prediction.md
akkudoktoreos/pvakkuadapt.md
akkudoktoreos/measurement.md
akkudoktoreos/integration.md
akkudoktoreos/logging.md
openapi.json
+770 -1
View File
@@ -8,7 +8,7 @@
"name": "Apache 2.0",
"url": "https://www.apache.org/licenses/LICENSE-2.0.html"
},
"version": "v0.3.0.dev2610301534325811"
"version": "v0.3.0.dev2611011668106370"
},
"paths": {
"/v1/measurement/battery-capacity/{battery_id}": {
@@ -330,6 +330,182 @@
}
}
},
"/v1/prediction/pv-adaptive/status": {
"get": {
"tags": [
"prediction"
],
"summary": "Status",
"description": "Status contains honest cold-start counts and matched per-lead error metrics.",
"operationId": "status_v1_prediction_pv_adaptive_status_get",
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"additionalProperties": true,
"type": "object",
"title": "Response Status V1 Prediction Pv Adaptive Status Get"
}
}
}
}
}
}
},
"/v1/prediction/pv-adaptive/samples": {
"put": {
"tags": [
"prediction"
],
"summary": "Samples",
"description": "Store original power readings in the bounded adaptive archive, without zero filling.",
"operationId": "samples_v1_prediction_pv_adaptive_samples_put",
"requestBody": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/MeasurementSample"
},
"type": "array",
"maxItems": 10000,
"title": "Values"
}
}
},
"required": true
},
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"additionalProperties": {
"type": "integer"
},
"type": "object",
"title": "Response Samples V1 Prediction Pv Adaptive Samples Put"
}
}
}
},
"422": {
"description": "Validation Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
}
}
}
}
},
"/v1/prediction/pv-adaptive/comparisons": {
"get": {
"tags": [
"prediction"
],
"summary": "Comparisons",
"description": "Export genuinely issued forecasts with later measured truth, issuance and lead.",
"operationId": "comparisons_v1_prediction_pv_adaptive_comparisons_get",
"parameters": [
{
"name": "days",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"maximum": 30,
"minimum": 1,
"default": 2,
"title": "Days"
}
},
{
"name": "limit",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"maximum": 20000,
"minimum": 1,
"default": 2000,
"title": "Limit"
}
},
{
"name": "since_ms",
"in": "query",
"required": false,
"schema": {
"type": "integer",
"minimum": 0,
"default": 0,
"title": "Since Ms"
}
}
],
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": true
},
"title": "Response Comparisons V1 Prediction Pv Adaptive Comparisons Get"
}
}
}
},
"422": {
"description": "Validation Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
}
}
}
}
},
"/v1/prediction/pv-adaptive/latest": {
"get": {
"tags": [
"prediction"
],
"summary": "Latest",
"description": "Latest complete output curve; validation uses the separate immutable archive.",
"operationId": "latest_v1_prediction_pv_adaptive_latest_get",
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"items": {
"additionalProperties": true,
"type": "object"
},
"type": "array",
"title": "Response Latest V1 Prediction Pv Adaptive Latest Get"
}
}
}
}
}
}
},
"/v1/admin/cache/clear": {
"post": {
"tags": [
@@ -14127,6 +14303,470 @@
"title": "PPBCStartInterruptionInstruction",
"description": "Represents an instruction to interrupt execution of a running power sequence.\n\nThis model defines a control instruction that interrupts the execution of an\nactive power sequence. It enables dynamic control over sequence execution,\nallowing temporary suspension of a sequence in response to changing system conditions\nor requirements, particularly for sequences marked as interruptible."
},
"PVForecastAdaptiveAkkudoktorCommonSettings": {
"properties": {
"backend": {
"type": "string",
"enum": [
"remote",
"local"
],
"title": "Backend",
"default": "local"
},
"resolution_minutes": {
"type": "integer",
"title": "Resolution Minutes",
"description": "Forecast resolution in minutes. 15 requests Open-Meteo's `minutely_15` block (natively resolved over Central Europe and North America, interpolated from hourly elsewhere); 60 requests the `hourly` block.",
"default": 15,
"examples": [
15,
60
]
},
"forecast_days": {
"anyOf": [
{
"type": "integer",
"maximum": 16.0,
"minimum": 1.0
},
{
"type": "null"
}
],
"title": "Forecast Days",
"description": "Forecast horizon in days (1-16). Leave empty to derive it from `prediction.hours`, which is what keeps the optimizer's tail horizon fed.",
"examples": [
null,
7
]
},
"past_days": {
"anyOf": [
{
"type": "integer",
"maximum": 92.0,
"minimum": 0.0
},
{
"type": "null"
}
],
"title": "Past Days",
"description": "Days of past data to request (0-92). Leave empty to derive it from `prediction.historic_hours`.",
"examples": [
null,
3
]
},
"weather_models": {
"items": {
"type": "string"
},
"type": "array",
"minItems": 1,
"title": "Weather Models",
"description": "Open-Meteo weather models to request. Listing more than one turns the input into a poor-man's ensemble: the members are averaged per variable, which is the cheapest reliable way to cut irradiance forecast error. Costs no extra API calls.",
"default": [
"best_match"
],
"examples": [
[
"best_match"
],
[
"icon_seamless",
"ecmwf_ifs025",
"gfs_seamless"
]
]
},
"transposition_model": {
"type": "string",
"title": "Transposition Model",
"description": "pvlib sky-diffuse transposition model: isotropic, klucher, haydavies, reindl, king or perez.",
"default": "perez",
"examples": [
"perez",
"haydavies"
]
},
"albedo": {
"type": "number",
"maximum": 1.0,
"minimum": 0.0,
"title": "Albedo",
"description": "Ground albedo used for planes that do not set their own.",
"default": 0.25,
"examples": [
0.25,
0.2
]
},
"inverter_efficiency": {
"type": "number",
"maximum": 1.0,
"exclusiveMinimum": 0.0,
"title": "Inverter Efficiency",
"description": "Nominal inverter efficiency (PVWatts eta_inv_nom).",
"default": 0.96,
"examples": [
0.96,
0.94
]
},
"temperature_coefficient": {
"type": "number",
"title": "Temperature Coefficient",
"description": "Module power temperature coefficient in %/degC (negative). Matches the `cellCoEff` the akkudoktor.net forecast uses.",
"default": -0.36,
"examples": [
-0.36,
-0.29
]
},
"apply_iam": {
"type": "boolean",
"title": "Apply Iam",
"description": "Apply the ASHRAE incidence-angle modifier to the beam component.",
"default": true,
"examples": [
true
]
},
"shift_to_interval_start": {
"type": "boolean",
"title": "Shift To Interval Start",
"description": "Open-Meteo stamps an interval mean with the interval END. EOS labels an interval by its START, so records are shifted back by one interval. Disable only to compare like-for-like against a provider that does not.",
"default": true,
"examples": [
true
]
},
"calibration_enabled": {
"type": "boolean",
"title": "Calibration Enabled",
"default": false
},
"calibration_days": {
"type": "integer",
"maximum": 92.0,
"minimum": 1.0,
"title": "Calibration Days",
"description": "Length of the measurement window used to fit the correction.",
"default": 30,
"examples": [
30,
14
]
},
"calibration_reference_days": {
"type": "integer",
"maximum": 92.0,
"minimum": 3.0,
"title": "Calibration Reference Days",
"description": "Lookback used to distinguish healthy production from outages or curtailment. If the calibration window contains too few healthy days, the most recent healthy days from this reference window are used.",
"default": 30,
"examples": [
30,
14
]
},
"calibration_outage_filter_enabled": {
"type": "boolean",
"title": "Calibration Outage Filter Enabled",
"description": "Exclude days whose measured production is far below the recent healthy plant level. This prevents inverter, battery and curtailment events from being learned as permanent PV model losses.",
"default": true,
"examples": [
true
]
},
"calibration_outage_threshold": {
"type": "number",
"exclusiveMaximum": 1.0,
"exclusiveMinimum": 0.0,
"title": "Calibration Outage Threshold",
"description": "A day is treated as unavailable when its measured/modelled energy ratio is below this fraction of the robust healthy reference ratio.",
"default": 0.55,
"examples": [
0.55,
0.5
]
},
"calibration_min_healthy_days": {
"type": "integer",
"maximum": 31.0,
"minimum": 1.0,
"title": "Calibration Min Healthy Days",
"description": "Minimum number of healthy days used for a fit. Older healthy days from the reference window are added when the recent window contains fewer.",
"default": 3,
"examples": [
3
]
},
"calibration_azimuth_bin_degrees": {
"type": "integer",
"maximum": 180.0,
"minimum": 0.0,
"title": "Calibration Azimuth Bin Degrees",
"description": "Width of the solar-azimuth bins for the correction. 0 fits a single global factor only.",
"default": 45,
"examples": [
45,
30,
15,
0
]
},
"calibration_prior_kwh": {
"type": "number",
"minimum": 0.0,
"title": "Calibration Prior Kwh",
"description": "Shrinkage strength: a bin needs this much modelled energy before its own factor outweighs the global one. Higher is more conservative.",
"default": 5.0,
"examples": [
5.0,
20.0
]
},
"calibration_min_factor": {
"type": "number",
"exclusiveMinimum": 0.0,
"title": "Calibration Min Factor",
"description": "Lower clamp on any fitted correction factor.",
"default": 0.5,
"examples": [
0.5
]
},
"calibration_max_factor": {
"type": "number",
"exclusiveMinimum": 0.0,
"title": "Calibration Max Factor",
"description": "Upper clamp on any fitted correction factor.",
"default": 1.5,
"examples": [
1.5
]
},
"regional_weather_model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Regional Weather Model",
"default": "icon_d2"
},
"regional_horizon_hours": {
"type": "number",
"maximum": 48.0,
"exclusiveMinimum": 0.0,
"title": "Regional Horizon Hours",
"default": 6
},
"learning_days": {
"type": "integer",
"maximum": 30.0,
"minimum": 7.0,
"title": "Learning Days",
"default": 21
},
"min_training_days": {
"type": "integer",
"maximum": 14.0,
"minimum": 2.0,
"title": "Min Training Days",
"default": 5
},
"max_ac_power_w": {
"type": "number",
"minimum": 0.0,
"title": "Max Ac Power W",
"default": 0
},
"nominal_total_peakpower_kw": {
"anyOf": [
{
"type": "number",
"exclusiveMinimum": 0.0
},
{
"type": "null"
}
],
"title": "Nominal Total Peakpower Kw",
"description": "Known installed nameplate total; fitted plane powers may be effective estimates.",
"examples": [
null
]
},
"diagnostics_days": {
"type": "integer",
"maximum": 30.0,
"minimum": 1.0,
"title": "Diagnostics Days",
"default": 7
},
"archive_retention_days": {
"type": "integer",
"maximum": 365.0,
"minimum": 31.0,
"title": "Archive Retention Days",
"default": 60
},
"nowcast_enabled": {
"type": "boolean",
"title": "Nowcast Enabled",
"default": true
},
"trend_enabled": {
"type": "boolean",
"title": "Trend Enabled",
"default": true
},
"solar_response_enabled": {
"type": "boolean",
"title": "Solar Response Enabled",
"description": "Learn empirical angular response relative to separate, eligible reference arrays; activate only after issued-candidate validation.",
"default": true
},
"weather_response_enabled": {
"type": "boolean",
"title": "Weather Response Enabled",
"description": "Archive received model weather and learn horizon/regime corrections; activate only after issued-candidate validation.",
"default": true
},
"correction_mode": {
"type": "string",
"enum": [
"auto",
"calibrated",
"local",
"trend"
],
"title": "Correction Mode",
"description": "Choose validated short-term candidates per group/lead, or explicitly fix a correction stage.",
"default": "auto"
},
"validation_days": {
"type": "integer",
"maximum": 30.0,
"minimum": 3.0,
"title": "Validation Days",
"default": 7
},
"min_validation_days": {
"type": "integer",
"maximum": 14.0,
"minimum": 2.0,
"title": "Min Validation Days",
"default": 3
},
"min_validation_samples": {
"type": "integer",
"minimum": 8.0,
"title": "Min Validation Samples",
"default": 32
},
"min_validation_improvement": {
"type": "number",
"maximum": 0.5,
"minimum": 0.0,
"title": "Min Validation Improvement",
"default": 0.02
},
"training_delay_minutes": {
"type": "integer",
"maximum": 120.0,
"minimum": 0.0,
"title": "Training Delay Minutes",
"default": 15
},
"total_measurement_key": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Total Measurement Key",
"description": "Optional whole-plant power channel when individual plane measurements are unavailable.",
"examples": [
null
]
},
"total_curtailment_sensitive": {
"type": "boolean",
"title": "Total Curtailment Sensitive",
"default": false
},
"total_curtailment_key": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Total Curtailment Key",
"examples": [
null
]
},
"total_learning_guard_key": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Total Learning Guard Key",
"examples": [
null
]
},
"total_learning_guard_max": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Total Learning Guard Max",
"examples": [
null
]
},
"total_learning_guard_max_gap_seconds": {
"type": "integer",
"maximum": 3600.0,
"minimum": 1.0,
"title": "Total Learning Guard Max Gap Seconds",
"default": 600
}
},
"type": "object",
"title": "PVForecastAdaptiveAkkudoktorCommonSettings",
"description": "Automatic defaults; configure measurements and reference flags on each plane."
},
"PVForecastAkkudoktorLocalCommonSettings": {
"properties": {
"backend": {
@@ -14409,6 +15049,10 @@
"$ref": "#/components/schemas/PVForecastAkkudoktorLocalCommonSettings",
"description": "Akkudoktor forecast backend and local calibration settings"
},
"adaptive": {
"$ref": "#/components/schemas/PVForecastAdaptiveAkkudoktorCommonSettings",
"description": "Adaptive Akkudoktor: configure string measurements and independent reference flags on the planes."
},
"pvforecastimport": {
"$ref": "#/components/schemas/PVForecastImportCommonSettings",
"description": "PV forecast import provider settings"
@@ -14538,6 +15182,10 @@
"$ref": "#/components/schemas/PVForecastAkkudoktorLocalCommonSettings",
"description": "Akkudoktor forecast backend and local calibration settings"
},
"adaptive": {
"$ref": "#/components/schemas/PVForecastAdaptiveAkkudoktorCommonSettings",
"description": "Adaptive Akkudoktor: configure string measurements and independent reference flags on the planes."
},
"pvforecastimport": {
"$ref": "#/components/schemas/PVForecastImportCommonSettings",
"description": "PV forecast import provider settings"
@@ -14902,6 +15550,127 @@
},
"PVForecastPlaneSetting": {
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Name",
"description": "Readable PV string name.",
"examples": [
null
]
},
"measurement_key": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Measurement Key",
"description": "Matching EOS power measurement key for adaptive learning.",
"examples": [
null
]
},
"nowcast_reference": {
"type": "boolean",
"title": "Nowcast Reference",
"description": "Use this independent array as a local weather reference.",
"default": false
},
"curtailment_sensitive": {
"type": "boolean",
"title": "Curtailment Sensitive",
"description": "Battery-full curtailment possible; exclude from unrestricted weather learning.",
"default": false
},
"parameters_estimated": {
"type": "boolean",
"title": "Parameters Estimated",
"description": "Geometry and peak power are effective fitted model estimates, not verified nameplate data.",
"default": false
},
"correction_group": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Correction Group",
"description": "Optional group of planes sharing short-term validation. Unset independent references share one group; other planes are separate.",
"examples": [
null
]
},
"curtailment_key": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Curtailment Key",
"description": "Optional measured flag: 0 explicitly unrestricted, 1 curtailed. Missing or stale flags never authorize learning a curtailment-sensitive plane.",
"examples": [
null
]
},
"learning_guard_key": {
"anyOf": [
{
"type": "string",
"maxLength": 128,
"minLength": 1
},
{
"type": "null"
}
],
"title": "Learning Guard Key",
"description": "Optional site-specific positive learning guard, e.g. battery SoC, as an alternative to a binary curtailment flag.",
"examples": [
null
]
},
"learning_guard_max": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Learning Guard Max",
"description": "Learn only while every supported guard value is below this bound. This is an explicit site assumption, not automatic curtailment detection.",
"examples": [
null
]
},
"learning_guard_max_gap_seconds": {
"type": "integer",
"maximum": 3600.0,
"minimum": 1.0,
"title": "Learning Guard Max Gap Seconds",
"default": 600
},
"surface_tilt": {
"anyOf": [
{
src/akkudoktoreos/core/database.py
+5 -1
View File
@@ -747,7 +747,7 @@ class SQLiteDatabase(DatabaseBackendABC):
ns = self._ns(namespace)
with self.conn:
with self.lock, self.conn:
# Ensure metadata table exists
self.conn.execute("""
CREATE TABLE IF NOT EXISTS metadata (
@@ -821,6 +821,9 @@ class SQLiteDatabase(DatabaseBackendABC):
with self.lock:
with self.conn:
# isolation_level=None disables implicit transactions. Without
# BEGIN each row commits separately and failed batches are partial.
self.conn.execute("BEGIN IMMEDIATE")
self.conn.executemany(
"INSERT OR REPLACE INTO records (namespace, key, value) VALUES (?, ?, ?)",
rows,
@@ -850,6 +853,7 @@ class SQLiteDatabase(DatabaseBackendABC):
with self.lock:
with self.conn:
self.conn.execute("BEGIN IMMEDIATE")
cursor = self.conn.executemany(
"DELETE FROM records WHERE namespace = ? AND key = ?",
rows,
src/akkudoktoreos/prediction/prediction.py
+7
View File
@@ -53,6 +53,9 @@ from akkudoktoreos.prediction.loadakkudoktor import (
from akkudoktoreos.prediction.loadimport import LoadImport
from akkudoktoreos.prediction.loadvrm import LoadVrm
from akkudoktoreos.prediction.predictionabc import PredictionContainer
from akkudoktoreos.prediction.pvakkuadapt import (
PVForecastAdaptiveAkkudoktor,
)
from akkudoktoreos.prediction.pvforecastakkudoktor import PVForecastAkkudoktor
from akkudoktoreos.prediction.pvforecastforecastsolar import PVForecastForecastSolar
from akkudoktoreos.prediction.pvforecasthomeassistant import PVForecastHomeAssistant
@@ -106,6 +109,7 @@ loadforecast_akkudoktor_adjusted = LoadAkkudoktorAdjusted()
loadforecast_vrm = LoadVrm()
loadforecast_import = LoadImport()
pvforecast_akkudoktor = PVForecastAkkudoktor()
pvforecast_adaptive_akkudoktor = PVForecastAdaptiveAkkudoktor()
pvforecast_vrm = PVForecastVrm()
pvforecast_homeassistant = PVForecastHomeAssistant()
pvforecast_pvlib = PVForecastPVLib()
@@ -140,6 +144,7 @@ PredictionProviderType = Union[
LoadImport,
LoadVrm,
PVForecastAkkudoktor,
PVForecastAdaptiveAkkudoktor,
PVForecastForecastSolar,
PVForecastHomeAssistant,
PVForecastImport,
@@ -180,6 +185,7 @@ def prediction_providers() -> list[PredictionProviderType]:
loadforecast_vrm, \
loadforecast_import, \
pvforecast_akkudoktor, \
pvforecast_adaptive_akkudoktor, \
pvforecast_vrm, \
pvforecast_homeassistant, \
pvforecast_pvlib, \
@@ -221,6 +227,7 @@ def prediction_providers() -> list[PredictionProviderType]:
loadforecast_import,
loadforecast_vrm,
pvforecast_akkudoktor,
pvforecast_adaptive_akkudoktor,
pvforecast_forecastsolar,
pvforecast_homeassistant,
pvforecast_import,
src/akkudoktoreos/prediction/pvakkuadapt.py
+1444
View File
File diff suppressed because it is too large Load Diff
src/akkudoktoreos/prediction/pvakkuadaptmodel.py
+304
View File
@@ -0,0 +1,304 @@
"""Small causal PV calibration model shared by the provider and offline backtest."""
from dataclasses import dataclass, field
import numpy as np
import pandas as pd
@dataclass
class AdaptivePVModel:
"""Learn energy bias and a smooth intraday shape from completed measurements.
Inputs are actual 15-minute or hourly interval-start timestamps and mean powers in W.
``available_at``
records when a measured interval became usable, including any ingestion lag.
This is statistical calibration, not identification of physical fault causes.
"""
timezone: str = "Europe/Berlin"
learning_days: int = 60
min_days: int = 4
half_life_days: float = 30.0
shape_prior_kwh: float = 2.0
min_factor: float = 0.2
max_factor: float = 3.0
interval_minutes: int = 15
allow_partial_days: bool = False
min_partial_day_samples: int = 8
gain: float = field(default=1.0, init=False)
shape: np.ndarray = field(default_factory=lambda: np.ones(12), init=False)
trained_days: int = field(default=0, init=False)
def fit(self, history: pd.DataFrame, cutoff: pd.Timestamp) -> "AdaptivePVModel":
"""Fit only completed previous local days available before ``cutoff``.
Global energy gain uses complete days. Shape uses valid paired intervals
from those days, with shrinkage and recency weighting. Missing values,
negative production, and explicitly invalid samples never become zeros.
With allow_partial_days, the caller must first censor every interval
without positive unrestricted-operation evidence. Suitable days then
require min_partial_day_samples daylight pairs rather than night data.
"""
self.gain, self.shape, self.trained_days = 1.0, np.ones(12), 0
cutoff = _aware(cutoff)
if self.interval_minutes not in (15, 60):
raise ValueError("Training intervals must be 15 or 60 minutes.")
if history.empty:
return self
history = history.sort_index()
history_index = _aware_index(history.index)
utc = history_index.tz_convert("UTC")
if not utc.equals(utc.floor(f"{self.interval_minutes}min")):
raise ValueError("Training timestamps must align with their actual interval.")
if history.index.has_duplicates:
raise ValueError("One training pair per interval is required.")
available = pd.to_datetime(history["available_at"], utc=True)
local_day = history_index.tz_convert(self.timezone).normalize()
today = cutoff.tz_convert(self.timezone).normalize()
valid = (
(available <= cutoff)
& (local_day < today)
& (history.index >= cutoff - pd.Timedelta(days=self.learning_days))
& np.isfinite(history["raw_power"])
& np.isfinite(history["measured_power"])
& (history["raw_power"] >= 0)
& (history["measured_power"] >= 0)
)
pairs = history.loc[valid, ["raw_power", "measured_power"]].copy()
if pairs.empty:
return self
pairs["day"] = _aware_index(pairs.index).tz_convert(self.timezone).normalize()
# Count against elapsed local days, including daylight-saving transitions.
days = pairs.groupby("day").agg(
raw=("raw_power", "sum"), actual=("measured_power", "sum"), n=("raw_power", "size")
)
expected = np.array(
[
(day + pd.DateOffset(days=1) - day).total_seconds() / (self.interval_minutes * 60)
for day in days.index
]
)
days = days[(days["n"].to_numpy() == expected) & (days["raw"] > 200.0)]
if self.allow_partial_days:
# The caller has positively identified unrestricted intervals.
# Unknown/curtailed intervals have already been removed, never zeroed.
daylight = pairs[pairs.raw_power > 50].groupby("day").size()
days = pairs.groupby("day").agg(
raw=("raw_power", "sum"), actual=("measured_power", "sum"), n=("raw_power", "size")
)
days = days[
(daylight.reindex(days.index, fill_value=0) >= self.min_partial_day_samples)
& (days.raw > 200)
]
self.trained_days = len(days)
if self.trained_days < self.min_days:
return self
pairs = pairs[pairs["day"].isin(days.index)]
age_days = np.array([(cutoff - day).total_seconds() / 86400 for day in days.index])
day_weights = np.exp2(-age_days / self.half_life_days)
self.gain = float(
np.clip(
np.dot(days["actual"], day_weights) / np.dot(days["raw"], day_weights),
self.min_factor,
self.max_factor,
)
)
local = _aware_index(pairs.index).tz_convert(self.timezone)
hours = local.hour.to_numpy() + local.minute.to_numpy() / 60 + self.interval_minutes / 120
weights = np.exp2(
(_aware_index(pairs.index) - cutoff).total_seconds().to_numpy()
/ 86400
/ self.half_life_days
)
prior_wh = self.shape_prior_kwh * 1000
for number, center in enumerate(np.arange(12) * 2 + 1):
distance = np.abs(hours - center)
distance = np.minimum(distance, 24 - distance)
kernel = np.maximum(0, 1 - distance / 2) * weights
raw_wh = float(np.dot(kernel, pairs["raw_power"])) * self.interval_minutes / 60
actual_wh = float(np.dot(kernel, pairs["measured_power"])) * self.interval_minutes / 60
self.shape[number] = np.clip(
(actual_wh + prior_wh * self.gain) / (raw_wh + prior_wh),
self.min_factor,
self.max_factor,
)
return self
def predict(
self, raw_power: pd.Series, capacity_w: float = 0.0, resolution_minutes: int = 15
) -> pd.Series:
"""Correct shape, conserving the independently learned energy per day."""
local = _aware_index(raw_power.index).tz_convert(self.timezone)
hours = local.hour.to_numpy() + local.minute.to_numpy() / 60 + resolution_minutes / 120
centers = np.arange(12) * 2 + 1
scales = np.interp(
hours,
np.r_[centers[-1] - 24, centers, centers[0] + 24],
np.r_[self.shape[-1], self.shape, self.shape[0]],
)
raw = raw_power.to_numpy(dtype=float)
days = local.normalize()
for day in days.unique():
mask = days == day
before = float(np.nansum(raw[mask]))
after = float(np.nansum(raw[mask] * scales[mask]))
if before > 0 and after > 0:
# Bounded multiplicative correction with energy constraint. Bisection
# avoids a renormalization violating the configured factor limits.
target = before * self.gain
lo, hi = 0.0, self.max_factor / max(float(scales[mask].min()), 1e-12)
for _ in range(40):
multiplier = (lo + hi) / 2
energy = np.nansum(
raw[mask]
* np.clip(scales[mask] * multiplier, self.min_factor, self.max_factor)
)
if energy < target:
lo = multiplier
else:
hi = multiplier
scales[mask] = np.clip(
scales[mask] * (lo + hi) / 2, self.min_factor, self.max_factor
)
corrected = raw_power * scales
return corrected.clip(lower=0, upper=capacity_w if capacity_w > 0 else None)
def apply_nowcast(
prediction: pd.Series,
history: pd.DataFrame,
cutoff: pd.Timestamp,
half_life_hours: float = 1.0,
max_age_minutes: int = 30,
capacity_w: float = 0.0,
) -> pd.Series:
"""Blend a recent measured/modelled ratio back to the forecast with lead time.
Require at least 30 minutes of actual support in the past hour. The short-term
correction does not survive overnight, is clamped, and expires with stale data.
History must contain ``calibrated_power``, not already nowcast predictions.
"""
cutoff = _aware(cutoff)
if history.empty:
return prediction.copy()
_aware_index(history.index)
available = pd.to_datetime(history["available_at"], utc=True)
usable = history[
(available <= cutoff)
& (history.index + pd.Timedelta(minutes=15) <= cutoff)
& (history.index >= cutoff - pd.Timedelta(hours=1))
]
usable = usable[
np.isfinite(usable["calibrated_power"])
& np.isfinite(usable["measured_power"])
& (usable["calibrated_power"] > 50)
& (usable["measured_power"] >= 0)
]
if len(usable) < 2:
return prediction.copy()
latest_end = usable.index.max() + pd.Timedelta(minutes=15)
if cutoff - latest_end > pd.Timedelta(minutes=max_age_minutes):
return prediction.copy()
ratio = float(
np.clip(usable["measured_power"].sum() / usable["calibrated_power"].sum(), 0.5, 1.5)
)
lead = np.maximum(0, (prediction.index - latest_end).total_seconds().to_numpy() / 3600)
scale = 1 + (ratio - 1) * np.exp2(-lead / half_life_hours)
same_day = _aware_index(prediction.index).tz_convert(cutoff.tz).date == cutoff.date()
scale = np.where((prediction.index >= cutoff) & same_day, scale, 1.0)
return (prediction * scale).clip(lower=0, upper=capacity_w if capacity_w > 0 else None)
def apply_local_nowcast(
prediction: pd.Series,
clear_sky_power: pd.Series,
history: pd.DataFrame,
cutoff: pd.Timestamp,
half_life_minutes: float = 30.0,
horizon_minutes: int = 120,
max_age_minutes: int = 30,
capacity_w: float = 0.0,
reference_interval_minutes: int = 15,
) -> pd.Series:
"""Carry a recent local weather residual toward the current weather forecast.
Reference measurements may be a non-curtailed subset of the plant. Normalize
their residual by their own clear-sky power, then transfer it to the whole
plant's clear-sky power. Require two contiguous complete reference intervals.
Missing, stale, unavailable or demand-limited references yield no correction.
``eligible`` is optional; callers must exclude curtailment using measurement
quality or provide explicit eligibility. This does not infer cloud motion.
"""
cutoff = _aware(cutoff)
prediction_index = _aware_index(prediction.index)
if reference_interval_minutes not in (5, 15):
raise ValueError("Local reference intervals must be 5 or 15 minutes.")
_aware_index(history.index, reference_interval_minutes)
if half_life_minutes <= 0 or horizon_minutes <= 0:
raise ValueError("Local nowcast decay and horizon must be positive.")
if history.index.has_duplicates:
raise ValueError("One reference measurement per interval is required.")
if history.empty:
return prediction.copy()
available = pd.to_datetime(history["available_at"], utc=True)
recent = (
history[
(available <= cutoff)
& (history.index + pd.Timedelta(minutes=reference_interval_minutes) <= cutoff)
& (history.index >= cutoff - pd.Timedelta(hours=1))
]
.sort_index()
.tail(2)
)
reference_index = _aware_index(recent.index, reference_interval_minutes)
if len(recent) != 2 or reference_index[1] - reference_index[0] != pd.Timedelta(
minutes=reference_interval_minutes
):
return prediction.copy()
end = reference_index[-1] + pd.Timedelta(minutes=reference_interval_minutes)
if cutoff - end > pd.Timedelta(minutes=max_age_minutes):
return prediction.copy()
if (
not np.isfinite(recent[["calibrated_power", "measured_power", "clear_sky_power"]])
.all()
.all()
):
return prediction.copy()
if (recent["measured_power"] < 0).any() or (recent["clear_sky_power"] < 150).any():
return prediction.copy()
if "eligible" in recent and not recent["eligible"].fillna(False).all():
return prediction.copy()
residual = (
(recent["measured_power"] - recent["calibrated_power"]) / recent["clear_sky_power"]
).clip(-1.5, 1.5)
# Fixed recency weighting: no trend extrapolation from a passing cloud.
local_error = float(np.dot(residual.to_numpy(), [0.35, 0.65]))
age = (
cutoff - (reference_index[-1] + pd.Timedelta(minutes=reference_interval_minutes / 2))
).total_seconds() / 60
lead = (prediction_index - cutoff).total_seconds().to_numpy() / 60
decay = np.exp2(-np.maximum(0, lead + age) / half_life_minutes)
# Fade to exactly zero, avoiding a step at the finite horizon.
taper = np.clip(2 * (1 - lead / horizon_minutes), 0, 1)
same_day = prediction_index.tz_convert(cutoff.tz).date == cutoff.date()
usable = (lead >= 0) & (lead < horizon_minutes) & same_day
envelope = clear_sky_power.reindex(prediction.index)
correction = envelope * np.where(usable, local_error * decay * taper, 0.0)
result = (prediction + correction).where(envelope.notna(), prediction)
return result.clip(lower=0, upper=capacity_w if capacity_w > 0 else None)
def _aware(value: pd.Timestamp) -> pd.Timestamp:
if value.tzinfo is None:
raise ValueError("Explicit timezone required.")
return value
def _aware_index(index: pd.Index, resolution_minutes: int = 15) -> pd.DatetimeIndex:
if not isinstance(index, pd.DatetimeIndex) or index.tz is None:
raise ValueError("Timezone-aware DatetimeIndex required.")
utc = index.tz_convert("UTC")
if not utc.equals(utc.floor(f"{resolution_minutes}min")):
raise ValueError(f"PV intervals must start on {resolution_minutes}-minute boundaries.")
return index
src/akkudoktoreos/prediction/pvakkuadaptquality.py
+148
View File
@@ -0,0 +1,148 @@
"""Causal selection of issued PV candidates on completed, independent target days."""
from typing import Any, cast
import numpy as np
import pandas as pd
METHODS = ("calibrated", "local", "trend")
def select_corrections(
rows: list[dict[str, Any]],
observations: pd.DataFrame,
groups: dict[str, list[str]],
cutoff: pd.Timestamp,
timezone: str,
reference_group: str | None = None,
validation_days: int = 7,
minimum_days: int = 3,
minimum_samples: int = 32,
improvement: float = 0.02,
enabled_methods: tuple[str, ...] = METHODS,
) -> dict[str, Any]:
"""Compare frozen candidates, never coefficients refitted to their own labels.
Each target interval gets one vote per lead, irrespective of polling rate.
Predictions issued today, unavailable labels and censored observations do
not validate a candidate. Ties and insufficient evidence retain calibration.
Legacy archives can validate only their explicitly recorded reference sum.
"""
today = cutoff.tz_convert(timezone).normalize()
start = today - pd.DateOffset(days=validation_days)
start_ms, end_ms = int(start.timestamp() * 1000), int(today.timestamp() * 1000)
if (
not isinstance(observations.index, pd.DatetimeIndex)
or observations.index.tz is None
or observations.index.has_duplicates
):
raise ValueError("Validation requires distinct, timezone-aware observation intervals.")
selected: dict[str, Any] = {}
for group, keys in groups.items():
comparisons = []
truth = {}
for timestamp, values_row in observations.iterrows():
target_time = cast(pd.Timestamp, timestamp)
if not start <= target_time < today:
continue
obs = cast("pd.Series[Any]", values_row)
if all(
pd.notna(obs.get(key))
and pd.notna(obs.get(key + "_eligible", True))
and bool(obs.get(key + "_eligible", True))
and pd.notna(obs.get(key + "_available"))
and obs[key + "_available"] <= cutoff
for key in keys
):
truth[int(target_time.timestamp() * 1000)] = (
sum(float(obs[key]) for key in keys),
str(target_time.tz_convert(timezone).date()),
)
for row in rows:
target_ms, issued_ms = row["target_ms"], row["issue_ms"]
lead = float(row["lead_minutes"])
if (
not start_ms <= target_ms < end_ms
or not issued_ms < target_ms
or not 0 <= lead < 120
):
continue
if target_ms not in truth:
continue
candidate = row.get("groups", {}).get(group)
if candidate is None:
if group != reference_group:
continue
candidate = {
"calibrated": row.get("reference_calibrated_w"),
"local": row.get("reference_local_w"),
"trend": row.get("reference_adaptive_w"),
}
values = [candidate.get(method) for method in METHODS]
if any(value is None or not np.isfinite(value) for value in values):
continue
actual, day = truth[target_ms]
if actual <= 50 and float(candidate["calibrated"]) <= 50:
continue
bucket = int(np.ceil(lead / 15) * 15)
comparisons.append(
dict(
target=row["target_ms"],
bucket=bucket,
distance=abs(lead - bucket + 7.5),
day=day,
**{method: abs(float(candidate[method]) - actual) for method in METHODS},
)
)
per_lead = {}
if comparisons:
frame = (
pd.DataFrame(comparisons)
.sort_values("distance")
.drop_duplicates(["target", "bucket"])
)
for bucket_label, part in frame.groupby("bucket"):
# Equal day weighting prevents a bright long day or faster poller
# from choosing a model for every other day.
errors = part.groupby("day")[list(METHODS)].mean().mean()
days = int(part.day.nunique())
enough = days >= minimum_days and len(part) >= minimum_samples
choice = "calibrated"
best = float(errors[choice])
for method in METHODS[1:]:
value = float(errors[method])
if method in enabled_methods and enough and value < best * (1 - improvement):
choice, best = method, value
per_lead[str(int(cast(int, bucket_label)))] = dict(
method=choice,
days=days,
samples=len(part),
validated=enough,
mae_w={method: float(errors[method]) for method in METHODS},
)
selected[group] = per_lead
return selected
def choose_correction(
calibrated: pd.Series,
local: pd.Series,
trend: pd.Series,
issue: pd.Timestamp,
validation: dict[str, Any],
mode: str = "auto",
) -> pd.Series:
"""Use a validated candidate per lead; outside two hours keep calibration."""
result = calibrated.copy()
leads = (pd.DatetimeIndex(calibrated.index) - issue).total_seconds().to_numpy() / 60
buckets = np.ceil(leads / 15).astype(int) * 15
candidates = {"calibrated": calibrated, "local": local, "trend": trend}
for bucket in np.unique(buckets):
method = mode
if mode == "auto":
method = validation.get(str(int(bucket)), {}).get("method", "calibrated")
if method not in candidates:
method = "calibrated"
mask = (buckets == bucket) & (leads >= 0) & (leads < 120)
result.loc[mask] = candidates[method].loc[mask]
return result
src/akkudoktoreos/prediction/pvakkuadaptresponse.py
+262
View File
@@ -0,0 +1,262 @@
"""Regularized angular response and issued-weather calibration with causal guards."""
from typing import Any, cast
import numpy as np
import pandas as pd
LEADS = ("2-6h", "6-24h", "24-48h")
METHODS = ("calibrated", "solar", "weather")
def _sum_powers(values: list[Any]) -> float:
if any(value is None for value in values):
return float("nan")
return float(np.sum(values))
def lead_bucket(minutes: float) -> str | None:
"""Separate intraday, next-day and second-day weather skill."""
if 120 <= minutes < 360:
return LEADS[0]
if 360 <= minutes < 1440:
return LEADS[1]
if 1440 <= minutes <= 2880:
return LEADS[2]
return None
def solar_basis(index: pd.DatetimeIndex, latitude: float, longitude: float) -> np.ndarray:
"""Smooth local angular basis at interval centers, independent of civil time."""
import pvlib
solar = pvlib.solarposition.get_solarposition(
index + pd.Timedelta(minutes=7.5), latitude, longitude
)
azimuth = solar.azimuth.to_numpy()
elevation = solar.apparent_elevation.to_numpy()
az = np.exp(-0.5 * ((azimuth[:, None] - np.arange(60, 301, 30)) / 22) ** 2)
el = np.exp(-0.5 * ((elevation[:, None] - np.arange(0, 81, 20)) / 14) ** 2)
basis = (az[:, :, None] * el[:, None, :]).reshape(len(index), 45)
return basis / np.maximum(basis.sum(axis=1, keepdims=True), 1e-9)
def fit_solar_response(
history: pd.DataFrame,
cutoff: pd.Timestamp,
timezone: str,
latitude: float,
longitude: float,
minimum_days: int,
) -> dict[str, Any]:
"""Learn differential array response after removing common reference weather.
This estimates repeatable angular residuals, not a measured geometric horizon.
Training requires eligible observations and a separate independent reference.
Unseen angular regions retain the prior response of one.
"""
result: dict[str, Any] = dict(coefficients=[0.0] * 45, days=0, samples=0, fitted=False)
if history.empty:
return result
today = cutoff.tz_convert(timezone).normalize()
valid = (
(history.index < today)
& (history.available_at <= cutoff)
& np.isfinite(history[["base", "actual", "common"]]).all(axis=1)
& (history.base > 80)
& (history.actual >= 0)
& history.common.between(0.35, 2.5)
)
pairs = history.loc[valid].copy()
days = pd.DatetimeIndex(pairs.index).tz_convert(timezone).normalize().nunique()
result.update(days=int(days), samples=len(pairs))
if days < minimum_days or len(pairs) < 48:
return result
basis = solar_basis(pd.DatetimeIndex(pairs.index), latitude, longitude)
target = (pairs.actual / pairs.base / pairs.common).clip(0.35, 1.8).to_numpy() - 1
age = (cutoff - pd.DatetimeIndex(pairs.index)).total_seconds().to_numpy() / 86400
weights = np.exp2(-age / 14) * np.minimum(1, pairs.base.to_numpy() / 800)
# Downweight outliers; strong identity prior shrinks sparsely covered knots.
weights *= np.minimum(1, 0.35 / np.maximum(np.abs(target), 1e-9))
matrix = basis.T @ (weights[:, None] * basis) + np.eye(45) * 8
coefficients = np.linalg.solve(matrix, basis.T @ (weights * target))
result.update(coefficients=np.clip(coefficients, -0.5, 0.5).tolist(), fitted=True)
return result
def solar_factors(basis: np.ndarray, model: dict[str, Any]) -> np.ndarray:
"""Bound a fitted empirical response; cold start is exactly identity."""
coefficients = np.asarray(model.get("coefficients", [0.0] * 45), dtype=float)
if coefficients.shape != (45,) or not np.isfinite(coefficients).all():
raise ValueError("Invalid adaptive solar response")
return np.clip(1 + basis @ coefficients, 0.5, 1.5)
def regime(clearness: float) -> str:
"""Use only forecast irradiance to identify a weather regime."""
return "cloudy" if clearness < 0.3 else "mixed" if clearness < 0.7 else "clear"
def paired_rows(
rows: list[dict[str, Any]],
observations: pd.DataFrame,
keys: list[str],
cutoff: pd.Timestamp,
timezone: str,
) -> list[dict[str, Any]]:
"""Join frozen inputs to causally available, positively eligible observations.
One target gets one vote per horizon, independent of the polling frequency.
All candidates and member models are compared on identical support.
"""
today = cutoff.tz_convert(timezone).normalize()
paired: dict[tuple[int, str], dict[str, Any]] = {}
centers = {"2-6h": 240, "6-24h": 900, "24-48h": 2160}
for row in rows:
target = pd.Timestamp(row["target_ms"], unit="ms", tz="UTC")
issued = pd.Timestamp(row["issued_ms"], unit="ms", tz="UTC")
lead = (target - issued).total_seconds() / 60
bucket = lead_bucket(lead)
if (
bucket is None
or target >= today
or target not in observations.index
or row.get("response_version") != 1
):
continue
truth = cast("pd.Series[Any]", observations.loc[target])
if not all(
pd.notna(truth.get(key))
and pd.notna(truth.get(key + "_eligible"))
and bool(truth.get(key + "_eligible", False))
and pd.notna(truth.get(key + "_available"))
and truth[key + "_available"] <= cutoff
for key in keys
):
continue
planes = row.get("planes", {})
if not all(key in planes for key in keys):
continue
values = {
method: _sum_powers([planes[key].get(method, np.nan) for key in keys])
for method in METHODS
}
actual = sum(float(truth[key]) for key in keys)
if not np.isfinite(list(values.values())).all() or max(actual, values["calibrated"]) < 50:
continue
members = {
name: _sum_powers([member.get(key, np.nan) for key in keys])
for name, member in row.get("members", {}).items()
}
sky = _sum_powers([planes[key].get("sky", np.nan) for key in keys])
record = dict(
target_ms=row["target_ms"],
bucket=bucket,
actual=actual,
day=str(target.tz_convert(timezone).date()),
members=members,
distance=abs(lead - centers[bucket]),
sky=sky,
**values,
)
identity = (row["target_ms"], bucket)
if identity not in paired or record["distance"] < paired[identity]["distance"]:
paired[identity] = record
return list(paired.values())
def fit_weather_response(
pairs: list[dict[str, Any]],
minimum_days: int,
minimum_samples: int,
) -> dict[str, Any]:
"""Rank actually issued model members and shrink regime bias toward identity."""
result = {}
for bucket in LEADS:
part = [row for row in pairs if row["bucket"] == bucket]
names = set.intersection(*(set(row["members"]) for row in part)) if part else set()
part = [
row
for row in part
if np.isfinite([row["members"][name] for name in names]).all()
and np.isfinite(row["sky"])
and row["sky"] > 50
]
days = len({row["day"] for row in part})
value: dict[str, Any] = dict(
days=days, samples=len(part), weights={}, regimes={}, fitted=False
)
if names and days >= minimum_days and len(part) >= minimum_samples:
errors = {}
for name in sorted(names):
frame = pd.DataFrame(
dict(day=row["day"], error=abs(row["members"][name] - row["actual"]))
for row in part
)
errors[name] = float(frame.groupby("day").error.mean().mean())
scores = {name: 1 / max(error, 20) ** 2 for name, error in errors.items()}
weights = {
name: 0.5 / len(names) + 0.5 * score / sum(scores.values())
for name, score in scores.items()
}
for row in part:
prediction = sum(weights[name] * row["members"][name] for name in names)
row["weather_prediction"] = prediction
row["regime"] = regime(prediction / row["sky"])
factors = {}
for name in ("cloudy", "mixed", "clear"):
subset = [row for row in part if row["regime"] == name]
if len({row["day"] for row in subset}) >= minimum_days:
predicted = sum(row["weather_prediction"] for row in subset)
measured = sum(row["actual"] for row in subset)
factors[name] = float(np.clip((measured + 8000) / (predicted + 8000), 0.5, 1.5))
value.update(weights=weights, regimes=factors, member_mae_w=errors, fitted=True)
result[bucket] = value
return result
def select_responses(
pairs: list[dict[str, Any]],
minimum_days: int,
minimum_samples: int,
improvement: float,
enabled_methods: tuple[str, ...] = METHODS,
) -> dict[str, Any]:
"""Activate only better frozen candidates, with day weighting and energy guard."""
result = {}
for bucket in LEADS:
part = pd.DataFrame(row for row in pairs if row["bucket"] == bucket)
value: dict[str, Any] = dict(
method="calibrated",
days=0,
samples=len(part),
validated=False,
mae_w={},
daily_energy_error_kwh={},
)
if not part.empty:
days = int(part.day.nunique())
errors = pd.DataFrame({method: part[method] - part.actual for method in METHODS})
daily_mae = errors.abs().groupby(part.day).mean()
mae = daily_mae.mean()
daily_energy = (errors.groupby(part.day).sum() * 0.25 / 1000).abs().mean()
enough = days >= minimum_days and len(part) >= minimum_samples
best = "calibrated"
for method in METHODS[1:]:
if (
method in enabled_methods
and enough
and mae[method] < mae[best] * (1 - improvement)
and daily_energy[method] <= daily_energy[best] * 1.02
and (daily_mae[method] < daily_mae[best]).mean() >= 0.6
):
best = method
value.update(
method=best,
days=days,
validated=enough,
mae_w=mae.to_dict(),
daily_energy_error_kwh=daily_energy.to_dict(),
)
result[bucket] = value
return result
src/akkudoktoreos/prediction/pvakkuadaptstore.py
+675
View File
@@ -0,0 +1,675 @@
"""Bounded, persistent issued PV forecasts and independently timed measurements."""
import asyncio
import hashlib
import json
import os
import sqlite3
import threading
from collections.abc import Coroutine
from contextlib import contextmanager
from pathlib import Path
from typing import Any, ClassVar, Iterator, TypeVar, cast
import numpy as np
import pandas as pd
from akkudoktoreos.core.coreabc import get_config, get_database
from akkudoktoreos.measurement.quality import MeasurementSample
T = TypeVar("T")
class _DatabaseWorker:
"""Use the existing asynchronous EOS database from numerical worker threads."""
_loop: ClassVar[asyncio.AbstractEventLoop | None] = None
_lock: ClassVar[threading.Lock] = threading.Lock()
_pid: ClassVar[int] = 0
@classmethod
def run(cls, operation: Coroutine[Any, Any, T]) -> T:
"""Run database frontend operations on a shared, process-local I/O loop."""
with cls._lock:
if cls._loop is None or cls._pid != os.getpid():
cls._loop = asyncio.new_event_loop()
cls._pid = os.getpid()
threading.Thread(
target=cls._loop.run_forever, daemon=True, name="pv-database"
).start()
loop = cls._loop
return asyncio.run_coroutine_threadsafe(operation, loop).result(timeout=120)
class PVLearningStore:
"""Persist immutable issue/target records through the configured EOS database.
SQLite is only an in-memory query index. Durable records live in the existing
EOS backend namespaces. NoDB intentionally learns in memory without persistence.
The old private SQLite file is read-only migration input and remains untouched.
"""
_keys = {
"samples": ("key", "time_ms"),
"forecasts": ("signature", "issue_ms", "target_ms"),
"baselines": ("signature", "target_ms"),
"latest": ("target_ms",),
"metadata": ("key",),
"weather": ("issue_ms",),
"longforecasts": ("signature", "issue_ms", "target_ms"),
}
def __init__(self, path: Path):
self.path = path
self.database = get_database()
self.persistent = get_config().database.provider not in (None, "NoDB")
self._lock = threading.RLock()
self._db = sqlite3.connect(":memory:", check_same_thread=False)
self._db.row_factory = sqlite3.Row
self._initializing = True
self._saved_sequence = 0
migrated = None
with self.connection() as db:
db.executescript("""
CREATE TABLE IF NOT EXISTS samples (
key TEXT NOT NULL, time_ms INTEGER NOT NULL, value REAL,
quality TEXT NOT NULL, available_ms INTEGER NOT NULL,
PRIMARY KEY(key,time_ms));
CREATE TABLE IF NOT EXISTS forecasts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
signature TEXT NOT NULL, issue_ms INTEGER NOT NULL, target_ms INTEGER NOT NULL,
lead_minutes REAL NOT NULL, payload TEXT NOT NULL,
actual_w REAL, reference_actual_w REAL, available_ms INTEGER,
updated_ms INTEGER NOT NULL,
UNIQUE(signature,issue_ms,target_ms));
CREATE INDEX IF NOT EXISTS forecast_target ON forecasts(signature,target_ms);
CREATE TABLE IF NOT EXISTS baselines (
signature TEXT NOT NULL, target_ms INTEGER NOT NULL,
issue_ms INTEGER NOT NULL, payload TEXT NOT NULL,
PRIMARY KEY(signature,target_ms));
CREATE TABLE IF NOT EXISTS latest (
target_ms INTEGER PRIMARY KEY, issue_ms INTEGER NOT NULL, payload TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, payload TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS weather (issue_ms INTEGER PRIMARY KEY, payload TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS longforecasts (
signature TEXT NOT NULL, issue_ms INTEGER NOT NULL, target_ms INTEGER NOT NULL,
payload TEXT NOT NULL, PRIMARY KEY(signature,issue_ms,target_ms));
""")
if self.persistent:
_DatabaseWorker.run(self.database.open(namespace="pvakkuadapt"))
migrated = _DatabaseWorker.run(self.database.get_metadata(namespace="pvakkuadapt"))
if path.is_file() and not migrated:
self._import_legacy()
else:
self._load_backend()
self._track_changes()
self._initializing = False
if self.persistent and not migrated:
if path.is_file():
for table, keys in self._keys.items():
expression = ",".join(keys)
self._db.execute(
f"INSERT OR REPLACE INTO _changes SELECT ?,json_array({expression}) FROM {table}", # noqa: S608
(table,),
) # noqa: S608
self._db.commit()
self._flush()
_DatabaseWorker.run(self.database.flush())
_DatabaseWorker.run(
self.database.set_metadata(
self.dump(
dict(
schema_version=1,
migrated_from=str(path) if path.is_file() else None,
counts=self.counts(),
)
).encode(),
namespace="pvakkuadapt",
)
)
_DatabaseWorker.run(self.database.flush())
def _import_legacy(self) -> None:
"""Read the old archive without changing data, IDs or availability timestamps."""
with sqlite3.connect(self.path.resolve().as_uri() + "?mode=ro", uri=True) as old:
old.row_factory = sqlite3.Row
for table in ("samples", "forecasts", "baselines", "latest", "metadata"):
cursor = old.execute("SELECT * FROM " + table) # noqa: S608 -- fixed table list
while batch := cursor.fetchmany(2000):
self._insert_rows(table, [dict(row) for row in batch])
sequence = old.execute(
"SELECT seq FROM sqlite_sequence WHERE name='forecasts'"
).fetchone()
if sequence:
self._restore_sequence(int(sequence[0]))
def _restore_sequence(self, sequence: int) -> None:
current = self._db.execute(
"SELECT seq FROM sqlite_sequence WHERE name='forecasts'"
).fetchone()
if current:
self._db.execute(
"UPDATE sqlite_sequence SET seq=? WHERE name='forecasts'",
(max(sequence, current[0]),),
)
else:
self._db.execute("INSERT INTO sqlite_sequence VALUES ('forecasts',?)", (sequence,))
self._db.commit()
def _insert_rows(self, table: str, rows: list[dict[str, Any]]) -> None:
if not rows:
return
if table not in self._keys:
raise ValueError("Unknown PV record type")
columns = tuple(rows[0])
allowed = {row[1] for row in self._db.execute("PRAGMA table_info(" + table + ")")}
if set(columns) != allowed:
raise ValueError("Invalid PV database record columns")
placeholders = ",".join("?" for _ in columns)
self._db.executemany(
f"INSERT OR REPLACE INTO {table} ({','.join(columns)}) VALUES ({placeholders})", # noqa: S608
[tuple(row[column] for column in columns) for row in rows],
)
self._db.commit()
async def _backend_rows(self, table: str) -> list[dict[str, Any]]:
return [
json.loads(self.database.deserialize_data(value))
async for _, value in self.database.iterate_records(namespace="pvakkuadapt." + table)
]
def _load_backend(self) -> None:
for table in self._keys:
self._insert_rows(table, _DatabaseWorker.run(self._backend_rows(table)))
sequence = _DatabaseWorker.run(
self.database.get_metadata(namespace="pvakkuadapt.forecasts")
)
if sequence:
self._restore_sequence(int(sequence))
self._saved_sequence = int(sequence)
def _track_changes(self) -> None:
self._db.execute(
"CREATE TABLE _changes (table_name TEXT, record_key TEXT, PRIMARY KEY(table_name,record_key))"
)
for table, keys in self._keys.items():
for event, source in (("INSERT", "NEW"), ("UPDATE", "NEW"), ("DELETE", "OLD")):
expression = ",".join(source + "." + key for key in keys)
self._db.execute(
f"CREATE TRIGGER track_{table}_{event} AFTER {event} ON {table} BEGIN " # noqa: S608
f"INSERT OR REPLACE INTO _changes VALUES ('{table}',json_array({expression})); END"
)
self._db.commit()
def _flush(self) -> None:
if self._initializing:
return
if self.persistent:
for table, keys in self._keys.items():
changed = self._db.execute(
"SELECT record_key FROM _changes WHERE table_name=?", (table,)
).fetchall()
writes, deletes = [], []
for (encoded_key,) in changed:
identity = json.loads(encoded_key)
row = self._db.execute(
f"SELECT * FROM {table} WHERE " + " AND ".join(k + "=?" for k in keys), # noqa: S608
identity,
).fetchone() # noqa: S608
key = encoded_key.encode()
if row is None:
deletes.append(key)
else:
writes.append(
(key, self.database.serialize_data(self.dump(dict(row)).encode()))
)
if writes:
count = _DatabaseWorker.run(
self.database.save_records(writes, namespace="pvakkuadapt." + table)
)
if count != len(writes):
raise RuntimeError("Incomplete PV database write")
if deletes:
_DatabaseWorker.run(
self.database.delete_records(deletes, namespace="pvakkuadapt." + table)
)
sequence = self._db.execute(
"SELECT seq FROM sqlite_sequence WHERE name='forecasts'"
).fetchone()
if sequence and sequence[0] > self._saved_sequence:
_DatabaseWorker.run(
self.database.set_metadata(
str(sequence[0]).encode(), namespace="pvakkuadapt.forecasts"
)
)
self._saved_sequence = int(sequence[0])
self._db.execute("DELETE FROM _changes")
self._db.commit()
def counts(self) -> dict[str, int]:
"""Domain record counts for migration verification and diagnostics."""
return {
table: self._db.execute("SELECT COUNT(*) FROM " + table).fetchone()[0] # noqa: S608
for table in self._keys
} # noqa: S608 -- fixed internal table names
def storage_status(self) -> dict[str, Any]:
"""Describe persistence explicitly rather than silently creating a private DB."""
return dict(
backend=get_config().database.provider or "NoDB",
persistent=self.persistent,
namespace="pvakkuadapt",
records=self.counts(),
)
@contextmanager
def connection(self) -> Iterator[sqlite3.Connection]:
"""Use short atomic transactions and bound lock waits across ingestion/readout."""
with self._lock:
if not self._initializing:
self._flush()
with self._db:
yield self._db
self._flush()
@staticmethod
def milliseconds(timestamp: pd.Timestamp) -> int:
"""Milliseconds from an explicitly zoned timestamp."""
value = pd.Timestamp(timestamp)
if value.tzinfo is None:
raise ValueError("Explicit timezone required.")
return int(value.timestamp() * 1000)
@staticmethod
def dump(payload: Any) -> str:
"""Serialize finite values; missing model inputs remain null."""
def clean(value: Any) -> Any:
if isinstance(value, dict):
return {key: clean(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [clean(item) for item in value]
if isinstance(value, pd.Timestamp):
return value.isoformat() if pd.notna(value) else None
if isinstance(value, (float, np.floating)):
return float(value) if np.isfinite(value) else None
if isinstance(value, np.integer):
return int(value)
if value is pd.NaT:
return None
return value
return json.dumps(clean(payload), allow_nan=False, separators=(",", ":"))
def metadata(self, key: str, value: Any = None) -> Any:
"""Read or atomically replace separately named model/status metadata."""
with self.connection() as db:
if value is not None:
db.execute(
"INSERT INTO metadata VALUES (?,?) ON CONFLICT(key) DO UPDATE SET payload=excluded.payload",
(key, self.dump(value)),
)
row = db.execute("SELECT payload FROM metadata WHERE key=?", (key,)).fetchone()
return json.loads(row[0]) if row else None
def import_samples(self, samples: list[MeasurementSample], now: pd.Timestamp) -> int:
"""Retain the first real arrival; a changed value gets its new availability."""
received = self.milliseconds(now)
values = [
(
sample.key,
self.milliseconds(pd.Timestamp(sample.date_time)),
sample.value,
sample.quality.status,
received,
)
for sample in samples
]
if any(time > received for _, time, *_ in values):
raise ValueError("Future PV measurements cannot be imported.")
with self.connection() as db:
db.executemany(
"""INSERT INTO samples VALUES (?,?,?,?,?)
ON CONFLICT(key,time_ms) DO UPDATE SET value=excluded.value, quality=excluded.quality,
available_ms=CASE WHEN samples.value IS excluded.value AND samples.quality=excluded.quality
THEN samples.available_ms ELSE excluded.available_ms END""",
values,
)
return len(values)
def samples(self, key: str, start: pd.Timestamp, cutoff: pd.Timestamp) -> list[dict[str, Any]]:
"""Exclude future source timestamps and samples received after the cutoff."""
with self.connection() as db:
return [
dict(row)
for row in db.execute(
"SELECT * FROM samples WHERE key=? AND time_ms>=? AND time_ms<=? AND available_ms<=? ORDER BY time_ms",
(
key,
self.milliseconds(start),
self.milliseconds(cutoff),
self.milliseconds(cutoff),
),
)
]
def remember(
self,
signature: str,
issue: pd.Timestamp,
rows: list[tuple[pd.Timestamp, dict[str, Any]]],
timezone: str,
) -> None:
"""Archive each short lead once and freeze previous-day physical baselines."""
issued = self.milliseconds(issue)
with self.connection() as db:
for timestamp, payload in rows:
target = self.milliseconds(timestamp)
if target < issued:
continue
text = self.dump({**payload, "signature": signature})
db.execute(
"INSERT INTO latest VALUES (?,?,?) ON CONFLICT(target_ms) DO UPDATE SET issue_ms=excluded.issue_ms,payload=excluded.payload",
(target, issued, text),
)
day_start = pd.Timestamp(timestamp).tz_convert(timezone).normalize()
existing = db.execute(
"SELECT issue_ms FROM baselines WHERE signature=? AND target_ms=?",
(signature, target),
).fetchone()
if existing is None or issued < self.milliseconds(day_start):
db.execute(
"INSERT INTO baselines VALUES (?,?,?,?) ON CONFLICT(signature,target_ms) DO UPDATE SET issue_ms=excluded.issue_ms,payload=excluded.payload",
(signature, target, issued, text),
)
lead = (target - issued) / 60000
if lead < 120:
db.execute(
"INSERT OR IGNORE INTO forecasts(signature,issue_ms,target_ms,lead_minutes,payload,updated_ms) VALUES (?,?,?,?,?,?)",
(signature, issued, target, lead, text, issued),
)
def archive_weather(self, data: dict[str, Any], received: pd.Timestamp) -> str:
"""Archive each received response once, retaining only genuinely future inputs.
Open-Meteo does not expose the numerical model's issuance here. The first
local receipt is recorded explicitly; returned past data never become
historical forecasts. Cache hits preserve that first receipt.
"""
digest = hashlib.sha256(self.dump(data).encode()).hexdigest()
with self.connection() as db:
if db.execute(
"SELECT 1 FROM weather WHERE json_extract(payload,'$.digest')=? LIMIT 1", (digest,)
).fetchone():
return digest
raw = data["minutely_15"]
times = pd.DatetimeIndex(pd.to_datetime(raw["time"], utc=True))
starts = times - pd.Timedelta(minutes=15)
keep = (starts >= received) & (starts <= received + pd.Timedelta(hours=48))
subset = {
key: [value for value, valid in zip(values, keep) if valid]
for key, values in raw.items()
}
payload = dict(
digest=digest,
received_at=received.isoformat(),
source_issue_at=None,
timestamp_semantics="interval_end",
minutely_15=subset,
units=data.get("minutely_15_units", {}),
elevation=data.get("elevation"),
)
db.execute(
"INSERT OR IGNORE INTO weather VALUES (?,?)",
(self.milliseconds(received), self.dump(payload)),
)
return digest
def remember_long(
self, signature: str, issue: pd.Timestamp, rows: list[tuple[pd.Timestamp, dict[str, Any]]]
) -> None:
"""Freeze an hourly snapshot per future target, bounding archive growth."""
issued = self.milliseconds(issue)
bucket = self.milliseconds(issue.floor("h"))
with self.connection() as db:
for timestamp, payload in rows:
target = self.milliseconds(timestamp)
if not issued < target <= issued + 48 * 3600000:
continue
db.execute(
"INSERT OR IGNORE INTO longforecasts VALUES (?,?,?,?)",
(
signature,
bucket,
target,
self.dump({**payload, "issued_ms": issued, "response_version": 1}),
),
)
def long_rows(
self, signature: str, start: pd.Timestamp, cutoff: pd.Timestamp
) -> list[dict[str, Any]]:
"""Read issued long-horizon candidates, with no reconstructed past weather."""
with self.connection() as db:
rows = db.execute(
"SELECT * FROM longforecasts WHERE signature=? AND target_ms>=? AND target_ms<? ORDER BY target_ms,issue_ms",
(signature, self.milliseconds(start), self.milliseconds(cutoff)),
).fetchall()
return [{**dict(row), **json.loads(row["payload"]), "payload": None} for row in rows]
def resolve(self, signature: str, observed: pd.DataFrame, now: pd.Timestamp) -> None:
"""Attach first complete observations, preserving the original forecast."""
updated = self.milliseconds(now)
with self.connection() as db:
for timestamp, values in observed.iterrows():
target = self.milliseconds(cast(pd.Timestamp, timestamp))
if target + 900000 > updated:
continue
if (
pd.notna(values.actual_w)
and np.isfinite(float(values.actual_w))
and pd.notna(values.available_at)
):
if self.milliseconds(values.available_at) <= updated:
db.execute(
"UPDATE forecasts SET actual_w=?,updated_ms=? WHERE signature=? AND target_ms=? AND actual_w IS NULL",
(float(values.actual_w), updated, signature, target),
)
if pd.isna(values.reference_actual_w) or not np.isfinite(
float(values.reference_actual_w)
):
continue
availability = self.milliseconds(values.available_at)
if availability > updated or target + 900000 > updated:
continue
actual = (
float(values.actual_w)
if pd.notna(values.actual_w) and np.isfinite(float(values.actual_w))
else None
)
db.execute(
"""UPDATE forecasts SET reference_actual_w=?,actual_w=?,available_ms=?,updated_ms=?
WHERE signature=? AND target_ms=? AND reference_actual_w IS NULL""",
(
float(values.reference_actual_w),
actual,
availability,
updated,
signature,
target,
),
)
if actual is not None:
db.execute(
"UPDATE forecasts SET actual_w=?,updated_ms=? WHERE signature=? AND target_ms=? AND actual_w IS NULL AND reference_actual_w IS NOT NULL",
(actual, updated, signature, target),
)
def training(self, signature: str, cutoff: pd.Timestamp, days: int = 21) -> pd.DataFrame:
"""Return archived inputs with independently available reference targets."""
end = self.milliseconds(cutoff)
with self.connection() as db:
rows = db.execute(
"SELECT * FROM forecasts WHERE signature=? AND issue_ms>=? AND available_ms<=? ORDER BY issue_ms,target_ms",
(signature, end - days * 86400000, end),
).fetchall()
result = []
for row in rows:
values = json.loads(row["payload"])
if values.get("features") is None or not values.get("reference_sky_w"):
continue
result.append(
{
**values["features"],
"issue": pd.Timestamp(row["issue_ms"], unit="ms", tz="UTC"),
"time": pd.Timestamp(row["target_ms"], unit="ms", tz="UTC"),
"lead_minutes": row["lead_minutes"],
"target_available_at": pd.Timestamp(row["available_ms"], unit="ms", tz="UTC"),
"target_residual": (
row["reference_actual_w"] - values["reference_calibrated_w"]
)
/ values["reference_sky_w"],
"reference_sky_w": values["reference_sky_w"],
}
)
return pd.DataFrame(result)
def baselines(
self, signature: str, start: pd.Timestamp, cutoff: pd.Timestamp
) -> list[tuple[pd.Timestamp, dict[str, Any]]]:
"""Return genuine issued physical baselines only, never revised weather."""
with self.connection() as db:
rows = db.execute(
"SELECT * FROM baselines WHERE signature=? AND target_ms>=? AND target_ms<? AND issue_ms<=target_ms ORDER BY target_ms",
(signature, self.milliseconds(start), self.milliseconds(cutoff)),
).fetchall()
return [
(pd.Timestamp(row["target_ms"], unit="ms", tz="UTC"), json.loads(row["payload"]))
for row in rows
]
def comparisons(
self, signature: str, start_ms: int, end_ms: int, limit: int = 2000, since_ms: int = 0
) -> list[dict[str, Any]]:
"""Export bounded matched forecasts, keeping issue time and lead for Grafana."""
with self.connection() as db:
rows = db.execute(
"SELECT * FROM forecasts WHERE signature=? AND target_ms>=? AND target_ms<? AND updated_ms>=? AND (reference_actual_w IS NOT NULL OR actual_w IS NOT NULL) ORDER BY updated_ms,id LIMIT ?",
(signature, start_ms, end_ms, since_ms, limit),
).fetchall()
return [{**dict(row), **json.loads(row["payload"]), "payload": None} for row in rows]
def validation_rows(
self, signature: str, start: pd.Timestamp, end: pd.Timestamp
) -> list[dict[str, Any]]:
"""Read the complete validation window; public export pagination is separate."""
with self.connection() as db:
rows = db.execute(
"SELECT issue_ms,target_ms,lead_minutes,payload FROM forecasts WHERE signature=? AND target_ms>=? AND target_ms<? ORDER BY target_ms,issue_ms",
(signature, self.milliseconds(start), self.milliseconds(end)),
).fetchall()
return [{**dict(row), **json.loads(row["payload"]), "payload": None} for row in rows]
def metric_rows(
self, signature: str, start: pd.Timestamp, end: pd.Timestamp
) -> list[dict[str, Any]]:
"""Full-window scalar projection avoids decoding every feature/model payload."""
with self.connection() as db:
rows = db.execute(
"""SELECT target_ms,lead_minutes,actual_w,reference_actual_w,
COALESCE(json_extract(payload,'$.algorithm_version'),1) AS algorithm_version,
json_extract(payload,'$.plant_raw_w') AS plant_raw_w,
json_extract(payload,'$.plant_calibrated_w') AS plant_calibrated_w,
json_extract(payload,'$.plant_local_w') AS plant_local_w,
json_extract(payload,'$.plant_adaptive_w') AS plant_adaptive_w,
json_extract(payload,'$.reference_raw_w') AS reference_raw_w,
json_extract(payload,'$.reference_calibrated_w') AS reference_calibrated_w,
json_extract(payload,'$.reference_local_w') AS reference_local_w,
json_extract(payload,'$.reference_adaptive_w') AS reference_adaptive_w
FROM forecasts WHERE signature=? AND target_ms>=? AND target_ms<?
AND (actual_w IS NOT NULL OR reference_actual_w IS NOT NULL)""",
(signature, self.milliseconds(start), self.milliseconds(end)),
).fetchall()
return [dict(row) for row in rows]
def daily_metrics(
self, signature: str, start: pd.Timestamp, end: pd.Timestamp, timezone: str
) -> list[dict[str, Any]]:
"""Evaluate frozen pre-midnight forecasts only against complete local days."""
begin, until = self.milliseconds(start), self.milliseconds(end)
with self.connection() as db:
rows = db.execute(
"""SELECT b.target_ms,b.issue_ms,b.payload,o.actual_w FROM baselines b
LEFT JOIN (SELECT target_ms,MAX(actual_w) actual_w FROM forecasts
WHERE signature=? AND target_ms>=? AND target_ms<? GROUP BY target_ms) o
ON b.target_ms=o.target_ms WHERE b.signature=? AND b.target_ms>=? AND b.target_ms<? ORDER BY b.target_ms""",
(signature, begin, until, signature, begin, until),
).fetchall()
days: dict[str, list[dict[str, Any]]] = {}
for row in rows:
time = pd.Timestamp(row["target_ms"], unit="ms", tz="UTC").tz_convert(timezone)
days.setdefault(str(time.date()), []).append(
{**dict(row), **json.loads(row["payload"])}
)
result = []
for day, part in days.items():
left = pd.Timestamp(day, tz=timezone)
right = left + pd.DateOffset(days=1)
expected = int((right - left).total_seconds() / 900)
complete = (
right <= end
and len(part) == expected
and all(
r["actual_w"] is not None and r["issue_ms"] < self.milliseconds(left)
for r in part
)
)
result.append(
dict(
day=day,
complete=bool(complete),
slots=len(part),
expected_slots=expected,
observed_slots=sum(r["actual_w"] is not None for r in part),
actual_kwh=sum(r["actual_w"] for r in part) * 0.25 / 1000 if complete else None,
**{
method + "_kwh": sum(r["plant_" + method + "_w"] for r in part)
* 0.25
/ 1000
if complete
and all(r.get("plant_" + method + "_w") is not None for r in part)
else None
for method in ("raw", "calibrated", "local", "adaptive")
},
)
)
return result
def latest(self, start_ms: int) -> list[dict[str, Any]]:
"""Current public curve, distinct from the immutable validation archive."""
with self.connection() as db:
rows = db.execute(
"SELECT * FROM latest WHERE target_ms>=? ORDER BY target_ms LIMIT 1600", (start_ms,)
).fetchall()
return [
{
"target_ms": row["target_ms"],
"issue_ms": row["issue_ms"],
**json.loads(row["payload"]),
}
for row in rows
]
def prune(self, cutoff: pd.Timestamp, days: int = 60) -> None:
"""Bound only this provider's private archive; ordinary EOS data is untouched."""
oldest = self.milliseconds(cutoff) - days * 86400000
with self.connection() as db:
for statement in (
"DELETE FROM samples WHERE time_ms<?",
"DELETE FROM forecasts WHERE target_ms<?",
"DELETE FROM baselines WHERE target_ms<?",
"DELETE FROM latest WHERE target_ms<?",
"DELETE FROM weather WHERE issue_ms<?",
"DELETE FROM longforecasts WHERE target_ms<?",
):
db.execute(statement, (oldest,))
src/akkudoktoreos/prediction/pvakkuadapttrend.py
+248
View File
@@ -0,0 +1,248 @@
"""Causal, lead-specific learning of local PV residual persistence and trends.
Training needs archived features from genuinely issued forecasts, plus the
later availability of their independent reference measurements. A revised
weather history cannot substitute for that archive.
"""
from dataclasses import dataclass, field
from typing import cast
import numpy as np
import pandas as pd
from akkudoktoreos.prediction.pvakkuadaptmodel import _aware, _aware_index
LEVEL_FEATURES = ("prior_residual", "level", "weather_change", "intercept")
TREND_FEATURES = ("ramp", "slope", "volatility", "consistent_ramp")
FEATURES = LEVEL_FEATURES + TREND_FEATURES
def local_trend_features(
history: pd.DataFrame,
future_reference: pd.Series,
future_reference_sky: pd.Series,
cutoff: pd.Timestamp,
reference_interval_minutes: int = 15,
) -> pd.DataFrame:
"""Describe four complete past intervals without reading future actuals.
Slopes describe deviations from the current weather model, normalized by the
reference's clear-sky power. They therefore exclude its regular daily solar
curve. All future quantities must come from a weather run available at cutoff.
An invalid latest interval disables features instead of selecting older values.
"""
cutoff = _aware(cutoff)
future_index = _aware_index(future_reference.index)
if reference_interval_minutes not in (5, 15):
raise ValueError("Reference resolution must be 5 or 15 minutes.")
_aware_index(history.index, reference_interval_minutes)
if history.index.has_duplicates or future_index.has_duplicates:
raise ValueError("Distinct reference and forecast timestamps required.")
result = pd.DataFrame(np.nan, index=future_index, columns=list(FEATURES))
result["feature_available_at"] = pd.Series(
pd.NaT, index=future_index, dtype="datetime64[ns, UTC]"
)
if history.empty:
return result
available = pd.to_datetime(history["available_at"], utc=True)
recent = (
history[
(available <= cutoff)
& (history.index + pd.Timedelta(minutes=reference_interval_minutes) <= cutoff)
& (history.index >= cutoff - pd.Timedelta(hours=2))
]
.sort_index()
.tail(4)
)
if len(recent) != 4:
return result
reference_index = _aware_index(recent.index, reference_interval_minutes)
interval = pd.Timedelta(minutes=reference_interval_minutes)
if not ((reference_index[1:] - reference_index[:-1]) == interval).all():
return result
if cutoff - (recent.index[-1] + interval) > pd.Timedelta(minutes=30):
return result
values = recent[["calibrated_power", "measured_power", "clear_sky_power"]]
if not np.isfinite(values.to_numpy()).all():
return result
if (values[["calibrated_power", "measured_power"]] < 0).any().any():
return result
if (values.clear_sky_power < 150).any():
return result
if "eligible" in recent and not recent.eligible.fillna(False).all():
return result
residual = (
((values.measured_power - values.calibrated_power) / values.clear_sky_power)
.clip(-1.5, 1.5)
.to_numpy(dtype=float)
)
level = float(np.dot(residual[-2:], [0.35, 0.65]))
changes = np.diff(residual) * 15 / reference_interval_minutes
ramp = float(changes[-1])
slope = float(np.dot(residual, [-0.3, -0.1, 0.1, 0.3]) * 15 / reference_interval_minutes)
volatility = float(np.std(changes))
consistent = ramp if np.sign(changes[-1]) == np.sign(changes[-2]) else 0.0
current_weather = float(values.calibrated_power.iloc[-1] / values.clear_sky_power.iloc[-1])
sky = future_reference_sky.reindex(future_index)
lead = (future_index - cutoff).total_seconds().to_numpy() / 60
age = (cutoff - (recent.index[-1] + interval / 2)).total_seconds() / 60
decay = np.exp2(-np.maximum(0, lead + age) / 30)
taper = np.clip(2 * (1 - lead / 120), 0, 1)
valid = (
(lead >= 0)
& (lead < 120)
& (future_index.tz_convert(cutoff.tz).date == cutoff.date())
& sky.ge(150).to_numpy()
& np.isfinite(future_reference.to_numpy())
& future_reference.ge(0).to_numpy()
)
rows = pd.DataFrame(
{
"prior_residual": level * decay * taper,
"level": level,
"weather_change": (current_weather - future_reference / sky).clip(-1.5, 1.5),
"intercept": 1.0,
"ramp": ramp,
"slope": slope,
"volatility": volatility,
"consistent_ramp": consistent,
"feature_available_at": pd.to_datetime(recent.available_at, utc=True).max(),
},
index=future_index,
)
result.loc[valid, :] = rows.loc[valid, :]
return result
@dataclass
class LocalPVTrendModel:
"""Learn a robust correction per forecast lead, shrinking to the old nowcast.
Each daily fit uses only issued samples whose target measurements were already
available on completed earlier local days. Refit coefficients from that archive
instead of training on corrected forecasts or curtailed whole-plant production.
"""
use_trend: bool = True
timezone: str = "Europe/Berlin"
learning_days: int = 21
half_life_days: float = 14.0
min_days: int = 5
min_samples: int = 100
ridge: float = 20.0
scale_features: bool = False
coefficients: dict[int, np.ndarray] = field(default_factory=dict, init=False)
training_counts: dict[int, int] = field(default_factory=dict, init=False)
@property
def feature_names(self) -> tuple[str, ...]:
"""Features shared with the level-only comparison and optional trends."""
return FEATURES if self.use_trend else LEVEL_FEATURES
def fit(self, samples: pd.DataFrame, cutoff: pd.Timestamp) -> "LocalPVTrendModel":
"""Fit using available labels from previous days, with recency and Huber weights."""
cutoff = _aware(cutoff)
self.coefficients, self.training_counts = {}, {}
if self.ridge <= 0 or self.half_life_days <= 0:
raise ValueError("Positive regularization and recency half-life required.")
if samples.empty:
return self
issued = pd.to_datetime(samples.issue, utc=True)
target_time = pd.to_datetime(samples.time, utc=True)
available = pd.to_datetime(samples.target_available_at, utc=True)
feature_available = pd.to_datetime(samples.feature_available_at, utc=True)
today = cutoff.tz_convert(self.timezone).normalize()
target_day = target_time.dt.tz_convert(self.timezone).dt.normalize()
valid = (
(issued < target_time)
& (feature_available <= issued)
& (available >= target_time + pd.Timedelta(minutes=15))
& (available <= cutoff)
& (target_day < today)
& (issued >= cutoff - pd.Timedelta(days=self.learning_days))
& pd.Series(
np.isfinite(
samples[[*self.feature_names, "target_residual", "reference_sky_w"]].to_numpy(
dtype=float
)
).all(axis=1),
index=samples.index,
)
& samples.reference_sky_w.ge(150)
& samples.lead_minutes.between(0, 120, inclusive="left")
)
selected = samples.loc[valid].copy()
if selected.empty:
return self
selected["bucket"] = np.ceil(selected.lead_minutes / 15).astype(int) * 15
prior = np.zeros(len(self.feature_names))
prior[0] = 1.0
for bucket, part in selected.groupby("bucket"):
bucket_number = cast(int, bucket)
count = len(part)
days = (
pd.to_datetime(part.time, utc=True).dt.tz_convert(self.timezone).dt.date.nunique()
)
self.training_counts[bucket_number] = count
if count < self.min_samples or days < self.min_days:
continue
x = part[list(self.feature_names)].to_numpy(dtype=float)
y = part.target_residual.clip(-1.5, 1.5).to_numpy(dtype=float)
age = (cutoff - pd.to_datetime(part.issue, utc=True)).dt.total_seconds() / 86400
base = np.exp2(-age.to_numpy() / self.half_life_days)
base *= np.minimum(1, part.reference_sky_w.to_numpy(dtype=float) / 1500)
weight = base.copy()
scale = np.ones(len(prior))
if self.scale_features:
# Penalize standardized feature contributions. A physically
# small slope must not be suppressed merely because of its units.
scale = np.maximum(np.sqrt(np.average(x * x, axis=0, weights=base)), 1e-3)
penalty = np.diag(scale * scale) * self.ridge
coef = prior.copy()
for _ in range(4):
coef = np.linalg.solve(
x.T @ (weight[:, None] * x) + penalty, x.T @ (weight * y) + penalty @ prior
)
error = np.abs(y - x @ coef)
weight = base * np.minimum(1, 0.15 / np.maximum(error, 1e-9))
self.coefficients[bucket_number] = coef
return self
def predict(
self,
prediction: pd.Series,
clear_sky_power: pd.Series,
features: pd.DataFrame,
cutoff: pd.Timestamp,
fallback: pd.Series,
capacity_w: float = 0.0,
) -> pd.Series:
"""Transfer learned reference residuals to the plant; otherwise keep fallback."""
cutoff = _aware(cutoff)
index = _aware_index(prediction.index)
result = fallback.reindex(index).copy()
rows = features.reindex(index)
sky = clear_sky_power.reindex(index)
leads = (index - cutoff).total_seconds().to_numpy() / 60
buckets = np.ceil(leads / 15).astype(int) * 15
available = pd.to_datetime(rows.feature_available_at, utc=True)
valid = (
(leads >= 0)
& (leads < 120)
& (index.tz_convert(cutoff.tz).date == cutoff.date())
& np.isfinite(rows[list(self.feature_names)].to_numpy(dtype=float)).all(axis=1)
& (available <= cutoff).to_numpy()
& np.isfinite(sky.to_numpy())
& np.isfinite(prediction.to_numpy())
)
for bucket, coef in self.coefficients.items():
mask = valid & (buckets == bucket)
if not mask.any():
continue
residual = rows.loc[mask, list(self.feature_names)].to_numpy(dtype=float) @ coef
residual = np.clip(residual, -1.5, 1.5)
# Smoothly remove any learned constant near the finite horizon.
taper = np.clip((120 - leads[mask]) / 30, 0, 1)
result.loc[mask] = prediction.loc[mask] + sky.loc[mask].clip(lower=0) * residual * taper
return result.clip(lower=0, upper=capacity_w if capacity_w > 0 else None)
src/akkudoktoreos/prediction/pvforecast.py
+66
View File
@@ -6,6 +6,9 @@ from pydantic import Field, computed_field, field_validator, model_validator
from akkudoktoreos.config.configabc import SettingsBaseModel
from akkudoktoreos.core.coreabc import get_prediction
from akkudoktoreos.prediction.pvakkuadapt import (
PVForecastAdaptiveAkkudoktorCommonSettings,
)
from akkudoktoreos.prediction.pvforecastabc import PVForecastProvider
from akkudoktoreos.prediction.pvforecastakkudoktorlocal import (
PVForecastAkkudoktorLocalCommonSettings,
@@ -32,6 +35,7 @@ def pvforecast_provider_ids() -> list[str]:
# Prediction may not be initialized
# Return at least provider used in example
return [
"PVForecastAdaptiveAkkudoktor",
"PVForecastAkkudoktor",
"PVForecastForecastSolar",
"PVForecastHomeAssistant",
@@ -52,6 +56,54 @@ def pvforecast_provider_ids() -> list[str]:
class PVForecastPlaneSetting(SettingsBaseModel):
"""PV Forecast Plane Configuration."""
name: Optional[str] = Field(
default=None, description="Readable PV string name.", examples=[None]
)
measurement_key: Optional[str] = Field(
default=None,
description="Matching EOS power measurement key for adaptive learning.",
examples=[None],
)
nowcast_reference: bool = Field(
default=False, description="Use this independent array as a local weather reference."
)
curtailment_sensitive: bool = Field(
default=False,
description="Battery-full curtailment possible; exclude from unrestricted weather learning.",
)
parameters_estimated: bool = Field(
default=False,
description="Geometry and peak power are effective fitted model estimates, not verified nameplate data.",
)
correction_group: Optional[str] = Field(
default=None,
examples=[None],
min_length=1,
max_length=128,
description="Optional group of planes sharing short-term validation. Unset independent references share one group; other planes are separate.",
)
curtailment_key: Optional[str] = Field(
default=None,
examples=[None],
min_length=1,
max_length=128,
description="Optional measured flag: 0 explicitly unrestricted, 1 curtailed. Missing or stale flags never authorize learning a curtailment-sensitive plane.",
)
learning_guard_key: Optional[str] = Field(
default=None,
examples=[None],
min_length=1,
max_length=128,
description="Optional site-specific positive learning guard, e.g. battery SoC, as an alternative to a binary curtailment flag.",
)
learning_guard_max: Optional[float] = Field(
default=None,
examples=[None],
allow_inf_nan=False,
description="Learn only while every supported guard value is below this bound. This is an explicit site assumption, not automatic curtailment detection.",
)
learning_guard_max_gap_seconds: int = Field(default=600, ge=1, le=3600)
# latitude: Optional[float] = Field(default=None, json_schema_extra={ "description": "Latitude in decimal degrees, between -90 and 90, north is positive (ISO 19115) (°)" })
surface_tilt: Optional[float] = Field(
default=30.0,
@@ -168,6 +220,15 @@ class PVForecastPlaneSetting(SettingsBaseModel):
@model_validator(mode="after")
def validate_list_length(self) -> Self:
if bool(self.learning_guard_key) != (self.learning_guard_max is not None):
raise ValueError("A learning guard requires both a key and an explicit upper bound.")
if self.curtailment_key and self.learning_guard_key:
raise ValueError("Use either a curtailment flag or a site-specific learning guard.")
if self.measurement_key and self.measurement_key in (
self.curtailment_key,
self.learning_guard_key,
):
raise ValueError("Power and learning-status measurements must use distinct keys.")
# Check if either attribute is set and add to active planes
if self.trackingtype == 2:
# Tilt angle from horizontal plane is ignored for two-axis tracking.
@@ -213,6 +274,11 @@ class PVForecastCommonSettings(SettingsBaseModel):
},
)
adaptive: PVForecastAdaptiveAkkudoktorCommonSettings = Field(
default_factory=PVForecastAdaptiveAkkudoktorCommonSettings,
description="Adaptive Akkudoktor: configure string measurements and independent reference flags on the planes.",
)
pvforecastimport: PVForecastImportCommonSettings = Field(
default_factory=PVForecastImportCommonSettings,
json_schema_extra={"description": "PV forecast import provider settings"},
src/akkudoktoreos/server/eos.py
+2
View File
@@ -80,6 +80,7 @@ from akkudoktoreos.server.rest.error import (
register_problem_handlers,
)
from akkudoktoreos.server.rest.measurement import router as measurement_router
from akkudoktoreos.server.rest.pvakkuadaptrest import router as pvadaptive_router
from akkudoktoreos.server.rest.starteosdash import supervise_eosdash
from akkudoktoreos.server.retentionmanager import RetentionManager
from akkudoktoreos.server.server import (
@@ -262,6 +263,7 @@ The genetic optimization API fields were renamed from German to English. For bac
# ----------------------
app.include_router(measurement_router)
app.include_router(pvadaptive_router)
register_problem_handlers(app)
src/akkudoktoreos/server/rest/pvakkuadaptrest.py
+93
View File
@@ -0,0 +1,93 @@
"""Learning status, issued comparisons and bounded raw PV ingestion for Grafana."""
import asyncio
from typing import Any
import pandas as pd
from fastapi import APIRouter, Body, HTTPException, Query
from akkudoktoreos.core.coreabc import get_prediction
from akkudoktoreos.measurement.quality import MeasurementSample
from akkudoktoreos.prediction.pvakkuadapt import (
PVForecastAdaptiveAkkudoktor,
)
router = APIRouter(prefix="/v1/prediction/pv-adaptive", tags=["prediction"])
def provider() -> PVForecastAdaptiveAkkudoktor:
"""Resolve the registered singleton without creating a separate predictor."""
return next(
p for p in get_prediction().providers if isinstance(p, PVForecastAdaptiveAkkudoktor)
)
@router.get("/status")
async def status() -> dict[str, Any]:
"""Status contains honest cold-start counts and matched per-lead error metrics."""
store = await asyncio.to_thread(provider().store)
value = await asyncio.to_thread(store.metadata, "status")
return value or dict(
state="initializing",
state_code=0,
training_days=0,
trained_leads=0,
message="Noch keine adaptive Prognose ausgegeben.",
metrics=[],
)
@router.put("/samples")
async def samples(values: list[MeasurementSample] = Body(max_length=10000)) -> dict[str, int]:
"""Store original power readings in the bounded adaptive archive, without zero filling."""
instance = provider()
allowed = instance._sample_keys()
if any(sample.key not in allowed for sample in values):
raise HTTPException(422, "Measurement key is not assigned to an adaptive PV string.")
if any(sample.value is not None and sample.value < 0 for sample in values):
raise HTTPException(422, "PV production must not be negative.")
flags = {
plane.curtailment_key for plane in instance._learning_planes() if plane.curtailment_key
}
if any(sample.key in flags and sample.value not in (None, 0, 1) for sample in values):
raise HTTPException(
422, "Curtailment flags must be 0 (unrestricted), 1 (curtailed), or null."
)
try:
store = await asyncio.to_thread(instance.store)
updated = await asyncio.to_thread(store.import_samples, values, pd.Timestamp.now(tz="UTC"))
return dict(updated=updated)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@router.get("/comparisons")
async def comparisons(
days: int = Query(default=2, ge=1, le=30),
limit: int = Query(default=2000, ge=1, le=20000),
since_ms: int = Query(default=0, ge=0),
) -> list[dict[str, Any]]:
"""Export genuinely issued forecasts with later measured truth, issuance and lead."""
instance = provider()
store = await asyncio.to_thread(instance.store)
state = await asyncio.to_thread(store.metadata, "status")
if not state:
return []
now = pd.Timestamp.now(tz="UTC")
return await asyncio.to_thread(
store.comparisons,
state["signature"],
store.milliseconds(now - pd.Timedelta(days=days)),
store.milliseconds(now),
limit,
since_ms,
)
@router.get("/latest")
async def latest() -> list[dict[str, Any]]:
"""Latest complete output curve; validation uses the separate immutable archive."""
instance = provider()
store = await asyncio.to_thread(instance.store)
start = pd.Timestamp.now(tz=instance.config.general.timezone).normalize()
return await asyncio.to_thread(store.latest, store.milliseconds(start))
tests/test_database.py
+24 -1
View File
@@ -10,7 +10,7 @@ import shutil
import tempfile
import time
from pathlib import Path
from typing import AsyncIterator, Optional, Type
from typing import Any, AsyncIterator, Optional, Type, cast
import pytest
import pytest_asyncio
@@ -876,3 +876,26 @@ class TestPerformance:
assert loaded == n
print(f"\nLoaded {n} records in {load_duration:.2f}s "
f"({n / load_duration:.0f} rec/s)")
@pytest.mark.asyncio
async def test_sqlite_bulk_operations_rollback_as_one_transaction(config_eos):
import sqlite3
config_eos.database.provider = "SQLite"
db = get_database()
await db.open()
try:
await db.save_records([(b"first", b"original"), (b"second", b"other")], namespace="atomic")
with pytest.raises(sqlite3.ProgrammingError):
await db.save_records(
[(b"first", b"changed"), (b"bad", cast(Any, object()))], namespace="atomic"
)
assert dict([row async for row in db.iterate_records(namespace="atomic")]) == {
b"first": b"original",
b"second": b"other",
}
with pytest.raises(sqlite3.ProgrammingError):
await db.delete_records([b"first", cast(Any, object())], namespace="atomic")
assert await db.count_records(namespace="atomic") == 2
finally:
await db.close()
tests/test_prediction.py
+16 -8
View File
@@ -27,6 +27,9 @@ from akkudoktoreos.prediction.prediction import (
Prediction,
PredictionCommonSettings,
)
from akkudoktoreos.prediction.pvakkuadapt import (
PVForecastAdaptiveAkkudoktor,
)
from akkudoktoreos.prediction.pvforecastakkudoktor import PVForecastAkkudoktor
from akkudoktoreos.prediction.pvforecastforecastsolar import PVForecastForecastSolar
from akkudoktoreos.prediction.pvforecasthomeassistant import PVForecastHomeAssistant
@@ -75,6 +78,7 @@ def forecast_providers():
LoadImport(),
LoadVrm(),
PVForecastAkkudoktor(),
PVForecastAdaptiveAkkudoktor(),
PVForecastForecastSolar(),
PVForecastHomeAssistant(),
PVForecastImport(),
@@ -138,19 +142,23 @@ def test_provider_sequence(prediction):
assert isinstance(prediction.providers[21], LoadImport)
assert isinstance(prediction.providers[22], LoadVrm)
assert isinstance(prediction.providers[23], PVForecastAkkudoktor)
assert isinstance(prediction.providers[24], PVForecastForecastSolar)
assert isinstance(prediction.providers[25], PVForecastHomeAssistant)
assert isinstance(prediction.providers[26], PVForecastImport)
assert isinstance(prediction.providers[27], PVForecastPVLib)
assert isinstance(prediction.providers[28], PVForecastPVNode)
assert isinstance(prediction.providers[29], PVForecastSolcast)
assert isinstance(prediction.providers[30], PVForecastVrm)
assert isinstance(prediction.providers[24], PVForecastAdaptiveAkkudoktor)
assert isinstance(prediction.providers[25], PVForecastForecastSolar)
assert isinstance(prediction.providers[26], PVForecastHomeAssistant)
assert isinstance(prediction.providers[27], PVForecastImport)
assert isinstance(prediction.providers[28], PVForecastPVLib)
assert isinstance(prediction.providers[29], PVForecastPVNode)
assert isinstance(prediction.providers[30], PVForecastSolcast)
assert isinstance(prediction.providers[31], PVForecastVrm)
def test_provider_by_id(prediction, forecast_providers):
"""Test that provider_by_id method returns the correct provider."""
for provider in forecast_providers:
assert prediction.provider_by_id(provider.provider_id()).provider_id() == provider.provider_id()
assert (
prediction.provider_by_id(provider.provider_id()).provider_id()
== provider.provider_id()
)
def test_prediction_repr(prediction):
tests/test_pvakkuadapt.py
+769
View File
@@ -0,0 +1,769 @@
"""Issued forecast persistence, causal arrivals, RC provider wiring and data gaps."""
import json
from typing import Any
from unittest.mock import patch
import numpy as np
import pandas as pd
import pytest
from akkudoktoreos.measurement.quality import MeasurementSample
from akkudoktoreos.prediction.pvakkuadapt import (
PVForecastAdaptiveAkkudoktor,
PVForecastAdaptiveAkkudoktorCommonSettings,
)
from akkudoktoreos.prediction.pvakkuadaptstore import PVLearningStore
from akkudoktoreos.utils.datetimeutil import to_datetime
def sample(key, time, value):
return MeasurementSample(
key=key, date_time=pd.Timestamp(time).to_pydatetime(), value=float(value)
)
def test_sample_arrival_is_preserved_and_revisions_get_later_availability(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
point = sample("micro", "2026-09-01T12:00Z", 1000)
first, later = pd.Timestamp("2026-09-01T12:01Z"), pd.Timestamp("2026-09-01T13:00Z")
store.import_samples([point], first)
store.import_samples([point], later)
assert store.samples("micro", pd.Timestamp("2026-09-01T11:00Z"), first)[0][
"available_ms"
] == store.milliseconds(first)
changed = sample("micro", point.date_time, 2000)
store.import_samples([changed], later)
assert store.samples("micro", pd.Timestamp("2026-09-01T11:00Z"), first) == []
def test_future_sample_rejected_before_storage(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
now = pd.Timestamp("2026-09-01T12:00Z")
with pytest.raises(ValueError, match="Future"):
store.import_samples([sample("micro", now + pd.Timedelta(minutes=1), 1)], now)
def test_issued_forecast_immutable_and_day_ahead_baseline_freezes(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
target = pd.Timestamp("2026-09-02T08:00Z")
before = target - pd.Timedelta(minutes=30)
original = dict(plant_raw_w=1000, features=None)
store.remember("a", before, [(target, original)], "Europe/Berlin")
store.remember("a", before, [(target, dict(plant_raw_w=9999, features=None))], "Europe/Berlin")
with store.connection() as db:
row = db.execute("SELECT payload FROM forecasts").fetchone()
assert json.loads(row[0])["plant_raw_w"] == 1000
assert store.baselines("a", before, target + pd.Timedelta(hours=1))[0][1]["plant_raw_w"] == 1000
@pytest.fixture
def provider(config_eos):
config_eos.merge_settings_from_dict(
dict(
general=dict(latitude=50.8588, longitude=7.3747, timezone="Europe/Berlin"),
pvforecast=dict(
provider="PVForecastAdaptiveAkkudoktor",
adaptive=dict(regional_weather_model=None, weather_models=["icon_seamless"]),
planes=[
dict(
name="Hybrid",
peakpower=3,
surface_tilt=20,
surface_azimuth=180,
measurement_key="hybrid",
curtailment_sensitive=True,
),
dict(
name="Micro",
peakpower=1,
surface_tilt=20,
surface_azimuth=180,
measurement_key="micro",
nowcast_reference=True,
),
],
),
measurement=dict(
keys=["hybrid", "micro"],
channels={
key: dict(
quantity="power", unit="W", integration_method="hold", max_gap_seconds=600
)
for key in ("hybrid", "micro")
},
),
)
)
return PVForecastAdaptiveAkkudoktor()
def test_measurement_closing_samples_and_ingestion_arrivals_are_causal(provider):
issue = pd.Timestamp("2026-09-18T12:20:00Z")
values = [
sample(key, time, 200)
for key in ("hybrid", "micro")
for time in pd.date_range(
issue.floor("15min") - pd.Timedelta(minutes=30), issue, freq="5min"
)
]
provider.store().import_samples(values, issue)
result = provider._measurements(issue, 1)
assert result.loc["2026-09-18T12:00Z", "reference_actual_w"] == pytest.approx(200)
before_arrival = provider._measurements(issue - pd.Timedelta(minutes=1), 1)
assert before_arrival.reference_actual_w.isna().all()
def test_missing_reference_does_not_become_zero_or_borrow_hybrid_power(provider):
issue = pd.Timestamp("2026-09-18T12:20:00Z")
values = [
sample("hybrid", time, 200)
for time in pd.date_range(issue - pd.Timedelta(hours=1), issue, freq="5min")
]
provider.store().import_samples(values, issue)
result = provider._measurements(issue, 1)
assert result.reference_actual_w.isna().all()
assert result.actual_w.isna().all()
def test_hybrid_never_trains_weather_gain(provider):
issue = pd.Timestamp("2026-09-18T12:20:00Z")
state = provider._models(provider._signature(), issue)
assert state["gains"]["hybrid"]["gain"] == 1
assert state["gains"]["hybrid"]["days"] == 0
assert state["coefficients"] == {}
assert provider.store().metadata("daily_models") == state
@pytest.mark.asyncio
async def test_cold_provider_generates_standard_power_and_status(provider):
from test_pvforecastakkudoktorlocal import synthetic_openmeteo
original = to_datetime
issue = pd.Timestamp("2025-06-15T12:05:00+02:00")
def clock(*args, **kwargs):
return (
original(issue.to_pydatetime())
if not args and kwargs.get("in_timezone")
else original(*args, **kwargs)
)
with (
patch("akkudoktoreos.prediction.pvakkuadapt.to_datetime", side_effect=clock),
patch.object(type(provider), "_request_local_forecast", return_value=synthetic_openmeteo()),
):
await provider._update_data()
status = provider.store().metadata("status")
assert status["state"] == "reference_missing"
assert status["training_days"] == 0
assert status["trained_leads"] == 0
assert status["references"] == ["micro"]
assert provider.record_keys.count("pvforecast_ac_power") == 1
future = await provider.key_to_series("pvforecast_ac_power")
assert future.max() > 500
assert np.isfinite(future.to_numpy()).all()
for row in provider.store().latest(provider.store().milliseconds(issue)):
assert row["plant_adaptive_w"] == pytest.approx(row["plant_calibrated_w"])
@pytest.mark.parametrize(
"patch",
[
dict(backend="remote"),
dict(resolution_minutes=60),
dict(calibration_enabled=True),
dict(min_training_days=10, learning_days=7),
],
)
def test_adaptive_settings_reject_inconsistent_modes(patch):
with pytest.raises(ValueError):
PVForecastAdaptiveAkkudoktorCommonSettings(**patch)
def test_pruning_is_scoped_to_private_archive(tmp_path):
path = tmp_path / "learning.sqlite3"
store = PVLearningStore(path)
store.import_samples(
[sample("micro", "2026-08-01T12:00Z", 100)], pd.Timestamp("2026-08-01T12:00Z")
)
store.metadata("profiles", dict(micro=dict(response=[1] * 46)))
store.prune(pd.Timestamp("2026-10-01T12:00Z"), 31)
assert (
store.samples("micro", pd.Timestamp("2026-07-01T12:00Z"), pd.Timestamp("2026-10-01T12:00Z"))
== []
)
assert store.metadata("profiles")
def test_labels_are_unavailable_before_true_receipt_and_actual_can_arrive_later(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
target = pd.Timestamp("2026-09-18T12:15Z")
issue = target - pd.Timedelta(minutes=15)
payload = dict(
features=dict(level=0.1), reference_sky_w=1000, reference_calibrated_w=500, plant_raw_w=1000
)
store.remember("a", issue, [(target, payload)], "Europe/Berlin")
received = target + pd.Timedelta(minutes=25)
observations = pd.DataFrame(
dict(reference_actual_w=[600], actual_w=[np.nan], available_at=[received]), index=[target]
)
store.resolve("a", observations, received)
assert store.training("a", received - pd.Timedelta(seconds=1)).empty
assert store.training("a", received).iloc[0].target_residual == pytest.approx(0.1)
observations["actual_w"] = 1500
store.resolve("a", observations, received + pd.Timedelta(minutes=2))
row = store.comparisons("a", store.milliseconds(issue), store.milliseconds(received))[0]
assert row["actual_w"] == 1500
assert row["reference_actual_w"] == 600
assert row["plant_raw_w"] == 1000
assert row["available_ms"] == store.milliseconds(received)
def test_regional_missing_tail_falls_back_without_filling_radiation_gaps(provider):
from test_pvforecastakkudoktorlocal import synthetic_openmeteo
data = synthetic_openmeteo()
raw = data["minutely_15"]
from akkudoktoreos.prediction.pvforecastakkudoktorlocal import OPENMETEO_VARIABLES
provider.config.pvforecast.adaptive.regional_weather_model = "icon_d2"
for variable in OPENMETEO_VARIABLES:
raw[variable + "_icon_d2"] = [None] * len(raw["time"])
frame = provider._weather_frame(data)
assert (frame.regional_weight == 0).all()
for variable in OPENMETEO_VARIABLES[3:]:
for member in list(raw):
if member == variable or member.startswith(variable + "_"):
raw[member][50] = None
frame = provider._weather_frame(data)
assert frame.iloc[50][list(OPENMETEO_VARIABLES[3:])].isna().all()
@pytest.mark.asyncio
async def test_native_measurements_keep_first_seen_availability(provider):
from types import SimpleNamespace
issue = pd.Timestamp("2026-09-18T12:20Z")
record = SimpleNamespace(
date_time=to_datetime("2026-09-18T12:00Z"),
configured_data=dict(micro=500),
sample_quality={},
)
async def records(*args):
yield record
with patch.object(type(provider.measurement), "db_iterate_records", side_effect=records):
await provider._sync_measurements(issue)
await provider._sync_measurements(issue + pd.Timedelta(minutes=1))
assert provider.store().samples("micro", issue - pd.Timedelta(hours=1), issue)[0][
"available_ms"
] == provider.store().milliseconds(issue)
def test_persisted_live_features_train_after_five_complete_days(tmp_path):
from akkudoktoreos.prediction.pvakkuadapttrend import FEATURES, LocalPVTrendModel
store = PVLearningStore(tmp_path / "learning.sqlite3")
for day in pd.date_range("2026-09-12T08:00Z", periods=5, freq="D"):
for offset in range(24):
issue = day + pd.Timedelta(minutes=offset * 15)
target = issue + pd.Timedelta(minutes=15)
features: dict[str, Any] = {key: 0.1 for key in FEATURES}
features["intercept"] = 1.0
features["feature_available_at"] = issue
store.remember(
"a",
issue,
[
(
target,
dict(features=features, reference_sky_w=1000, reference_calibrated_w=500),
)
],
"Europe/Berlin",
)
received = target + pd.Timedelta(minutes=20)
observed = pd.DataFrame(
dict(reference_actual_w=[650], actual_w=[1000], available_at=[received]),
index=[target],
)
store.resolve("a", observed, received)
cutoff = pd.Timestamp("2026-09-18T12:00Z")
trained = LocalPVTrendModel(scale_features=True).fit(store.training("a", cutoff), cutoff)
assert trained.training_counts[15] == 120
assert 15 in trained.coefficients
assert np.isfinite(trained.coefficients[15]).all()
@pytest.fixture
def adaptive_client(provider, monkeypatch):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from akkudoktoreos.server.rest import pvakkuadaptrest
monkeypatch.setattr(pvakkuadaptrest, "provider", lambda: provider)
app = FastAPI()
app.include_router(pvakkuadaptrest.router)
with TestClient(app) as client:
yield client
def test_api_exposes_honest_cold_start_and_persists_samples(adaptive_client, provider):
provider.config.database.provider = "SQLite"
prefix = "/v1/prediction/pv-adaptive"
status = adaptive_client.get(prefix + "/status")
assert status.status_code == 200
assert status.json()["state"] == "initializing"
assert status.json()["training_days"] == 0
assert adaptive_client.get(prefix + "/latest").json() == []
assert adaptive_client.get(prefix + "/comparisons").json() == []
now = pd.Timestamp.now(tz="UTC")
reading = sample("micro", now - pd.Timedelta(minutes=5), 250)
response = adaptive_client.put(prefix + "/samples", json=[reading.model_dump(mode="json")])
assert response.status_code == 200
assert response.json() == {"updated": 1}
stored = PVLearningStore(provider.store().path)
assert (
stored.samples("micro", now - pd.Timedelta(hours=1), pd.Timestamp.now(tz="UTC"))[0]["value"]
== 250
)
@pytest.mark.parametrize("case", ["unknown", "negative", "future", "timezone_missing"])
def test_api_rejects_invalid_sample_batches_atomically(adaptive_client, provider, case):
now = pd.Timestamp.now(tz="UTC")
valid = sample("micro", now - pd.Timedelta(minutes=10), 250).model_dump(mode="json")
invalid = dict(valid)
if case == "unknown":
invalid["key"] = "not_a_pv_plane"
elif case == "negative":
invalid["value"] = -1
elif case == "future":
invalid["date_time"] = (now + pd.Timedelta(hours=1)).isoformat()
else:
invalid["date_time"] = "2026-01-01T12:00:00"
response = adaptive_client.put("/v1/prediction/pv-adaptive/samples", json=[valid, invalid])
assert response.status_code == 422
assert (
provider.store().samples("micro", now - pd.Timedelta(hours=1), pd.Timestamp.now(tz="UTC"))
== []
)
def test_curtailment_flag_requires_complete_fresh_zero_support(provider):
plane = provider.config.pvforecast.planes[0]
plane.curtailment_key = "unrestricted_flag"
issue = pd.Timestamp("2026-09-18T12:20Z")
times = pd.date_range("2026-09-18T11:30Z", issue, freq="5min")
provider.store().import_samples(
[
sample(key, t, 200 if key != "unrestricted_flag" else 0)
for key in ("hybrid", "micro", "unrestricted_flag")
for t in times
],
issue,
)
measured = provider._measurements(issue, 1)
assert bool(measured.loc["2026-09-18T12:00Z", "hybrid_eligible"])
provider.store().import_samples([sample("unrestricted_flag", "2026-09-18T12:05Z", 1)], issue)
measured = provider._measurements(issue, 1)
assert not bool(measured.loc["2026-09-18T12:00Z", "hybrid_eligible"])
assert measured.loc["2026-09-18T12:00Z", "hybrid"] == 200
def test_curtailment_flag_missing_does_not_authorize_learning(provider):
provider.config.pvforecast.planes[0].curtailment_key = "missing_flag"
issue = pd.Timestamp("2026-09-18T12:20Z")
provider.store().import_samples(
[sample("hybrid", t, 200) for t in pd.date_range("2026-09-18T11:30Z", issue, freq="5min")],
issue,
)
measured = provider._measurements(issue, 1)
assert not measured.hybrid_eligible.any()
def test_explicit_site_guard_excludes_high_values_and_requires_support(provider):
from akkudoktoreos.prediction.pvforecast import PVForecastPlaneSetting
provider.config.pvforecast.planes[0] = PVForecastPlaneSetting(
**{
**provider.config.pvforecast.planes[0].model_dump(),
"learning_guard_key": "battery_soc_fraction",
"learning_guard_max": 0.85,
"learning_guard_max_gap_seconds": 1800,
}
)
issue = pd.Timestamp("2026-09-18T12:20Z")
times = pd.date_range("2026-09-18T11:30Z", issue, freq="5min")
provider.store().import_samples(
[sample("hybrid", t, 200) for t in times]
+ [sample("battery_soc_fraction", t, 0.6) for t in times],
issue,
)
assert provider._measurements(issue, 1).loc["2026-09-18T12:00Z", "hybrid_eligible"]
provider.store().import_samples(
[sample("battery_soc_fraction", "2026-09-18T12:05Z", 0.9)], issue
)
assert not provider._measurements(issue, 1).loc["2026-09-18T12:00Z", "hybrid_eligible"]
@pytest.mark.parametrize(
"fields",
[
{"learning_guard_key": "soc"},
{"learning_guard_max": 0.9},
{"learning_guard_key": "soc", "learning_guard_max": 0.9, "curtailment_key": "flag"},
{"measurement_key": "power", "curtailment_key": "power"},
],
)
def test_invalid_learning_guard_config_rejected(fields):
from akkudoktoreos.prediction.pvforecast import PVForecastPlaneSetting
with pytest.raises(ValueError):
PVForecastPlaneSetting(**fields)
def test_new_quality_options_preserve_legacy_physical_signature(provider):
original = provider._signature()
provider.config.pvforecast.planes[0].correction_group = "new_inverter"
provider.config.pvforecast.planes[0].curtailment_key = "new_status_flag"
provider.config.pvforecast.adaptive.correction_mode = "calibrated"
assert provider._signature() == original
def test_total_channel_and_no_independent_reference_are_supported(provider):
from akkudoktoreos.measurement.measurement import MeasurementChannelSettings
provider.config.pvforecast.adaptive.total_measurement_key = "whole_plant"
provider.config.pvforecast.adaptive.total_curtailment_sensitive = True
for plane in provider.config.pvforecast.planes:
plane.measurement_key = None
provider.config.measurement.channels["whole_plant"] = MeasurementChannelSettings(
quantity="power", unit="W", integration_method="hold", max_gap_seconds=600
)
issue = pd.Timestamp("2026-09-18T12:20Z")
provider.store().import_samples(
[
sample("whole_plant", t, 500)
for t in pd.date_range("2026-09-18T11:30Z", issue, freq="5min")
],
issue,
)
measured = provider._measurements(issue, 1)
assert measured.loc["2026-09-18T12:00Z", "actual_w"] == 500
assert measured.reference_actual_w.isna().all()
assert provider._groups() == {"plant": ["whole_plant"]}
@pytest.mark.asyncio
async def test_calibrated_ac_is_limited_per_plane_and_dc_is_not_reconstructed(provider):
from test_pvforecastakkudoktorlocal import synthetic_openmeteo
issue = pd.Timestamp("2026-09-18T12:05Z")
provider.config.pvforecast.planes[1].inverter_paco = 800
index = pd.date_range(issue.floor("15min") - pd.Timedelta(hours=2), periods=20, freq="15min")
raw = pd.DataFrame(
dict(
hybrid=3000.0,
micro=800.0,
hybrid_dc=5000.0,
micro_dc=1000.0,
weather_weight=0.0,
solar_elevation=30.0,
),
index=index,
)
models = dict(
gains={
k: dict(
gain=1.5 if k == "micro" else 1.0, shape=[1.0] * 12, days=5, curtailed=k == "hybrid"
)
for k in ("hybrid", "micro")
},
coefficients={},
training_counts={},
fitted_at=issue.isoformat(),
selection={},
)
original = to_datetime
def clock(*args, **kwargs):
return (
original(issue.to_pydatetime())
if not args and kwargs.get("in_timezone")
else original(*args, **kwargs)
)
with (
patch("akkudoktoreos.prediction.pvakkuadapt.to_datetime", side_effect=clock),
patch.object(type(provider), "_physical_frames", return_value=[raw, raw.copy()]),
patch.object(type(provider), "_request_local_forecast", return_value=synthetic_openmeteo()),
patch.object(type(provider), "_models", return_value=models),
):
await provider._update_data()
forecast = await provider.key_to_series("pvforecast_ac_power")
dc = await provider.key_to_series("pvforecast_dc_power")
assert forecast.max() <= 3800
assert dc.max() == pytest.approx(6500)
def test_missing_reference_still_resolves_whole_plant_metrics(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
issue = pd.Timestamp("2026-09-01T10:00Z")
target = issue + pd.Timedelta(minutes=15)
store.remember(
"a",
issue,
[
(
target,
dict(
plant_raw_w=1000,
plant_calibrated_w=900,
plant_local_w=900,
plant_adaptive_w=900,
),
)
],
"UTC",
)
available = target + pd.Timedelta(minutes=20)
store.resolve(
"a",
pd.DataFrame(
dict(actual_w=[800], reference_actual_w=[np.nan], available_at=[available]),
index=[target],
),
available,
)
rows = store.metric_rows("a", issue, available)
assert rows[0]["actual_w"] == 800
assert (
store.comparisons("a", store.milliseconds(issue), store.milliseconds(available))[0][
"actual_w"
]
== 800
)
def test_metrics_use_full_window_independently_of_public_export_limit(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
target = pd.Timestamp("2026-09-02T12:00Z")
stamp = store.milliseconds(target)
payload = store.dump(dict(plant_raw_w=1000, reference_raw_w=100))
with store.connection() as db:
db.executemany(
"INSERT INTO forecasts(signature,issue_ms,target_ms,lead_minutes,payload,actual_w,updated_ms) VALUES (?,?,?,?,?,?,?)",
[
("a", stamp - 100000 + i, stamp, 1, payload, 500, stamp + 900000)
for i in range(20005)
],
)
start, end = target - pd.Timedelta(hours=1), target + pd.Timedelta(hours=1)
assert (
len(store.comparisons("a", store.milliseconds(start), store.milliseconds(end), limit=20000))
== 20000
)
assert len(store.metric_rows("a", start, end)) == 20005
def test_daily_fit_waits_for_previous_day_closing_samples(provider):
first = provider._models(provider._signature(), pd.Timestamp("2026-09-18T12:00Z"))
before = provider._models(provider._signature(), pd.Timestamp("2026-09-19T00:02+02:00"))
assert before["fitted_at"] == first["fitted_at"]
after = provider._models(provider._signature(), pd.Timestamp("2026-09-19T00:16+02:00"))
assert after["day"] == "2026-09-19"
assert after["after_delay"] is True
def test_day_ahead_metrics_count_dst_intervals_and_never_zero_fill(tmp_path):
store = PVLearningStore(tmp_path / "learning.sqlite3")
start = pd.Timestamp("2026-10-25", tz="Europe/Berlin")
end = start + pd.DateOffset(days=1)
targets = pd.date_range(start, end, freq="15min", inclusive="left")
payload = store.dump({f"plant_{m}_w": 2000 for m in ("raw", "calibrated", "local", "adaptive")})
with store.connection() as db:
db.executemany(
"INSERT INTO baselines VALUES (?,?,?,?)",
[
(
"a",
store.milliseconds(t),
store.milliseconds(start - pd.Timedelta(hours=1)),
payload,
)
for t in targets
],
)
db.executemany(
"INSERT INTO forecasts(signature,issue_ms,target_ms,lead_minutes,payload,actual_w,updated_ms) VALUES (?,?,?,?,?,?,?)",
[
(
"a",
store.milliseconds(t - pd.Timedelta(minutes=15)),
store.milliseconds(t),
15,
payload,
1000,
store.milliseconds(end),
)
for t in targets
],
)
rows = store.daily_metrics("a", start, end, "Europe/Berlin")
assert rows[0]["complete"] is True
assert rows[0]["expected_slots"] == 100
assert rows[0]["actual_kwh"] == 25
assert rows[0]["adaptive_kwh"] == 50
with store.connection() as db:
db.execute(
"UPDATE forecasts SET actual_w=NULL WHERE target_ms=?",
(store.milliseconds(targets[3]),),
)
assert store.daily_metrics("a", start, end, "Europe/Berlin")[0]["actual_kwh"] is None
def test_censored_calibration_uses_only_authorized_partial_days():
from akkudoktoreos.prediction.pvakkuadaptmodel import AdaptivePVModel
index = pd.DatetimeIndex(
[
t
for d in pd.date_range("2026-09-01T10:00Z", periods=5, freq="D")
for t in pd.date_range(d, periods=8, freq="15min")
]
)
history = pd.DataFrame(
dict(raw_power=1000.0, measured_power=600.0, available_at=index + pd.Timedelta(minutes=20)),
index=index,
)
cutoff = pd.Timestamp("2026-09-07T00:00Z")
model = AdaptivePVModel(min_days=5).fit(history, cutoff)
assert model.trained_days == 0
partial = AdaptivePVModel(min_days=5, allow_partial_days=True).fit(history, cutoff)
assert partial.trained_days == 5
assert partial.gain == pytest.approx(0.6)
@pytest.mark.parametrize("backend", ["SQLite", "LMDB"])
def test_eos_backend_migrates_archive_and_preserves_identity(config_eos, tmp_path, backend):
import hashlib
import sqlite3
from akkudoktoreos.prediction.pvakkuadaptstore import _DatabaseWorker
path = tmp_path / "legacy.sqlite3"
store = PVLearningStore(path)
target = pd.Timestamp("2026-09-02T08:00Z")
issue = target - pd.Timedelta(minutes=30)
store.import_samples([sample("micro", issue, 123)], issue)
store.remember("stable", issue, [(target, dict(plant_raw_w=1000))], "Europe/Berlin")
with sqlite3.connect(path) as legacy:
store._db.backup(legacy)
before = hashlib.sha256(path.read_bytes()).hexdigest()
config_eos.database.provider = backend
migrated = PVLearningStore(path)
_DatabaseWorker.run(
migrated.database.save_records([(b"other", b"untouched")], namespace="another-provider")
)
reopened = PVLearningStore(path)
assert reopened.counts() == migrated.counts()
assert reopened.storage_status()["backend"] == backend
assert reopened.samples("micro", issue, target)[0]["available_ms"] == store.milliseconds(issue)
with reopened.connection() as db:
assert dict(db.execute("SELECT * FROM forecasts").fetchone()) == dict(
store._db.execute("SELECT * FROM forecasts").fetchone()
)
reopened.remember("stable", issue, [(target, dict(plant_raw_w=9999))], "Europe/Berlin")
assert (
json.loads(reopened._db.execute("SELECT payload FROM forecasts").fetchone()[0])[
"plant_raw_w"
]
== 1000
)
reopened.prune(target + pd.Timedelta(days=61))
assert reopened.counts()["forecasts"] == 0
after_prune = PVLearningStore(path)
new_target = target + pd.Timedelta(days=62)
after_prune.remember(
"stable",
new_target - pd.Timedelta(minutes=30),
[(new_target, dict(plant_raw_w=1000))],
"Europe/Berlin",
)
assert after_prune._db.execute("SELECT id FROM forecasts").fetchone()[0] > 1
assert _DatabaseWorker.run(reopened.database.count_records(namespace="another-provider")) == 1
assert hashlib.sha256(path.read_bytes()).hexdigest() == before
_DatabaseWorker.run(reopened.database.close())
def test_nodb_is_explicitly_volatile_and_creates_no_private_file(config_eos, tmp_path):
path = tmp_path / "absent" / "learning.sqlite3"
store = PVLearningStore(path)
store.metadata("test", {"value": 1})
assert store.storage_status()["persistent"] is False
assert not path.exists()
assert PVLearningStore(path).metadata("test") is None
def test_long_snapshots_and_received_weather_are_immutable(tmp_path):
store = PVLearningStore(tmp_path / "unused.sqlite3")
issued = pd.Timestamp("2026-09-01T12:01Z")
target = issued.ceil("h") + pd.Timedelta(hours=3)
data = dict(
minutely_15=dict(
time=["2026-09-01T12:00Z", "2026-09-01T13:15Z"], shortwave_radiation=[999, 600]
)
)
digest = store.archive_weather(data, issued)
assert store.archive_weather(data, issued + pd.Timedelta(minutes=20)) == digest
assert store.counts()["weather"] == 1
payload = json.loads(store._db.execute("SELECT payload FROM weather").fetchone()[0])
assert payload["source_issue_at"] is None
assert payload["received_at"] == issued.isoformat()
assert payload["minutely_15"]["shortwave_radiation"] == [600]
store.remember_long("a", issued, [(target, {"planes": {"micro": {"calibrated": 100}}})])
store.remember_long(
"a",
issued + pd.Timedelta(minutes=5),
[(target, {"planes": {"micro": {"calibrated": 999}}})],
)
rows = store.long_rows("a", issued, target + pd.Timedelta(hours=1))
assert len(rows) == 1
assert rows[0]["planes"]["micro"]["calibrated"] == 100
assert rows[0]["issued_ms"] == store.milliseconds(issued)
def test_backend_write_failure_is_reported_and_pending_changes_retry(config_eos, tmp_path):
from unittest.mock import AsyncMock
from akkudoktoreos.prediction.pvakkuadaptstore import _DatabaseWorker
config_eos.database.provider = "SQLite"
store = PVLearningStore(tmp_path / "no-legacy.sqlite3")
with patch.object(type(store.database), "save_records", new=AsyncMock(return_value=0)):
with pytest.raises(RuntimeError, match="Incomplete"):
store.metadata("pending", {"value": 123})
assert store.metadata("pending") == {"value": 123}
assert PVLearningStore(store.path).metadata("pending") == {"value": 123}
_DatabaseWorker.run(store.database.close())
def test_long_solar_candidate_limits_ac_and_keeps_physical_dc(provider):
provider.config.pvforecast.adaptive.weather_response_enabled = False
provider.config.pvforecast.planes[0].inverter_paco = 3000
provider.config.pvforecast.planes[1].inverter_paco = 800
issue = pd.Timestamp("2026-09-18T06:00Z")
index = pd.date_range("2026-09-18T12:00Z", periods=4, freq="15min")
ac = pd.DataFrame(dict(hybrid=3000., micro=800.), index=index)
dc = pd.DataFrame(dict(hybrid=5000., micro=1200.), index=index)
response = dict(solar={key:dict(coefficients=[.5]*45) for key in ("hybrid", "micro")}, weather={})
candidates, members, dc_candidates = provider._long_candidates({}, ac, ac, dc, {}, response, issue)
assert not members
assert candidates["solar"].hybrid.max() == 3000
assert candidates["solar"].micro.max() == 800
np.testing.assert_allclose(dc_candidates["solar"].hybrid, 7500)
np.testing.assert_allclose(dc_candidates["solar"].micro, 1800)
tests/test_pvakkuadaptquality.py
+137
View File
@@ -0,0 +1,137 @@
"""Group/horizon selection on genuinely issued, independent-day predictions."""
import numpy as np
import pandas as pd
import pytest
from akkudoktoreos.prediction.pvakkuadaptquality import (
choose_correction,
select_corrections,
)
def archive():
rows, index = [], []
for day in pd.date_range("2026-09-01T08:00Z", periods=4, freq="D"):
for slot in range(12):
time = day + pd.Timedelta(minutes=slot * 15)
index.append(time)
for lead in (7, 52):
rows.append(
dict(
target_ms=int(time.timestamp() * 1000),
issue_ms=int((time - pd.Timedelta(minutes=lead)).timestamp() * 1000),
lead_minutes=lead,
groups={
"roof": {
"calibrated": 1000,
"local": 800 if lead == 7 else 1100,
"trend": 700 if lead == 7 else 1200,
}
},
)
)
observed = pd.DataFrame(
dict(
renamed_sensor=700,
renamed_sensor_eligible=True,
renamed_sensor_available=pd.DatetimeIndex(index) + pd.Timedelta(minutes=20),
),
index=pd.DatetimeIndex(index),
)
return rows, observed
def test_selects_per_group_and_horizon_with_no_site_names():
rows, observed = archive()
result = select_corrections(
rows, observed, {"roof": ["renamed_sensor"]}, pd.Timestamp("2026-09-06T00:00Z"), "UTC"
)
assert result["roof"]["15"]["method"] == "trend"
assert result["roof"]["60"]["method"] == "calibrated"
assert result["roof"]["15"]["days"] == 4
@pytest.mark.parametrize("problem", ["future_labels", "curtailed", "missing", "today"])
def test_missing_censored_or_unavailable_truth_cannot_select_complex_model(problem):
rows, observed = archive()
cutoff = pd.Timestamp("2026-09-06T00:00Z")
if problem == "future_labels":
observed.renamed_sensor_available = cutoff + pd.Timedelta(minutes=1)
if problem == "curtailed":
observed.renamed_sensor_eligible = False
if problem == "missing":
observed.renamed_sensor = np.nan
if problem == "today":
cutoff = pd.Timestamp("2026-09-01T12:00Z")
assert (
select_corrections(rows, observed, {"roof": ["renamed_sensor"]}, cutoff, "UTC")["roof"]
== {}
)
def test_repeated_polling_cannot_manufacture_sample_size():
rows, observed = archive()
rows = [r for r in rows if r["target_ms"] == rows[0]["target_ms"]] * 1000
result = select_corrections(
rows, observed, {"roof": ["renamed_sensor"]}, pd.Timestamp("2026-09-06T00:00Z"), "UTC"
)
assert result["roof"]["15"]["samples"] == 1
assert result["roof"]["15"]["method"] == "calibrated"
def test_unknown_group_and_insufficient_history_preserve_calibration():
rows, observed = archive()
result = select_corrections(
rows,
observed,
{"different_roof": ["renamed_sensor"]},
pd.Timestamp("2026-09-06T00:00Z"),
"UTC",
)
assert result["different_roof"] == {}
def test_choice_expires_after_two_hours_and_cold_start_is_simple():
issue = pd.Timestamp("2026-09-06T08:00Z")
idx = pd.date_range(issue + pd.Timedelta(minutes=15), periods=12, freq="15min")
base = pd.Series(1000.0, index=idx)
local = base * 0.8
trend = base * 0.7
selected = choose_correction(
base, local, trend, issue, {"15": {"method": "trend"}, "60": {"method": "local"}}
)
assert selected.iloc[0] == 700
assert selected.iloc[3] == 800
assert selected.iloc[-1] == 1000
pd.testing.assert_series_equal(choose_correction(base, local, trend, issue, {}), base)
def test_legacy_archive_can_only_validate_explicit_reference_sum():
rows, observed = archive()
for r in rows:
r.pop("groups")
r.update(reference_calibrated_w=1000, reference_local_w=800, reference_adaptive_w=700)
result = select_corrections(
rows,
observed,
{"roof": ["renamed_sensor"], "other": ["renamed_sensor"]},
pd.Timestamp("2026-09-06T00:00Z"),
"UTC",
reference_group="roof",
)
assert result["roof"]["15"]["method"] == "trend"
assert result["other"] == {}
def test_disabled_trend_candidate_cannot_be_selected():
rows, observed = archive()
result = select_corrections(
rows,
observed,
{"roof": ["renamed_sensor"]},
pd.Timestamp("2026-09-06T00:00Z"),
"UTC",
enabled_methods=("calibrated", "local"),
)
assert result["roof"]["15"]["method"] == "local"
tests/test_pvakkuadaptresponse.py
+124
View File
@@ -0,0 +1,124 @@
"""Causal solar response, model weather weighting and frozen-candidate selection."""
import numpy as np
import pandas as pd
from akkudoktoreos.prediction.pvakkuadaptresponse import (
fit_solar_response,
fit_weather_response,
paired_rows,
select_responses,
solar_basis,
solar_factors,
)
def test_solar_response_learns_repeatable_shape_without_learning_common_cloud_error():
index = pd.date_range("2026-09-01T06:00Z", "2026-09-10T16:00Z", freq="15min")
index = index[(index.hour >= 6) & (index.hour < 17)]
basis = solar_basis(index, 51.0, 7.4)
shade = 1 - 0.35 * basis[:, 5:20].sum(axis=1)
clouds = 0.55 + 0.4 * (np.sin(np.arange(len(index)) * 0.4) + 1) / 2
history = pd.DataFrame(
dict(
base=1000,
actual=1000 * clouds * shade,
common=clouds,
available_at=index + pd.Timedelta(minutes=20),
),
index=index,
)
cutoff = pd.Timestamp("2026-09-11T00:00Z")
fit = fit_solar_response(history, cutoff, "Europe/Berlin", 51.0, 7.4, 5)
assert fit["fitted"] and fit["days"] >= 5
future = pd.date_range("2026-09-12T06:00Z", periods=40, freq="15min")
future_basis = solar_basis(future, 51.0, 7.4)
truth = 1 - 0.35 * future_basis[:, 5:20].sum(axis=1)
predicted = solar_factors(future_basis, fit)
assert np.mean(abs(predicted - truth)) < np.mean(abs(1 - truth)) * 0.6
history["actual"] = 1000 * clouds
neutral = fit_solar_response(history, cutoff, "Europe/Berlin", 51.0, 7.4, 5)
assert np.allclose(solar_factors(future_basis, neutral), 1)
history["available_at"] = cutoff + pd.Timedelta(days=1)
assert not fit_solar_response(history, cutoff, "Europe/Berlin", 51.0, 7.4, 5)["fitted"]
def test_response_validation_needs_independent_days_and_does_not_reward_frequency():
pairs = [
dict(
bucket="6-24h",
day=f"2026-09-0{day}",
actual=1000.0,
calibrated=1300.0,
solar=1100.0,
weather=1050.0,
)
for day in range(1, 5)
for _ in range(10)
]
choice = select_responses(pairs, 3, 32, 0.02)["6-24h"]
assert choice["method"] == "weather"
assert choice["validated"] and choice["days"] == 4
assert select_responses(pairs[:10] * 50, 3, 32, 0.02)["6-24h"]["method"] == "calibrated"
assert select_responses(pairs, 3, 32, 0.02, ("calibrated",))["6-24h"]["method"] == "calibrated"
# A better MAE alone must not worsen energy bias.
pairs = [
dict(
bucket="6-24h",
day=f"2026-09-0{day}",
actual=1000.0,
calibrated=1000.0 + sign * 300,
solar=1100.0,
weather=1100.0,
)
for day in range(1, 5)
for sign in (-1, 1)
for _ in range(5)
]
assert select_responses(pairs, 3, 32, 0.02)["6-24h"]["method"] == "calibrated"
def test_paired_rows_reject_late_missing_and_curtailment_labels_and_deduplicate():
time = pd.Timestamp("2026-09-01T12:00Z")
truth = pd.DataFrame(
dict(
micro=[1000.0], micro_eligible=[True], micro_available=[time + pd.Timedelta(minutes=20)]
),
index=[time],
)
row = dict(
target_ms=int(time.timestamp() * 1000),
issued_ms=int((time - pd.Timedelta(hours=4)).timestamp() * 1000),
planes=dict(micro=dict(calibrated=1200.0, solar=1100.0, weather=1000.0, sky=1500.0)),
members=dict(icon_d2=dict(micro=None)),
response_version=1,
)
cutoff = pd.Timestamp("2026-09-02T00:00Z")
paired = paired_rows([row] * 20, truth, ["micro"], cutoff, "Europe/Berlin")
assert len(paired) == 1
assert np.isnan(paired[0]["members"]["icon_d2"])
assert not fit_weather_response(paired, 3, 32)["2-6h"]["fitted"]
truth["micro_eligible"] = False
assert paired_rows([row], truth, ["micro"], cutoff, "Europe/Berlin") == []
truth["micro_eligible"] = True
truth["micro_available"] = cutoff + pd.Timedelta(days=1)
assert paired_rows([row], truth, ["micro"], cutoff, "Europe/Berlin") == []
def test_weather_weights_are_trained_by_horizon_on_matching_members():
pairs = [
dict(
bucket="6-24h",
day=f"2026-09-0{day}",
actual=500.0,
sky=1500.0,
members=dict(icon_d2=600.0, icon_seamless=1000.0),
)
for day in range(1, 6)
for _ in range(10)
]
fitted = fit_weather_response(pairs, 5, 32)
assert fitted["6-24h"]["fitted"]
assert fitted["6-24h"]["weights"]["icon_d2"] > fitted["6-24h"]["weights"]["icon_seamless"]
assert not fitted["24-48h"]["fitted"]
assert abs(sum(fitted["6-24h"]["weights"].values()) - 1) < 1e-9
tests/test_pvakkuadapttrend.py
+137
View File
@@ -0,0 +1,137 @@
"""Causal trend features, online label availability and trend generalization."""
import numpy as np
import pandas as pd
import pytest
from akkudoktoreos.prediction.pvakkuadaptmodel import apply_local_nowcast
from akkudoktoreos.prediction.pvakkuadapttrend import (
FEATURES,
LocalPVTrendModel,
local_trend_features,
)
def case():
issue = pd.Timestamp("2026-09-18T12:05:00+02:00")
past = pd.date_range(issue.floor("15min") - pd.Timedelta(hours=1), periods=4, freq="15min")
hist = pd.DataFrame(
dict(
measured_power=[800, 900, 1100, 1500],
calibrated_power=1200.0,
clear_sky_power=2000.0,
eligible=True,
available_at=past + pd.Timedelta(minutes=20),
),
index=past,
)
future = pd.date_range(issue.ceil("15min"), periods=10, freq="15min")
return issue, hist, pd.Series(1200.0, index=future), pd.Series(2000.0, index=future)
def test_features_ignore_future_actuals_and_describe_a_rising_weather_residual():
issue, hist, pred, sky = case()
expected = local_trend_features(hist, pred, sky, issue)
assert expected.ramp.iloc[0] > 0
assert expected.slope.iloc[0] > 0
assert expected.feature_available_at.iloc[0] <= issue
hist.loc[issue.ceil("15min")] = [999999, 1200, 2000, True, issue + pd.Timedelta(hours=1)]
pd.testing.assert_frame_equal(local_trend_features(hist, pred, sky, issue), expected)
@pytest.mark.parametrize("bad", ["gap", "quality", "nan", "stale", "dim"])
def test_invalid_latest_reference_never_produces_a_trend(bad):
issue, hist, pred, sky = case()
if bad == "gap":
hist = hist.drop(hist.index[1])
elif bad == "quality":
hist.loc[hist.index[-1], "eligible"] = False
elif bad == "nan":
hist.loc[hist.index[-1], "measured_power"] = np.nan
elif bad == "stale":
hist.index -= pd.Timedelta(hours=1)
hist.available_at -= pd.Timedelta(hours=1)
else:
hist.loc[hist.index[-1], "clear_sky_power"] = 50
assert local_trend_features(hist, pred, sky, issue)[list(FEATURES)].isna().all().all()
def training():
rng = np.random.default_rng(42)
issue = pd.date_range("2026-09-01T08:05:00Z", periods=200, freq="1h")
frame = pd.DataFrame(rng.uniform(-0.2, 0.2, (len(issue), len(FEATURES))), columns=FEATURES)
frame["intercept"] = 1.0
frame["issue"] = issue
frame["time"] = issue + pd.Timedelta(minutes=10)
frame["lead_minutes"] = 10.0
frame["target_available_at"] = issue + pd.Timedelta(minutes=30)
frame["feature_available_at"] = issue
frame["reference_sky_w"] = 2000.0
frame["target_residual"] = 0.7 * frame.level + 0.8 * frame.ramp
return frame
def test_model_learns_slope_on_unseen_samples_while_level_only_cannot():
samples = training()
cutoff = pd.Timestamp("2026-09-15T00:00:00Z")
trend = LocalPVTrendModel(ridge=0.01).fit(samples.iloc[:160], cutoff)
level = LocalPVTrendModel(use_trend=False, ridge=0.01).fit(samples.iloc[:160], cutoff)
test = samples.iloc[160:]
trend_error = np.abs(
test[list(trend.feature_names)] @ trend.coefficients[15] - test.target_residual
)
level_error = np.abs(
test[list(level.feature_names)] @ level.coefficients[15] - test.target_residual
)
assert trend_error.mean() < 0.1 * level_error.mean()
def test_scaling_allows_learning_small_normalized_features_with_regularization():
samples = training()
cutoff = pd.Timestamp("2026-09-15T00:00:00Z")
model = LocalPVTrendModel(ridge=20, scale_features=True).fit(samples.iloc[:160], cutoff)
test = samples.iloc[160:]
error = np.abs(test[list(model.feature_names)] @ model.coefficients[15] - test.target_residual)
assert error.mean() < 0.02
def test_later_labels_and_later_feature_arrivals_cannot_change_coefficients():
samples = training()
cutoff = pd.Timestamp("2026-09-15T00:00:00Z")
expected = LocalPVTrendModel().fit(samples, cutoff).coefficients[15]
poison = samples.copy()
poison.target_residual = 1.5
poison.target_available_at = cutoff + pd.Timedelta(minutes=1)
combined = pd.concat([samples, poison])
np.testing.assert_allclose(LocalPVTrendModel().fit(combined, cutoff).coefficients[15], expected)
poison.target_available_at = poison.time + pd.Timedelta(minutes=20)
poison.feature_available_at = poison.issue + pd.Timedelta(minutes=1)
combined = pd.concat([samples, poison])
np.testing.assert_allclose(LocalPVTrendModel().fit(combined, cutoff).coefficients[15], expected)
def test_today_targets_do_not_train_and_cold_start_preserves_old_correction():
issue, hist, pred, sky = case()
model = LocalPVTrendModel().fit(training().iloc[:50], issue)
assert not model.coefficients
features = local_trend_features(hist, pred, sky, issue)
fallback = apply_local_nowcast(pred, sky, hist, issue)
pd.testing.assert_series_equal(model.predict(pred, sky, features, issue, fallback), fallback)
samples = training()
cutoff = pd.Timestamp("2026-09-09T12:00:00Z")
earlier = samples[pd.to_datetime(samples.time, utc=True).dt.date < cutoff.date()]
expected = LocalPVTrendModel().fit(earlier, cutoff).coefficients[15]
samples.loc[samples.index[-8:], "target_residual"] = 1.5
np.testing.assert_allclose(LocalPVTrendModel().fit(samples, cutoff).coefficients[15], expected)
def test_predict_respects_capacity_feature_arrival_and_expiry():
issue, hist, pred, sky = case()
features = local_trend_features(hist, pred, sky, issue)
model = LocalPVTrendModel()
model.coefficients = {lead: np.array([0, 0, 0, 10, 0, 0, 0, 0]) for lead in range(15, 121, 15)}
result = model.predict(pred, sky, features, issue, pred, capacity_w=1500)
assert result.iloc[0] == 1500
assert result.iloc[-1] == pred.iloc[-1]
features.loc[features.index[0], "feature_available_at"] = issue + pd.Timedelta(minutes=1)
assert model.predict(pred, sky, features, issue, pred).iloc[0] == pred.iloc[0]
tests/test_pvforecastpvlib.py
+3 -1
View File
@@ -1,6 +1,7 @@
"""Tests for PVForecastPVLib prediction provider."""
import os
from pathlib import Path
from unittest.mock import MagicMock, patch
@@ -487,7 +488,8 @@ class TestCECDatabase:
#
# Force new modification time.
#
path.touch()
previous = path.stat()
os.utime(path, ns=(previous.st_atime_ns, previous.st_mtime_ns + 1_000_000_000))
second = _load_cec_database(path)