fix: prevent exception when load prediction data is missing (#925)

Validate solution prediction data before processing.
If required prediction data is missing, the prediction is skipped
instead of raising an exception.

Introduce a new configuration file saving policy to improve loading robustness:
- Exclude computed fields
- Exclude fields set to their default values
- Exclude fields with value None
- Use field aliases
- Recursively remove empty dictionaries and lists
- Ensure general.version is always present and correctly set

When loading older configuration files, computed fields are now stripped
before migration. This further improves backward compatibility and loading
robustness.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

src/akkudoktoreos/optimization/genetic/geneticparams.py

@@ -218,17 +218,6 @@ class GeneticOptimizationParameters(
         if cls.config.optimization.genetic.generations is None:
             logger.info("Genetic generations unknown - defaulting to 400.")
             cls.config.optimization.genetic.generations = 400
-        if cls.config.optimization.genetic.penalties is None:
-            logger.info("Genetic penalties unknown - defaulting to demo config.")
-            cls.config.optimization.genetic.penalties = {"ev_soc_miss": 10}
-        if "ev_soc_miss" not in cls.config.optimization.genetic.penalties:
-            logger.info("ev_soc_miss penalty function parameter unknown - defaulting to 10.")
-            cls.config.optimization.genetic.penalties["ev_soc_miss"] = 10
-        if "ac_charge_break_even" not in cls.config.optimization.genetic.penalties:
-            # Default multiplier 1.0: penalty equals the exact economic loss in € from
-            # charging at a price that cannot be recovered given the round-trip efficiency
-            # and the best available future discharge price (after free PV energy is used).
-            cls.config.optimization.genetic.penalties["ac_charge_break_even"] = 1.0
 
         # Get start solution from last run
         start_solution = None

src/akkudoktoreos/optimization/genetic/geneticsolution.py

@@ -2,6 +2,7 @@
 
 from typing import Any, Optional
 
+import numpy as np
 import pandas as pd
 from loguru import logger
 from pydantic import Field, field_validator
@@ -460,91 +461,67 @@ class GeneticSolution(ConfigMixin, GeneticParametersBaseModel):
         )
         pred = get_prediction()
 
-        if "pvforecast_ac_power" in pred.record_keys:
-            prediction["pvforecast_ac_energy_wh"] = (
-                pred.key_to_array(
-                    key="pvforecast_ac_power",
+        for pred_key, pred_fill_method, pred_solution_key, pred_solution_factor in [
+            (
+                "pvforecast_ac_power",
+                "linear",
+                "pvforecast_ac_energy_wh",
+                power_to_energy_per_interval_factor,
+            ),
+            (
+                "pvforecast_dc_power",
+                "linear",
+                "pvforecast_dc_energy_wh",
+                power_to_energy_per_interval_factor,
+            ),
+            (
+                "elecprice_marketprice_wh",
+                "ffill",
+                "elec_price_amt_kwh",
+                1000.0,
+            ),
+            (
+                "feed_in_tariff_wh",
+                "linear",
+                "feed_in_tariff_amt_kwh",
+                1000.0,
+            ),
+            (
+                "weather_temp_air",
+                "linear",
+                "weather_air_temp_celcius",
+                1.0,
+            ),
+            (
+                "loadforecast_power_w",
+                "linear",
+                "loadforecast_energy_wh",
+                power_to_energy_per_interval_factor,
+            ),
+            (
+                "loadakkudoktor_std_power_w",
+                "linear",
+                "loadakkudoktor_std_energy_wh",
+                power_to_energy_per_interval_factor,
+            ),
+            (
+                "loadakkudoktor_mean_power_w",
+                "linear",
+                "loadakkudoktor_mean_energy_wh",
+                power_to_energy_per_interval_factor,
+            ),
+        ]:
+            if pred_key in pred.record_keys:
+                array = pred.key_to_array(
+                    key=pred_key,
                     start_datetime=start_datetime,
                     end_datetime=end_datetime,
                     interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
+                    fill_method=pred_fill_method,
                 )
-                * power_to_energy_per_interval_factor
-            ).tolist()
-        if "pvforecast_dc_power" in pred.record_keys:
-            prediction["pvforecast_dc_energy_wh"] = (
-                pred.key_to_array(
-                    key="pvforecast_dc_power",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
-                )
-                * power_to_energy_per_interval_factor
-            ).tolist()
-        if "elecprice_marketprice_wh" in pred.record_keys:
-            prediction["elec_price_amt_kwh"] = (
-                pred.key_to_array(
-                    key="elecprice_marketprice_wh",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="ffill",
-                )
-                * 1000
-            ).tolist()
-        if "feed_in_tariff_wh" in pred.record_keys:
-            prediction["feed_in_tariff_amt_kwh"] = (
-                pred.key_to_array(
-                    key="feed_in_tariff_wh",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
-                )
-                * 1000
-            ).tolist()
-        if "weather_temp_air" in pred.record_keys:
-            prediction["weather_air_temp_celcius"] = pred.key_to_array(
-                key="weather_temp_air",
-                start_datetime=start_datetime,
-                end_datetime=end_datetime,
-                interval=to_duration(f"{interval_hours} hours"),
-                fill_method="linear",
-            ).tolist()
-        if "loadforecast_power_w" in pred.record_keys:
-            prediction["loadforecast_energy_wh"] = (
-                pred.key_to_array(
-                    key="loadforecast_power_w",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
-                )
-                * power_to_energy_per_interval_factor
-            ).tolist()
-        if "loadakkudoktor_std_power_w" in pred.record_keys:
-            prediction["loadakkudoktor_std_energy_wh"] = (
-                pred.key_to_array(
-                    key="loadakkudoktor_std_power_w",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
-                )
-                * power_to_energy_per_interval_factor
-            ).tolist()
-        if "loadakkudoktor_mean_power_w" in pred.record_keys:
-            prediction["loadakkudoktor_mean_energy_wh"] = (
-                pred.key_to_array(
-                    key="loadakkudoktor_mean_power_w",
-                    start_datetime=start_datetime,
-                    end_datetime=end_datetime,
-                    interval=to_duration(f"{interval_hours} hours"),
-                    fill_method="linear",
-                )
-                * power_to_energy_per_interval_factor
-            ).tolist()
+                # 'key_to_array()' creates None values array if no data records are available.
+                if array is not None and array.size > 0 and not np.any(pd.isna(array)):
+                    prediction[pred_solution_key] = (array * pred_solution_factor).tolist()
 
         optimization_solution = OptimizationSolution(
             id=f"optimization-genetic@{to_datetime(as_string=True)}",

src/akkudoktoreos/optimization/optimization.py

@@ -41,8 +41,11 @@ class GeneticCommonSettings(SettingsBaseModel):
         },
     )
 
-    penalties: Optional[dict[str, Union[float, int, str]]] = Field(
-        default=None,
+    penalties: dict[str, Union[float, int, str]] = Field(
+        default_factory=lambda: {
+            "ev_soc_miss": 10,
+            "ac_charge_break_even": 1.0,
+        },
         json_schema_extra={
             "description": "A dictionary of penalty function parameters consisting of a penalty function parameter name and the associated value.",
             "examples": [