fix: load data for automatic optimization (#731)

Automatic optimization used to take the adjusted load data even if there were no
measurements leading to 0 load values.

Split LoadAkkudoktor into LoadAkkudoktor and LoadAkkudoktorAdjusted. This allows
to select load data either purely from the load data database or load data additionally
adjusted by load measurements. Some value names have been adapted to denote
also the unit of a value.

For better load bug squashing the optimization solution data availability was
improved. For better data visbility prediction data can now be distinguished from
solution data in the generic optimization solution.

Some predictions that may be of interest to understand the solution were added.

Documentation was updated to resemble the addition load prediction provider and
the value name changes.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

src/akkudoktoreos/prediction/loadabc.py

@@ -15,12 +15,8 @@ from akkudoktoreos.prediction.predictionabc import PredictionProvider, Predictio
 class LoadDataRecord(PredictionRecord):
     """Represents a load data record containing various load attributes at a specific datetime."""
 
-    load_mean: Optional[float] = Field(default=None, description="Predicted load mean value (W).")
-    load_std: Optional[float] = Field(
-        default=None, description="Predicted load standard deviation (W)."
-    )
-    load_mean_adjusted: Optional[float] = Field(
-        default=None, description="Predicted load mean value adjusted by load measurement (W)."
+    loadforecast_power_w: Optional[float] = Field(
+        default=None, description="Predicted load mean value (W)."
     )
 
 

src/akkudoktoreos/prediction/loadakkudoktor.py

@@ -7,26 +7,97 @@ from loguru import logger
 from pydantic import Field
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
-from akkudoktoreos.prediction.loadabc import LoadProvider
+from akkudoktoreos.prediction.loadabc import LoadDataRecord, LoadProvider
 from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration
 
 
 class LoadAkkudoktorCommonSettings(SettingsBaseModel):
     """Common settings for load data import from file."""
 
-    loadakkudoktor_year_energy: Optional[float] = Field(
+    loadakkudoktor_year_energy_kwh: Optional[float] = Field(
         default=None, description="Yearly energy consumption (kWh).", examples=[40421]
     )
 
 
+class LoadAkkudoktorDataRecord(LoadDataRecord):
+    """Represents a load data record with extra fields for LoadAkkudoktor."""
+
+    loadakkudoktor_mean_power_w: Optional[float] = Field(
+        default=None, description="Predicted load mean value (W)."
+    )
+
+    loadakkudoktor_std_power_w: Optional[float] = Field(
+        default=None, description="Predicted load standard deviation (W)."
+    )
+
+
 class LoadAkkudoktor(LoadProvider):
     """Fetch Load forecast data from Akkudoktor load profiles."""
 
+    records: list[LoadAkkudoktorDataRecord] = Field(
+        default_factory=list, description="List of LoadAkkudoktorDataRecord records"
+    )
+
     @classmethod
     def provider_id(cls) -> str:
         """Return the unique identifier for the LoadAkkudoktor provider."""
         return "LoadAkkudoktor"
 
+    def load_data(self) -> np.ndarray:
+        """Loads data from the Akkudoktor load file."""
+        load_file = self.config.package_root_path.joinpath("data/load_profiles.npz")
+        data_year_energy = None
+        try:
+            file_data = np.load(load_file)
+            profile_data = np.array(
+                list(zip(file_data["yearly_profiles"], file_data["yearly_profiles_std"]))
+            )
+            # Calculate values in W by relative profile data and yearly consumption given in kWh
+            data_year_energy = (
+                profile_data
+                * self.config.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy_kwh
+                * 1000
+            )
+        except FileNotFoundError:
+            error_msg = f"Error: File {load_file} not found."
+            logger.error(error_msg)
+            raise FileNotFoundError(error_msg)
+        except Exception as e:
+            error_msg = f"An error occurred while loading data: {e}"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+        return data_year_energy
+
+    def _update_data(self, force_update: Optional[bool] = False) -> None:
+        """Adds the load means and standard deviations."""
+        data_year_energy = self.load_data()
+        # We provide prediction starting at start of day, to be compatible to old system.
+        # End date for prediction is prediction hours from now.
+        date = self.ems_start_datetime.start_of("day")
+        end_date = self.ems_start_datetime.add(hours=self.config.prediction.hours)
+        while compare_datetimes(date, end_date).lt:
+            # Extract mean (index 0) and standard deviation (index 1) for the given day and hour
+            # Day indexing starts at 0, -1 because of that
+            hourly_stats = data_year_energy[date.day_of_year - 1, :, date.hour]
+            values = {
+                "loadforecast_power_w": hourly_stats[0],
+                "loadakkudoktor_mean_power_w": hourly_stats[0],
+                "loadakkudoktor_std_power_w": hourly_stats[1],
+            }
+            self.update_value(date, values)
+            date += to_duration("1 hour")
+        # We are working on fresh data (no cache), report update time
+        self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
+
+
+class LoadAkkudoktorAdjusted(LoadAkkudoktor):
+    """Fetch Load forecast data from Akkudoktor load profiles with adjustment by measurements."""
+
+    @classmethod
+    def provider_id(cls) -> str:
+        """Return the unique identifier for the LoadAkkudoktor provider."""
+        return "LoadAkkudoktorAdjusted"
+
     def _calculate_adjustment(self, data_year_energy: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
         """Calculate weekday and week end adjustment from total load measurement data.
 
@@ -79,31 +150,6 @@ class LoadAkkudoktor(LoadProvider):
 
         return (weekday_adjust, weekend_adjust)
 
-    def load_data(self) -> np.ndarray:
-        """Loads data from the Akkudoktor load file."""
-        load_file = self.config.package_root_path.joinpath("data/load_profiles.npz")
-        data_year_energy = None
-        try:
-            file_data = np.load(load_file)
-            profile_data = np.array(
-                list(zip(file_data["yearly_profiles"], file_data["yearly_profiles_std"]))
-            )
-            # Calculate values in W by relative profile data and yearly consumption given in kWh
-            data_year_energy = (
-                profile_data
-                * self.config.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy
-                * 1000
-            )
-        except FileNotFoundError:
-            error_msg = f"Error: File {load_file} not found."
-            logger.error(error_msg)
-            raise FileNotFoundError(error_msg)
-        except Exception as e:
-            error_msg = f"An error occurred while loading data: {e}"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-        return data_year_energy
-
     def _update_data(self, force_update: Optional[bool] = False) -> None:
         """Adds the load means and standard deviations."""
         data_year_energy = self.load_data()
@@ -117,8 +163,8 @@ class LoadAkkudoktor(LoadProvider):
             # Day indexing starts at 0, -1 because of that
             hourly_stats = data_year_energy[date.day_of_year - 1, :, date.hour]
             values = {
-                "load_mean": hourly_stats[0],
-                "load_std": hourly_stats[1],
+                "loadakkudoktor_mean_power_w": hourly_stats[0],
+                "loadakkudoktor_std_power_w": hourly_stats[1],
             }
             if date.day_of_week < 5:
                 # Monday to Friday (0..4)
@@ -126,7 +172,7 @@ class LoadAkkudoktor(LoadProvider):
             else:
                 # Saturday, Sunday (5, 6)
                 value_adjusted = hourly_stats[0] + weekend_adjust[date.hour]
-            values["load_mean_adjusted"] = max(0, value_adjusted)
+            values["loadforecast_power_w"] = max(0, value_adjusted)
             self.update_value(date, values)
             date += to_duration("1 hour")
         # We are working on fresh data (no cache), report update time

src/akkudoktoreos/prediction/loadvrm.py

@@ -78,7 +78,7 @@ class LoadVrm(LoadProvider):
         return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone)
 
     def _update_data(self, force_update: Optional[bool] = False) -> None:
-        """Fetch and store VRM load forecast as load_mean and related values."""
+        """Fetch and store VRM load forecast as loadforecast_power_w and related values."""
         start_date = self.ems_start_datetime.start_of("day")
         end_date = self.ems_start_datetime.add(hours=self.config.prediction.hours)
         start_ts = int(start_date.timestamp())
@@ -87,19 +87,19 @@ class LoadVrm(LoadProvider):
         logger.info(f"Updating Load forecast from VRM: {start_date} to {end_date}")
         vrm_forecast_data = self._request_forecast(start_ts, end_ts)
 
-        load_mean_data = []
+        loadforecast_power_w_data = []
         for timestamp, value in vrm_forecast_data.records.vrm_consumption_fc:
             date = self._ts_to_datetime(timestamp)
             rounded_value = round(value, 2)
 
             self.update_value(
                 date,
-                {"load_mean": rounded_value, "load_std": 0.0, "load_mean_adjusted": rounded_value},
+                {"loadforecast_power_w": rounded_value},
             )
 
-            load_mean_data.append((date, rounded_value))
+            loadforecast_power_w_data.append((date, rounded_value))
 
-        logger.debug(f"Updated load_mean with {len(load_mean_data)} entries.")
+        logger.debug(f"Updated loadforecast_power_w with {len(loadforecast_power_w_data)} entries.")
         self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
 
 

src/akkudoktoreos/prediction/prediction.py

@@ -36,7 +36,10 @@ from akkudoktoreos.prediction.elecpriceenergycharts import ElecPriceEnergyCharts
 from akkudoktoreos.prediction.elecpriceimport import ElecPriceImport
 from akkudoktoreos.prediction.feedintarifffixed import FeedInTariffFixed
 from akkudoktoreos.prediction.feedintariffimport import FeedInTariffImport
-from akkudoktoreos.prediction.loadakkudoktor import LoadAkkudoktor
+from akkudoktoreos.prediction.loadakkudoktor import (
+    LoadAkkudoktor,
+    LoadAkkudoktorAdjusted,
+)
 from akkudoktoreos.prediction.loadimport import LoadImport
 from akkudoktoreos.prediction.loadvrm import LoadVrm
 from akkudoktoreos.prediction.predictionabc import PredictionContainer
@@ -94,6 +97,7 @@ class Prediction(PredictionContainer):
             FeedInTariffFixed,
             FeedInTariffImport,
             LoadAkkudoktor,
+            LoadAkkudoktorAdjusted,
             LoadVrm,
             LoadImport,
             PVForecastAkkudoktor,
@@ -112,9 +116,10 @@ elecprice_energy_charts = ElecPriceEnergyCharts()
 elecprice_import = ElecPriceImport()
 feedintariff_fixed = FeedInTariffFixed()
 feedintariff_import = FeedInTariffImport()
-load_akkudoktor = LoadAkkudoktor()
-load_vrm = LoadVrm()
-load_import = LoadImport()
+loadforecast_akkudoktor = LoadAkkudoktor()
+loadforecast_akkudoktor_adjusted = LoadAkkudoktorAdjusted()
+loadforecast_vrm = LoadVrm()
+loadforecast_import = LoadImport()
 pvforecast_akkudoktor = PVForecastAkkudoktor()
 pvforecast_vrm = PVForecastVrm()
 pvforecast_import = PVForecastImport()
@@ -134,9 +139,10 @@ def get_prediction() -> Prediction:
             elecprice_import,
             feedintariff_fixed,
             feedintariff_import,
-            load_akkudoktor,
-            load_vrm,
-            load_import,
+            loadforecast_akkudoktor,
+            loadforecast_akkudoktor_adjusted,
+            loadforecast_vrm,
+            loadforecast_import,
             pvforecast_akkudoktor,
             pvforecast_vrm,
             pvforecast_import,