From b01bb1c61c00b0b348095919bc08bce67b3c0ccd Mon Sep 17 00:00:00 2001 From: Bobby Noelte Date: Sat, 1 Nov 2025 00:49:11 +0100 Subject: [PATCH] fix: load data for automatic optimization (#731) Automatic optimization used to take the adjusted load data even if there were no measurements leading to 0 load values. Split LoadAkkudoktor into LoadAkkudoktor and LoadAkkudoktorAdjusted. This allows to select load data either purely from the load data database or load data additionally adjusted by load measurements. Some value names have been adapted to denote also the unit of a value. For better load bug squashing the optimization solution data availability was improved. For better data visbility prediction data can now be distinguished from solution data in the generic optimization solution. Some predictions that may be of interest to understand the solution were added. Documentation was updated to resemble the addition load prediction provider and the value name changes. Signed-off-by: Bobby Noelte --- docs/_generated/config.md | 4 +- docs/_generated/openapi.md | 4 +- docs/akkudoktoreos/prediction.md | 44 ++-- openapi.json | 21 +- single_test_optimization.py | 2 +- single_test_prediction.py | 8 +- src/akkudoktoreos/config/configmigrate.py | 5 +- src/akkudoktoreos/core/dataabc.py | 8 +- src/akkudoktoreos/core/pydantic.py | 2 +- .../optimization/genetic/geneticparams.py | 8 +- .../optimization/genetic/geneticsolution.py | 132 +++++++++--- .../optimization/optimization.py | 19 +- src/akkudoktoreos/prediction/loadabc.py | 8 +- .../prediction/loadakkudoktor.py | 106 +++++++--- src/akkudoktoreos/prediction/loadvrm.py | 10 +- src/akkudoktoreos/prediction/prediction.py | 20 +- src/akkudoktoreos/server/dash/plan.py | 188 ++++++++++++++---- src/akkudoktoreos/server/dash/prediction.py | 24 +-- src/akkudoktoreos/server/eos.py | 14 +- tests/test_configmigrate.py | 4 +- tests/test_loadakkudoktor.py | 78 +++++--- tests/test_loadvrm.py | 4 +- tests/test_prediction.py | 23 ++- tests/test_system.py | 2 +- tests/testdata/eos_config_andreas_now.json | 2 +- tests/testdata/eosserver_config_1.json | 2 +- 26 files changed, 515 insertions(+), 227 deletions(-) diff --git a/docs/_generated/config.md b/docs/_generated/config.md index b645e36..42fffeb 100644 --- a/docs/_generated/config.md +++ b/docs/_generated/config.md @@ -1262,7 +1262,7 @@ Validators: | Name | Type | Read-Only | Default | Description | | ---- | ---- | --------- | ------- | ----------- | -| loadakkudoktor_year_energy | `Optional[float]` | `rw` | `None` | Yearly energy consumption (kWh). | +| loadakkudoktor_year_energy_kwh | `Optional[float]` | `rw` | `None` | Yearly energy consumption (kWh). | ::: #### Example Input/Output @@ -1274,7 +1274,7 @@ Validators: "load": { "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": 40421.0 + "loadakkudoktor_year_energy_kwh": 40421.0 } } } diff --git a/docs/_generated/openapi.md b/docs/_generated/openapi.md index b29cbb8..b4f7782 100644 --- a/docs/_generated/openapi.md +++ b/docs/_generated/openapi.md @@ -24,7 +24,7 @@ If no prediction values are available the missing ones at the start of the serie filled with the first available prediction value. Note: - Use '/v1/prediction/list?key=load_mean_adjusted' instead. + Use '/v1/prediction/list?key=loadforecast_power_w' instead. Load energy meter readings to be added to EOS measurement by: '/v1/measurement/value' or '/v1/measurement/series' or @@ -68,7 +68,7 @@ Note: Set LoadAkkudoktor as provider, then update data with '/v1/prediction/update' and then request data with - '/v1/prediction/list?key=load_mean' instead. + '/v1/prediction/list?key=loadforecast_power_w' instead. ``` **Parameters**: diff --git a/docs/akkudoktoreos/prediction.md b/docs/akkudoktoreos/prediction.md index 4acf043..e4d0163 100644 --- a/docs/akkudoktoreos/prediction.md +++ b/docs/akkudoktoreos/prediction.md @@ -194,7 +194,7 @@ Configuration options: Prediction keys: -- `load_mean`: Predicted load mean value (W). +- `loadforecast_power_w`: Predicted load mean value (W). - `load_std`: Predicted load standard deviation (W). - `load_mean_adjusted`: Predicted load mean value adjusted by load measurement (W). @@ -208,15 +208,27 @@ Configuration options: - `LoadVrm`: Retrieves data from the VRM API by Victron Energy. - `LoadImport`: Imports from a file or JSON string. - - `provider_settings.loadakkudoktor_year_energy`: Yearly energy consumption (kWh). - - `provider_settings.loadimport_file_path`: Path to the file to import load forecast data from. - - `provider_settings.loadimport_json`: JSON string, dictionary of load forecast value lists. + - `provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy_kwh`: Yearly energy consumption (kWh). + - `provider_settings.LoadVRM.load_vrm_token`: API token. + - `provider_settings.LoadVRM.load_vrm_idsite`: load_vrm_idsite. + - `provider_settings.LoadImport.loadimport_file_path`: Path to the file to import load forecast data from. + - `provider_settings.LoadImport.loadimport_json`: JSON string, dictionary of load forecast value lists. ### LoadAkkudoktor Provider -The `LoadAkkudoktor` provider retrieves generic load data from a local database and tailors it to -align with the annual energy consumption specified in the `loadakkudoktor_year_energy` configuration -option. +The `LoadAkkudoktor` provider retrieves generic load data from the local database and scales +it to match the annual energy consumption specified in the +`LoadAkkudoktor.loadakkudoktor_year_energy` configuration option. + +### LoadAkkudoktorAdjusted Provider + +The `LoadAkkudoktorAdjusted` provider retrieves generic load data from the local database and scales +it to match the annual energy consumption specified in the +`LoadAkkudoktor.loadakkudoktor_year_energy` configuration option. In addition, the provider refines +the forecast by incorporating available measured load data, ensuring a more realistic and +site-specific consumption profile. + +For details on how to supply load measurements, see the [Measurements](measurement-page) section. ### LoadVrm Provider @@ -225,13 +237,17 @@ To receive forecasts, the system data must be configured under Dynamic ESS in th To query the forecasts, an API token is required, which can also be created in the VRM portal under Preferences. This token must be stored in the EOS configuration along with the VRM-Installations-ID. -```python +```json { "load": { "provider": "LoadVrm", "provider_settings": { - "load_vrm_token": "dummy-token", - "load_vrm_idsite": 12345 + "LoadVRM": { + "load_vrm_token": "dummy-token", + "load_vrm_idsite": 12345 + } + } + } } ``` @@ -530,7 +546,7 @@ Prediction keys: - `weather_temp_air`: Temperature (°C) - `weather_total_clouds`: Total Clouds (% Sky Obscured) - `weather_visibility`: Visibility (m) -- `weather_wind_direction`: "Wind Direction (°) +- `weather_wind_direction`: Wind Direction (°) - `weather_wind_speed`: Wind Speed (kmph) Configuration options: @@ -562,7 +578,7 @@ The provider provides forecast data for the following prediction keys: - `weather_temp_air`: Temperature (°C) - `weather_total_clouds`: Total Clouds (% Sky Obscured) - `weather_visibility`: Visibility (m) -- `weather_wind_direction`: "Wind Direction (°) +- `weather_wind_direction`: Wind Direction (°) - `weather_wind_speed`: Wind Speed (kmph) ### ClearOutside Provider @@ -592,7 +608,7 @@ The provider provides forecast data for the following prediction keys: - `weather_temp_air`: Temperature (°C) - `weather_total_clouds`: Total Clouds (% Sky Obscured) - `weather_visibility`: Visibility (m) -- `weather_wind_direction`: "Wind Direction (°) +- `weather_wind_direction`: Wind Direction (°) - `weather_wind_speed`: Wind Speed (kmph) ### WeatherImport Provider @@ -623,7 +639,7 @@ The prediction keys for the weather forecast data are: - `weather_temp_air`: Temperature (°C) - `weather_total_clouds`: Total Clouds (% Sky Obscured) - `weather_visibility`: Visibility (m) -- `weather_wind_direction`: "Wind Direction (°) +- `weather_wind_direction`: Wind Direction (°) - `weather_wind_speed`: Wind Speed (kmph) The PV forecast data must be provided in one of the formats described in diff --git a/openapi.json b/openapi.json index cd579e9..87715ed 100644 --- a/openapi.json +++ b/openapi.json @@ -1735,7 +1735,7 @@ "prediction" ], "summary": "Fastapi Gesamtlast", - "description": "Deprecated: Total Load Prediction with adjustment.\n\nEndpoint to handle total load prediction adjusted by latest measured data.\n\nTotal load prediction starts at 00.00.00 today and is provided for 48 hours.\nIf no prediction values are available the missing ones at the start of the series are\nfilled with the first available prediction value.\n\nNote:\n Use '/v1/prediction/list?key=load_mean_adjusted' instead.\n Load energy meter readings to be added to EOS measurement by:\n '/v1/measurement/value' or\n '/v1/measurement/series' or\n '/v1/measurement/dataframe' or\n '/v1/measurement/data'", + "description": "Deprecated: Total Load Prediction with adjustment.\n\nEndpoint to handle total load prediction adjusted by latest measured data.\n\nTotal load prediction starts at 00.00.00 today and is provided for 48 hours.\nIf no prediction values are available the missing ones at the start of the series are\nfilled with the first available prediction value.\n\nNote:\n Use '/v1/prediction/list?key=loadforecast_power_w' instead.\n Load energy meter readings to be added to EOS measurement by:\n '/v1/measurement/value' or\n '/v1/measurement/series' or\n '/v1/measurement/dataframe' or\n '/v1/measurement/data'", "operationId": "fastapi_gesamtlast_gesamtlast_post", "requestBody": { "content": { @@ -1781,7 +1781,7 @@ "prediction" ], "summary": "Fastapi Gesamtlast Simple", - "description": "Deprecated: Total Load Prediction.\n\nEndpoint to handle total load prediction.\n\nTotal load prediction starts at 00.00.00 today and is provided for 48 hours.\nIf no prediction values are available the missing ones at the start of the series are\nfilled with the first available prediction value.\n\nArgs:\n year_energy (float): Yearly energy consumption in Wh.\n\nNote:\n Set LoadAkkudoktor as provider, then update data with\n '/v1/prediction/update'\n and then request data with\n '/v1/prediction/list?key=load_mean' instead.", + "description": "Deprecated: Total Load Prediction.\n\nEndpoint to handle total load prediction.\n\nTotal load prediction starts at 00.00.00 today and is provided for 48 hours.\nIf no prediction values are available the missing ones at the start of the series are\nfilled with the first available prediction value.\n\nArgs:\n year_energy (float): Yearly energy consumption in Wh.\n\nNote:\n Set LoadAkkudoktor as provider, then update data with\n '/v1/prediction/update'\n and then request data with\n '/v1/prediction/list?key=loadforecast_power_w' instead.", "operationId": "fastapi_gesamtlast_simple_gesamtlast_simple_get", "parameters": [ { @@ -5099,7 +5099,7 @@ }, "LoadAkkudoktorCommonSettings": { "properties": { - "loadakkudoktor_year_energy": { + "loadakkudoktor_year_energy_kwh": { "anyOf": [ { "type": "number" @@ -5108,7 +5108,7 @@ "type": "null" } ], - "title": "Loadakkudoktor Year Energy", + "title": "Loadakkudoktor Year Energy Kwh", "description": "Yearly energy consumption (kWh).", "examples": [ 40421 @@ -5849,9 +5849,13 @@ "title": "Total Costs Amt", "description": "The total costs [money amount]." }, - "data": { + "prediction": { "$ref": "#/components/schemas/PydanticDateTimeDataFrame", - "description": "Datetime data frame with time series optimization data per optimization interval:- load_energy_wh: Load of all energy consumers in wh- grid_energy_wh: Grid energy feed in (negative) or consumption (positive) in wh- pv_prediction_energy_wh: PV energy prediction (positive) in wh- elec_price_prediction_amt_kwh: Electricity price prediction in money per kwh- costs_amt: Costs in money amount- revenue_amt: Revenue in money amount- losses_energy_wh: Energy losses in wh- _operation_mode_id: Operation mode id of the device.- _operation_mode_factor: Operation mode factor of the device.- _soc_factor: State of charge of a battery/ electric vehicle device as factor of total capacity.- _energy_wh: Energy consumption (positive) of a device in wh." + "description": "Datetime data frame with time series prediction data per optimization interval:- pv_energy_wh: PV energy prediction (positive) in wh- elec_price_amt_kwh: Electricity price prediction in money per kwh- feed_in_tariff_amt_kwh: Feed in tariff prediction in money per kwh- weather_temp_air_celcius: Temperature in \u00b0C- loadforecast_energy_wh: Load mean energy prediction in wh- loadakkudoktor_std_energy_wh: Load energy standard deviation prediction in wh- loadakkudoktor_mean_energy_wh: Load mean energy prediction in wh" + }, + "solution": { + "$ref": "#/components/schemas/PydanticDateTimeDataFrame", + "description": "Datetime data frame with time series solution data per optimization interval:- load_energy_wh: Load of all energy consumers in wh- grid_energy_wh: Grid energy feed in (negative) or consumption (positive) in wh- costs_amt: Costs in money amount- revenue_amt: Revenue in money amount- losses_energy_wh: Energy losses in wh- _operation_mode_id: Operation mode id of the device.- _operation_mode_factor: Operation mode factor of the device.- _soc_factor: State of charge of a battery/ electric vehicle device as factor of total capacity.- _energy_wh: Energy consumption (positive) of a device in wh." } }, "type": "object", @@ -5861,7 +5865,8 @@ "total_losses_energy_wh", "total_revenues_amt", "total_costs_amt", - "data" + "prediction", + "solution" ], "title": "OptimizationSolution", "description": "General Optimization Solution." @@ -7066,7 +7071,7 @@ }, "type": "object", "title": "PydanticDateTimeData", - "description": "Pydantic model for time series data with consistent value lengths.\n\nThis model validates a dictionary where:\n- Keys are strings representing data series names\n- Values are lists of numeric or string values\n- Special keys 'start_datetime' and 'interval' can contain string values\nfor time series indexing\n- All value lists must have the same length\n\nExample:\n {\n \"start_datetime\": \"2024-01-01 00:00:00\", # optional\n \"interval\": \"1 Hour\", # optional\n \"load_mean\": [20.5, 21.0, 22.1],\n \"load_min\": [18.5, 19.0, 20.1]\n }" + "description": "Pydantic model for time series data with consistent value lengths.\n\nThis model validates a dictionary where:\n- Keys are strings representing data series names\n- Values are lists of numeric or string values\n- Special keys 'start_datetime' and 'interval' can contain string values\nfor time series indexing\n- All value lists must have the same length\n\nExample:\n {\n \"start_datetime\": \"2024-01-01 00:00:00\", # optional\n \"interval\": \"1 Hour\", # optional\n \"loadforecast_power_w\": [20.5, 21.0, 22.1],\n \"load_min\": [18.5, 19.0, 20.1]\n }" }, "PydanticDateTimeDataFrame": { "properties": { diff --git a/single_test_optimization.py b/single_test_optimization.py index 642c5cb..8d989f3 100755 --- a/single_test_optimization.py +++ b/single_test_optimization.py @@ -101,7 +101,7 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: "provider": "LoadAkkudoktor", "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": 5000, # Energy consumption per year in kWh + "loadakkudoktor_year_energy_kwh": 5000, # Energy consumption per year in kWh }, }, }, diff --git a/single_test_prediction.py b/single_test_prediction.py index af1c745..41398c1 100644 --- a/single_test_prediction.py +++ b/single_test_prediction.py @@ -148,11 +148,11 @@ def run_prediction(provider_id: str, verbose: bool = False) -> str: forecast = "elecprice" elif provider_id in ("FeedInTariffFixed",): settings = config_feedintarifffixed() - forecast = "elecprice" + forecast = "feedintariff" elif provider_id in ("LoadAkkudoktor",): - settings = config_elecprice() - forecast = "load" - settings["load"]["loadakkudoktor_year_energy"] = 1000 + settings = config_load() + forecast = "loadforecast" + settings["load"]["LoadAkkudoktor"]["loadakkudoktor_year_energy_wh"] = 1000 else: raise ValueError(f"Unknown provider '{provider_id}'.") settings[forecast]["provider"] = provider_id diff --git a/src/akkudoktoreos/config/configmigrate.py b/src/akkudoktoreos/config/configmigrate.py index 1367757..aeecc87 100644 --- a/src/akkudoktoreos/config/configmigrate.py +++ b/src/akkudoktoreos/config/configmigrate.py @@ -24,7 +24,7 @@ MIGRATION_MAP: Dict[str, Union[str, Tuple[str, Callable[[Any], Any]], None]] = { "elecprice/provider_settings/import_json": "elecprice/provider_settings/ElecPriceImport/import_json", "load/provider_settings/import_file_path": "load/provider_settings/LoadImport/import_file_path", "load/provider_settings/import_json": "load/provider_settings/LoadImport/import_json", - "load/provider_settings/loadakkudoktor_year_energy": "load/provider_settings/LoadAkkudoktor/loadakkudoktor_year_energy", + "load/provider_settings/loadakkudoktor_year_energy": "load/provider_settings/LoadAkkudoktor/loadakkudoktor_year_energy_kwh", "load/provider_settings/load_vrm_idsite": "load/provider_settings/LoadVrm/load_vrm_idsite", "load/provider_settings/load_vrm_token": "load/provider_settings/LoadVrm/load_vrm_token", "logging/level": "logging/console_level", @@ -123,6 +123,9 @@ def migrate_config_file(config_file: Path, backup_file: Path) -> bool: old_value = _get_json_nested_value(config_data, old_path) if old_value is None: + migrated_source_paths.add(old_path.strip("/")) + mapped_count += 1 + logger.debug(f"✅ Migrated mapped '{old_path}' → 'None'") continue try: diff --git a/src/akkudoktoreos/core/dataabc.py b/src/akkudoktoreos/core/dataabc.py index 7cd5245..31968d9 100644 --- a/src/akkudoktoreos/core/dataabc.py +++ b/src/akkudoktoreos/core/dataabc.py @@ -1664,11 +1664,11 @@ class DataImportMixin: { "start_datetime": "2024-11-10 00:00:00" "interval": "30 minutes" - "load_mean": [20.5, 21.0, 22.1], + "loadforecast_power_w": [20.5, 21.0, 22.1], "other_xyz: [10.5, 11.0, 12.1], } ``` - and `key_prefix = "load"`, only the "load_mean" key will be processed even though + and `key_prefix = "load"`, only the "loadforecast_power_w" key will be processed even though both keys are in the record. """ # Try pandas dataframe with orient="split" @@ -1738,11 +1738,11 @@ class DataImportMixin: Given a JSON file with the following content: ```json { - "load_mean": [20.5, 21.0, 22.1], + "loadforecast_power_w": [20.5, 21.0, 22.1], "other_xyz: [10.5, 11.0, 12.1], } ``` - and `key_prefix = "load"`, only the "load_mean" key will be processed even though + and `key_prefix = "load"`, only the "loadforecast_power_w" key will be processed even though both keys are in the record. """ with import_file_path.open("r", encoding="utf-8", newline=None) as import_file: diff --git a/src/akkudoktoreos/core/pydantic.py b/src/akkudoktoreos/core/pydantic.py index b3bddd7..5b4511c 100644 --- a/src/akkudoktoreos/core/pydantic.py +++ b/src/akkudoktoreos/core/pydantic.py @@ -735,7 +735,7 @@ class PydanticDateTimeData(RootModel): { "start_datetime": "2024-01-01 00:00:00", # optional "interval": "1 Hour", # optional - "load_mean": [20.5, 21.0, 22.1], + "loadforecast_power_w": [20.5, 21.0, 22.1], "load_min": [18.5, 19.0, 20.1] } """ diff --git a/src/akkudoktoreos/optimization/genetic/geneticparams.py b/src/akkudoktoreos/optimization/genetic/geneticparams.py index bb4429e..c4e49db 100644 --- a/src/akkudoktoreos/optimization/genetic/geneticparams.py +++ b/src/akkudoktoreos/optimization/genetic/geneticparams.py @@ -301,8 +301,8 @@ class GeneticOptimizationParameters( # Retry continue try: - load_mean_adjusted = cls.prediction.key_to_array( - key="load_mean_adjusted", + loadforecast_power_w = cls.prediction.key_to_array( + key="loadforecast_power_w", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, interval=interval, @@ -319,7 +319,7 @@ class GeneticOptimizationParameters( "provider": "LoadAkkudoktor", "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": "1000", + "loadakkudoktor_year_energy_kwh": "3000", }, }, }, @@ -607,7 +607,7 @@ class GeneticOptimizationParameters( pv_prognose_wh=pvforecast_ac_power, strompreis_euro_pro_wh=elecprice_marketprice_wh, einspeiseverguetung_euro_pro_wh=feed_in_tariff_wh, - gesamtlast=load_mean_adjusted, + gesamtlast=loadforecast_power_w, preis_euro_pro_wh_akku=battery_lcos_kwh / 1000, ), temperature_forecast=weather_temp_air, diff --git a/src/akkudoktoreos/optimization/genetic/geneticsolution.py b/src/akkudoktoreos/optimization/genetic/geneticsolution.py index cc392ed..2ac6fc7 100644 --- a/src/akkudoktoreos/optimization/genetic/geneticsolution.py +++ b/src/akkudoktoreos/optimization/genetic/geneticsolution.py @@ -231,6 +231,7 @@ class GeneticSolution(GeneticParametersBaseModel): config = get_config() start_datetime = get_ems().start_datetime interval_hours = 1 + power_to_energy_per_interval_factor = 1.0 # --- Create index based on list length and interval --- n_points = len(self.result.Kosten_Euro_pro_Stunde) @@ -241,11 +242,9 @@ class GeneticSolution(GeneticParametersBaseModel): ) end_datetime = start_datetime.add(hours=n_points) - # Fill data into dataframe with correct column names + # Fill solution into dataframe with correct column names # - load_energy_wh: Load of all energy consumers in wh" # - grid_energy_wh: Grid energy feed in (negative) or consumption (positive) in wh" - # - pv_prediction_energy_wh: PV energy prediction (positive) in wh" - # - elec_price_prediction_amt_kwh: Electricity price prediction in money per kwh" # - costs_amt: Costs in money amount" # - revenue_amt: Revenue in money amount" # - losses_energy_wh: Energy losses in wh" @@ -254,7 +253,7 @@ class GeneticSolution(GeneticParametersBaseModel): # - _soc_factor: State of charge of a battery/ electric vehicle device as factor of total capacity." # - _energy_wh: Energy consumption (positive) of a device in wh." - data = pd.DataFrame( + solution = pd.DataFrame( { "date_time": time_index, "load_energy_wh": self.result.Last_Wh_pro_Stunde, @@ -269,7 +268,7 @@ class GeneticSolution(GeneticParametersBaseModel): ) # Add battery data - data["battery1_soc_factor"] = [v / 100 for v in self.result.akku_soc_pro_stunde] + solution["battery1_soc_factor"] = [v / 100 for v in self.result.akku_soc_pro_stunde] operation: dict[str, list[float]] = {} for hour, rate in enumerate(self.ac_charge): if hour >= n_points: @@ -290,13 +289,13 @@ class GeneticSolution(GeneticParametersBaseModel): operation[mode_key].append(0.0) operation[factor_key].append(0.0) for key in operation.keys(): - data[key] = operation[key] + solution[key] = operation[key] - # Add EV battery data + # Add EV battery solution if self.eauto_obj: if self.eautocharge_hours_float is None: # Electric vehicle is full enough. No load times. - data[f"{self.eauto_obj.device_id}_soc_factor"] = [ + solution[f"{self.eauto_obj.device_id}_soc_factor"] = [ self.eauto_obj.initial_soc_percentage / 100.0 ] * n_points # operation modes @@ -305,13 +304,13 @@ class GeneticSolution(GeneticParametersBaseModel): mode_key = f"{self.eauto_obj.device_id}_{mode.lower()}_op_mode" factor_key = f"{self.eauto_obj.device_id}_{mode.lower()}_op_factor" if mode == operation_mode: - data[mode_key] = [1.0] * n_points - data[factor_key] = [1.0] * n_points + solution[mode_key] = [1.0] * n_points + solution[factor_key] = [1.0] * n_points else: - data[mode_key] = [0.0] * n_points - data[factor_key] = [0.0] * n_points + solution[mode_key] = [0.0] * n_points + solution[factor_key] = [0.0] * n_points else: - data[f"{self.eauto_obj.device_id}_soc_factor"] = [ + solution[f"{self.eauto_obj.device_id}_soc_factor"] = [ v / 100 for v in self.result.EAuto_SoC_pro_Stunde ] operation = {} @@ -334,18 +333,30 @@ class GeneticSolution(GeneticParametersBaseModel): operation[mode_key].append(0.0) operation[factor_key].append(0.0) for key in operation.keys(): - data[key] = operation[key] + solution[key] = operation[key] # Add home appliance data if self.washingstart: - data["homeappliance1_energy_wh"] = self.result.Home_appliance_wh_per_hour + solution["homeappliance1_energy_wh"] = self.result.Home_appliance_wh_per_hour - # Add important predictions that are not already available from the GenericSolution - prediction = get_prediction() - power_to_energy_per_interval_factor = 1.0 - if "pvforecast_ac_power" in prediction.record_keys: - data["pv_prediction_energy_wh"] = ( - prediction.key_to_array( + # Fill prediction into dataframe with correct column names + # - pvforecast_ac_energy_wh_energy_wh: PV energy prediction (positive) in wh + # - elec_price_amt_kwh: Electricity price prediction in money per kwh + # - weather_temp_air_celcius: Temperature in °C" + # - loadforecast_energy_wh: Load energy prediction in wh + # - loadakkudoktor_std_energy_wh: Load energy standard deviation prediction in wh + # - loadakkudoktor_mean_energy_wh: Load mean energy prediction in wh + prediction = pd.DataFrame( + { + "date_time": time_index, + }, + index=time_index, + ) + pred = get_prediction() + + if "pvforecast_ac_power" in pred.record_keys: + prediction["pvforecast_ac_energy_wh"] = ( + pred.key_to_array( key="pvforecast_ac_power", start_datetime=start_datetime, end_datetime=end_datetime, @@ -354,18 +365,82 @@ class GeneticSolution(GeneticParametersBaseModel): ) * power_to_energy_per_interval_factor ).tolist() - if "weather_temp_air" in prediction.record_keys: - data["weather_temp_air"] = ( - prediction.key_to_array( - key="weather_temp_air", + if "pvforecast_dc_power" in pred.record_keys: + prediction["pvforecast_dc_energy_wh"] = ( + pred.key_to_array( + key="pvforecast_dc_power", start_datetime=start_datetime, end_datetime=end_datetime, interval=to_duration(f"{interval_hours} hours"), fill_method="linear", ) + * power_to_energy_per_interval_factor + ).tolist() + if "elecprice_marketprice_wh" in pred.record_keys: + prediction["elec_price_amt_kwh"] = ( + pred.key_to_array( + key="elecprice_marketprice_wh", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="ffill", + ) + * 1000 + ).tolist() + if "feed_in_tariff_wh" in pred.record_keys: + prediction["feed_in_tariff_amt_kwh"] = ( + pred.key_to_array( + key="feed_in_tariff_wh", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="linear", + ) + * 1000 + ).tolist() + if "weather_temp_air" in pred.record_keys: + prediction["weather_air_temp_celcius"] = pred.key_to_array( + key="weather_temp_air", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="linear", + ).tolist() + if "loadforecast_power_w" in pred.record_keys: + prediction["loadforecast_energy_wh"] = ( + pred.key_to_array( + key="loadforecast_power_w", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="linear", + ) + * power_to_energy_per_interval_factor + ).tolist() + if "loadakkudoktor_std_power_w" in pred.record_keys: + prediction["loadakkudoktor_std_energy_wh"] = ( + pred.key_to_array( + key="loadakkudoktor_std_power_w", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="linear", + ) + * power_to_energy_per_interval_factor + ).tolist() + if "loadakkudoktor_mean_power_w" in pred.record_keys: + prediction["loadakkudoktor_mean_energy_wh"] = ( + pred.key_to_array( + key="loadakkudoktor_mean_power_w", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=to_duration(f"{interval_hours} hours"), + fill_method="linear", + ) + * power_to_energy_per_interval_factor ).tolist() - solution = OptimizationSolution( + optimization_solution = OptimizationSolution( id=f"optimization-genetic@{to_datetime(as_string=True)}", generated_at=to_datetime(), comment="Optimization solution derived from GeneticSolution.", @@ -374,10 +449,11 @@ class GeneticSolution(GeneticParametersBaseModel): total_losses_energy_wh=self.result.Gesamt_Verluste, total_revenues_amt=self.result.Gesamteinnahmen_Euro, total_costs_amt=self.result.Gesamtkosten_Euro, - data=PydanticDateTimeDataFrame.from_dataframe(data), + prediction=PydanticDateTimeDataFrame.from_dataframe(prediction), + solution=PydanticDateTimeDataFrame.from_dataframe(solution), ) - return solution + return optimization_solution def energy_management_plan(self) -> EnergyManagementPlan: """Provide the genetic solution as an energy management plan.""" diff --git a/src/akkudoktoreos/optimization/optimization.py b/src/akkudoktoreos/optimization/optimization.py index 82c44ab..3644b8d 100644 --- a/src/akkudoktoreos/optimization/optimization.py +++ b/src/akkudoktoreos/optimization/optimization.py @@ -110,13 +110,24 @@ class OptimizationSolution(PydanticBaseModel): total_costs_amt: float = Field(description="The total costs [money amount].") - data: PydanticDateTimeDataFrame = Field( + prediction: PydanticDateTimeDataFrame = Field( description=( - "Datetime data frame with time series optimization data per optimization interval:" + "Datetime data frame with time series prediction data per optimization interval:" + "- pv_energy_wh: PV energy prediction (positive) in wh" + "- elec_price_amt_kwh: Electricity price prediction in money per kwh" + "- feed_in_tariff_amt_kwh: Feed in tariff prediction in money per kwh" + "- weather_temp_air_celcius: Temperature in °C" + "- loadforecast_energy_wh: Load mean energy prediction in wh" + "- loadakkudoktor_std_energy_wh: Load energy standard deviation prediction in wh" + "- loadakkudoktor_mean_energy_wh: Load mean energy prediction in wh" + ) + ) + + solution: PydanticDateTimeDataFrame = Field( + description=( + "Datetime data frame with time series solution data per optimization interval:" "- load_energy_wh: Load of all energy consumers in wh" "- grid_energy_wh: Grid energy feed in (negative) or consumption (positive) in wh" - "- pv_prediction_energy_wh: PV energy prediction (positive) in wh" - "- elec_price_prediction_amt_kwh: Electricity price prediction in money per kwh" "- costs_amt: Costs in money amount" "- revenue_amt: Revenue in money amount" "- losses_energy_wh: Energy losses in wh" diff --git a/src/akkudoktoreos/prediction/loadabc.py b/src/akkudoktoreos/prediction/loadabc.py index de0df8d..c7f5a87 100644 --- a/src/akkudoktoreos/prediction/loadabc.py +++ b/src/akkudoktoreos/prediction/loadabc.py @@ -15,12 +15,8 @@ from akkudoktoreos.prediction.predictionabc import PredictionProvider, Predictio class LoadDataRecord(PredictionRecord): """Represents a load data record containing various load attributes at a specific datetime.""" - load_mean: Optional[float] = Field(default=None, description="Predicted load mean value (W).") - load_std: Optional[float] = Field( - default=None, description="Predicted load standard deviation (W)." - ) - load_mean_adjusted: Optional[float] = Field( - default=None, description="Predicted load mean value adjusted by load measurement (W)." + loadforecast_power_w: Optional[float] = Field( + default=None, description="Predicted load mean value (W)." ) diff --git a/src/akkudoktoreos/prediction/loadakkudoktor.py b/src/akkudoktoreos/prediction/loadakkudoktor.py index 9c5759c..640449c 100644 --- a/src/akkudoktoreos/prediction/loadakkudoktor.py +++ b/src/akkudoktoreos/prediction/loadakkudoktor.py @@ -7,26 +7,97 @@ from loguru import logger from pydantic import Field from akkudoktoreos.config.configabc import SettingsBaseModel -from akkudoktoreos.prediction.loadabc import LoadProvider +from akkudoktoreos.prediction.loadabc import LoadDataRecord, LoadProvider from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration class LoadAkkudoktorCommonSettings(SettingsBaseModel): """Common settings for load data import from file.""" - loadakkudoktor_year_energy: Optional[float] = Field( + loadakkudoktor_year_energy_kwh: Optional[float] = Field( default=None, description="Yearly energy consumption (kWh).", examples=[40421] ) +class LoadAkkudoktorDataRecord(LoadDataRecord): + """Represents a load data record with extra fields for LoadAkkudoktor.""" + + loadakkudoktor_mean_power_w: Optional[float] = Field( + default=None, description="Predicted load mean value (W)." + ) + + loadakkudoktor_std_power_w: Optional[float] = Field( + default=None, description="Predicted load standard deviation (W)." + ) + + class LoadAkkudoktor(LoadProvider): """Fetch Load forecast data from Akkudoktor load profiles.""" + records: list[LoadAkkudoktorDataRecord] = Field( + default_factory=list, description="List of LoadAkkudoktorDataRecord records" + ) + @classmethod def provider_id(cls) -> str: """Return the unique identifier for the LoadAkkudoktor provider.""" return "LoadAkkudoktor" + def load_data(self) -> np.ndarray: + """Loads data from the Akkudoktor load file.""" + load_file = self.config.package_root_path.joinpath("data/load_profiles.npz") + data_year_energy = None + try: + file_data = np.load(load_file) + profile_data = np.array( + list(zip(file_data["yearly_profiles"], file_data["yearly_profiles_std"])) + ) + # Calculate values in W by relative profile data and yearly consumption given in kWh + data_year_energy = ( + profile_data + * self.config.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy_kwh + * 1000 + ) + except FileNotFoundError: + error_msg = f"Error: File {load_file} not found." + logger.error(error_msg) + raise FileNotFoundError(error_msg) + except Exception as e: + error_msg = f"An error occurred while loading data: {e}" + logger.error(error_msg) + raise ValueError(error_msg) + return data_year_energy + + def _update_data(self, force_update: Optional[bool] = False) -> None: + """Adds the load means and standard deviations.""" + data_year_energy = self.load_data() + # We provide prediction starting at start of day, to be compatible to old system. + # End date for prediction is prediction hours from now. + date = self.ems_start_datetime.start_of("day") + end_date = self.ems_start_datetime.add(hours=self.config.prediction.hours) + while compare_datetimes(date, end_date).lt: + # Extract mean (index 0) and standard deviation (index 1) for the given day and hour + # Day indexing starts at 0, -1 because of that + hourly_stats = data_year_energy[date.day_of_year - 1, :, date.hour] + values = { + "loadforecast_power_w": hourly_stats[0], + "loadakkudoktor_mean_power_w": hourly_stats[0], + "loadakkudoktor_std_power_w": hourly_stats[1], + } + self.update_value(date, values) + date += to_duration("1 hour") + # We are working on fresh data (no cache), report update time + self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) + + +class LoadAkkudoktorAdjusted(LoadAkkudoktor): + """Fetch Load forecast data from Akkudoktor load profiles with adjustment by measurements.""" + + @classmethod + def provider_id(cls) -> str: + """Return the unique identifier for the LoadAkkudoktor provider.""" + return "LoadAkkudoktorAdjusted" + def _calculate_adjustment(self, data_year_energy: np.ndarray) -> tuple[np.ndarray, np.ndarray]: """Calculate weekday and week end adjustment from total load measurement data. @@ -79,31 +150,6 @@ class LoadAkkudoktor(LoadProvider): return (weekday_adjust, weekend_adjust) - def load_data(self) -> np.ndarray: - """Loads data from the Akkudoktor load file.""" - load_file = self.config.package_root_path.joinpath("data/load_profiles.npz") - data_year_energy = None - try: - file_data = np.load(load_file) - profile_data = np.array( - list(zip(file_data["yearly_profiles"], file_data["yearly_profiles_std"])) - ) - # Calculate values in W by relative profile data and yearly consumption given in kWh - data_year_energy = ( - profile_data - * self.config.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy - * 1000 - ) - except FileNotFoundError: - error_msg = f"Error: File {load_file} not found." - logger.error(error_msg) - raise FileNotFoundError(error_msg) - except Exception as e: - error_msg = f"An error occurred while loading data: {e}" - logger.error(error_msg) - raise ValueError(error_msg) - return data_year_energy - def _update_data(self, force_update: Optional[bool] = False) -> None: """Adds the load means and standard deviations.""" data_year_energy = self.load_data() @@ -117,8 +163,8 @@ class LoadAkkudoktor(LoadProvider): # Day indexing starts at 0, -1 because of that hourly_stats = data_year_energy[date.day_of_year - 1, :, date.hour] values = { - "load_mean": hourly_stats[0], - "load_std": hourly_stats[1], + "loadakkudoktor_mean_power_w": hourly_stats[0], + "loadakkudoktor_std_power_w": hourly_stats[1], } if date.day_of_week < 5: # Monday to Friday (0..4) @@ -126,7 +172,7 @@ class LoadAkkudoktor(LoadProvider): else: # Saturday, Sunday (5, 6) value_adjusted = hourly_stats[0] + weekend_adjust[date.hour] - values["load_mean_adjusted"] = max(0, value_adjusted) + values["loadforecast_power_w"] = max(0, value_adjusted) self.update_value(date, values) date += to_duration("1 hour") # We are working on fresh data (no cache), report update time diff --git a/src/akkudoktoreos/prediction/loadvrm.py b/src/akkudoktoreos/prediction/loadvrm.py index 569f6ab..bb200b5 100644 --- a/src/akkudoktoreos/prediction/loadvrm.py +++ b/src/akkudoktoreos/prediction/loadvrm.py @@ -78,7 +78,7 @@ class LoadVrm(LoadProvider): return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone) def _update_data(self, force_update: Optional[bool] = False) -> None: - """Fetch and store VRM load forecast as load_mean and related values.""" + """Fetch and store VRM load forecast as loadforecast_power_w and related values.""" start_date = self.ems_start_datetime.start_of("day") end_date = self.ems_start_datetime.add(hours=self.config.prediction.hours) start_ts = int(start_date.timestamp()) @@ -87,19 +87,19 @@ class LoadVrm(LoadProvider): logger.info(f"Updating Load forecast from VRM: {start_date} to {end_date}") vrm_forecast_data = self._request_forecast(start_ts, end_ts) - load_mean_data = [] + loadforecast_power_w_data = [] for timestamp, value in vrm_forecast_data.records.vrm_consumption_fc: date = self._ts_to_datetime(timestamp) rounded_value = round(value, 2) self.update_value( date, - {"load_mean": rounded_value, "load_std": 0.0, "load_mean_adjusted": rounded_value}, + {"loadforecast_power_w": rounded_value}, ) - load_mean_data.append((date, rounded_value)) + loadforecast_power_w_data.append((date, rounded_value)) - logger.debug(f"Updated load_mean with {len(load_mean_data)} entries.") + logger.debug(f"Updated loadforecast_power_w with {len(loadforecast_power_w_data)} entries.") self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) diff --git a/src/akkudoktoreos/prediction/prediction.py b/src/akkudoktoreos/prediction/prediction.py index 6a4f829..e55f5ba 100644 --- a/src/akkudoktoreos/prediction/prediction.py +++ b/src/akkudoktoreos/prediction/prediction.py @@ -36,7 +36,10 @@ from akkudoktoreos.prediction.elecpriceenergycharts import ElecPriceEnergyCharts from akkudoktoreos.prediction.elecpriceimport import ElecPriceImport from akkudoktoreos.prediction.feedintarifffixed import FeedInTariffFixed from akkudoktoreos.prediction.feedintariffimport import FeedInTariffImport -from akkudoktoreos.prediction.loadakkudoktor import LoadAkkudoktor +from akkudoktoreos.prediction.loadakkudoktor import ( + LoadAkkudoktor, + LoadAkkudoktorAdjusted, +) from akkudoktoreos.prediction.loadimport import LoadImport from akkudoktoreos.prediction.loadvrm import LoadVrm from akkudoktoreos.prediction.predictionabc import PredictionContainer @@ -94,6 +97,7 @@ class Prediction(PredictionContainer): FeedInTariffFixed, FeedInTariffImport, LoadAkkudoktor, + LoadAkkudoktorAdjusted, LoadVrm, LoadImport, PVForecastAkkudoktor, @@ -112,9 +116,10 @@ elecprice_energy_charts = ElecPriceEnergyCharts() elecprice_import = ElecPriceImport() feedintariff_fixed = FeedInTariffFixed() feedintariff_import = FeedInTariffImport() -load_akkudoktor = LoadAkkudoktor() -load_vrm = LoadVrm() -load_import = LoadImport() +loadforecast_akkudoktor = LoadAkkudoktor() +loadforecast_akkudoktor_adjusted = LoadAkkudoktorAdjusted() +loadforecast_vrm = LoadVrm() +loadforecast_import = LoadImport() pvforecast_akkudoktor = PVForecastAkkudoktor() pvforecast_vrm = PVForecastVrm() pvforecast_import = PVForecastImport() @@ -134,9 +139,10 @@ def get_prediction() -> Prediction: elecprice_import, feedintariff_fixed, feedintariff_import, - load_akkudoktor, - load_vrm, - load_import, + loadforecast_akkudoktor, + loadforecast_akkudoktor_adjusted, + loadforecast_vrm, + loadforecast_import, pvforecast_akkudoktor, pvforecast_vrm, pvforecast_import, diff --git a/src/akkudoktoreos/server/dash/plan.py b/src/akkudoktoreos/server/dash/plan.py index eabcb8d..4831a7a 100644 --- a/src/akkudoktoreos/server/dash/plan.py +++ b/src/akkudoktoreos/server/dash/plan.py @@ -7,6 +7,7 @@ from bokeh.plotting import figure from loguru import logger from monsterui.franken import ( Card, + CardTitle, Details, Div, DivLAligned, @@ -33,10 +34,33 @@ from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime # bar width for 1 hour bars (time given in millseconds) BAR_WIDTH_1HOUR = 1000 * 60 * 60 + +# Tailwind compatible color palette +color_palette = { + "red-500": "#EF4444", # red-500 + "orange-500": "#F97316", # orange-500 + "amber-500": "#F59E0B", # amber-500 + "yellow-500": "#EAB308", # yellow-500 + "lime-500": "#84CC16", # lime-500 + "green-500": "#22C55E", # green-500 + "emerald-500": "#10B981", # emerald-500 + "teal-500": "#14B8A6", # teal-500 + "cyan-500": "#06B6D4", # cyan-500 + "sky-500": "#0EA5E9", # sky-500 + "blue-500": "#3B82F6", # blue-500 + "indigo-500": "#6366F1", # indigo-500 + "violet-500": "#8B5CF6", # violet-500 + "purple-500": "#A855F7", # purple-500 + "pink-500": "#EC4899", # pink-500 + "rose-500": "#F43F5E", # rose-500 +} +colors = list(color_palette.keys()) + # Current state of solution displayed solution_visible: dict[str, bool] = { - "pv_prediction_energy_wh": True, - "elec_price_prediction_amt_kwh": True, + "pv_energy_wh": True, + "elec_price_amt_kwh": True, + "feed_in_tariff_amt_kwh": True, } solution_color: dict[str, str] = {} @@ -75,6 +99,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti Args: data (Optional[dict]): Incoming data containing action and category for processing. """ + global colors, color_palette category = "solution" dark = False if data and data.get("category", None) == category: @@ -86,11 +111,34 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti if data and data.get("dark", None) == "true": dark = True - df = solution.data.to_dataframe() + df = solution.solution.to_dataframe() if df.empty or len(df.columns) <= 1: - raise ValueError(f"DataFrame is empty or missing plottable columns: {list(df.columns)}") + raise ValueError( + f"Solution DataFrame is empty or missing plottable columns: {list(df.columns)}" + ) if "date_time" not in df.columns: - raise ValueError(f"DataFrame is missing column 'date_time': {list(df.columns)}") + raise ValueError(f"Solution DataFrame is missing column 'date_time': {list(df.columns)}") + solution_columns = list(df.columns) + instruction_columns = [ + instruction + for instruction in solution_columns + if instruction.endswith("op_mode") or instruction.endswith("op_factor") + ] + solution_columns = [x for x in solution_columns if x not in instruction_columns] + + prediction_df = solution.prediction.to_dataframe() + if prediction_df.empty or len(prediction_df.columns) <= 1: + raise ValueError( + f"Prediction DataFrame is empty or missing plottable columns: {list(prediction_df.columns)}" + ) + if "date_time" not in prediction_df.columns: + raise ValueError( + f"Prediction DataFrame is missing column 'date_time': {list(prediction_df.columns)}" + ) + prediction_columns = list(prediction_df.columns) + + prediction_columns_to_join = prediction_df.columns.difference(df.columns) + df = df.join(prediction_df[prediction_columns_to_join], how="inner") # Remove time offset from UTC to get naive local time and make bokey plot in local time dst_offsets = df.index.map(lambda x: x.dst().total_seconds() / 3600) @@ -192,7 +240,6 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti # Create line renderers for each column renderers = {} - colors = ["black", "blue", "cyan", "green", "orange", "pink", "purple"] for i, col in enumerate(sorted(df.columns)): # Exclude some columns that are currently not used or are covered by others @@ -218,24 +265,24 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti solution_visible[col] = visible if col in solution_color: color = solution_color[col] - elif col == "pv_prediction_energy_wh": - color = "yellow" + elif col == "pv_energy_wh": + color = "yellow-500" solution_color[col] = color - elif col == "elec_price_prediction_amt_kwh": - color = "red" + elif col == "elec_price_amt_kwh": + color = "red-500" solution_color[col] = color else: color = colors[i % len(colors)] solution_color[col] = color if visible: - if col == "pv_prediction_energy_wh": + if col == "pv_energy_wh": r = plot.vbar( x="date_time", top=col, source=source, width=BAR_WIDTH_1HOUR * 0.8, legend_label=col, - color=color, + color=color_palette[color], level="underlay", ) elif col.endswith("energy_wh"): @@ -245,7 +292,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti mode="before", source=source, legend_label=col, - color=color, + color=color_palette[color], ) elif col.endswith("factor"): r = plot.step( @@ -254,7 +301,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti mode="before", source=source, legend_label=col, - color=color, + color=color_palette[color], y_range_name="factor", ) elif col.endswith("mode"): @@ -264,7 +311,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti mode="before", source=source, legend_label=col, - color=color, + color=color_palette[color], y_range_name="factor", ) elif col.endswith("amt_kwh"): @@ -274,7 +321,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti mode="before", source=source, legend_label=col, - color=color, + color=color_palette[color], y_range_name="amt_kwh", ) elif col.endswith("amt"): @@ -284,7 +331,7 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti mode="before", source=source, legend_label=col, - color=color, + color=color_palette[color], y_range_name="amt", ) else: @@ -298,34 +345,93 @@ def SolutionCard(solution: OptimizationSolution, config: SettingsEOS, data: Opti # --- CheckboxGroup to toggle datasets --- Checkbox = Grid( - *[ - LabelCheckboxX( - label=renderer, - id=f"{renderer}-visible", - name=f"{renderer}-visible", - value="true", - checked=solution_visible[renderer], - hx_post="/eosdash/plan", - hx_target="#page-content", - hx_swap="innerHTML", - hx_vals='js:{ "category": "solution", "action": "visible", "renderer": ' - + '"' - + f"{renderer}" - + '", ' - + '"dark": window.matchMedia("(prefers-color-scheme: dark)").matches ' - + "}", - lbl_cls=f"text-{solution_color[renderer]}-500", - ) - for renderer in list(renderers.keys()) - ], - cols=2, + Card( + Grid( + *[ + LabelCheckboxX( + label=renderer, + id=f"{renderer}-visible", + name=f"{renderer}-visible", + value="true", + checked=solution_visible[renderer], + hx_post="/eosdash/plan", + hx_target="#page-content", + hx_swap="innerHTML", + hx_vals='js:{ "category": "solution", "action": "visible", "renderer": ' + + '"' + + f"{renderer}" + + '", ' + + '"dark": window.matchMedia("(prefers-color-scheme: dark)").matches ' + + "}", + lbl_cls=f"text-{solution_color[renderer]}", + ) + for renderer in list(renderers.keys()) + if renderer in prediction_columns + ], + cols=2, + ), + header=CardTitle("Prediction"), + ), + Card( + Grid( + *[ + LabelCheckboxX( + label=renderer, + id=f"{renderer}-visible", + name=f"{renderer}-visible", + value="true", + checked=solution_visible[renderer], + hx_post="/eosdash/plan", + hx_target="#page-content", + hx_swap="innerHTML", + hx_vals='js:{ "category": "solution", "action": "visible", "renderer": ' + + '"' + + f"{renderer}" + + '", ' + + '"dark": window.matchMedia("(prefers-color-scheme: dark)").matches ' + + "}", + lbl_cls=f"text-{solution_color[renderer]}", + ) + for renderer in list(renderers.keys()) + if renderer in solution_columns + ], + cols=2, + ), + header=CardTitle("Solution"), + ), + Card( + Grid( + *[ + LabelCheckboxX( + label=renderer, + id=f"{renderer}-visible", + name=f"{renderer}-visible", + value="true", + checked=solution_visible[renderer], + hx_post="/eosdash/plan", + hx_target="#page-content", + hx_swap="innerHTML", + hx_vals='js:{ "category": "solution", "action": "visible", "renderer": ' + + '"' + + f"{renderer}" + + '", ' + + '"dark": window.matchMedia("(prefers-color-scheme: dark)").matches ' + + "}", + lbl_cls=f"text-{solution_color[renderer]}", + ) + for renderer in list(renderers.keys()) + if renderer in instruction_columns + ], + cols=2, + ), + header=CardTitle("Instruction"), + ), + cols=1, ) return Grid( Bokeh(plot), - Card( - Checkbox, - ), + Checkbox, cls="w-full space-y-3 space-x-3", ) diff --git a/src/akkudoktoreos/server/dash/prediction.py b/src/akkudoktoreos/server/dash/prediction.py index ec491e6..7e86d78 100644 --- a/src/akkudoktoreos/server/dash/prediction.py +++ b/src/akkudoktoreos/server/dash/prediction.py @@ -153,9 +153,9 @@ def WeatherIrradianceForecast( def LoadForecast(predictions: pd.DataFrame, config: dict, date_time_tz: str, dark: bool) -> FT: source = ColumnDataSource(predictions) provider = config["load"]["provider"] - if provider == "LoadAkkudoktor": + if provider == "LoadAkkudoktorAdjusted": year_energy = config["load"]["provider_settings"]["LoadAkkudoktor"][ - "loadakkudoktor_year_energy" + "loadakkudoktor_year_energy_kwh" ] provider = f"{provider}, {year_energy} kWh" @@ -168,8 +168,8 @@ def LoadForecast(predictions: pd.DataFrame, config: dict, date_time_tz: str, dar height=400, ) # Add secondary y-axis for stddev - stddev_min = predictions["load_std"].min() - stddev_max = predictions["load_std"].max() + stddev_min = predictions["loadakkudoktor_std_power_w"].min() + stddev_max = predictions["loadakkudoktor_std_power_w"].max() plot.extra_y_ranges["stddev"] = Range1d(start=stddev_min - 5, end=stddev_max + 5) y2_axis = LinearAxis(y_range_name="stddev", axis_label="Load Standard Deviation [W]") y2_axis.axis_label_text_color = "green" @@ -177,21 +177,21 @@ def LoadForecast(predictions: pd.DataFrame, config: dict, date_time_tz: str, dar plot.line( "date_time", - "load_mean", + "loadforecast_power_w", source=source, - legend_label="Load mean value", + legend_label="Load forcast value (adjusted by measurement)", color="red", ) plot.line( "date_time", - "load_mean_adjusted", + "loadakkudoktor_mean_power_w", source=source, - legend_label="Load adjusted by measurement", + legend_label="Load mean value", color="blue", ) plot.line( "date_time", - "load_std", + "loadakkudoktor_std_power_w", source=source, legend_label="Load standard deviation", color="green", @@ -233,9 +233,9 @@ def Prediction(eos_host: str, eos_port: Union[str, int], data: Optional[dict] = "weather_ghi", "weather_dni", "weather_dhi", - "load_mean", - "load_std", - "load_mean_adjusted", + "loadforecast_power_w", + "loadakkudoktor_std_power_w", + "loadakkudoktor_mean_power_w", ], } result = requests.get(f"{server}/v1/prediction/dataframe", params=params, timeout=10) diff --git a/src/akkudoktoreos/server/eos.py b/src/akkudoktoreos/server/eos.py index 40de4e6..696782f 100755 --- a/src/akkudoktoreos/server/eos.py +++ b/src/akkudoktoreos/server/eos.py @@ -1243,7 +1243,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: filled with the first available prediction value. Note: - Use '/v1/prediction/list?key=load_mean_adjusted' instead. + Use '/v1/prediction/list?key=loadforecast_power_w' instead. Load energy meter readings to be added to EOS measurement by: '/v1/measurement/value' or '/v1/measurement/series' or @@ -1255,10 +1255,10 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: "hours": request.hours, }, "load": { - "provider": "LoadAkkudoktor", + "provider": "LoadAkkudoktorAdjusted", "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": request.year_energy, + "loadakkudoktor_year_energy_kwh": request.year_energy, }, }, }, @@ -1317,7 +1317,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: end_datetime = start_datetime.add(days=2) try: prediction_list = prediction_eos.key_to_array( - key="load_mean_adjusted", + key="loadforecast_power_w", start_datetime=start_datetime, end_datetime=end_datetime, ).tolist() @@ -1347,14 +1347,14 @@ async def fastapi_gesamtlast_simple(year_energy: float) -> list[float]: Set LoadAkkudoktor as provider, then update data with '/v1/prediction/update' and then request data with - '/v1/prediction/list?key=load_mean' instead. + '/v1/prediction/list?key=loadforecast_power_w' instead. """ settings = SettingsEOS( load=LoadCommonSettings( provider="LoadAkkudoktor", provider_settings=LoadCommonProviderSettings( LoadAkkudoktor=LoadAkkudoktorCommonSettings( - loadakkudoktor_year_energy=year_energy / 1000, # Convert to kWh + loadakkudoktor_year_energy_kwh=year_energy / 1000, # Convert to kWh ), ), ) @@ -1378,7 +1378,7 @@ async def fastapi_gesamtlast_simple(year_energy: float) -> list[float]: end_datetime = start_datetime.add(days=2) try: prediction_list = prediction_eos.key_to_array( - key="load_mean", + key="loadforecast_power_w", start_datetime=start_datetime, end_datetime=end_datetime, ).tolist() diff --git a/tests/test_configmigrate.py b/tests/test_configmigrate.py index e97def3..cb04603 100644 --- a/tests/test_configmigrate.py +++ b/tests/test_configmigrate.py @@ -155,7 +155,7 @@ class TestConfigMigration: assert configmigrate.mapped_count >= 1, f"No mapped migrations for {old_file.name}" assert configmigrate.auto_count >= 1, f"No automatic migrations for {old_file.name}" - assert len(configmigrate.skipped_paths) <= 7, ( + assert len(configmigrate.skipped_paths) <= 3, ( f"Too many skipped paths in {old_file.name}: {configmigrate.skipped_paths}" ) @@ -174,7 +174,7 @@ class TestConfigMigration: errors = _dict_contains(new_data, expected_data) assert not errors, ( f"Migrated config for {old_file.name} is missing or mismatched fields:\n" + - "\n".join(errors) + "\n".join(errors) + f"\n{new_data}" ) # --- Compare migrated result with migration map --- diff --git a/tests/test_loadakkudoktor.py b/tests/test_loadakkudoktor.py index 4b31ccb..6d61a95 100644 --- a/tests/test_loadakkudoktor.py +++ b/tests/test_loadakkudoktor.py @@ -8,20 +8,39 @@ from akkudoktoreos.core.ems import get_ems from akkudoktoreos.measurement.measurement import MeasurementDataRecord, get_measurement from akkudoktoreos.prediction.loadakkudoktor import ( LoadAkkudoktor, + LoadAkkudoktorAdjusted, LoadAkkudoktorCommonSettings, ) from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration @pytest.fixture -def provider(config_eos): +def loadakkudoktor(config_eos): """Fixture to initialise the LoadAkkudoktor instance.""" settings = { "load": { "provider": "LoadAkkudoktor", "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": "1000", + "loadakkudoktor_year_energy_kwh": "1000", + }, + }, + }, + } + config_eos.merge_settings_from_dict(settings) + assert config_eos.load.provider == "LoadAkkudoktor" + assert config_eos.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy_kwh == 1000 + return LoadAkkudoktor() + +@pytest.fixture +def loadakkudoktoradjusted(config_eos): + """Fixture to initialise the LoadAkkudoktorAdjusted instance.""" + settings = { + "load": { + "provider": "LoadAkkudoktorAdjusted", + "provider_settings": { + "LoadAkkudoktor": { + "loadakkudoktor_year_energy_kwh": "1000", }, }, }, @@ -30,10 +49,9 @@ def provider(config_eos): } } config_eos.merge_settings_from_dict(settings) - assert config_eos.load.provider == "LoadAkkudoktor" - assert config_eos.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy == 1000 - return LoadAkkudoktor() - + assert config_eos.load.provider == "LoadAkkudoktorAdjusted" + assert config_eos.load.provider_settings.LoadAkkudoktor.loadakkudoktor_year_energy_kwh == 1000 + return LoadAkkudoktorAdjusted() @pytest.fixture def measurement_eos(): @@ -72,23 +90,23 @@ def mock_load_profiles_file(tmp_path): def test_loadakkudoktor_settings_validator(): - """Test the field validator for `loadakkudoktor_year_energy`.""" - settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy=1234) - assert isinstance(settings.loadakkudoktor_year_energy, float) - assert settings.loadakkudoktor_year_energy == 1234.0 + """Test the field validator for `loadakkudoktor_year_energy_kwh`.""" + settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234) + assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) + assert settings.loadakkudoktor_year_energy_kwh == 1234.0 - settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy=1234.56) - assert isinstance(settings.loadakkudoktor_year_energy, float) - assert settings.loadakkudoktor_year_energy == 1234.56 + settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234.56) + assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) + assert settings.loadakkudoktor_year_energy_kwh == 1234.56 -def test_loadakkudoktor_provider_id(provider): +def test_loadakkudoktor_provider_id(loadakkudoktor): """Test the `provider_id` class method.""" - assert provider.provider_id() == "LoadAkkudoktor" + assert loadakkudoktor.provider_id() == "LoadAkkudoktor" @patch("akkudoktoreos.prediction.loadakkudoktor.np.load") -def test_load_data_from_mock(mock_np_load, mock_load_profiles_file, provider): +def test_load_data_from_mock(mock_np_load, mock_load_profiles_file, loadakkudoktor): """Test the `load_data` method.""" # Mock numpy load to return data similar to what would be in the file mock_np_load.return_value = { @@ -97,19 +115,19 @@ def test_load_data_from_mock(mock_np_load, mock_load_profiles_file, provider): } # Test data loading - data_year_energy = provider.load_data() + data_year_energy = loadakkudoktor.load_data() assert data_year_energy is not None assert data_year_energy.shape == (365, 2, 24) -def test_load_data_from_file(provider): +def test_load_data_from_file(loadakkudoktor): """Test `load_data` loads data from the profiles file.""" - data_year_energy = provider.load_data() + data_year_energy = loadakkudoktor.load_data() assert data_year_energy is not None @patch("akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktor.load_data") -def test_update_data(mock_load_data, provider): +def test_update_data(mock_load_data, loadakkudoktor): """Test the `_update` method.""" mock_load_data.return_value = np.random.rand(365, 2, 24) @@ -118,27 +136,27 @@ def test_update_data(mock_load_data, provider): ems_eos.set_start_datetime(pendulum.datetime(2024, 1, 1)) # Assure there are no prediction records - provider.clear() - assert len(provider) == 0 + loadakkudoktor.clear() + assert len(loadakkudoktor) == 0 # Execute the method - provider._update_data() + loadakkudoktor._update_data() # Validate that update_value is called - assert len(provider) > 0 + assert len(loadakkudoktor) > 0 -def test_calculate_adjustment(provider, measurement_eos): +def test_calculate_adjustment(loadakkudoktoradjusted, measurement_eos): """Test `_calculate_adjustment` for various scenarios.""" data_year_energy = np.random.rand(365, 2, 24) # Call the method and validate results - weekday_adjust, weekend_adjust = provider._calculate_adjustment(data_year_energy) + weekday_adjust, weekend_adjust = loadakkudoktoradjusted._calculate_adjustment(data_year_energy) assert weekday_adjust.shape == (24,) assert weekend_adjust.shape == (24,) data_year_energy = np.zeros((365, 2, 24)) - weekday_adjust, weekend_adjust = provider._calculate_adjustment(data_year_energy) + weekday_adjust, weekend_adjust = loadakkudoktoradjusted._calculate_adjustment(data_year_energy) assert weekday_adjust.shape == (24,) expected = np.array( @@ -203,13 +221,13 @@ def test_calculate_adjustment(provider, measurement_eos): np.testing.assert_array_equal(weekend_adjust, expected) -def test_provider_adjustments_with_mock_data(provider): +def test_provider_adjustments_with_mock_data(loadakkudoktoradjusted): """Test full integration of adjustments with mock data.""" with patch( - "akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktor._calculate_adjustment" + "akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktorAdjusted._calculate_adjustment" ) as mock_adjust: mock_adjust.return_value = (np.zeros(24), np.zeros(24)) # Test execution - provider._update_data() + loadakkudoktoradjusted._update_data() assert mock_adjust.called diff --git a/tests/test_loadvrm.py b/tests/test_loadvrm.py index d7d42de..918b068 100644 --- a/tests/test_loadvrm.py +++ b/tests/test_loadvrm.py @@ -62,11 +62,11 @@ def test_update_data_calls_update_value(load_vrm_instance): expected_calls = [ call( pendulum.datetime(2025, 1, 1, 0, 0, 0, tz='Europe/Berlin'), - {"load_mean": 100.5, "load_std": 0.0, "load_mean_adjusted": 100.5} + {"loadforecast_power_w": 100.5,} ), call( pendulum.datetime(2025, 1, 1, 1, 0, 0, tz='Europe/Berlin'), - {"load_mean": 101.2, "load_std": 0.0, "load_mean_adjusted": 101.2} + {"loadforecast_power_w": 101.2,} ), ] diff --git a/tests/test_prediction.py b/tests/test_prediction.py index 8e86ec8..18af477 100644 --- a/tests/test_prediction.py +++ b/tests/test_prediction.py @@ -6,7 +6,10 @@ from akkudoktoreos.prediction.elecpriceenergycharts import ElecPriceEnergyCharts from akkudoktoreos.prediction.elecpriceimport import ElecPriceImport from akkudoktoreos.prediction.feedintarifffixed import FeedInTariffFixed from akkudoktoreos.prediction.feedintariffimport import FeedInTariffImport -from akkudoktoreos.prediction.loadakkudoktor import LoadAkkudoktor +from akkudoktoreos.prediction.loadakkudoktor import ( + LoadAkkudoktor, + LoadAkkudoktorAdjusted, +) from akkudoktoreos.prediction.loadimport import LoadImport from akkudoktoreos.prediction.loadvrm import LoadVrm from akkudoktoreos.prediction.prediction import ( @@ -38,6 +41,7 @@ def forecast_providers(): FeedInTariffFixed(), FeedInTariffImport(), LoadAkkudoktor(), + LoadAkkudoktorAdjusted(), LoadVrm(), LoadImport(), PVForecastAkkudoktor(), @@ -83,14 +87,15 @@ def test_provider_sequence(prediction): assert isinstance(prediction.providers[3], FeedInTariffFixed) assert isinstance(prediction.providers[4], FeedInTariffImport) assert isinstance(prediction.providers[5], LoadAkkudoktor) - assert isinstance(prediction.providers[6], LoadVrm) - assert isinstance(prediction.providers[7], LoadImport) - assert isinstance(prediction.providers[8], PVForecastAkkudoktor) - assert isinstance(prediction.providers[9], PVForecastVrm) - assert isinstance(prediction.providers[10], PVForecastImport) - assert isinstance(prediction.providers[11], WeatherBrightSky) - assert isinstance(prediction.providers[12], WeatherClearOutside) - assert isinstance(prediction.providers[13], WeatherImport) + assert isinstance(prediction.providers[6], LoadAkkudoktorAdjusted) + assert isinstance(prediction.providers[7], LoadVrm) + assert isinstance(prediction.providers[8], LoadImport) + assert isinstance(prediction.providers[9], PVForecastAkkudoktor) + assert isinstance(prediction.providers[10], PVForecastVrm) + assert isinstance(prediction.providers[11], PVForecastImport) + assert isinstance(prediction.providers[12], WeatherBrightSky) + assert isinstance(prediction.providers[13], WeatherClearOutside) + assert isinstance(prediction.providers[14], WeatherImport) def test_provider_by_id(prediction, forecast_providers): diff --git a/tests/test_system.py b/tests/test_system.py index 868d595..aa635e9 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -157,7 +157,7 @@ class TestSystem: result = requests.post(f"{server}/v1/prediction/update/LoadAkkudoktor") assert result.status_code == HTTPStatus.OK - result = requests.get(f"{server}/v1/prediction/series?key=load_mean") + result = requests.get(f"{server}/v1/prediction/series?key=loadforecast_power_w") assert result.status_code == HTTPStatus.OK data = result.json() diff --git a/tests/testdata/eos_config_andreas_now.json b/tests/testdata/eos_config_andreas_now.json index 3110997..e7847a5 100644 --- a/tests/testdata/eos_config_andreas_now.json +++ b/tests/testdata/eos_config_andreas_now.json @@ -58,7 +58,7 @@ "load": { "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": 13000 + "loadakkudoktor_year_energy_kwh": 13000 } } }, diff --git a/tests/testdata/eosserver_config_1.json b/tests/testdata/eosserver_config_1.json index f7a0871..614bf7f 100644 --- a/tests/testdata/eosserver_config_1.json +++ b/tests/testdata/eosserver_config_1.json @@ -15,7 +15,7 @@ "provider": "LoadImport", "provider_settings": { "LoadAkkudoktor": { - "loadakkudoktor_year_energy": 20000 + "loadakkudoktor_year_energy_kwh": 20000 } } },