Mirrors/EOS
1
0
Fork 0
mirror of https://github.com/Akkudoktor-EOS/EOS.git synced 2026-07-21 09:18:12 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

fix: move data management to async (#1015)

Browse Source 
FAstAPI is an async framework. Data may be imported and exported, load and save, set and get
asynchronously. Prevent interleaving data operations to corrupt the data. In the previous design
sync and async data access was intermixed leading to data corruption.

The basic data classes DataSequence and DataContainer and the derived classes like Provider and
Measurement now are async. Data access is protected by several async locks.

To support the async design of the data classes the database interface became async.

The energy management is also adapted to the new async design. Optimization is still off-loaded
to another thread, but the prepration for the optimization and the post optimization actions now
follow the async design.

Adapter operations are now also protected by async locks.

Tests were adapted to the async design and new tests were created.

Besides this major fix several other improvements and fixes are included in this PR.

* fix: key_to_dict/list/array only regard data records with key value set.

  Before the exclusion of no value data records was only done if the dropna flag was set.

* fix: test for visual result pdf generation

  Due to updates in the library the generated charts text was a little bit different.
  Adapt the test to create the comaprison pdf in the test data durectory and
  update the reference pdf.

* chore: Remove MutableMapping from DataSequence and DataContainer.

  Mutable Mapping does not fit to the now async design.

* chore: Add NoDB database backend

  This backend implements the full database backend interface but performs
  no actual persistence. It is intended for configurations where database
  persistence is disabled (`provider=None`).

* chore: Improve measurement data import testing with real world scenarios.

  Added two new endpoints to support testing.

* chore: Add mermaid to supported documentation tools

* chore: Add documentation about async design

* chore: Add documentation about generic data handling

  Covers the basics of measurement and prediction time series data handling.

* chore: Add empty lines around markdown lists.

* chore: sync pre-commit config to updated package versions

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
This commit is contained in:
Bobby Noelte
2026-07-15 16:38:53 +02:00
committed by GitHub
parent 38011780c5
commit eb9e966de9
99 changed files with 11971 additions and 5981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/akkudoktoreos/prediction/elecpriceakkudoktor.py
View File

@@ -135,7 +135,7 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
clean_history = self._cap_outliers(history)
return np.full(hours, np.median(clean_history))
def _update_data(
async def _update_data(
self, force_update: Optional[bool] = False
) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Update forecast data in the ElecPriceDataRecord format.
@@ -170,10 +170,10 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
series_data.at[orig_datetime] = price_wh
# Update values using key_from_series
self.key_from_series("elecprice_marketprice_wh", series_data)
await self.key_from_series("elecprice_marketprice_wh", series_data)
# Generate history array for prediction
history = self.key_to_array(
history = await self.key_to_array(
key="elecprice_marketprice_wh", end_datetime=highest_orig_datetime, fill_method="linear"
)
@@ -213,9 +213,9 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
for i in range(len(prediction))
],
)
self.key_from_series("elecprice_marketprice_wh", prediction_series)
await self.key_from_series("elecprice_marketprice_wh", prediction_series)
# history2 = self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002
# history2 = await self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002
# return history, history2, prediction # for debug main

10
src/akkudoktoreos/prediction/elecpriceenergycharts.py
View File

@@ -195,7 +195,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
clean_history = self._cap_outliers(history)
return np.full(hours, np.median(clean_history))
def _update_data(
async def _update_data(
self, force_update: Optional[bool] = False
) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Update forecast data in the ElecPriceDataRecord format.
@@ -217,7 +217,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
# Determine if update is needed and how many days
past_days = 35
if self.highest_orig_datetime:
history_series = self.key_to_series(
history_series = await self.key_to_series(
key="elecprice_marketprice_wh", start_datetime=self.ems_start_datetime
)
# If history lower, then start_datetime
@@ -244,14 +244,14 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
# Parse and store data
series_data = self._parse_data(energy_charts_data)
self.highest_orig_datetime = series_data.index.max()
self.key_from_series("elecprice_marketprice_wh", series_data)
await self.key_from_series("elecprice_marketprice_wh", series_data)
else:
logger.info(
f"No Update ElecPriceEnergyCharts is needed, last in history: {self.highest_orig_datetime}"
)
# Generate history array for prediction
history = self.key_to_array(
history = await self.key_to_array(
key="elecprice_marketprice_wh",
end_datetime=self.highest_orig_datetime,
fill_method="linear",
@@ -293,4 +293,4 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
for i in range(len(prediction))
],
)
self.key_from_series("elecprice_marketprice_wh", prediction_series)
await self.key_from_series("elecprice_marketprice_wh", prediction_series)

4
src/akkudoktoreos/prediction/elecpricefixed.py
View File

@@ -59,7 +59,7 @@ class ElecPriceFixed(ElecPriceProvider):
"""Return the unique identifier for the ElecPriceFixed provider."""
return "ElecPriceFixed"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update electricity price data from fixed schedule.
Generates electricity prices based on the configured time windows
@@ -106,6 +106,6 @@ class ElecPriceFixed(ElecPriceProvider):
# Convert kWh → Wh and store one entry per interval step.
for idx, price_kwh in enumerate(prices_kwh):
current_dt = start_datetime.add(seconds=idx * interval_seconds)
self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0)
await self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0)
logger.debug(f"Successfully generated {len(prices_kwh)} fixed electricity price entries")

8
src/akkudoktoreos/prediction/elecpriceimport.py
View File

@@ -63,14 +63,16 @@ class ElecPriceImport(ElecPriceProvider, PredictionImportProvider):
"""Return the unique identifier for the ElecPriceImport provider."""
return "ElecPriceImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.elecprice.elecpriceimport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.elecprice.elecpriceimport.import_file_path,
key_prefix="elecprice",
)
if self.config.elecprice.elecpriceimport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.elecprice.elecpriceimport.import_json,
key_prefix="elecprice",
)

4
src/akkudoktoreos/prediction/feedintarifffixed.py
View File

@@ -34,7 +34,7 @@ class FeedInTariffFixed(FeedInTariffProvider):
"""Return the unique identifier for the FeedInTariffFixed provider."""
return "FeedInTariffFixed"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
error_msg = "Feed in tariff not provided"
try:
feed_in_tariff = (
@@ -47,4 +47,4 @@ class FeedInTariffFixed(FeedInTariffProvider):
logger.error(error_msg)
raise ValueError(error_msg)
feed_in_tariff_wh = feed_in_tariff / 1000
self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh)
await self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh)

8
src/akkudoktoreos/prediction/feedintariffimport.py
View File

@@ -64,17 +64,19 @@ class FeedInTariffImport(FeedInTariffProvider, PredictionImportProvider):
"""Return the unique identifier for the FeedInTariffImport provider."""
return "FeedInTariffImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.feedintariff.provider_settings.FeedInTariffImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.feedintariff.provider_settings.FeedInTariffImport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.provider_settings.FeedInTariffImport.import_file_path,
key_prefix="feedintariff",
)
if self.config.feedintariff.provider_settings.FeedInTariffImport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.feedintariff.provider_settings.FeedInTariffImport.import_json,
key_prefix="feedintariff",
)

29
src/akkudoktoreos/prediction/loadakkudoktor.py
View File

@@ -68,7 +68,7 @@ class LoadAkkudoktor(LoadProvider):
raise ValueError(error_msg)
return data_year_energy
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Adds the load means and standard deviations."""
data_year_energy = self.load_data()
# We provide prediction starting at start of day, to be compatible to old system.
@@ -84,7 +84,7 @@ class LoadAkkudoktor(LoadProvider):
"loadakkudoktor_mean_power_w": hourly_stats[0],
"loadakkudoktor_std_power_w": hourly_stats[1],
}
self.update_value(date, values)
await self.update_value(date, values)
date += to_duration("1 hour")
# We are working on fresh data (no cache), report update time
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
@@ -98,7 +98,9 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
"""Return the unique identifier for the LoadAkkudoktor provider."""
return "LoadAkkudoktorAdjusted"
def _calculate_adjustment(self, data_year_energy: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
async def _calculate_adjustment(
self, data_year_energy: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
"""Calculate weekday and week end adjustment from total load measurement data.
Returns:
@@ -110,19 +112,22 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
weekend_adjust = np.zeros(24)
weekend_adjust_weight = np.zeros(24)
if self.measurement.max_datetime is None:
max_dt = await self.measurement.max_datetime()
if max_dt is None:
# No measurements - return 0 adjustment
return (weekday_adjust, weekday_adjust)
min_dt = await self.measurement.min_datetime()
# compare predictions with real measurement - try to use last 7 days
compare_start = self.measurement.max_datetime - to_duration("7 days")
if compare_datetimes(compare_start, self.measurement.min_datetime).lt:
compare_start = max_dt - to_duration("7 days")
if compare_datetimes(compare_start, min_dt).lt:
# Not enough measurements for 7 days - use what is available
compare_start = self.measurement.min_datetime
compare_end = self.measurement.max_datetime
compare_start = min_dt
compare_end = max_dt
compare_interval = to_duration("1 hour")
load_total_kwh_array = self.measurement.load_total_kwh(
load_total_kwh_array = await self.measurement.load_total_kwh(
start_datetime=compare_start,
end_datetime=compare_end,
interval=compare_interval,
@@ -159,10 +164,10 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
return (weekday_adjust, weekend_adjust)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Adds the load means and standard deviations."""
data_year_energy = self.load_data()
weekday_adjust, weekend_adjust = self._calculate_adjustment(data_year_energy)
weekday_adjust, weekend_adjust = await self._calculate_adjustment(data_year_energy)
# We provide prediction starting at start of day, to be compatible to old system.
# End date for prediction is prediction hours from now.
date = self.ems_start_datetime.start_of("day")
@@ -182,7 +187,7 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
# Saturday, Sunday (5, 6)
value_adjusted = hourly_stats[0] + weekend_adjust[date.hour]
values["loadforecast_power_w"] = max(0, value_adjusted)
self.update_value(date, values)
await self.update_value(date, values)
date += to_duration("1 hour")
# We are working on fresh data (no cache), report update time
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)

10
src/akkudoktoreos/prediction/loadimport.py
View File

@@ -62,8 +62,12 @@ class LoadImport(LoadProvider, PredictionImportProvider):
"""Return the unique identifier for the LoadImport provider."""
return "LoadImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.load.loadimport.import_file_path:
self.import_from_file(self.config.load.loadimport.import_file_path, key_prefix="load")
await self._import_from_file(
self.config.load.loadimport.import_file_path, key_prefix="load"
)
if self.config.load.loadimport.import_json:
self.import_from_json(self.config.load.loadimport.import_json, key_prefix="load")
await self._import_from_json(self.config.load.loadimport.import_json, key_prefix="load")

9
src/akkudoktoreos/prediction/loadvrm.py
View File

@@ -84,7 +84,7 @@ class LoadVrm(LoadProvider):
"""Convert UNIX ms timestamp to timezone-aware datetime."""
return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Fetch and store VRM load forecast as loadforecast_power_w and related values."""
if self.enabled is False:
logger.info("LoadVrm is disabled, skipping update.")
@@ -102,7 +102,7 @@ class LoadVrm(LoadProvider):
date = self._ts_to_datetime(timestamp)
rounded_value = round(value, 2)
self.update_value(
await self.update_value(
date,
{"loadforecast_power_w": rounded_value},
)
@@ -111,8 +111,3 @@ class LoadVrm(LoadProvider):
logger.debug(f"Updated loadforecast_power_w with {len(loadforecast_power_w_data)} entries.")
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
if __name__ == "__main__":
lv = LoadVrm()
lv._update_data()

2
src/akkudoktoreos/prediction/prediction.py
View File

@@ -14,7 +14,7 @@ Example:
# Create singleton prediction instance with prediction providers
from akkudoktoreos.prediction.prediction import prediction
prediction.update_data()
await prediction.update_data()
print("Prediction:", prediction)
Classes:

6
src/akkudoktoreos/prediction/predictionabc.py
View File

@@ -225,7 +225,7 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider):
hours = max(self.config.prediction.hours, self.config.prediction_historic_hours, 24)
return to_duration(hours * 3600)
def update_data(
async def update_data(
self,
force_enable: Optional[bool] = False,
force_update: Optional[bool] = False,
@@ -243,10 +243,10 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider):
return
# Delete outdated records before updating
self.delete_by_datetime(end_datetime=self.keep_datetime)
await self.delete_by_datetime(end_datetime=self.keep_datetime)
# Call the custom update logic
self._update_data(force_update=force_update)
await self._update_data(force_update=force_update)
class PredictionImportProvider(PredictionProvider, DataImportProvider):

20
src/akkudoktoreos/prediction/pvforecastakkudoktor.py
View File

@@ -52,10 +52,10 @@ Example:
forecast = PVForecastAkkudoktor(settings=config)
# Get an actual forecast
forecast.update_data()
await forecast.update_data()
# Update the AC power measurement for a specific date and time
forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0)
await forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0)
# Report the DC and AC power forecast along with AC measurements
print(forecast.report_ac_power_and_measurement())
@@ -286,7 +286,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
return akkudoktor_data
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the PVForecastAkkudoktorDataRecord format.
Retrieves data from Akkudoktor. The processed data is inserted into the sequence as
@@ -341,7 +341,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
"pvforecastakkudoktor_temp_air": forecast_values[0].temperature,
}
self.update_value(dt, data)
await self.update_value(dt, data)
if len(self) < self.config.prediction.hours:
raise ValueError(
@@ -385,7 +385,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
# Example of how to use the PVForecastAkkudoktor class
if __name__ == "__main__":
async def main() -> None:
"""Main execution block to demonstrate the use of the PVForecastAkkudoktor class.
Sets up the forecast configuration fields, fetches PV power forecast data,
@@ -441,12 +441,18 @@ if __name__ == "__main__":
forecast = PVForecastAkkudoktor()
# Get an actual forecast
forecast.update_data()
await forecast.update_data()
# Update the AC power measurement for a specific date and time
forecast.update_value(
await forecast.update_value(
to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0
)
# Report the DC and AC power forecast along with AC measurements
print(forecast.report_ac_power_and_measurement())
if __name__ == "__main__":
import asyncio
asyncio.run(main())

8
src/akkudoktoreos/prediction/pvforecastimport.py
View File

@@ -64,17 +64,19 @@ class PVForecastImport(PVForecastProvider, PredictionImportProvider):
"""Return the unique identifier for the PVForecastImport provider."""
return "PVForecastImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.pvforecast.provider_settings.PVForecastImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.pvforecast.provider_settings.PVForecastImport.import_file_path is not None:
self.import_from_file(
await self._import_from_file(
self.config.pvforecast.provider_settings.PVForecastImport.import_file_path,
key_prefix="pvforecast",
)
if self.config.pvforecast.provider_settings.PVForecastImport.import_json is not None:
self.import_from_json(
await self._import_from_json(
self.config.pvforecast.provider_settings.PVForecastImport.import_json,
key_prefix="pvforecast",
)

10
src/akkudoktoreos/prediction/pvforecastvrm.py
View File

@@ -83,7 +83,7 @@ class PVForecastVrm(PVForecastProvider):
"""Convert UNIX ms timestamp to timezone-aware datetime."""
return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the PVForecastDataRecord format."""
if self.enabled is False:
logger.info("PVForecastVrm is disabled, skipping update.")
@@ -101,16 +101,10 @@ class PVForecastVrm(PVForecastProvider):
date = self._ts_to_datetime(timestamp)
dc_power = round(value, 2)
ac_power = round(dc_power * 0.96, 2)
self.update_value(
await self.update_value(
date, {"pvforecast_dc_power": dc_power, "pvforecast_ac_power": ac_power}
)
pv_forecast.append((date, dc_power))
logger.debug(f"Updated pvforecast_dc_power with {len(pv_forecast)} entries.")
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
# Example usage
if __name__ == "__main__":
pv = PVForecastVrm()
pv._update_data()

26
src/akkudoktoreos/prediction/weatherbrightsky.py
View File

@@ -111,7 +111,7 @@ class WeatherBrightSky(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return brightsky_data
def _description_to_series(self, description: str) -> pd.Series:
async def _description_to_series(self, description: str) -> pd.Series:
"""Retrieve a pandas Series corresponding to a weather data description.
This method fetches the key associated with the provided description
@@ -132,10 +132,10 @@ class WeatherBrightSky(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
series = self.key_to_series(key)
series = await self.key_to_series(key)
return series
def _description_from_series(self, description: str, data: pd.Series) -> None:
async def _description_from_series(self, description: str, data: pd.Series) -> None:
"""Update a weather data with a pandas Series based on its description.
This method fetches the key associated with the provided description
@@ -154,9 +154,9 @@ class WeatherBrightSky(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
self.key_from_series(key, data)
await self.key_from_series(key, data)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the WeatherDataRecord format.
Retrieves data from BrightSky, maps each BrightSky field to the corresponding
@@ -197,31 +197,31 @@ class WeatherBrightSky(WeatherProvider):
else:
value = value * corr_factor
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)
# Converting the cloud cover into Irradiance (GHI, DNI, DHI)
description = "Total Clouds (% Sky Obscured)"
cloud_cover = self._description_to_series(description)
cloud_cover = await self._description_to_series(description)
ghi, dni, dhi = self.estimate_irradiance_from_cloud_cover(
self.config.general.latitude, self.config.general.longitude, cloud_cover
)
description = "Global Horizontal Irradiance (W/m2)"
ghi = pd.Series(data=ghi, index=cloud_cover.index)
self._description_from_series(description, ghi)
await self._description_from_series(description, ghi)
description = "Direct Normal Irradiance (W/m2)"
dni = pd.Series(data=dni, index=cloud_cover.index)
self._description_from_series(description, dni)
await self._description_from_series(description, dni)
description = "Diffuse Horizontal Irradiance (W/m2)"
dhi = pd.Series(data=dhi, index=cloud_cover.index)
self._description_from_series(description, dhi)
await self._description_from_series(description, dhi)
# Add Preciptable Water (PWAT) with a PVLib method.
key = WeatherDataRecord.key_from_description("Temperature (°C)")
assert key # noqa: S101
temperature = self.key_to_array(
temperature = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -234,7 +234,7 @@ class WeatherBrightSky(WeatherProvider):
return
key = WeatherDataRecord.key_from_description("Relative Humidity (%)")
assert key # noqa: S101
humidity = self.key_to_array(
humidity = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -258,4 +258,4 @@ class WeatherBrightSky(WeatherProvider):
),
)
description = "Preciptable Water (cm)"
self._description_from_series(description, pwat)
await self._description_from_series(description, pwat)

6
src/akkudoktoreos/prediction/weatherclearoutside.py
View File

@@ -97,7 +97,7 @@ class WeatherClearOutside(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return response
def _update_data(self, force_update: Optional[bool] = None) -> None:
async def _update_data(self, force_update: Optional[bool] = None) -> None:
"""Scrape weather forecast data from ClearOutside's website.
This method requests weather forecast data from ClearOutside based on latitude
@@ -202,7 +202,7 @@ class WeatherClearOutside(WeatherProvider):
raise ValueError(error_msg)
# Delete all records that will be newly added
self.delete_by_datetime(start_datetime=forecast_start_datetime)
await self.delete_by_datetime(start_datetime=forecast_start_datetime)
# Collect weather data, loop over all days
for day, p_day in enumerate(p_days):
@@ -341,4 +341,4 @@ class WeatherClearOutside(WeatherProvider):
if corr_factor:
value = value * corr_factor
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)

8
src/akkudoktoreos/prediction/weatherimport.py
View File

@@ -64,17 +64,19 @@ class WeatherImport(WeatherProvider, PredictionImportProvider):
"""Return the unique identifier for the WeatherImport provider."""
return "WeatherImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.weather.provider_settings.WeatherImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.weather.provider_settings.WeatherImport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.weather.provider_settings.WeatherImport.import_file_path,
key_prefix="weather",
)
if self.config.weather.provider_settings.WeatherImport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.weather.provider_settings.WeatherImport.import_json,
key_prefix="weather",
)

20
src/akkudoktoreos/prediction/weatheropenmeteo.py
View File

@@ -197,7 +197,7 @@ class WeatherOpenMeteo(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return openmeteo_data
def _description_to_series(self, description: str) -> pd.Series:
async def _description_to_series(self, description: str) -> pd.Series:
"""Retrieve a pandas Series corresponding to a weather data description.
This method fetches the key associated with the provided description
@@ -218,10 +218,10 @@ class WeatherOpenMeteo(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
series = self.key_to_series(key)
series = await self.key_to_series(key)
return series
def _description_from_series(self, description: str, data: pd.Series) -> None:
async def _description_from_series(self, description: str, data: pd.Series) -> None:
"""Update a weather data with a pandas Series based on its description.
This method fetches the key associated with the provided description
@@ -240,9 +240,9 @@ class WeatherOpenMeteo(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
self.key_from_series(key, data)
await self.key_from_series(key, data)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the WeatherDataRecord format.
Retrieves data from Open-Meteo, maps each Open-Meteo field to the corresponding
@@ -299,11 +299,11 @@ class WeatherOpenMeteo(WeatherProvider):
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)
# Check whether radiation values exist (for logging)
description_ghi = "Global Horizontal Irradiance (W/m2)"
ghi_series = self._description_to_series(description_ghi)
ghi_series = await self._description_to_series(description_ghi)
if ghi_series.isnull().all():
logger.warning("No GHI data received from Open-Meteo")
@@ -315,7 +315,7 @@ class WeatherOpenMeteo(WeatherProvider):
# Add Precipitable Water (PWAT) using PVLib method
key = WeatherDataRecord.key_from_description("Temperature (°C)")
assert key # noqa: S101
temperature = self.key_to_array(
temperature = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -329,7 +329,7 @@ class WeatherOpenMeteo(WeatherProvider):
key = WeatherDataRecord.key_from_description("Relative Humidity (%)")
assert key # noqa: S101
humidity = self.key_to_array(
humidity = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -354,4 +354,4 @@ class WeatherOpenMeteo(WeatherProvider):
),
)
description = "Precipitable Water (cm)"
self._description_from_series(description, pwat)
await self._description_from_series(description, pwat)