mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2025-10-11 11:56:17 +00:00
Fix2 config and predictions revamp. (#281)
measurement:
- Add new measurement class to hold real world measurements.
- Handles load meter readings, grid import and export meter readings.
- Aggregates load meter readings aka. measurements to total load.
- Can import measurements from files, pandas datetime series,
pandas datetime dataframes, simple daetime arrays and
programmatically.
- Maybe expanded to other measurement values.
- Should be used for load prediction adaptions by real world
measurements.
core/coreabc:
- Add mixin class to access measurements
core/pydantic:
- Add pydantic models for pandas datetime series and dataframes.
- Add pydantic models for simple datetime array
core/dataabc:
- Provide DataImport mixin class for generic import handling.
Imports from JSON string and files. Imports from pandas datetime dataframes
and simple datetime arrays. Signature of import method changed to
allow import datetimes to be given programmatically and by data content.
- Use pydantic models for datetime series, dataframes, arrays
- Validate generic imports by pydantic models
- Provide new attributes min_datetime and max_datetime for DataSequence.
- Add parameter dropna to drop NAN/ None values when creating lists, pandas series
or numpy array from DataSequence.
config/config:
- Add common settings for the measurement module.
predictions/elecpriceakkudoktor:
- Use mean values of last 7 days to fill prediction values not provided by
akkudoktor.net (only provides 24 values).
prediction/loadabc:
- Extend the generic prediction keys by 'load_total_adjusted' for load predictions
that adjust the predicted total load by measured load values.
prediction/loadakkudoktor:
- Extend the Akkudoktor load prediction by load adjustment using measured load
values.
prediction/load_aggregator:
- Module removed. Load aggregation is now handled by the measurement module.
prediction/load_corrector:
- Module removed. Load correction (aka. adjustment of load prediction by
measured load energy) is handled by the LoadAkkudoktor prediction and
the generic 'load_mean_adjusted' prediction key.
prediction/load_forecast:
- Module removed. Functionality now completely handled by the LoadAkkudoktor
prediction.
utils/cacheutil:
- Use pydantic.
- Fix potential bug in ttl (time to live) duration handling.
utils/datetimeutil:
- Added missing handling of pendulum.DateTime and pendulum.Duration instances
as input. Handled before as datetime.datetime and datetime.timedelta.
utils/visualize:
- Move main to generate_example_report() for better testing support.
server/server:
- Added new configuration option server_fastapi_startup_server_fasthtml
to make startup of FastHTML server by FastAPI server conditional.
server/fastapi_server:
- Add APIs for measurements
- Improve APIs to provide or take pandas datetime series and
datetime dataframes controlled by pydantic model.
- Improve APIs to provide or take simple datetime data arrays
controlled by pydantic model.
- Move fastAPI server API to v1 for new APIs.
- Update pre v1 endpoints to use new prediction and measurement capabilities.
- Only start FastHTML server if 'server_fastapi_startup_server_fasthtml'
config option is set.
tests:
- Adapt import tests to changed import method signature
- Adapt server test to use the v1 API
- Extend the dataabc test to test for array generation from data
with several data interval scenarios.
- Extend the datetimeutil test to also test for correct handling
of to_datetime() providing now().
- Adapt LoadAkkudoktor test for new adjustment calculation.
- Adapt visualization test to use example report function instead of visualize.py
run as process.
- Removed test_load_aggregator. Functionality is now tested in test_measurement.
- Added tests for measurement module
docs:
- Remove sphinxcontrib-openapi as it prevents build of documentation.
"site-packages/sphinxcontrib/openapi/openapi31.py", line 305, in _get_type_from_schema
for t in schema["anyOf"]: KeyError: 'anyOf'"
Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
This commit is contained in:
Bobby Noelte
GitHub
@@ -8,13 +8,15 @@ format, enabling consistent access to forecasted and historical electricity pric
|
||||
|
||||
from typing import Any, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
from pydantic import ValidationError
|
||||
from numpydantic import NDArray, Shape
|
||||
from pydantic import Field, ValidationError
|
||||
|
||||
from akkudoktoreos.core.pydantic import PydanticBaseModel
|
||||
from akkudoktoreos.prediction.elecpriceabc import ElecPriceDataRecord, ElecPriceProvider
|
||||
from akkudoktoreos.utils.cacheutil import cache_in_file
|
||||
from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime
|
||||
from akkudoktoreos.utils.cacheutil import CacheFileStore, cache_in_file
|
||||
from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration
|
||||
from akkudoktoreos.utils.logutil import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -63,6 +65,20 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
|
||||
_update_data(): Processes and updates forecast data from Akkudoktor in ElecPriceDataRecord format.
|
||||
"""
|
||||
|
||||
elecprice_8days: NDArray[Shape["24, 8"], float] = Field(
|
||||
default=np.full((24, 8), np.nan),
|
||||
description="Hourly electricity prices for the last 7 days and today (€/KWh). "
|
||||
"A NumPy array of 24 elements, each representing the hourly prices "
|
||||
"of the last 7 days (index 0..6, Monday..Sunday) and today (index 7).",
|
||||
)
|
||||
elecprice_8days_weights_day_of_week: NDArray[Shape["7, 8"], float] = Field(
|
||||
default=np.full((7, 8), np.nan),
|
||||
description="Daily electricity price weights for the last 7 days and today. "
|
||||
"A NumPy array of 7 elements (Monday..Sunday), each representing "
|
||||
"the daily price weights of the last 7 days (index 0..6, Monday..Sunday) "
|
||||
"and today (index 7).",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def provider_id(cls) -> str:
|
||||
"""Return the unique identifier for the Akkudoktor provider."""
|
||||
@@ -84,6 +100,50 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
|
||||
raise ValueError(error_msg)
|
||||
return akkudoktor_data
|
||||
|
||||
def _calculate_weighted_mean(self, day_of_week: int, hour: int) -> float:
|
||||
"""Calculate the weighted mean price for given day_of_week and hour.
|
||||
|
||||
Args:
|
||||
day_of_week (int). The day of week to calculate the mean for (0=Monday..6).
|
||||
hour (int): The hour week to calculate the mean for (0..23).
|
||||
|
||||
Returns:
|
||||
price_weihgted_mead (float): Weighted mean price for given day_of:week and hour.
|
||||
"""
|
||||
if np.isnan(self.elecprice_8days_weights_day_of_week[0][0]):
|
||||
# Weights not initialized - do now
|
||||
|
||||
# Priority of day: 1=most .. 7=least
|
||||
priority_of_day = np.array(
|
||||
# Available Prediction days /
|
||||
# M,Tu,We,Th,Fr,Sa,Su,Today/ Forecast day_of_week
|
||||
[
|
||||
[1, 2, 3, 4, 5, 6, 7, 1], # Monday
|
||||
[3, 1, 2, 4, 5, 6, 7, 1], # Tuesday
|
||||
[4, 2, 1, 3, 5, 6, 7, 1], # Wednesday
|
||||
[5, 4, 2, 1, 3, 6, 7, 1], # Thursday
|
||||
[5, 4, 3, 2, 1, 6, 7, 1], # Friday
|
||||
[7, 6, 5, 4, 2, 1, 3, 1], # Saturday
|
||||
[7, 6, 5, 4, 3, 2, 1, 1], # Sunday
|
||||
]
|
||||
)
|
||||
# Take priorities above to decrease relevance in 2s exponential
|
||||
self.elecprice_8days_weights_day_of_week = 2 / (2**priority_of_day)
|
||||
|
||||
# Compute the weighted mean for day_of_week and hour
|
||||
prices_of_hour = self.elecprice_8days[hour]
|
||||
if np.isnan(prices_of_hour).all():
|
||||
# No prediction prices available for this hour - use mean value of all prices
|
||||
price_weighted_mean = np.nanmean(self.elecprice_marketprice_8day)
|
||||
else:
|
||||
weights = self.elecprice_8days_weights_day_of_week[day_of_week]
|
||||
prices_of_hour_masked: NDArray[Shape["24"]] = np.ma.MaskedArray(
|
||||
prices_of_hour, mask=np.isnan(prices_of_hour)
|
||||
)
|
||||
price_weighted_mean = np.ma.average(prices_of_hour_masked, weights=weights)
|
||||
|
||||
return float(price_weighted_mean)
|
||||
|
||||
@cache_in_file(with_ttl="1 hour")
|
||||
def _request_forecast(self) -> AkkudoktorElecPrice:
|
||||
"""Fetch electricity price forecast data from Akkudoktor API.
|
||||
@@ -98,13 +158,13 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
|
||||
ValueError: If the API response does not include expected `electricity price` data.
|
||||
"""
|
||||
source = "https://api.akkudoktor.net"
|
||||
date = to_datetime(self.start_datetime, as_string="Y-M-D")
|
||||
# Try to take data from 7 days back for prediction - usually only some hours back are available
|
||||
date = to_datetime(self.start_datetime - to_duration("7 days"), as_string="Y-M-D")
|
||||
last_date = to_datetime(self.end_datetime, as_string="Y-M-D")
|
||||
response = requests.get(
|
||||
f"{source}/prices?date={date}&last_date={last_date}&tz={self.config.timezone}"
|
||||
)
|
||||
url = f"{source}/prices?date={date}&last_date={last_date}&tz={self.config.timezone}"
|
||||
response = requests.get(url)
|
||||
logger.debug(f"Response from {url}: {response}")
|
||||
response.raise_for_status() # Raise an error for bad responses
|
||||
logger.debug(f"Response from {source}: {response}")
|
||||
akkudoktor_data = self._validate_data(response.content)
|
||||
# We are working on fresh data (no cache), report update time
|
||||
self.update_datetime = to_datetime(in_timezone=self.config.timezone)
|
||||
@@ -131,38 +191,66 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
|
||||
f"but only {values_len} data sets are given in forecast data."
|
||||
)
|
||||
|
||||
previous_price = akkudoktor_data.values[0].marketpriceEurocentPerKWh
|
||||
# Get cached 8day values
|
||||
elecprice_cache_file = CacheFileStore().get(key="ElecPriceAkkudoktor8dayCache")
|
||||
if elecprice_cache_file is None:
|
||||
# Cache does not exist - create it
|
||||
elecprice_cache_file = CacheFileStore().create(
|
||||
key="ElecPriceAkkudoktor8dayCache",
|
||||
until_datetime=to_datetime("infinity"),
|
||||
suffix=".npy",
|
||||
)
|
||||
np.save(elecprice_cache_file, self.elecprice_8days)
|
||||
elecprice_cache_file.seek(0)
|
||||
self.elecprice_8days = np.load(elecprice_cache_file)
|
||||
|
||||
for i in range(values_len):
|
||||
original_datetime = akkudoktor_data.values[i].start
|
||||
dt = to_datetime(original_datetime, in_timezone=self.config.timezone)
|
||||
akkudoktor_value = akkudoktor_data.values[i]
|
||||
|
||||
if compare_datetimes(dt, self.start_datetime).le:
|
||||
if compare_datetimes(dt, self.start_datetime).lt:
|
||||
# forecast data is too old
|
||||
previous_price = akkudoktor_data.values[i].marketpriceEurocentPerKWh
|
||||
self.elecprice_8days[dt.hour, dt.day_of_week] = (
|
||||
akkudoktor_value.marketpriceEurocentPerKWh
|
||||
)
|
||||
continue
|
||||
self.elecprice_8days[dt.hour, 7] = akkudoktor_value.marketpriceEurocentPerKWh
|
||||
|
||||
record = ElecPriceDataRecord(
|
||||
date_time=dt,
|
||||
elecprice_marketprice=akkudoktor_data.values[i].marketpriceEurocentPerKWh,
|
||||
elecprice_marketprice=akkudoktor_value.marketpriceEurocentPerKWh,
|
||||
)
|
||||
self.append(record)
|
||||
|
||||
# Update 8day cache
|
||||
elecprice_cache_file.seek(0)
|
||||
np.save(elecprice_cache_file, self.elecprice_8days)
|
||||
|
||||
# Check for new/ valid forecast data
|
||||
if len(self) == 0:
|
||||
# Got no valid forecast data
|
||||
return
|
||||
|
||||
# Assure price starts at start_time
|
||||
if compare_datetimes(self[0].date_time, self.start_datetime).gt:
|
||||
while compare_datetimes(self[0].date_time, self.start_datetime).gt:
|
||||
# Repeat the mean on the 8 day array to cover the missing hours
|
||||
dt = self[0].date_time.subtract(hours=1) # type: ignore
|
||||
value = self._calculate_weighted_mean(dt.day_of_week, dt.hour)
|
||||
|
||||
record = ElecPriceDataRecord(
|
||||
date_time=self.start_datetime,
|
||||
elecprice_marketprice=previous_price,
|
||||
date_time=dt,
|
||||
elecprice_marketprice=value,
|
||||
)
|
||||
self.insert(0, record)
|
||||
# Assure price ends at end_time
|
||||
if compare_datetimes(self[-1].date_time, self.end_datetime).lt:
|
||||
while compare_datetimes(self[-1].date_time, self.end_datetime).lt:
|
||||
# Repeat the mean on the 8 day array to cover the missing hours
|
||||
dt = self[-1].date_time.add(hours=1) # type: ignore
|
||||
value = self._calculate_weighted_mean(dt.day_of_week, dt.hour)
|
||||
|
||||
record = ElecPriceDataRecord(
|
||||
date_time=self.end_datetime,
|
||||
elecprice_marketprice=self[-1].elecprice_marketprice,
|
||||
date_time=dt,
|
||||
elecprice_marketprice=value,
|
||||
)
|
||||
self.append(record)
|
||||
# If some of the hourly values are missing, they will be interpolated when using
|
||||
# `key_to_array`.
|
||||
|
||||
Reference in New Issue
Block a user