Add Documentation 2 (#334)

Add documentation that covers:

- configuration
- prediction

Add Python scripts that support automatic documentation generation for
configuration data defined with pydantic.

Adapt EOS configuration to provide more methods for REST API and
automatic documentation generation.

Adapt REST API to allow for EOS configuration file load and save.
Sort REST API on generation of openapi markdown for docs.

Move logutil to core/logging to allow configuration of logging by standard config.

Make Akkudoktor predictions always start extraction of prediction data at start of day.
Previously extraction started at actual hour. This is to support the code that assumes
prediction data to start at start of day.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

src/akkudoktoreos/prediction/elecpriceabc.py

@@ -9,8 +9,8 @@ from typing import List, Optional
 
 from pydantic import Field, computed_field
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/elecpriceakkudoktor.py

@@ -13,11 +13,11 @@ import requests
 from numpydantic import NDArray, Shape
 from pydantic import Field, ValidationError
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.core.pydantic import PydanticBaseModel
 from akkudoktoreos.prediction.elecpriceabc import ElecPriceDataRecord, ElecPriceProvider
 from akkudoktoreos.utils.cacheutil import CacheFileStore, cache_in_file
 from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 
@@ -218,17 +218,14 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
                 akkudoktor_value.marketpriceEurocentPerKWh / (100 * 1000) + charges_kwh / 1000
             )
 
-            if compare_datetimes(dt, self.start_datetime).lt:
-                # forecast data is too old
+            # We provide prediction starting at start of day, to be compatible to old system.
+            if compare_datetimes(dt, self.start_datetime.start_of("day")).lt:
+                # forecast data is too old - older than start_datetime with time set to 00:00:00
                 self.elecprice_8days[dt.hour, dt.day_of_week] = price_wh
                 continue
             self.elecprice_8days[dt.hour, 7] = price_wh
 
-            record = ElecPriceDataRecord(
-                date_time=dt,
-                elecprice_marketprice_wh=price_wh,
-            )
-            self.append(record)
+            self.update_value(dt, "elecprice_marketprice_wh", price_wh)
 
         # Update 8day cache
         elecprice_cache_file.seek(0)

src/akkudoktoreos/prediction/elecpriceimport.py

@@ -12,9 +12,9 @@ from typing import Optional, Union
 from pydantic import Field, field_validator
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.elecpriceabc import ElecPriceProvider
 from akkudoktoreos.prediction.predictionabc import PredictionImportProvider
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/load.py

@@ -5,12 +5,14 @@ from typing import Optional
 from pydantic import Field
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
-from akkudoktoreos.utils.logutil import get_logger
+from akkudoktoreos.core.logging import get_logger
 
 logger = get_logger(__name__)
 
 
 class LoadCommonSettings(SettingsBaseModel):
+    """Common settings for loaod forecast providers."""
+
     load_provider: Optional[str] = Field(
         default=None, description="Load provider id of provider to be used."
     )

src/akkudoktoreos/prediction/loadabc.py

@@ -9,8 +9,8 @@ from typing import List, Optional
 
 from pydantic import Field
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/loadakkudoktor.py

@@ -1,15 +1,14 @@
 """Retrieves load forecast data from Akkudoktor load profiles."""
 
-from pathlib import Path
 from typing import Optional
 
 import numpy as np
 from pydantic import Field
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.loadabc import LoadProvider
 from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 
@@ -84,7 +83,7 @@ class LoadAkkudoktor(LoadProvider):
 
     def load_data(self) -> np.ndarray:
         """Loads data from the Akkudoktor load file."""
-        load_file = Path(__file__).parent.parent.joinpath("data/load_profiles.npz")
+        load_file = self.config.package_root_path.joinpath("data/load_profiles.npz")
         data_year_energy = None
         try:
             file_data = np.load(load_file)
@@ -107,23 +106,25 @@ class LoadAkkudoktor(LoadProvider):
         """Adds the load means and standard deviations."""
         data_year_energy = self.load_data()
         weekday_adjust, weekend_adjust = self._calculate_adjustment(data_year_energy)
-        date = self.start_datetime
-        for i in range(self.config.prediction_hours):
+        # We provide prediction starting at start of day, to be compatible to old system.
+        # End date for prediction is prediction hours from now.
+        date = self.start_datetime.start_of("day")
+        end_date = self.start_datetime.add(hours=self.config.prediction_hours)
+        while compare_datetimes(date, end_date).lt:
             # Extract mean (index 0) and standard deviation (index 1) for the given day and hour
             # Day indexing starts at 0, -1 because of that
             hourly_stats = data_year_energy[date.day_of_year - 1, :, date.hour]
-            self.update_value(date, "load_mean", hourly_stats[0])
-            self.update_value(date, "load_std", hourly_stats[1])
+            values = {
+                "load_mean": hourly_stats[0],
+                "load_std": hourly_stats[1],
+            }
             if date.day_of_week < 5:
                 # Monday to Friday (0..4)
-                self.update_value(
-                    date, "load_mean_adjusted", hourly_stats[0] + weekday_adjust[date.hour]
-                )
+                values["load_mean_adjusted"] = hourly_stats[0] + weekday_adjust[date.hour]
             else:
                 # Saturday, Sunday (5, 6)
-                self.update_value(
-                    date, "load_mean_adjusted", hourly_stats[0] + weekend_adjust[date.hour]
-                )
+                values["load_mean_adjusted"] = hourly_stats[0] + weekend_adjust[date.hour]
+            self.update_value(date, values)
             date += to_duration("1 hour")
         # We are working on fresh data (no cache), report update time
         self.update_datetime = to_datetime(in_timezone=self.config.timezone)

src/akkudoktoreos/prediction/loadimport.py

@@ -12,9 +12,9 @@ from typing import Optional, Union
 from pydantic import Field, field_validator
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.loadabc import LoadProvider
 from akkudoktoreos.prediction.predictionabc import PredictionImportProvider
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/predictionabc.py

@@ -22,8 +22,8 @@ from akkudoktoreos.core.dataabc import (
     DataRecord,
     DataSequence,
 )
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.utils.datetimeutil import to_duration
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/pvforecast.py

@@ -5,7 +5,7 @@ from typing import Any, ClassVar, List, Optional
 from pydantic import Field, computed_field
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
-from akkudoktoreos.utils.logutil import get_logger
+from akkudoktoreos.core.logging import get_logger
 
 logger = get_logger(__name__)
 
@@ -43,7 +43,7 @@ class PVForecastCommonSettings(SettingsBaseModel):
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
     pvforecast0_loss: Optional[float] = Field(
-        default=None, description="Sum of PV system losses in percent"
+        default=14.0, description="Sum of PV system losses in percent"
     )
     pvforecast0_trackingtype: Optional[int] = Field(
         default=None,
@@ -98,7 +98,9 @@ class PVForecastCommonSettings(SettingsBaseModel):
         default="free",
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
-    pvforecast1_loss: Optional[float] = Field(0, description="Sum of PV system losses in percent")
+    pvforecast1_loss: Optional[float] = Field(
+        default=14.0, description="Sum of PV system losses in percent"
+    )
     pvforecast1_trackingtype: Optional[int] = Field(
         default=None,
         description="Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south.",
@@ -152,7 +154,9 @@ class PVForecastCommonSettings(SettingsBaseModel):
         default="free",
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
-    pvforecast2_loss: Optional[float] = Field(0, description="Sum of PV system losses in percent")
+    pvforecast2_loss: Optional[float] = Field(
+        default=14.0, description="Sum of PV system losses in percent"
+    )
     pvforecast2_trackingtype: Optional[int] = Field(
         default=None,
         description="Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south.",
@@ -206,7 +210,9 @@ class PVForecastCommonSettings(SettingsBaseModel):
         default="free",
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
-    pvforecast3_loss: Optional[float] = Field(0, description="Sum of PV system losses in percent")
+    pvforecast3_loss: Optional[float] = Field(
+        default=14.0, description="Sum of PV system losses in percent"
+    )
     pvforecast3_trackingtype: Optional[int] = Field(
         default=None,
         description="Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south.",
@@ -260,7 +266,9 @@ class PVForecastCommonSettings(SettingsBaseModel):
         default="free",
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
-    pvforecast4_loss: Optional[float] = Field(0, description="Sum of PV system losses in percent")
+    pvforecast4_loss: Optional[float] = Field(
+        default=14.0, description="Sum of PV system losses in percent"
+    )
     pvforecast4_trackingtype: Optional[int] = Field(
         default=None,
         description="Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south.",
@@ -314,7 +322,9 @@ class PVForecastCommonSettings(SettingsBaseModel):
         default="free",
         description="Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated.",
     )
-    pvforecast5_loss: Optional[float] = Field(0, description="Sum of PV system losses in percent")
+    pvforecast5_loss: Optional[float] = Field(
+        default=14.0, description="Sum of PV system losses in percent"
+    )
     pvforecast5_trackingtype: Optional[int] = Field(
         default=None,
         description="Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south.",

src/akkudoktoreos/prediction/pvforecastabc.py

@@ -9,8 +9,8 @@ from typing import List, Optional
 
 from pydantic import Field
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/pvforecastakkudoktor.py

@@ -68,6 +68,7 @@ from typing import Any, List, Optional, Union
 import requests
 from pydantic import Field, ValidationError, computed_field
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.core.pydantic import PydanticBaseModel
 from akkudoktoreos.prediction.pvforecastabc import (
     PVForecastDataRecord,
@@ -75,7 +76,6 @@ from akkudoktoreos.prediction.pvforecastabc import (
 )
 from akkudoktoreos.utils.cacheutil import cache_in_file
 from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 
@@ -283,27 +283,21 @@ class PVForecastAkkudoktor(PVForecastProvider):
             original_datetime = akkudoktor_data.values[0][i].datetime
             dt = to_datetime(original_datetime, in_timezone=self.config.timezone)
 
-            # iso_datetime = parser.parse(original_datetime).isoformat()  # Konvertiere zu ISO-Format
-            # print()
-            # Optional: 2 Stunden abziehen, um die Zeitanpassung zu testen
-            # adjusted_datetime = parser.parse(original_datetime) - timedelta(hours=2)
-            # print(f"Angepasste Zeitstempel: {adjusted_datetime.isoformat()}")
-
-            if compare_datetimes(dt, self.start_datetime).lt:
+            # We provide prediction starting at start of day, to be compatible to old system.
+            if compare_datetimes(dt, self.start_datetime.start_of("day")).lt:
                 # forecast data is too old
                 continue
 
             sum_dc_power = sum(values[i].dcPower for values in akkudoktor_data.values)
             sum_ac_power = sum(values[i].power for values in akkudoktor_data.values)
 
-            record = PVForecastAkkudoktorDataRecord(
-                date_time=dt,  # Verwende angepassten Zeitstempel
-                pvforecast_dc_power=sum_dc_power,
-                pvforecast_ac_power=sum_ac_power,
-                pvforecastakkudoktor_wind_speed_10m=akkudoktor_data.values[0][i].windspeed_10m,
-                pvforecastakkudoktor_temp_air=akkudoktor_data.values[0][i].temperature,
-            )
-            self.append(record)
+            data = {
+                "pvforecast_dc_power": sum_dc_power,
+                "pvforecast_ac_power": sum_ac_power,
+                "pvforecastakkudoktor_wind_speed_10m": akkudoktor_data.values[0][i].windspeed_10m,
+                "pvforecastakkudoktor_temp_air": akkudoktor_data.values[0][i].temperature,
+            }
+            self.update_value(dt, data)
 
         if len(self) < self.config.prediction_hours:
             raise ValueError(

src/akkudoktoreos/prediction/pvforecastimport.py

@@ -12,9 +12,9 @@ from typing import Optional, Union
 from pydantic import Field, field_validator
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionImportProvider
 from akkudoktoreos.prediction.pvforecastabc import PVForecastProvider
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/weatherabc.py

@@ -14,8 +14,8 @@ import pandas as pd
 import pvlib
 from pydantic import Field
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/weatherbrightsky.py

@@ -13,10 +13,10 @@ import pandas as pd
 import pvlib
 import requests
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.weatherabc import WeatherDataRecord, WeatherProvider
 from akkudoktoreos.utils.cacheutil import cache_in_file
 from akkudoktoreos.utils.datetimeutil import to_datetime
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/weatherclearoutside.py

@@ -19,10 +19,10 @@ import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.weatherabc import WeatherDataRecord, WeatherProvider
 from akkudoktoreos.utils.cacheutil import cache_in_file
 from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration, to_timezone
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)
 

src/akkudoktoreos/prediction/weatherimport.py

@@ -12,9 +12,9 @@ from typing import Optional, Union
 from pydantic import Field, field_validator
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.predictionabc import PredictionImportProvider
 from akkudoktoreos.prediction.weatherabc import WeatherProvider
-from akkudoktoreos.utils.logutil import get_logger
 
 logger = get_logger(__name__)