Fix BrightSky weather prediction

- Get weather data with fully specified end_date datetime argument to not miss data.
- Make preciptable water records generation robust against missing temperature
  or humidity values.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

src/akkudoktoreos/core/dataabc.py

@@ -811,7 +811,8 @@ class DataSequence(DataBase, MutableSequence):
         dates, values = self.key_to_lists(
             key=key, start_datetime=start_datetime, end_datetime=end_datetime, dropna=dropna
         )
-        return pd.Series(data=values, index=pd.DatetimeIndex(dates), name=key)
+        series = pd.Series(data=values, index=pd.DatetimeIndex(dates), name=key)
+        return series
 
     def key_from_series(self, key: str, series: pd.Series) -> None:
         """Update the DataSequence from a Pandas Series.

src/akkudoktoreos/prediction/weatherbrightsky.py

@@ -7,7 +7,7 @@ format, enabling consistent access to forecasted and historical weather attribut
 """
 
 import json
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
 import pandas as pd
 import pvlib
@@ -16,14 +16,14 @@ import requests
 from akkudoktoreos.core.cache import cache_in_file
 from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.prediction.weatherabc import WeatherDataRecord, WeatherProvider
-from akkudoktoreos.utils.datetimeutil import to_datetime
+from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration
 
 logger = get_logger(__name__)
 
 
-WheaterDataBrightSkyMapping: List[Tuple[str, Optional[str], Optional[float]]] = [
+WheaterDataBrightSkyMapping: List[Tuple[str, Optional[str], Optional[Union[str, float]]]] = [
     # brightsky_key, description, corr_factor
-    ("timestamp", "DateTime", None),
+    ("timestamp", "DateTime", "to datetime in timezone"),
     ("precipitation", "Precipitation Amount (mm)", 1),
     ("pressure_msl", "Pressure (mb)", 1),
     ("sunshine", None, None),
@@ -96,8 +96,8 @@ class WeatherBrightSky(WeatherProvider):
             ValueError: If the API response does not include expected `weather` data.
         """
         source = "https://api.brightsky.dev"
-        date = to_datetime(self.start_datetime, as_string="YYYY-MM-DD")
+        date = to_datetime(self.start_datetime, as_string=True)
-        last_date = to_datetime(self.end_datetime, as_string="YYYY-MM-DD")
+        last_date = to_datetime(self.end_datetime, as_string=True)
         response = requests.get(
             f"{source}/weather?lat={self.config.general.latitude}&lon={self.config.general.longitude}&date={date}&last_date={last_date}&tz={self.config.general.timezone}"
         )
@@ -133,7 +133,8 @@ class WeatherBrightSky(WeatherProvider):
             error_msg = f"No WeatherDataRecord key for '{description}'"
             logger.error(error_msg)
             raise ValueError(error_msg)
-        return self.key_to_series(key)
+        series = self.key_to_series(key)
+        return series
 
     def _description_from_series(self, description: str, data: pd.Series) -> None:
         """Update a weather data with a pandas Series based on its description.
@@ -170,7 +171,7 @@ class WeatherBrightSky(WeatherProvider):
         brightsky_data = self._request_forecast(force_update=force_update)  # type: ignore
 
         # Get key mapping from description
-        brightsky_key_mapping: Dict[str, Tuple[Optional[str], Optional[float]]] = {}
+        brightsky_key_mapping: Dict[str, Tuple[Optional[str], Optional[Union[str, float]]]] = {}
         for brightsky_key, description, corr_factor in WheaterDataBrightSkyMapping:
             if description is None:
                 brightsky_key_mapping[brightsky_key] = (None, None)
@@ -192,7 +193,10 @@ class WeatherBrightSky(WeatherProvider):
                 value = brightsky_record[brightsky_key]
                 corr_factor = item[1]
                 if value and corr_factor:
-                    value = value * corr_factor
+                    if corr_factor == "to datetime in timezone":
+                        value = to_datetime(value, in_timezone=self.config.general.timezone)
+                    else:
+                        value = value * corr_factor
                 setattr(weather_record, key, value)
             self.insert_by_datetime(weather_record)
 
@@ -216,14 +220,30 @@ class WeatherBrightSky(WeatherProvider):
         self._description_from_series(description, dhi)
 
         # Add Preciptable Water (PWAT) with a PVLib method.
-        description = "Temperature (°C)"
+        key = WeatherDataRecord.key_from_description("Temperature (°C)")
-        temperature = self._description_to_series(description)
+        assert key
-
+        temperature = self.key_to_array(
-        description = "Relative Humidity (%)"
+            key=key,
-        humidity = self._description_to_series(description)
+            start_datetime=self.start_datetime,
-
+            end_datetime=self.end_datetime,
+            interval=to_duration("1 hour"),
+        )
+        key = WeatherDataRecord.key_from_description("Relative Humidity (%)")
+        assert key
+        humidity = self.key_to_array(
+            key=key,
+            start_datetime=self.start_datetime,
+            end_datetime=self.end_datetime,
+            interval=to_duration("1 hour"),
+        )
+        data = pvlib.atmosphere.gueymard94_pw(temperature, humidity)
         pwat = pd.Series(
-            data=pvlib.atmosphere.gueymard94_pw(temperature, humidity), index=temperature.index
+            data=data,
+            index=pd.DatetimeIndex(
+                pd.date_range(
+                    start=self.start_datetime, end=self.end_datetime, freq="1h", inclusive="left"
+                )
+            ),
         )
         description = "Preciptable Water (cm)"
         self._description_from_series(description, pwat)

src/akkudoktoreos/server/eos.py

@@ -7,6 +7,7 @@ import os
 import signal
 import subprocess
 import sys
+import traceback
 from contextlib import asynccontextmanager
 from pathlib import Path
 from typing import Annotated, Any, AsyncGenerator, Dict, List, Optional, Union
@@ -844,7 +845,11 @@ def fastapi_prediction_update(
     try:
         prediction_eos.update_data(force_update=force_update, force_enable=force_enable)
     except Exception as e:
-        raise HTTPException(status_code=400, detail=f"Error on prediction update: {e}")
+        trace = "".join(traceback.TracebackException.from_exception(e).format())
+        raise HTTPException(
+            status_code=400,
+            detail=f"Error on prediction update: {e}{trace}",
+        )
     return Response()
 
 
@@ -868,7 +873,9 @@ def fastapi_prediction_update_provider(
     try:
         provider.update_data(force_update=force_update, force_enable=force_enable)
     except Exception as e:
-        raise HTTPException(status_code=400, detail=f"Error on update of provider: {e}")
+        raise HTTPException(
+            status_code=400, detail=f"Error on update of provider '{provider_id}': {e}"
+        )
     return Response()
 
 

tests/test_weatherbrightsky.py

@@ -162,10 +162,7 @@ def test_update_data(mock_get, provider, sample_brightsky_1_json, cache_store):
 
     # Assert: Verify the result is as expected
     mock_get.assert_called_once()
-    assert len(provider) == 338
+    assert len(provider) == 50
-
-    # with open(FILE_TESTDATA_WEATHERBRIGHTSKY_2_JSON, "w") as f_out:
-    #    f_out.write(provider.to_json())
 
 
 # ------------------------------------------------
@@ -188,3 +185,8 @@ def test_brightsky_development_forecast_data(provider, config_eos, is_system_tes
 
     with FILE_TESTDATA_WEATHERBRIGHTSKY_1_JSON.open("w", encoding="utf-8", newline="\n") as f_out:
         json.dump(brightsky_data, f_out, indent=4)
+
+    provider.update_data(force_enable=True, force_update=True)
+
+    with FILE_TESTDATA_WEATHERBRIGHTSKY_2_JSON.open("w", encoding="utf-8", newline="\n") as f_out:
+        f_out.write(provider.model_dump_json(indent=4))