fix: logging, prediction update, multiple bugs (#584)

* Fix logging configuration issues that made logging stop operation. Switch to Loguru
  logging (from Python logging). Enable console and file logging with different log levels.
  Add logging documentation.

* Fix logging configuration and EOS configuration out of sync. Added tracking support
  for nested value updates of Pydantic models. This used to update the logging configuration
  when the EOS configurationm for logging is changed. Should keep logging config and EOS
  config in sync as long as all changes to the EOS logging configuration are done by
  set_nested_value(), which is the case for the REST API.

* Fix energy management task looping endlessly after the second update when trying to update
  the last_update datetime.

* Fix get_nested_value() to correctly take values from the dicts in a Pydantic model instance.

* Fix usage of model classes instead of model instances in nested value access when evaluation
  the value type that is associated to each key.

* Fix illegal json format in prediction documentation for PVForecastAkkudoktor provider.

* Fix documentation qirks and add EOS Connect to integrations.

* Support deprecated fields in configuration in documentation generation and EOSdash.

* Enhance EOSdash demo to show BrightSky humidity data (that is often missing)

* Update documentation reference to German EOS installation videos.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

src/akkudoktoreos/server/dash/admin.py

@@ -10,6 +10,7 @@ from typing import Any, Optional, Union
 
 import requests
 from fasthtml.common import Select
+from loguru import logger
 from monsterui.foundations import stringify
 from monsterui.franken import (  # Select, TODO: Select from FrankenUI does not work - using Select from FastHTML instead
     H3,
@@ -29,13 +30,10 @@ from monsterui.franken import (  # Select, TODO: Select from FrankenUI does not
 )
 from platformdirs import user_config_dir
 
-from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.server.dash.components import Error, Success
 from akkudoktoreos.server.dash.configuration import get_nested_value
 from akkudoktoreos.utils.datetimeutil import to_datetime
 
-logger = get_logger(__name__)
-
 # Directory to export files to, or to import files from
 export_import_directory = Path(user_config_dir("net.akkudoktor.eosdash", "akkudoktor"))
 

src/akkudoktoreos/server/dash/components.py

@@ -100,6 +100,7 @@ def ConfigCard(
     value: str,
     default: str,
     description: str,
+    deprecated: Optional[Union[str, bool]],
     update_error: Optional[str],
     update_value: Optional[str],
     update_open: Optional[bool],
@@ -118,6 +119,7 @@ def ConfigCard(
         value (str): The current value of the configuration.
         default (str): The default value of the configuration.
         description (str): A description of the configuration.
+        deprecated (Optional[Union[str, bool]]): The deprecated marker of the configuration.
         update_error (Optional[str]): The error message, if any, during the update process.
         update_value (Optional[str]): The value to be updated, if different from the current value.
         update_open (Optional[bool]): A flag indicating whether the update section of the card
@@ -131,6 +133,9 @@ def ConfigCard(
         update_value = value
     if not update_open:
         update_open = False
+    if deprecated:
+        if isinstance(deprecated, bool):
+            deprecated = "Deprecated"
     return Card(
         Details(
             Summary(
@@ -151,13 +156,21 @@ def ConfigCard(
             Grid(
                 P(description),
                 P(config_type),
-            ),
+            )
+            if not deprecated
+            else None,
+            Grid(
+                P(deprecated),
+                P("DEPRECATED!"),
+            )
+            if deprecated
+            else None,
             # Default
             Grid(
                 DivRAligned(P("default")),
                 P(default),
             )
-            if read_only == "rw"
+            if read_only == "rw" and not deprecated
             else None,
             # Set value
             Grid(
@@ -172,7 +185,7 @@ def ConfigCard(
                     ),
                 ),
             )
-            if read_only == "rw"
+            if read_only == "rw" and not deprecated
             else None,
             # Last error
             Grid(

src/akkudoktoreos/server/dash/configuration.py

@@ -2,6 +2,7 @@ import json
 from typing import Any, Dict, List, Optional, Sequence, TypeVar, Union
 
 import requests
+from loguru import logger
 from monsterui.franken import (
     H3,
     H4,
@@ -22,13 +23,10 @@ from pydantic.fields import ComputedFieldInfo, FieldInfo
 from pydantic_core import PydanticUndefined
 
 from akkudoktoreos.config.config import ConfigEOS
-from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.core.pydantic import PydanticBaseModel
 from akkudoktoreos.prediction.pvforecast import PVForecastPlaneSetting
 from akkudoktoreos.server.dash.components import ConfigCard
 
-logger = get_logger(__name__)
-
 T = TypeVar("T")
 
 # Latest configuration update results
@@ -166,7 +164,7 @@ def configuration(
                     if found_basic:
                         continue
 
-                    config = {}
+                    config: dict[str, Optional[Any]] = {}
                     config["name"] = ".".join(values_prefix + parent_types)
                     config["value"] = json.dumps(
                         get_nested_value(values, values_prefix + parent_types, "<unknown>")
@@ -175,6 +173,9 @@ def configuration(
                     config["description"] = (
                         subfield_info.description if subfield_info.description else ""
                     )
+                    config["deprecated"] = (
+                        subfield_info.deprecated if subfield_info.deprecated else None
+                    )
                     if isinstance(subfield_info, ComputedFieldInfo):
                         config["read-only"] = "ro"
                         type_description = str(subfield_info.return_type)
@@ -319,6 +320,7 @@ def ConfigPlanesCard(
                     config["value"],
                     config["default"],
                     config["description"],
+                    config["deprecated"],
                     update_error,
                     update_value,
                     update_open,
@@ -457,6 +459,7 @@ def Configuration(
         if (
             config["type"]
             == "Optional[list[akkudoktoreos.prediction.pvforecast.PVForecastPlaneSetting]]"
+            and not config["deprecated"]
         ):
             # Special configuration for PV planes
             rows.append(
@@ -482,6 +485,7 @@ def Configuration(
                     config["value"],
                     config["default"],
                     config["description"],
+                    config["deprecated"],
                     update_error,
                     update_value,
                     update_open,

src/akkudoktoreos/server/dash/demo.py

@@ -8,7 +8,6 @@ from bokeh.models import ColumnDataSource, LinearAxis, Range1d
 from bokeh.plotting import figure
 from monsterui.franken import FT, Grid, P
 
-from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.core.pydantic import PydanticDateTimeDataFrame
 from akkudoktoreos.server.dash.bokeh import Bokeh
 
@@ -17,8 +16,6 @@ FILE_DEMOCONFIG = DIR_DEMODATA.joinpath("democonfig.json")
 if not FILE_DEMOCONFIG.exists():
     raise ValueError(f"File does not exist: {FILE_DEMOCONFIG}")
 
-logger = get_logger(__name__)
-
 # bar width for 1 hour bars (time given in millseconds)
 BAR_WIDTH_1HOUR = 1000 * 60 * 60
 
@@ -76,25 +73,37 @@ def DemoElectricityPriceForecast(predictions: pd.DataFrame, config: dict) -> FT:
     return Bokeh(plot)
 
 
-def DemoWeatherTempAir(predictions: pd.DataFrame, config: dict) -> FT:
+def DemoWeatherTempAirHumidity(predictions: pd.DataFrame, config: dict) -> FT:
     source = ColumnDataSource(predictions)
     provider = config["weather"]["provider"]
 
     plot = figure(
         x_axis_type="datetime",
-        y_range=Range1d(
-            predictions["weather_temp_air"].min() - 1.0, predictions["weather_temp_air"].max() + 1.0
-        ),
-        title=f"Air Temperature Prediction ({provider})",
+        title=f"Air Temperature and Humidity Prediction ({provider})",
         x_axis_label="Datetime",
         y_axis_label="Temperature [°C]",
         sizing_mode="stretch_width",
         height=400,
     )
+    # Add secondary y-axis for humidity
+    plot.extra_y_ranges["humidity"] = Range1d(start=-5, end=105)
+    y2_axis = LinearAxis(y_range_name="humidity", axis_label="Relative Humidity [%]")
+    y2_axis.axis_label_text_color = "green"
+    plot.add_layout(y2_axis, "left")
+
     plot.line(
         "date_time", "weather_temp_air", source=source, legend_label="Air Temperature", color="blue"
     )
 
+    plot.line(
+        "date_time",
+        "weather_relative_humidity",
+        source=source,
+        legend_label="Relative Humidity [%]",
+        color="green",
+        y_range_name="humidity",
+    )
+
     return Bokeh(plot)
 
 
@@ -150,7 +159,10 @@ def DemoLoad(predictions: pd.DataFrame, config: dict) -> FT:
         sizing_mode="stretch_width",
         height=400,
     )
-    plot.extra_y_ranges["stddev"] = Range1d(0, 1000)
+    # Add secondary y-axis for stddev
+    stddev_min = predictions["load_std"].min()
+    stddev_max = predictions["load_std"].max()
+    plot.extra_y_ranges["stddev"] = Range1d(start=stddev_min - 5, end=stddev_max + 5)
     y2_axis = LinearAxis(y_range_name="stddev", axis_label="Load Standard Deviation [W]")
     y2_axis.axis_label_text_color = "green"
     plot.add_layout(y2_axis, "left")
@@ -230,6 +242,7 @@ def Demo(eos_host: str, eos_port: Union[str, int]) -> str:
             "keys": [
                 "pvforecast_ac_power",
                 "elecprice_marketprice_kwh",
+                "weather_relative_humidity",
                 "weather_temp_air",
                 "weather_ghi",
                 "weather_dni",
@@ -260,7 +273,7 @@ def Demo(eos_host: str, eos_port: Union[str, int]) -> str:
     return Grid(
         DemoPVForecast(predictions, democonfig),
         DemoElectricityPriceForecast(predictions, democonfig),
-        DemoWeatherTempAir(predictions, democonfig),
+        DemoWeatherTempAirHumidity(predictions, democonfig),
         DemoWeatherIrradiance(predictions, democonfig),
         DemoLoad(predictions, democonfig),
         cols_max=2,

src/akkudoktoreos/server/dash/footer.py

@@ -1,14 +1,13 @@
 from typing import Optional, Union
 
 import requests
+from loguru import logger
 from monsterui.daisy import Loading, LoadingT
 from monsterui.franken import A, ButtonT, DivFullySpaced, P
 from requests.exceptions import RequestException
 
 from akkudoktoreos.config.config import get_config
-from akkudoktoreos.core.logging import get_logger
 
-logger = get_logger(__name__)
 config_eos = get_config()
 
 

src/akkudoktoreos/server/eos.py

@@ -25,11 +25,13 @@ from fastapi.responses import (
     RedirectResponse,
     Response,
 )
+from loguru import logger
 
 from akkudoktoreos.config.config import ConfigEOS, SettingsEOS, get_config
 from akkudoktoreos.core.cache import CacheFileStore
 from akkudoktoreos.core.ems import get_ems
-from akkudoktoreos.core.logging import get_logger
+from akkudoktoreos.core.logabc import LOGGING_LEVELS
+from akkudoktoreos.core.logging import read_file_log, track_logging_config
 from akkudoktoreos.core.pydantic import (
     PydanticBaseModel,
     PydanticDateTimeData,
@@ -56,7 +58,6 @@ from akkudoktoreos.server.server import (
 )
 from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration
 
-logger = get_logger(__name__)
 config_eos = get_config()
 measurement_eos = get_measurement()
 prediction_eos = get_prediction()
@@ -66,6 +67,14 @@ ems_eos = get_ems()
 args = None
 
 
+# ----------------------
+# Logging configuration
+# ----------------------
+
+logger.remove()
+track_logging_config(config_eos, "logging", None, None)
+config_eos.track_nested_value("/logging", track_logging_config)
+
 # ----------------------
 # EOSdash server startup
 # ----------------------
@@ -177,18 +186,32 @@ def cache_save() -> dict:
     return CacheFileStore().save_store()
 
 
-@repeat_every(seconds=float(config_eos.cache.cleanup_interval))
+def cache_cleanup_on_exception(e: Exception) -> None:
+    logger.error("Cache cleanup task caught an exception: {}", e, exc_info=True)
+
+
+@repeat_every(
+    seconds=float(config_eos.cache.cleanup_interval),
+    on_exception=cache_cleanup_on_exception,
+)
 def cache_cleanup_task() -> None:
     """Repeating task to clear cache from expired cache files."""
+    logger.debug("Clear cache")
     cache_clear()
 
 
+def energy_management_on_exception(e: Exception) -> None:
+    logger.error("Energy management task caught an exception: {}", e, exc_info=True)
+
+
 @repeat_every(
     seconds=10,
     wait_first=config_eos.ems.startup_delay,
+    on_exception=energy_management_on_exception,
 )
 def energy_management_task() -> None:
     """Repeating task for energy management."""
+    logger.debug("Check EMS run")
     ems_eos.manage_energy()
 
 
@@ -219,6 +242,7 @@ async def server_shutdown_task() -> None:
         os.kill(pid, signal.SIGTERM)  # type: ignore[attr-defined,unused-ignore]
 
     logger.info(f"🚀 EOS terminated, PID {pid}")
+    sys.exit(0)
 
 
 @asynccontextmanager
@@ -544,6 +568,52 @@ def fastapi_config_get_key(
         raise HTTPException(status_code=400, detail=str(e))
 
 
+@app.get("/v1/logging/log", tags=["logging"])
+async def fastapi_logging_get_log(
+    limit: int = Query(100, description="Maximum number of log entries to return."),
+    level: Optional[str] = Query(None, description="Filter by log level (e.g., INFO, ERROR)."),
+    contains: Optional[str] = Query(None, description="Filter logs containing this substring."),
+    regex: Optional[str] = Query(None, description="Filter logs by matching regex in message."),
+    from_time: Optional[str] = Query(
+        None, description="Start time (ISO format) for filtering logs."
+    ),
+    to_time: Optional[str] = Query(None, description="End time (ISO format) for filtering logs."),
+    tail: bool = Query(False, description="If True, returns the most recent lines (tail mode)."),
+) -> JSONResponse:
+    """Get structured log entries from the EOS log file.
+
+    Filters and returns log entries based on the specified query parameters. The log
+    file is expected to contain newline-delimited JSON entries.
+
+    Args:
+        limit (int): Maximum number of entries to return.
+        level (Optional[str]): Filter logs by severity level (e.g., DEBUG, INFO).
+        contains (Optional[str]): Return only logs that include this string in the message.
+        regex (Optional[str]): Return logs that match this regular expression in the message.
+        from_time (Optional[str]): ISO 8601 timestamp to filter logs not older than this.
+        to_time (Optional[str]): ISO 8601 timestamp to filter logs not newer than this.
+        tail (bool): If True, fetch the most recent log entries (like `tail`).
+
+    Returns:
+        JSONResponse: A JSON list of log entries.
+    """
+    log_path = config_eos.logging.file_path
+    try:
+        logs = read_file_log(
+            log_path=log_path,
+            limit=limit,
+            level=level,
+            contains=contains,
+            regex=regex,
+            from_time=from_time,
+            to_time=to_time,
+            tail=tail,
+        )
+        return JSONResponse(content=logs)
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+
+
 @app.get("/v1/measurement/keys", tags=["measurement"])
 def fastapi_measurement_keys_get() -> list[str]:
     """Get a list of available measurement keys."""
@@ -1245,45 +1315,46 @@ Did you want to connect to <a href="{url}" class="back-button">EOSdash</a>?
     return RedirectResponse(url="/docs", status_code=303)
 
 
-def run_eos(host: str, port: int, log_level: str, access_log: bool, reload: bool) -> None:
+def run_eos(host: str, port: int, log_level: str, reload: bool) -> None:
     """Run the EOS server with the specified configurations.
 
-    This function starts the EOS server using the Uvicorn ASGI server. It accepts
-    arguments for the host, port, log level, access log, and reload options. The
-    log level is converted to lowercase to ensure compatibility with Uvicorn's
-    expected log level format. If an error occurs while attempting to bind the
-    server to the specified host and port, an error message is logged and the
-    application exits.
+    Starts the EOS server using the Uvicorn ASGI server. Logs an error and exits if
+    binding to the host and port fails.
 
-    Parameters:
-    host (str): The hostname to bind the server to.
-    port (int): The port number to bind the server to.
-    log_level (str): The log level for the server. Options include "critical", "error",
-                     "warning", "info", "debug", and "trace".
-    access_log (bool): Whether to enable or disable the access log. Set to True to enable.
-    reload (bool): Whether to enable or disable auto-reload. Set to True for development.
+    Args:
+        host (str): Hostname to bind the server to.
+        port (int): Port number to bind the server to.
+        log_level (str): Log level for the server. One of
+            "critical", "error", "warning", "info", "debug", or "trace".
+        reload (bool): Enable or disable auto-reload. True for development.
 
     Returns:
-    None
+        None
     """
     # Make hostname Windows friendly
     if host == "0.0.0.0" and os.name == "nt":  # noqa: S104
         host = "localhost"
 
+    # Setup console logging level using nested value
+    # - triggers logging configuration by track_logging_config
+    if log_level.upper() in LOGGING_LEVELS:
+        config_eos.set_nested_value("logging/console_level", log_level.upper())
+
     # Wait for EOS port to be free - e.g. in case of restart
     wait_for_port_free(port, timeout=120, waiting_app_name="EOS")
 
     try:
+        # Let uvicorn run the fastAPI app
         uvicorn.run(
             "akkudoktoreos.server.eos:app",
             host=host,
             port=port,
-            log_level=log_level.lower(),  # Convert log_level to lowercase
-            access_log=access_log,
+            log_level="info",
+            access_log=True,
             reload=reload,
         )
     except Exception as e:
-        logger.error(f"Could not bind to host {host}:{port}. Error: {e}")
+        logger.exception("Failed to start uvicorn server.")
         raise e
 
 
@@ -1298,7 +1369,7 @@ def main() -> None:
     Command-line Arguments:
     --host (str): Host for the EOS server (default: value from config).
     --port (int): Port for the EOS server (default: value from config).
-    --log_level (str): Log level for the server. Options: "critical", "error", "warning", "info", "debug", "trace" (default: "info").
+    --log_level (str): Log level for the server console. Options: "critical", "error", "warning", "info", "debug", "trace" (default: "info").
     --access_log (bool): Enable or disable access log. Options: True or False (default: False).
     --reload (bool): Enable or disable auto-reload. Useful for development. Options: True or False (default: False).
     """
@@ -1322,14 +1393,8 @@ def main() -> None:
     parser.add_argument(
         "--log_level",
         type=str,
-        default="info",
-        help='Log level for the server. Options: "critical", "error", "warning", "info", "debug", "trace" (default: "info")',
-    )
-    parser.add_argument(
-        "--access_log",
-        type=bool,
-        default=False,
-        help="Enable or disable access log. Options: True or False (default: True)",
+        default="none",
+        help='Log level for the server console. Options: "critical", "error", "warning", "info", "debug", "trace" (default: "none")',
     )
     parser.add_argument(
         "--reload",
@@ -1344,7 +1409,7 @@ def main() -> None:
     port = args.port if args.port else 8503
 
     try:
-        run_eos(host, port, args.log_level, args.access_log, args.reload)
+        run_eos(host, port, args.log_level, args.reload)
     except:
         sys.exit(1)
 

src/akkudoktoreos/server/eosdash.py

@@ -8,10 +8,10 @@ from typing import Optional
 import psutil
 import uvicorn
 from fasthtml.common import FileResponse, JSONResponse
+from loguru import logger
 from monsterui.core import FastHTML, Theme
 
 from akkudoktoreos.config.config import get_config
-from akkudoktoreos.core.logging import get_logger
 
 # Pages
 from akkudoktoreos.server.dash.admin import Admin
@@ -23,7 +23,6 @@ from akkudoktoreos.server.dash.footer import Footer
 from akkudoktoreos.server.dash.hello import Hello
 from akkudoktoreos.server.server import get_default_host, wait_for_port_free
 
-logger = get_logger(__name__)
 config_eos = get_config()
 
 # The favicon for EOSdash

src/akkudoktoreos/server/server.py

@@ -6,12 +6,10 @@ import time
 from typing import Optional, Union
 
 import psutil
+from loguru import logger
 from pydantic import Field, IPvAnyAddress, field_validator
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
-from akkudoktoreos.core.logging import get_logger
-
-logger = get_logger(__name__)
 
 
 def get_default_host() -> str: