Improve Configuration and Prediction Usability (#220)

* Update utilities in utils submodule.
* Add base configuration modules.
* Add server base configuration modules.
* Add devices base configuration modules.
* Add optimization base configuration modules.
* Add utils base configuration modules.
* Add prediction abstract and base classes plus tests.
* Add PV forecast to prediction submodule.
   The PV forecast modules are adapted from the class_pvforecast module and
   replace it.
* Add weather forecast to prediction submodule.
   The modules provide classes and methods to retrieve, manage, and process weather forecast data
   from various sources. Includes are structured representations of weather data and utilities
   for fetching forecasts for specific locations and time ranges.
   BrightSky and ClearOutside are currently supported.
* Add electricity price forecast to prediction submodule.
* Adapt fastapi server to base config and add fasthtml server.
* Add ems to core submodule.
* Adapt genetic to config.
* Adapt visualize to config.
* Adapt common test fixtures to config.
* Add load forecast to prediction submodule.
* Add core abstract and base classes.
* Adapt single test optimization to config.
* Adapt devices to config.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>

tests/conftest.py

@@ -1,31 +1,41 @@
 import logging
-import os
+import shutil
 import subprocess
 import sys
-import time
+import tempfile
 from pathlib import Path
+from typing import Optional
 
+import pendulum
+import platformdirs
 import pytest
 from xprocess import ProcessStarter
 
-from akkudoktoreos.config import EOS_DIR, AppConfig, load_config
+from akkudoktoreos.config.config import get_config
+from akkudoktoreos.utils.logutil import get_logger
+
+logger = get_logger(__name__)
 
 
-@pytest.fixture(name="tmp_config")
-def load_config_tmp(tmp_path: Path) -> AppConfig:
-    """Creates an AppConfig from default.config.json with a tmp output directory."""
-    config = load_config(tmp_path)
-    config.directories.output = str(tmp_path)
-    return config
+@pytest.fixture()
+def disable_debug_logging(scope="session", autouse=True):
+    """Automatically disable debug logging for all tests."""
+    original_levels = {}
+    root_logger = logging.getLogger()
 
+    original_levels[root_logger] = root_logger.level
+    root_logger.setLevel(logging.INFO)
+
+    for logger_name, logger in logging.root.manager.loggerDict.items():
+        if isinstance(logger, logging.Logger):
+            original_levels[logger] = logger.level
+            if logger.level <= logging.DEBUG:
+                logger.setLevel(logging.INFO)
 
-@pytest.fixture(autouse=True)
-def disable_debug_logging():
-    # Temporarily set logging level higher than DEBUG
-    logging.disable(logging.DEBUG)
     yield
-    # Re-enable logging back to its original state after the test
-    logging.disable(logging.NOTSET)
+
+    for logger, level in original_levels.items():
+        logger.setLevel(level)
 
 
 def pytest_addoption(parser):
@@ -39,6 +49,84 @@ def is_full_run(request):
     yield bool(request.config.getoption("--full-run"))
 
 
+@pytest.fixture
+def reset_config(disable_debug_logging):
+    """Fixture to reset EOS config to default values."""
+    config_eos = get_config()
+    config_eos.reset_settings()
+    config_eos.reset_to_defaults()
+    return config_eos
+
+
+@pytest.fixture
+def config_default_dirs():
+    """Fixture that provides a list of directories to be used as config dir."""
+    config_eos = get_config()
+    # Default config directory from platform user config directory
+    config_default_dir_user = Path(platformdirs.user_config_dir(config_eos.APP_NAME))
+    # Default config directory from current working directory
+    config_default_dir_cwd = Path.cwd()
+    # Default config directory from default config file
+    config_default_dir_default = Path(__file__).parent.parent.joinpath("src/akkudoktoreos/data")
+    return config_default_dir_user, config_default_dir_cwd, config_default_dir_default
+
+
+@pytest.fixture
+def stash_config_file(config_default_dirs):
+    """Fixture to temporarily stash away an existing config file during a test.
+
+    If the specified config file exists, it moves the file to a temporary directory.
+    The file is restored to its original location after the test.
+
+    Keep right most in fixture parameter list to assure application at last.
+
+    Returns:
+        Path: Path to the stashed config file.
+    """
+    config_eos = get_config()
+    config_default_dir_user, config_default_dir_cwd, _ = config_default_dirs
+
+    config_file_path_user = config_default_dir_user.joinpath(config_eos.CONFIG_FILE_NAME)
+    config_file_path_cwd = config_default_dir_cwd.joinpath(config_eos.CONFIG_FILE_NAME)
+
+    original_config_file_user = None
+    original_config_file_cwd = None
+    if config_file_path_user.exists():
+        original_config_file_user = config_file_path_user
+    if config_file_path_cwd.exists():
+        original_config_file_cwd = config_file_path_cwd
+
+    temp_dir = tempfile.TemporaryDirectory()
+    temp_file_user = None
+    temp_file_cwd = None
+
+    # If the file exists, move it to the temporary directory
+    if original_config_file_user:
+        temp_file_user = Path(temp_dir.name) / f"user.{original_config_file_user.name}"
+        shutil.move(original_config_file_user, temp_file_user)
+        assert not original_config_file_user.exists()
+        logger.debug(f"Stashed: '{original_config_file_user}'")
+    if original_config_file_cwd:
+        temp_file_cwd = Path(temp_dir.name) / f"cwd.{original_config_file_cwd.name}"
+        shutil.move(original_config_file_cwd, temp_file_cwd)
+        assert not original_config_file_cwd.exists()
+        logger.debug(f"Stashed: '{original_config_file_cwd}'")
+
+    # Yield the temporary file path to the test
+    yield temp_file_user, temp_file_cwd
+
+    # Cleanup after the test
+    if temp_file_user:
+        # Restore the file to its original location
+        shutil.move(temp_file_user, original_config_file_user)
+        assert original_config_file_user.exists()
+    if temp_file_cwd:
+        # Restore the file to its original location
+        shutil.move(temp_file_cwd, original_config_file_cwd)
+        assert original_config_file_cwd.exists()
+    temp_dir.cleanup()
+
+
 @pytest.fixture
 def server(xprocess, tmp_path: Path):
     """Fixture to start the server.
@@ -50,13 +138,13 @@ def server(xprocess, tmp_path: Path):
         # assure server to be installed
         try:
             subprocess.run(
-                [sys.executable, "-c", "import akkudoktoreos.server"],
+                [sys.executable, "-c", "import akkudoktoreos.server.fastapi_server"],
                 check=True,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
             )
         except subprocess.CalledProcessError:
-            project_dir = Path(__file__).parent.parent
+            project_dir = Path(__file__).parent.parent.parent
             subprocess.run(
                 [sys.executable, "-m", "pip", "install", "-e", project_dir],
                 check=True,
@@ -66,11 +154,15 @@ def server(xprocess, tmp_path: Path):
 
         # command to start server process
         args = [sys.executable, "-m", "akkudoktoreos.server.fastapi_server"]
-        env = {EOS_DIR: f"{tmp_path}", **os.environ.copy()}
+        config_eos = get_config()
+        settings = {
+            "data_folder_path": tmp_path,
+        }
+        config_eos.merge_settings_from_dict(settings)
 
         # startup pattern
         pattern = "Application startup complete."
-        # search the first 30 lines for the startup pattern, if not found
+        # search this number of lines for the startup pattern, if not found
         # a RuntimeError will be raised informing the user
         max_read_lines = 30
 
@@ -81,7 +173,7 @@ def server(xprocess, tmp_path: Path):
         terminate_on_interrupt = True
 
     # ensure process is running and return its logfile
-    logfile = xprocess.ensure("eos", Starter)
+    pid, logfile = xprocess.ensure("eos", Starter)
 
     # create url/port info to the server
     url = "http://127.0.0.1:8503"
@@ -92,26 +184,26 @@ def server(xprocess, tmp_path: Path):
 
 
 @pytest.fixture
-def other_timezone():
-    """Fixture to temporarily change the timezone.
+def set_other_timezone():
+    """Temporarily sets a timezone for Pendulum during a test.
 
-    Restores the original timezone after the test.
+    Resets to the original timezone after the test completes.
     """
-    original_tz = os.environ.get("TZ", None)
+    original_timezone = pendulum.local_timezone()
 
-    other_tz = "Atlantic/Canary"
-    if original_tz == other_tz:
-        other_tz = "Asia/Singapore"
+    default_other_timezone = "Atlantic/Canary"
+    if default_other_timezone == original_timezone:
+        default_other_timezone = "Asia/Singapore"
 
-    # Change the timezone to another
-    os.environ["TZ"] = other_tz
-    time.tzset()  # For Unix/Linux to apply the timezone change
+    def _set_timezone(other_timezone: Optional[str] = None) -> str:
+        if other_timezone is None:
+            other_timezone = default_other_timezone
+        pendulum.set_local_timezone(other_timezone)
+        assert pendulum.local_timezone() == other_timezone
+        return other_timezone
 
-    yield os.environ["TZ"]  # Yield control back to the test case
+    yield _set_timezone
 
-    # Restore the original timezone after the test
-    if original_tz:
-        os.environ["TZ"] = original_tz
-    else:
-        del os.environ["TZ"]
-    time.tzset()  # Re-apply the original timezone
+    # Restore the original timezone
+    pendulum.set_local_timezone(original_timezone)
+    assert pendulum.local_timezone() == original_timezone