perf: import heavy optional dependencies on first use (#1377)

A GENETIC server with external (import) providers and EOSdash disabled
loads several large packages at start that it never uses, which costs
a noticeable part of the memory on 512 MB boards:

- matplotlib via the GENETIC/GENETIC0 PDF report modules (server/eos.py)
- statsmodels via the Holt-Winters fallback in prediction/priceabc.py
- pvlib via the weather base class and the pvlib-based providers
- bs4/lxml via the ClearOutside provider
- scipy via the self-consumption interpolator singletons, which were
  loaded at import time
- the GENETIC0 optimizer, imported by core/ems.py
- tzfpy, whose polygon data the computed general.timezone loads

Import these where they are used. The interpolator tables are loaded
on first use (thread-safe). Nothing changes in what is computed.

general.timezone is computed from latitude/longitude and cannot be set.
Add general.timezone_override (IANA name, validated with pendulum):
when set, general.timezone returns it and the lookup is skipped. When
unset (default) the lookup is unchanged.

Tests that patched the moved names now patch them at their source.
A new test starts the server module in a subprocess and checks that
none of these modules is loaded.

Co-authored-by: Christin <christin@BigBrother.bikini.bottom.zone>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_lazy_imports.py

@@ -0,0 +1,111 @@
+"""Heavy optional dependencies must not be imported by a plain server start.
+
+A GENETIC server with external (import) providers and EOSdash disabled should not
+load the PDF report stack, the price fallback forecast, pvlib-based providers,
+the HTML scraper, the GENETIC0 optimizer or the timezone polygon lookup. Each of
+these costs several to tens of MB of resident memory on small devices.
+"""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from akkudoktoreos.config.config import GeneralSettings
+
+LAZY_MODULES = [
+    "matplotlib",
+    "statsmodels",
+    "pvlib",
+    "bs4",
+    "lxml",
+    "scipy",
+    "tzfpy",
+    "akkudoktoreos.optimization.genetic0.genetic0",
+    "akkudoktoreos.optimization.genetic0.genetic0visualize",
+    "akkudoktoreos.optimization.genetic.geneticvisualize",
+]
+
+PROBE = """
+import json
+import sys
+
+import akkudoktoreos.server.eos  # noqa: F401
+from akkudoktoreos.core.coreabc import get_config, get_prediction, singletons_init
+
+config = get_config(init=True)
+singletons_init()
+get_prediction()
+assert config.general.timezone == "Europe/Berlin", config.general.timezone
+loaded = sorted(name for name in {modules} if name in sys.modules)
+print("LOADED_MODULES=" + json.dumps(loaded))
+"""
+
+
+def _loaded_modules(tmp_path: Path, extra_env: dict[str, str]) -> list[str]:
+    env = {key: value for key, value in os.environ.items() if not key.startswith("EOS_")}
+    env.update(
+        {
+            "EOS_DIR": str(tmp_path),
+            "EOS_CONFIG_DIR": str(tmp_path),
+            "EOS_GENERAL__DATA_FOLDER_PATH": str(tmp_path / "data"),
+            "EOS_SERVER__STARTUP_EOSDASH": "false",
+            "EOS_OPTIMIZATION__ALGORITHM": "GENETIC",
+            "EOS_LOGGING__CONSOLE_LEVEL": "ERROR",
+        }
+    )
+    env.update(extra_env)
+    result = subprocess.run(  # noqa: S603
+        [sys.executable, "-c", PROBE.format(modules=LAZY_MODULES)],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr[-4000:]
+    marker = "LOADED_MODULES="
+    lines = [line for line in result.stdout.splitlines() if line.startswith(marker)]
+    assert len(lines) == 1, result.stdout[-4000:]
+    return json.loads(lines[0][len(marker) :])
+
+
+def test_server_start_does_not_import_heavy_optional_modules(tmp_path: Path):
+    loaded = _loaded_modules(tmp_path, {"EOS_GENERAL__TIMEZONE_OVERRIDE": "Europe/Berlin"})
+    assert loaded == []
+
+
+def test_timezone_lookup_still_used_without_override(tmp_path: Path):
+    # Default latitude/longitude are in Berlin, so the lookup gives the same name.
+    loaded = _loaded_modules(tmp_path, {})
+    assert loaded == ["tzfpy"]
+
+
+def test_timezone_override_semantics():
+    general = GeneralSettings()
+    assert general.timezone_override is None
+    assert general.timezone == "Europe/Berlin"
+
+    general = GeneralSettings(latitude=40.7128, longitude=-74.0060)
+    assert general.timezone == "America/New_York"
+
+    general = GeneralSettings(
+        latitude=40.7128, longitude=-74.0060, timezone_override="Europe/Berlin"
+    )
+    assert general.timezone == "Europe/Berlin"
+
+    # Without coordinates the override still applies.
+    general = GeneralSettings(latitude=None, longitude=None, timezone_override="Asia/Tokyo")
+    assert general.timezone == "Asia/Tokyo"
+    assert GeneralSettings(latitude=None, longitude=None).timezone is None
+
+    assert GeneralSettings(timezone_override="Etc/UTC").timezone == "UTC"
+
+
+def test_timezone_override_rejects_unknown_names():
+    with pytest.raises(ValueError):
+        GeneralSettings(timezone_override="Mars/Olympus_Mons")

tests/test_optimization_pdf_api.py

@@ -9,6 +9,7 @@ import pytest
 from fastapi.testclient import TestClient
 from pypdf import PdfReader
 
+from akkudoktoreos.optimization.genetic import geneticvisualize
 from akkudoktoreos.optimization.genetic.geneticsolution import GeneticSolution
 from akkudoktoreos.server import eos
 from akkudoktoreos.utils.datetimeutil import to_datetime
@@ -70,7 +71,8 @@ async def test_render_is_offloaded_and_uses_an_owned_copy(config_eos, monkeypatc
         solution.ac_charge[0] = 0.123
         return b"%PDF-isolated-render"
 
-    monkeypatch.setattr(eos, "genetic_prepare_visualize", render)
+    # The PDF stack is imported on first use, so patch it at its source.
+    monkeypatch.setattr(geneticvisualize, "genetic_prepare_visualize", render)
     response = await eos.fastapi_energy_management_optimization_solution_genetic_pdf_get()
     assert response.body == b"%PDF-isolated-render"
     assert captured[0][0] != caller_thread

tests/test_optimize_run_result.py

@@ -9,6 +9,7 @@ import pytest
 
 from akkudoktoreos.core import ems as ems_module
 from akkudoktoreos.core.emsettings import EnergyManagementMode
+from akkudoktoreos.optimization.genetic0 import genetic0 as genetic0_module
 from akkudoktoreos.optimization.genetic0.genetic0params import (
     Genetic0OptimizationParameters,
 )
@@ -16,6 +17,15 @@ from akkudoktoreos.optimization.optimization import OptimizationAlgorithm
 from akkudoktoreos.utils.datetimeutil import to_datetime
 
 
+def _patch_optimizer(monkeypatch, prefix: str, constructor: Any) -> None:
+    """Replace the optimizer class that EnergyManagement.run() instantiates.
+
+    GENETIC0 is imported on first use inside run(), so it is patched at its source.
+    """
+    module = genetic0_module if prefix == "Genetic0" else ems_module
+    monkeypatch.setattr(module, prefix + "Optimization", constructor)
+
+
 @pytest.fixture
 def offline_ems(monkeypatch):
     """Run the actual EMS orchestration with no real adapters or prediction IO."""
@@ -110,7 +120,7 @@ async def test_optimization_routes_only_selected_algorithm(
         constructor = Mock()
         constructor.return_value.optimize_ems.return_value = solution
         constructors[prefix] = constructor
-        monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
+        _patch_optimizer(monkeypatch, prefix, constructor)
         prepare = AsyncMock(return_value=sentinel_parameters)
         preparers[prefix] = prepare
         monkeypatch.setattr(
@@ -170,8 +180,8 @@ async def test_optimization_routes_only_selected_algorithm(
 @pytest.mark.parametrize("mode", [EnergyManagementMode.DISABLED, EnergyManagementMode.PREDICTION])
 async def test_non_optimization_modes_never_optimize(monkeypatch, offline_ems, mode):
     constructors = [Mock(), Mock()]
-    monkeypatch.setattr(ems_module, "GeneticOptimization", constructors[0])
-    monkeypatch.setattr(ems_module, "Genetic0Optimization", constructors[1])
+    _patch_optimizer(monkeypatch, "Genetic", constructors[0])
+    _patch_optimizer(monkeypatch, "Genetic0", constructors[1])
     offline_ems.config.ems.mode = mode
     await ems_module.EnergyManagement.run(offline_ems)
     for constructor in constructors:
@@ -187,7 +197,7 @@ async def test_non_optimization_modes_never_optimize(monkeypatch, offline_ems, m
 @pytest.mark.parametrize("prefix", ["Genetic", "Genetic0"])
 async def test_missing_preparation_does_not_dispatch_controls(monkeypatch, offline_ems, prefix):
     constructor = Mock()
-    monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
+    _patch_optimizer(monkeypatch, prefix, constructor)
     prepare = AsyncMock(return_value=None)
     monkeypatch.setattr(getattr(ems_module, prefix + "OptimizationParameters"), "prepare", prepare)
     await ems_module.EnergyManagement.run(
@@ -273,7 +283,7 @@ async def test_failed_legacy_http_run_never_reports_previous_solution(
         native.optimization_solution.side_effect = error
     else:
         native.energy_management_plan.side_effect = error
-    monkeypatch.setattr(ems_module, "Genetic0Optimization", constructor)
+    _patch_optimizer(monkeypatch, "Genetic0", constructor)
     offline_ems.run = MethodType(cls.run, offline_ems)
     offline_ems.genetic0_solution = cls.genetic0_solution
     monkeypatch.setattr(eos, "get_ems", lambda: offline_ems)
@@ -311,7 +321,7 @@ async def test_conversion_failure_preserves_consistent_previous_results(
         solution.energy_management_plan.side_effect = RuntimeError("synthetic plan failure")
     constructor = Mock()
     constructor.return_value.optimize_ems.return_value = solution
-    monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
+    _patch_optimizer(monkeypatch, prefix, constructor)
     supplied_parameters: dict[str, Any] = {suffix + "_parameters": object()}
     result = await cls.run(
         offline_ems,

tests/test_pvforecastpvlib.py

@@ -1308,8 +1308,8 @@ class TestCalculation:
         assert isinstance(result, pd.DataFrame)
         assert result.empty
 
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.Location")
+    @patch("pvlib.modelchain.ModelChain")
+    @patch("pvlib.location.Location")
     def test_location_created(
         self,
         mock_location,
@@ -1352,7 +1352,7 @@ class TestCalculation:
 
         mock_location.assert_called_once()
 
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
+    @patch("pvlib.modelchain.ModelChain")
     def test_modelchain_run_called(
         self,
         mock_modelchain,
@@ -1395,7 +1395,7 @@ class TestCalculation:
 
         mc.run_model.assert_called_once_with(weather)
 
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
+    @patch("pvlib.modelchain.ModelChain")
     def test_negative_power_clipped(
         self,
         mock_modelchain,
@@ -1444,7 +1444,7 @@ class TestCalculation:
         assert (result["pv_dc_power"] >= 0).all()
         assert (result["ac_power"] >= 0).all()
 
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
+    @patch("pvlib.modelchain.ModelChain")
     def test_dataframe_columns(
         self,
         mock_modelchain,
@@ -1489,7 +1489,7 @@ class TestCalculation:
         assert "pv_dc_power" in result.columns
         assert "ac_power" in result.columns
 
-    @patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
+    @patch("pvlib.modelchain.ModelChain")
     def test_result_index_preserved(
         self,
         mock_modelchain,