perf: import heavy optional dependencies on first use (#1377)

A GENETIC server with external (import) providers and EOSdash disabled
loads several large packages at start that it never uses, which costs
a noticeable part of the memory on 512 MB boards:

- matplotlib via the GENETIC/GENETIC0 PDF report modules (server/eos.py)
- statsmodels via the Holt-Winters fallback in prediction/priceabc.py
- pvlib via the weather base class and the pvlib-based providers
- bs4/lxml via the ClearOutside provider
- scipy via the self-consumption interpolator singletons, which were
  loaded at import time
- the GENETIC0 optimizer, imported by core/ems.py
- tzfpy, whose polygon data the computed general.timezone loads

Import these where they are used. The interpolator tables are loaded
on first use (thread-safe). Nothing changes in what is computed.

general.timezone is computed from latitude/longitude and cannot be set.
Add general.timezone_override (IANA name, validated with pendulum):
when set, general.timezone returns it and the lookup is skipped. When
unset (default) the lookup is unchanged.

Tests that patched the moved names now patch them at their source.
A new test starts the server module in a subprocess and checks that
none of these modules is loaded.

Co-authored-by: Christin <christin@BigBrother.bikini.bottom.zone>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/akkudoktoreos/config/config.py

@@ -17,6 +17,7 @@ from copy import deepcopy
 from pathlib import Path
 from typing import Any, Callable, ClassVar, Optional, Type, Union
 
+import pendulum
 import pydantic_settings
 from loguru import logger
 from platformdirs import user_config_dir, user_data_dir
@@ -53,7 +54,11 @@ from akkudoktoreos.prediction.pvforecast import PVForecastCommonSettings
 from akkudoktoreos.prediction.weather import WeatherCommonSettings
 from akkudoktoreos.server.rest.cli import cli_argument_parser
 from akkudoktoreos.server.server import ServerCommonSettings
-from akkudoktoreos.utils.datetimeutil import to_datetime, to_timezone
+from akkudoktoreos.utils.datetimeutil import (
+    _canonicalize_tz_name,
+    to_datetime,
+    to_timezone,
+)
 from akkudoktoreos.utils.utils import UtilsCommonSettings
 
 
@@ -189,11 +194,37 @@ class GeneralSettings(SettingsBaseModel):
         json_schema_extra={"description": "Longitude in decimal degrees within -180 to 180 (°)"},
     )
 
+    timezone_override: Optional[str] = Field(
+        default=None,
+        json_schema_extra={
+            "description": (
+                "IANA timezone name, e.g. 'Europe/Berlin'. If set, 'timezone' is this "
+                "value instead of the timezone looked up from latitude and longitude, "
+                "and the lookup data (~25 MB) is never loaded. Defaults to None (look up)."
+            ),
+            "examples": [None, "Europe/Berlin"],
+        },
+    )
+
+    @field_validator("timezone_override", mode="after")
+    @classmethod
+    def validate_timezone_override(cls, value: Optional[str]) -> Optional[str]:
+        """Ensure the timezone override is a known IANA timezone name."""
+        if value is None:
+            return None
+        try:
+            name = pendulum.timezone(value).name
+        except Exception as e:
+            raise ValueError(f"Unknown timezone '{value}'.") from e
+        return _canonicalize_tz_name(name)
+
     # Computed fields
     @computed_field  # type: ignore[prop-decorator]
     @property
     def timezone(self) -> Optional[str]:
-        """Computed timezone based on latitude and longitude."""
+        """Timezone: 'timezone_override' if set, else looked up from latitude and longitude."""
+        if self.timezone_override is not None:
+            return self.timezone_override
         if self.latitude and self.longitude:
             return to_timezone(location=(self.latitude, self.longitude), as_string=True)
         return None

src/akkudoktoreos/core/ems.py

@@ -17,7 +17,6 @@ from akkudoktoreos.core.coreabc import (
 from akkudoktoreos.core.emplan import EnergyManagementPlan
 from akkudoktoreos.core.emsettings import EnergyManagementMode
 from akkudoktoreos.core.pydantic import PydanticBaseModel
-from akkudoktoreos.optimization.genetic0.genetic0 import Genetic0Optimization
 from akkudoktoreos.optimization.genetic0.genetic0params import (
     Genetic0OptimizationParameters,
 )
@@ -428,6 +427,12 @@ class EnergyManagement(
 
                 # --- Optimization (CPU-bound → MUST offload) ---
                 try:
+                    # Imported on first use: a server that runs GENETIC never needs
+                    # the GENETIC0 optimizer and its simulation.
+                    from akkudoktoreos.optimization.genetic0.genetic0 import (
+                        Genetic0Optimization,
+                    )
+
                     genetic0_optimization = Genetic0Optimization(
                         verbose=bool(self.config.server.verbose),
                         fixed_seed=genetic0_seed,

src/akkudoktoreos/optimization/genetic0/genetic0loadinterpolator.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python
 import pickle
+import threading
 from pathlib import Path
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
-from scipy.interpolate import RegularGridInterpolator
+
+if TYPE_CHECKING:
+    # Unpickling the table imports scipy when it is first loaded.
+    from scipy.interpolate import RegularGridInterpolator
 
 from akkudoktoreos.core.cache import cache_energy_management
 from akkudoktoreos.core.coreabc import SingletonMixin
@@ -92,9 +97,15 @@ class Genetic0LoadInterpolator(SelfConsumptionProbabilityInterpolator, Singleton
         super().__init__(filename)
 
 
-# Initialize the Energy Management System, it is a singleton.
-genetic0_load_interpolator = Genetic0LoadInterpolator()
+# Created on first use: loading the table imports scipy.
+_genetic0_load_interpolator: Optional[Genetic0LoadInterpolator] = None
+_genetic0_load_interpolator_lock = threading.Lock()
 
 
 def get_genetic0_load_interpolator() -> Genetic0LoadInterpolator:
-    return genetic0_load_interpolator
+    global _genetic0_load_interpolator
+    if _genetic0_load_interpolator is None:
+        with _genetic0_load_interpolator_lock:
+            if _genetic0_load_interpolator is None:
+                _genetic0_load_interpolator = Genetic0LoadInterpolator()
+    return _genetic0_load_interpolator

src/akkudoktoreos/prediction/interpolator.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python
 import pickle
+import threading
 from pathlib import Path
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
-from scipy.interpolate import RegularGridInterpolator
+
+if TYPE_CHECKING:
+    # Unpickling the table imports scipy when it is first loaded.
+    from scipy.interpolate import RegularGridInterpolator
 
 from akkudoktoreos.core.cache import cache_energy_management
 from akkudoktoreos.core.coreabc import SingletonMixin
@@ -170,9 +175,15 @@ class EOSLoadInterpolator(SelfConsumptionProbabilityInterpolator, SingletonMixin
         super().__init__(filename)
 
 
-# Initialize the Energy Management System, it is a singleton.
-eos_load_interpolator = EOSLoadInterpolator()
+# Created on first use: loading the table imports scipy.
+_eos_load_interpolator: Optional[EOSLoadInterpolator] = None
+_eos_load_interpolator_lock = threading.Lock()
 
 
 def get_eos_load_interpolator() -> EOSLoadInterpolator:
-    return eos_load_interpolator
+    global _eos_load_interpolator
+    if _eos_load_interpolator is None:
+        with _eos_load_interpolator_lock:
+            if _eos_load_interpolator is None:
+                _eos_load_interpolator = EOSLoadInterpolator()
+    return _eos_load_interpolator

src/akkudoktoreos/prediction/priceabc.py

@@ -6,7 +6,6 @@ from typing import Generic, cast
 import numpy as np
 import pandas as pd
 from loguru import logger
-from statsmodels.tsa.holtwinters import ExponentialSmoothing
 
 from akkudoktoreos.core.coreabc import PredictionMixin
 from akkudoktoreos.core.dataabc import DataRecordT
@@ -114,6 +113,9 @@ class PricePredictionProviderBase(
                 f"{seasonal_periods}: got {len(history)}, "
                 f"need at least {required_observations}"
             )
+        # statsmodels is only needed for this fallback forecast; import it on use.
+        from statsmodels.tsa.holtwinters import ExponentialSmoothing
+
         clean_history = self._cap_outliers(history)
         model = ExponentialSmoothing(
             clean_history, seasonal="add", seasonal_periods=seasonal_periods

src/akkudoktoreos/prediction/pvforecastakkudoktorlocal.py

@@ -38,7 +38,6 @@ from typing import Any, Literal, Optional, Self
 import numpy as np
 import pandas as pd
 import pendulum
-import pvlib
 import requests
 from loguru import logger
 from pydantic import Field, field_validator, model_validator
@@ -525,6 +524,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
     @staticmethod
     def _tracked_orientation(plane: Any, solpos: pd.DataFrame) -> tuple[Any, Any]:
         """Return (surface_tilt, surface_azimuth) honouring the plane's tracking type."""
+        import pvlib
+
         tilt = float(plane.surface_tilt if plane.surface_tilt is not None else 30.0)
         azimuth = float(plane.surface_azimuth if plane.surface_azimuth is not None else 180.0)
         tracking = plane.trackingtype
@@ -575,6 +576,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
         airmass: pd.Series,
     ) -> tuple[pd.Series, pd.Series]:
         """Run the pvlib chain for one plane, returning (dc_power_w, ac_power_w)."""
+        import pvlib
+
         settings = self._settings
 
         peakpower_kw = plane.peakpower
@@ -673,6 +676,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
                 fittable. Pass False to obtain the raw model output, which is what the
                 fit itself compares against.
         """
+        import pvlib
+
         settings = self._settings
         weather = self._weather_frame(data)
         if weather.empty:

src/akkudoktoreos/prediction/pvforecastpvlib.py

@@ -8,11 +8,6 @@ from typing import ClassVar, Literal, Optional, Union
 import numpy as np
 import pandas as pd
 from loguru import logger
-from pvlib.location import Location
-from pvlib.modelchain import ModelChain
-from pvlib.pvsystem import PVSystem, retrieve_sam
-from pvlib.solarposition import get_solarposition
-from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
 
 from akkudoktoreos.config.configabc import SettingsBaseModel
 from akkudoktoreos.core.coreabc import PredictionMixin, get_config
@@ -57,6 +52,8 @@ def _update_cec_database() -> None:
 
     Script taken from https://github.com/davidusb-geek/emhass/blob/master/scripts/save_pvlib_module_inverter_database.py
     """
+    from pvlib.pvsystem import retrieve_sam
+
     data_path = Path(__file__).parent.parent / "data"
 
     logger.info("Reading original outdated database bundled with PVLib")
@@ -269,6 +266,13 @@ class PVForecastPVLib(PredictionMixin, PVForecastProvider):
         Note:
             Taken from emhass
         """
+        # pvlib is imported on first use, so a server without this provider does
+        # not load it.
+        from pvlib.location import Location
+        from pvlib.modelchain import ModelChain
+        from pvlib.pvsystem import PVSystem
+        from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
+
         # Validate weather data
         required = [
             "temp_air",
@@ -418,6 +422,8 @@ class PVForecastPVLib(PredictionMixin, PVForecastProvider):
         :param longitude: Longitude of the PV system.
         :return: DataFrame with added solar elevation and azimuth.
         """
+        from pvlib.solarposition import get_solarposition
+
         df = df.copy()
         solpos = get_solarposition(df.index, latitude, longitude)
         df["solar_elevation"] = solpos["elevation"]

src/akkudoktoreos/prediction/weatherabc.py

@@ -11,7 +11,6 @@ from typing import List, Optional
 
 import numpy as np
 import pandas as pd
-import pvlib
 from pydantic import Field
 
 from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
@@ -170,6 +169,8 @@ class WeatherProvider(PredictionProvider[WeatherDataRecord]):
             This method is based on the implementation from PVLib and is adapted from
             https://github.com/davidusb-geek/emhass/blob/master/src/emhass/forecast.py (MIT License).
         """
+        import pvlib
+
         # Adjust offset percentage to scaling factor
         offset_fraction = offset / 100.0
 
@@ -212,4 +213,6 @@ class WeatherProvider(PredictionProvider[WeatherDataRecord]):
     def estimate_preciptable_water(
         cls, temperature: pd.Series, relative_humidity: pd.Series
     ) -> pd.Series:
+        import pvlib
+
         return pvlib.atmosphere.gueymard94_pw(temperature, relative_humidity)

src/akkudoktoreos/prediction/weatherbrightsky.py

@@ -11,7 +11,6 @@ from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
-import pvlib
 import requests
 from loguru import logger
 
@@ -166,6 +165,8 @@ class WeatherBrightSky(WeatherProvider):
 
         The final mapped and processed data is inserted into the sequence as `WeatherDataRecord`.
         """
+        import pvlib
+
         # Get BrightSky weather data for the given coordinates
         brightsky_data = self._request_forecast(force_update=force_update)  # type: ignore
 

src/akkudoktoreos/prediction/weatherclearoutside.py

@@ -17,7 +17,6 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import pandas as pd
 import requests
-from bs4 import BeautifulSoup
 from loguru import logger
 
 from akkudoktoreos.core.cache import cache_in_file
@@ -140,7 +139,9 @@ class WeatherClearOutside(WeatherProvider):
         # Get ClearOutside web content - either from site or cached
         response = self._request_forecast(force_update=force_update)  # type: ignore
 
-        # Scrape the data
+        # Scrape the data (bs4/lxml are only needed by this provider)
+        from bs4 import BeautifulSoup
+
         soup = BeautifulSoup(response.content, "html.parser")
 
         # Find generation data

src/akkudoktoreos/prediction/weatheropenmeteo.py

@@ -10,7 +10,6 @@ from typing import Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
-import pvlib
 import requests
 from loguru import logger
 
@@ -251,6 +250,8 @@ class WeatherOpenMeteo(WeatherProvider):
         are used directly without additional calculation. The final mapped and processed
         data is inserted into the sequence as `WeatherDataRecord`.
         """
+        import pvlib
+
         # Retrieve Open-Meteo weather data for the given coordinates
         openmeteo_data = self._request_forecast(force_update=force_update)  # type: ignore
 

src/akkudoktoreos/server/eos.py

@@ -62,14 +62,8 @@ from akkudoktoreos.optimization.genetic0.genetic0solution import (
     Genetic0Solution,
     Genetic0SolutionLegacy,
 )
-from akkudoktoreos.optimization.genetic0.genetic0visualize import (
-    genetic0_prepare_visualize,
-)
 from akkudoktoreos.optimization.genetic.configrequest import ConfigOptimizationRequest
 from akkudoktoreos.optimization.genetic.geneticsolution import GeneticSolution
-from akkudoktoreos.optimization.genetic.geneticvisualize import (
-    genetic_prepare_visualize,
-)
 from akkudoktoreos.optimization.optimization import (
     OptimizationAlgorithm,
     OptimizationSolution,
@@ -1893,6 +1887,11 @@ async def fastapi_energy_management_optimization_solution_genetic_pdf_get() -> R
             detail="Can not get the 'GENETIC' optimization solution.",
         )
     snapshot = retained.model_copy(deep=True)
+    # matplotlib is only needed for the PDF report; import it on first use.
+    from akkudoktoreos.optimization.genetic.geneticvisualize import (
+        genetic_prepare_visualize,
+    )
+
     pdf = await asyncio.to_thread(genetic_prepare_visualize, solution=snapshot)
     return Response(
         content=pdf,
@@ -2314,6 +2313,11 @@ def get_pdf() -> Response:
             detail="Can not get the 'GENETIC0' optimization solution.",
         )
 
+    # matplotlib is only needed for the PDF report; import it on first use.
+    from akkudoktoreos.optimization.genetic0.genetic0visualize import (
+        genetic0_prepare_visualize,
+    )
+
     pdf = genetic0_prepare_visualize(solution=genetic0_solution)
 
     return Response(

src/akkudoktoreos/utils/datetimeutil.py

@@ -67,7 +67,6 @@ from pydantic import (
     GetCoreSchemaHandler,
 )
 from pydantic_core import core_schema
-from tzfpy import get_tz
 
 if TYPE_CHECKING:
     # The Pydantic adapters validate Pendulum values; arithmetic and factory
@@ -1414,6 +1413,10 @@ def to_timezone(
             lat, lon = location
             if not (-90 <= lat <= 90 and -180 <= lon <= 180):
                 raise ValueError(f"Invalid latitude/longitude: {lat}, {lon}")
+            # tzfpy loads its timezone polygons (~25 MB) on first use; import it
+            # only when a location lookup is actually needed.
+            from tzfpy import get_tz
+
             tz_name = get_tz(lon, lat)
             if not tz_name:
                 raise ValueError(