mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2026-10-12 17:21:55 +00:00
perf: import heavy optional dependencies on first use (#1377)
A GENETIC server with external (import) providers and EOSdash disabled loads several large packages at start that it never uses, which costs a noticeable part of the memory on 512 MB boards: - matplotlib via the GENETIC/GENETIC0 PDF report modules (server/eos.py) - statsmodels via the Holt-Winters fallback in prediction/priceabc.py - pvlib via the weather base class and the pvlib-based providers - bs4/lxml via the ClearOutside provider - scipy via the self-consumption interpolator singletons, which were loaded at import time - the GENETIC0 optimizer, imported by core/ems.py - tzfpy, whose polygon data the computed general.timezone loads Import these where they are used. The interpolator tables are loaded on first use (thread-safe). Nothing changes in what is computed. general.timezone is computed from latitude/longitude and cannot be set. Add general.timezone_override (IANA name, validated with pendulum): when set, general.timezone returns it and the lookup is skipped. When unset (default) the lookup is unchanged. Tests that patched the moved names now patch them at their source. A new test starts the server module in a subprocess and checks that none of these modules is loaded. Co-authored-by: Christin <christin@BigBrother.bikini.bottom.zone> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Christin
Claude Opus 5.5
parent
6d26252c6a
commit
3651d33915
@@ -38,6 +38,8 @@ compatibility changes and a short test procedure. These changes are not yet a ta
|
||||
and coverage information. A capacity estimate does not overwrite configured capacity.
|
||||
- Optional local, calibratable forecasting within `PVForecastAkkudoktor`; the remote
|
||||
backend remains the default.
|
||||
- Optional `general.timezone_override`: an IANA timezone name used as `general.timezone`
|
||||
instead of the lookup from latitude/longitude. Unset (default), the lookup is unchanged.
|
||||
- New PV forecast providers giving operators more cloud forecast sources to choose from in
|
||||
addition to Akkudoktor, VRM and Import:
|
||||
- `PVForecastPVNode` — native 15-minute forecasts from the pvnode.com API.
|
||||
@@ -60,6 +62,9 @@ compatibility changes and a short test procedure. These changes are not yet a ta
|
||||
values. Runtime forecast energy is Wh per slot and prices are currency per Wh.
|
||||
- Runtime configuration changes now take priority over environment and file values for
|
||||
explicitly updated keys; command-line settings retain higher priority.
|
||||
- Lower idle memory: matplotlib (PDF reports), statsmodels (price fallback forecast), pvlib,
|
||||
bs4/lxml, scipy (self-consumption table), the GENETIC0 optimizer and the timezone lookup
|
||||
data are imported on first use instead of at server start. Results are unchanged.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@@ -210,7 +210,8 @@
|
||||
"data_folder_path": "/home/user/.local/share/net.akkudoktoreos.net",
|
||||
"data_output_subpath": "output",
|
||||
"latitude": 52.52,
|
||||
"longitude": 13.405
|
||||
"longitude": 13.405,
|
||||
"timezone_override": null
|
||||
},
|
||||
"load": {
|
||||
"provider": "LoadAkkudoktor",
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
| home_assistant_addon | `EOS_GENERAL__HOME_ASSISTANT_ADDON` | `bool` | `rw` | `required` | EOS is running as home assistant add-on. |
|
||||
| latitude | `EOS_GENERAL__LATITUDE` | `Optional[float]` | `rw` | `52.52` | Latitude in decimal degrees between -90 and 90. North is positive (ISO 19115) (°) |
|
||||
| longitude | `EOS_GENERAL__LONGITUDE` | `Optional[float]` | `rw` | `13.405` | Longitude in decimal degrees within -180 to 180 (°) |
|
||||
| timezone | | `Optional[str]` | `ro` | `N/A` | Computed timezone based on latitude and longitude. |
|
||||
| timezone | | `Optional[str]` | `ro` | `N/A` | Timezone: 'timezone_override' if set, else looked up from latitude and longitude. |
|
||||
| timezone_override | `EOS_GENERAL__TIMEZONE_OVERRIDE` | `Optional[str]` | `rw` | `None` | IANA timezone name, e.g. 'Europe/Berlin'. If set, 'timezone' is this value instead of the timezone looked up from latitude and longitude, and the lookup data (~25 MB) is never loaded. Defaults to None (look up). |
|
||||
| version | `EOS_GENERAL__VERSION` | `Optional[str]` | `rw` | `None` | Configuration file version. |
|
||||
:::
|
||||
<!-- pyml enable line-length -->
|
||||
@@ -36,7 +37,8 @@
|
||||
"data_folder_path": "/home/user/.local/share/net.akkudoktoreos.net",
|
||||
"data_output_subpath": "output",
|
||||
"latitude": 52.52,
|
||||
"longitude": 13.405
|
||||
"longitude": 13.405,
|
||||
"timezone_override": null
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -57,6 +59,7 @@
|
||||
"data_output_subpath": "output",
|
||||
"latitude": 52.52,
|
||||
"longitude": 13.405,
|
||||
"timezone_override": null,
|
||||
"timezone": "Europe/Berlin",
|
||||
"data_output_path": "/home/user/.local/share/net.akkudoktoreos.net/output",
|
||||
"config_folder_path": "/home/user/.config/net.akkudoktoreos.net",
|
||||
|
||||
+33
-1
@@ -6737,6 +6737,22 @@
|
||||
"title": "Longitude",
|
||||
"description": "Longitude in decimal degrees within -180 to 180 (\u00b0)",
|
||||
"default": 13.405
|
||||
},
|
||||
"timezone_override": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Timezone Override",
|
||||
"description": "IANA timezone name, e.g. 'Europe/Berlin'. If set, 'timezone' is this value instead of the timezone looked up from latitude and longitude, and the lookup data (~25 MB) is never loaded. Defaults to None (look up).",
|
||||
"examples": [
|
||||
null,
|
||||
"Europe/Berlin"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@@ -6829,6 +6845,22 @@
|
||||
"description": "Longitude in decimal degrees within -180 to 180 (\u00b0)",
|
||||
"default": 13.405
|
||||
},
|
||||
"timezone_override": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Timezone Override",
|
||||
"description": "IANA timezone name, e.g. 'Europe/Berlin'. If set, 'timezone' is this value instead of the timezone looked up from latitude and longitude, and the lookup data (~25 MB) is never loaded. Defaults to None (look up).",
|
||||
"examples": [
|
||||
null,
|
||||
"Europe/Berlin"
|
||||
]
|
||||
},
|
||||
"timezone": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -6839,7 +6871,7 @@
|
||||
}
|
||||
],
|
||||
"title": "Timezone",
|
||||
"description": "Computed timezone based on latitude and longitude.",
|
||||
"description": "Timezone: 'timezone_override' if set, else looked up from latitude and longitude.",
|
||||
"readOnly": true
|
||||
},
|
||||
"data_output_path": {
|
||||
|
||||
@@ -17,6 +17,7 @@ from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, ClassVar, Optional, Type, Union
|
||||
|
||||
import pendulum
|
||||
import pydantic_settings
|
||||
from loguru import logger
|
||||
from platformdirs import user_config_dir, user_data_dir
|
||||
@@ -53,7 +54,11 @@ from akkudoktoreos.prediction.pvforecast import PVForecastCommonSettings
|
||||
from akkudoktoreos.prediction.weather import WeatherCommonSettings
|
||||
from akkudoktoreos.server.rest.cli import cli_argument_parser
|
||||
from akkudoktoreos.server.server import ServerCommonSettings
|
||||
from akkudoktoreos.utils.datetimeutil import to_datetime, to_timezone
|
||||
from akkudoktoreos.utils.datetimeutil import (
|
||||
_canonicalize_tz_name,
|
||||
to_datetime,
|
||||
to_timezone,
|
||||
)
|
||||
from akkudoktoreos.utils.utils import UtilsCommonSettings
|
||||
|
||||
|
||||
@@ -189,11 +194,37 @@ class GeneralSettings(SettingsBaseModel):
|
||||
json_schema_extra={"description": "Longitude in decimal degrees within -180 to 180 (°)"},
|
||||
)
|
||||
|
||||
timezone_override: Optional[str] = Field(
|
||||
default=None,
|
||||
json_schema_extra={
|
||||
"description": (
|
||||
"IANA timezone name, e.g. 'Europe/Berlin'. If set, 'timezone' is this "
|
||||
"value instead of the timezone looked up from latitude and longitude, "
|
||||
"and the lookup data (~25 MB) is never loaded. Defaults to None (look up)."
|
||||
),
|
||||
"examples": [None, "Europe/Berlin"],
|
||||
},
|
||||
)
|
||||
|
||||
@field_validator("timezone_override", mode="after")
|
||||
@classmethod
|
||||
def validate_timezone_override(cls, value: Optional[str]) -> Optional[str]:
|
||||
"""Ensure the timezone override is a known IANA timezone name."""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
name = pendulum.timezone(value).name
|
||||
except Exception as e:
|
||||
raise ValueError(f"Unknown timezone '{value}'.") from e
|
||||
return _canonicalize_tz_name(name)
|
||||
|
||||
# Computed fields
|
||||
@computed_field # type: ignore[prop-decorator]
|
||||
@property
|
||||
def timezone(self) -> Optional[str]:
|
||||
"""Computed timezone based on latitude and longitude."""
|
||||
"""Timezone: 'timezone_override' if set, else looked up from latitude and longitude."""
|
||||
if self.timezone_override is not None:
|
||||
return self.timezone_override
|
||||
if self.latitude and self.longitude:
|
||||
return to_timezone(location=(self.latitude, self.longitude), as_string=True)
|
||||
return None
|
||||
|
||||
@@ -17,7 +17,6 @@ from akkudoktoreos.core.coreabc import (
|
||||
from akkudoktoreos.core.emplan import EnergyManagementPlan
|
||||
from akkudoktoreos.core.emsettings import EnergyManagementMode
|
||||
from akkudoktoreos.core.pydantic import PydanticBaseModel
|
||||
from akkudoktoreos.optimization.genetic0.genetic0 import Genetic0Optimization
|
||||
from akkudoktoreos.optimization.genetic0.genetic0params import (
|
||||
Genetic0OptimizationParameters,
|
||||
)
|
||||
@@ -428,6 +427,12 @@ class EnergyManagement(
|
||||
|
||||
# --- Optimization (CPU-bound → MUST offload) ---
|
||||
try:
|
||||
# Imported on first use: a server that runs GENETIC never needs
|
||||
# the GENETIC0 optimizer and its simulation.
|
||||
from akkudoktoreos.optimization.genetic0.genetic0 import (
|
||||
Genetic0Optimization,
|
||||
)
|
||||
|
||||
genetic0_optimization = Genetic0Optimization(
|
||||
verbose=bool(self.config.server.verbose),
|
||||
fixed_seed=genetic0_seed,
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
import pickle
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import numpy as np
|
||||
from scipy.interpolate import RegularGridInterpolator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Unpickling the table imports scipy when it is first loaded.
|
||||
from scipy.interpolate import RegularGridInterpolator
|
||||
|
||||
from akkudoktoreos.core.cache import cache_energy_management
|
||||
from akkudoktoreos.core.coreabc import SingletonMixin
|
||||
@@ -92,9 +97,15 @@ class Genetic0LoadInterpolator(SelfConsumptionProbabilityInterpolator, Singleton
|
||||
super().__init__(filename)
|
||||
|
||||
|
||||
# Initialize the Energy Management System, it is a singleton.
|
||||
genetic0_load_interpolator = Genetic0LoadInterpolator()
|
||||
# Created on first use: loading the table imports scipy.
|
||||
_genetic0_load_interpolator: Optional[Genetic0LoadInterpolator] = None
|
||||
_genetic0_load_interpolator_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_genetic0_load_interpolator() -> Genetic0LoadInterpolator:
|
||||
return genetic0_load_interpolator
|
||||
global _genetic0_load_interpolator
|
||||
if _genetic0_load_interpolator is None:
|
||||
with _genetic0_load_interpolator_lock:
|
||||
if _genetic0_load_interpolator is None:
|
||||
_genetic0_load_interpolator = Genetic0LoadInterpolator()
|
||||
return _genetic0_load_interpolator
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
#!/usr/bin/env python
|
||||
import pickle
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import numpy as np
|
||||
from scipy.interpolate import RegularGridInterpolator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Unpickling the table imports scipy when it is first loaded.
|
||||
from scipy.interpolate import RegularGridInterpolator
|
||||
|
||||
from akkudoktoreos.core.cache import cache_energy_management
|
||||
from akkudoktoreos.core.coreabc import SingletonMixin
|
||||
@@ -170,9 +175,15 @@ class EOSLoadInterpolator(SelfConsumptionProbabilityInterpolator, SingletonMixin
|
||||
super().__init__(filename)
|
||||
|
||||
|
||||
# Initialize the Energy Management System, it is a singleton.
|
||||
eos_load_interpolator = EOSLoadInterpolator()
|
||||
# Created on first use: loading the table imports scipy.
|
||||
_eos_load_interpolator: Optional[EOSLoadInterpolator] = None
|
||||
_eos_load_interpolator_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_eos_load_interpolator() -> EOSLoadInterpolator:
|
||||
return eos_load_interpolator
|
||||
global _eos_load_interpolator
|
||||
if _eos_load_interpolator is None:
|
||||
with _eos_load_interpolator_lock:
|
||||
if _eos_load_interpolator is None:
|
||||
_eos_load_interpolator = EOSLoadInterpolator()
|
||||
return _eos_load_interpolator
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import Generic, cast
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from loguru import logger
|
||||
from statsmodels.tsa.holtwinters import ExponentialSmoothing
|
||||
|
||||
from akkudoktoreos.core.coreabc import PredictionMixin
|
||||
from akkudoktoreos.core.dataabc import DataRecordT
|
||||
@@ -114,6 +113,9 @@ class PricePredictionProviderBase(
|
||||
f"{seasonal_periods}: got {len(history)}, "
|
||||
f"need at least {required_observations}"
|
||||
)
|
||||
# statsmodels is only needed for this fallback forecast; import it on use.
|
||||
from statsmodels.tsa.holtwinters import ExponentialSmoothing
|
||||
|
||||
clean_history = self._cap_outliers(history)
|
||||
model = ExponentialSmoothing(
|
||||
clean_history, seasonal="add", seasonal_periods=seasonal_periods
|
||||
|
||||
@@ -38,7 +38,6 @@ from typing import Any, Literal, Optional, Self
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pendulum
|
||||
import pvlib
|
||||
import requests
|
||||
from loguru import logger
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
@@ -525,6 +524,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
|
||||
@staticmethod
|
||||
def _tracked_orientation(plane: Any, solpos: pd.DataFrame) -> tuple[Any, Any]:
|
||||
"""Return (surface_tilt, surface_azimuth) honouring the plane's tracking type."""
|
||||
import pvlib
|
||||
|
||||
tilt = float(plane.surface_tilt if plane.surface_tilt is not None else 30.0)
|
||||
azimuth = float(plane.surface_azimuth if plane.surface_azimuth is not None else 180.0)
|
||||
tracking = plane.trackingtype
|
||||
@@ -575,6 +576,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
|
||||
airmass: pd.Series,
|
||||
) -> tuple[pd.Series, pd.Series]:
|
||||
"""Run the pvlib chain for one plane, returning (dc_power_w, ac_power_w)."""
|
||||
import pvlib
|
||||
|
||||
settings = self._settings
|
||||
|
||||
peakpower_kw = plane.peakpower
|
||||
@@ -673,6 +676,8 @@ class PVForecastAkkudoktorLocal(PVForecastProvider[PVForecastDataRecordT]):
|
||||
fittable. Pass False to obtain the raw model output, which is what the
|
||||
fit itself compares against.
|
||||
"""
|
||||
import pvlib
|
||||
|
||||
settings = self._settings
|
||||
weather = self._weather_frame(data)
|
||||
if weather.empty:
|
||||
|
||||
@@ -8,11 +8,6 @@ from typing import ClassVar, Literal, Optional, Union
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from loguru import logger
|
||||
from pvlib.location import Location
|
||||
from pvlib.modelchain import ModelChain
|
||||
from pvlib.pvsystem import PVSystem, retrieve_sam
|
||||
from pvlib.solarposition import get_solarposition
|
||||
from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
|
||||
|
||||
from akkudoktoreos.config.configabc import SettingsBaseModel
|
||||
from akkudoktoreos.core.coreabc import PredictionMixin, get_config
|
||||
@@ -57,6 +52,8 @@ def _update_cec_database() -> None:
|
||||
|
||||
Script taken from https://github.com/davidusb-geek/emhass/blob/master/scripts/save_pvlib_module_inverter_database.py
|
||||
"""
|
||||
from pvlib.pvsystem import retrieve_sam
|
||||
|
||||
data_path = Path(__file__).parent.parent / "data"
|
||||
|
||||
logger.info("Reading original outdated database bundled with PVLib")
|
||||
@@ -269,6 +266,13 @@ class PVForecastPVLib(PredictionMixin, PVForecastProvider):
|
||||
Note:
|
||||
Taken from emhass
|
||||
"""
|
||||
# pvlib is imported on first use, so a server without this provider does
|
||||
# not load it.
|
||||
from pvlib.location import Location
|
||||
from pvlib.modelchain import ModelChain
|
||||
from pvlib.pvsystem import PVSystem
|
||||
from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
|
||||
|
||||
# Validate weather data
|
||||
required = [
|
||||
"temp_air",
|
||||
@@ -418,6 +422,8 @@ class PVForecastPVLib(PredictionMixin, PVForecastProvider):
|
||||
:param longitude: Longitude of the PV system.
|
||||
:return: DataFrame with added solar elevation and azimuth.
|
||||
"""
|
||||
from pvlib.solarposition import get_solarposition
|
||||
|
||||
df = df.copy()
|
||||
solpos = get_solarposition(df.index, latitude, longitude)
|
||||
df["solar_elevation"] = solpos["elevation"]
|
||||
|
||||
@@ -11,7 +11,6 @@ from typing import List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pvlib
|
||||
from pydantic import Field
|
||||
|
||||
from akkudoktoreos.prediction.predictionabc import PredictionProvider, PredictionRecord
|
||||
@@ -170,6 +169,8 @@ class WeatherProvider(PredictionProvider[WeatherDataRecord]):
|
||||
This method is based on the implementation from PVLib and is adapted from
|
||||
https://github.com/davidusb-geek/emhass/blob/master/src/emhass/forecast.py (MIT License).
|
||||
"""
|
||||
import pvlib
|
||||
|
||||
# Adjust offset percentage to scaling factor
|
||||
offset_fraction = offset / 100.0
|
||||
|
||||
@@ -212,4 +213,6 @@ class WeatherProvider(PredictionProvider[WeatherDataRecord]):
|
||||
def estimate_preciptable_water(
|
||||
cls, temperature: pd.Series, relative_humidity: pd.Series
|
||||
) -> pd.Series:
|
||||
import pvlib
|
||||
|
||||
return pvlib.atmosphere.gueymard94_pw(temperature, relative_humidity)
|
||||
|
||||
@@ -11,7 +11,6 @@ from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pvlib
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
@@ -166,6 +165,8 @@ class WeatherBrightSky(WeatherProvider):
|
||||
|
||||
The final mapped and processed data is inserted into the sequence as `WeatherDataRecord`.
|
||||
"""
|
||||
import pvlib
|
||||
|
||||
# Get BrightSky weather data for the given coordinates
|
||||
brightsky_data = self._request_forecast(force_update=force_update) # type: ignore
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from akkudoktoreos.core.cache import cache_in_file
|
||||
@@ -140,7 +139,9 @@ class WeatherClearOutside(WeatherProvider):
|
||||
# Get ClearOutside web content - either from site or cached
|
||||
response = self._request_forecast(force_update=force_update) # type: ignore
|
||||
|
||||
# Scrape the data
|
||||
# Scrape the data (bs4/lxml are only needed by this provider)
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(response.content, "html.parser")
|
||||
|
||||
# Find generation data
|
||||
|
||||
@@ -10,7 +10,6 @@ from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pvlib
|
||||
import requests
|
||||
from loguru import logger
|
||||
|
||||
@@ -251,6 +250,8 @@ class WeatherOpenMeteo(WeatherProvider):
|
||||
are used directly without additional calculation. The final mapped and processed
|
||||
data is inserted into the sequence as `WeatherDataRecord`.
|
||||
"""
|
||||
import pvlib
|
||||
|
||||
# Retrieve Open-Meteo weather data for the given coordinates
|
||||
openmeteo_data = self._request_forecast(force_update=force_update) # type: ignore
|
||||
|
||||
|
||||
@@ -62,14 +62,8 @@ from akkudoktoreos.optimization.genetic0.genetic0solution import (
|
||||
Genetic0Solution,
|
||||
Genetic0SolutionLegacy,
|
||||
)
|
||||
from akkudoktoreos.optimization.genetic0.genetic0visualize import (
|
||||
genetic0_prepare_visualize,
|
||||
)
|
||||
from akkudoktoreos.optimization.genetic.configrequest import ConfigOptimizationRequest
|
||||
from akkudoktoreos.optimization.genetic.geneticsolution import GeneticSolution
|
||||
from akkudoktoreos.optimization.genetic.geneticvisualize import (
|
||||
genetic_prepare_visualize,
|
||||
)
|
||||
from akkudoktoreos.optimization.optimization import (
|
||||
OptimizationAlgorithm,
|
||||
OptimizationSolution,
|
||||
@@ -1893,6 +1887,11 @@ async def fastapi_energy_management_optimization_solution_genetic_pdf_get() -> R
|
||||
detail="Can not get the 'GENETIC' optimization solution.",
|
||||
)
|
||||
snapshot = retained.model_copy(deep=True)
|
||||
# matplotlib is only needed for the PDF report; import it on first use.
|
||||
from akkudoktoreos.optimization.genetic.geneticvisualize import (
|
||||
genetic_prepare_visualize,
|
||||
)
|
||||
|
||||
pdf = await asyncio.to_thread(genetic_prepare_visualize, solution=snapshot)
|
||||
return Response(
|
||||
content=pdf,
|
||||
@@ -2314,6 +2313,11 @@ def get_pdf() -> Response:
|
||||
detail="Can not get the 'GENETIC0' optimization solution.",
|
||||
)
|
||||
|
||||
# matplotlib is only needed for the PDF report; import it on first use.
|
||||
from akkudoktoreos.optimization.genetic0.genetic0visualize import (
|
||||
genetic0_prepare_visualize,
|
||||
)
|
||||
|
||||
pdf = genetic0_prepare_visualize(solution=genetic0_solution)
|
||||
|
||||
return Response(
|
||||
|
||||
@@ -67,7 +67,6 @@ from pydantic import (
|
||||
GetCoreSchemaHandler,
|
||||
)
|
||||
from pydantic_core import core_schema
|
||||
from tzfpy import get_tz
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# The Pydantic adapters validate Pendulum values; arithmetic and factory
|
||||
@@ -1414,6 +1413,10 @@ def to_timezone(
|
||||
lat, lon = location
|
||||
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
|
||||
raise ValueError(f"Invalid latitude/longitude: {lat}, {lon}")
|
||||
# tzfpy loads its timezone polygons (~25 MB) on first use; import it
|
||||
# only when a location lookup is actually needed.
|
||||
from tzfpy import get_tz
|
||||
|
||||
tz_name = get_tz(lon, lat)
|
||||
if not tz_name:
|
||||
raise ValueError(
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Heavy optional dependencies must not be imported by a plain server start.
|
||||
|
||||
A GENETIC server with external (import) providers and EOSdash disabled should not
|
||||
load the PDF report stack, the price fallback forecast, pvlib-based providers,
|
||||
the HTML scraper, the GENETIC0 optimizer or the timezone polygon lookup. Each of
|
||||
these costs several to tens of MB of resident memory on small devices.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from akkudoktoreos.config.config import GeneralSettings
|
||||
|
||||
LAZY_MODULES = [
|
||||
"matplotlib",
|
||||
"statsmodels",
|
||||
"pvlib",
|
||||
"bs4",
|
||||
"lxml",
|
||||
"scipy",
|
||||
"tzfpy",
|
||||
"akkudoktoreos.optimization.genetic0.genetic0",
|
||||
"akkudoktoreos.optimization.genetic0.genetic0visualize",
|
||||
"akkudoktoreos.optimization.genetic.geneticvisualize",
|
||||
]
|
||||
|
||||
PROBE = """
|
||||
import json
|
||||
import sys
|
||||
|
||||
import akkudoktoreos.server.eos # noqa: F401
|
||||
from akkudoktoreos.core.coreabc import get_config, get_prediction, singletons_init
|
||||
|
||||
config = get_config(init=True)
|
||||
singletons_init()
|
||||
get_prediction()
|
||||
assert config.general.timezone == "Europe/Berlin", config.general.timezone
|
||||
loaded = sorted(name for name in {modules} if name in sys.modules)
|
||||
print("LOADED_MODULES=" + json.dumps(loaded))
|
||||
"""
|
||||
|
||||
|
||||
def _loaded_modules(tmp_path: Path, extra_env: dict[str, str]) -> list[str]:
|
||||
env = {key: value for key, value in os.environ.items() if not key.startswith("EOS_")}
|
||||
env.update(
|
||||
{
|
||||
"EOS_DIR": str(tmp_path),
|
||||
"EOS_CONFIG_DIR": str(tmp_path),
|
||||
"EOS_GENERAL__DATA_FOLDER_PATH": str(tmp_path / "data"),
|
||||
"EOS_SERVER__STARTUP_EOSDASH": "false",
|
||||
"EOS_OPTIMIZATION__ALGORITHM": "GENETIC",
|
||||
"EOS_LOGGING__CONSOLE_LEVEL": "ERROR",
|
||||
}
|
||||
)
|
||||
env.update(extra_env)
|
||||
result = subprocess.run( # noqa: S603
|
||||
[sys.executable, "-c", PROBE.format(modules=LAZY_MODULES)],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr[-4000:]
|
||||
marker = "LOADED_MODULES="
|
||||
lines = [line for line in result.stdout.splitlines() if line.startswith(marker)]
|
||||
assert len(lines) == 1, result.stdout[-4000:]
|
||||
return json.loads(lines[0][len(marker) :])
|
||||
|
||||
|
||||
def test_server_start_does_not_import_heavy_optional_modules(tmp_path: Path):
|
||||
loaded = _loaded_modules(tmp_path, {"EOS_GENERAL__TIMEZONE_OVERRIDE": "Europe/Berlin"})
|
||||
assert loaded == []
|
||||
|
||||
|
||||
def test_timezone_lookup_still_used_without_override(tmp_path: Path):
|
||||
# Default latitude/longitude are in Berlin, so the lookup gives the same name.
|
||||
loaded = _loaded_modules(tmp_path, {})
|
||||
assert loaded == ["tzfpy"]
|
||||
|
||||
|
||||
def test_timezone_override_semantics():
|
||||
general = GeneralSettings()
|
||||
assert general.timezone_override is None
|
||||
assert general.timezone == "Europe/Berlin"
|
||||
|
||||
general = GeneralSettings(latitude=40.7128, longitude=-74.0060)
|
||||
assert general.timezone == "America/New_York"
|
||||
|
||||
general = GeneralSettings(
|
||||
latitude=40.7128, longitude=-74.0060, timezone_override="Europe/Berlin"
|
||||
)
|
||||
assert general.timezone == "Europe/Berlin"
|
||||
|
||||
# Without coordinates the override still applies.
|
||||
general = GeneralSettings(latitude=None, longitude=None, timezone_override="Asia/Tokyo")
|
||||
assert general.timezone == "Asia/Tokyo"
|
||||
assert GeneralSettings(latitude=None, longitude=None).timezone is None
|
||||
|
||||
assert GeneralSettings(timezone_override="Etc/UTC").timezone == "UTC"
|
||||
|
||||
|
||||
def test_timezone_override_rejects_unknown_names():
|
||||
with pytest.raises(ValueError):
|
||||
GeneralSettings(timezone_override="Mars/Olympus_Mons")
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from pypdf import PdfReader
|
||||
|
||||
from akkudoktoreos.optimization.genetic import geneticvisualize
|
||||
from akkudoktoreos.optimization.genetic.geneticsolution import GeneticSolution
|
||||
from akkudoktoreos.server import eos
|
||||
from akkudoktoreos.utils.datetimeutil import to_datetime
|
||||
@@ -70,7 +71,8 @@ async def test_render_is_offloaded_and_uses_an_owned_copy(config_eos, monkeypatc
|
||||
solution.ac_charge[0] = 0.123
|
||||
return b"%PDF-isolated-render"
|
||||
|
||||
monkeypatch.setattr(eos, "genetic_prepare_visualize", render)
|
||||
# The PDF stack is imported on first use, so patch it at its source.
|
||||
monkeypatch.setattr(geneticvisualize, "genetic_prepare_visualize", render)
|
||||
response = await eos.fastapi_energy_management_optimization_solution_genetic_pdf_get()
|
||||
assert response.body == b"%PDF-isolated-render"
|
||||
assert captured[0][0] != caller_thread
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
|
||||
from akkudoktoreos.core import ems as ems_module
|
||||
from akkudoktoreos.core.emsettings import EnergyManagementMode
|
||||
from akkudoktoreos.optimization.genetic0 import genetic0 as genetic0_module
|
||||
from akkudoktoreos.optimization.genetic0.genetic0params import (
|
||||
Genetic0OptimizationParameters,
|
||||
)
|
||||
@@ -16,6 +17,15 @@ from akkudoktoreos.optimization.optimization import OptimizationAlgorithm
|
||||
from akkudoktoreos.utils.datetimeutil import to_datetime
|
||||
|
||||
|
||||
def _patch_optimizer(monkeypatch, prefix: str, constructor: Any) -> None:
|
||||
"""Replace the optimizer class that EnergyManagement.run() instantiates.
|
||||
|
||||
GENETIC0 is imported on first use inside run(), so it is patched at its source.
|
||||
"""
|
||||
module = genetic0_module if prefix == "Genetic0" else ems_module
|
||||
monkeypatch.setattr(module, prefix + "Optimization", constructor)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def offline_ems(monkeypatch):
|
||||
"""Run the actual EMS orchestration with no real adapters or prediction IO."""
|
||||
@@ -110,7 +120,7 @@ async def test_optimization_routes_only_selected_algorithm(
|
||||
constructor = Mock()
|
||||
constructor.return_value.optimize_ems.return_value = solution
|
||||
constructors[prefix] = constructor
|
||||
monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
|
||||
_patch_optimizer(monkeypatch, prefix, constructor)
|
||||
prepare = AsyncMock(return_value=sentinel_parameters)
|
||||
preparers[prefix] = prepare
|
||||
monkeypatch.setattr(
|
||||
@@ -170,8 +180,8 @@ async def test_optimization_routes_only_selected_algorithm(
|
||||
@pytest.mark.parametrize("mode", [EnergyManagementMode.DISABLED, EnergyManagementMode.PREDICTION])
|
||||
async def test_non_optimization_modes_never_optimize(monkeypatch, offline_ems, mode):
|
||||
constructors = [Mock(), Mock()]
|
||||
monkeypatch.setattr(ems_module, "GeneticOptimization", constructors[0])
|
||||
monkeypatch.setattr(ems_module, "Genetic0Optimization", constructors[1])
|
||||
_patch_optimizer(monkeypatch, "Genetic", constructors[0])
|
||||
_patch_optimizer(monkeypatch, "Genetic0", constructors[1])
|
||||
offline_ems.config.ems.mode = mode
|
||||
await ems_module.EnergyManagement.run(offline_ems)
|
||||
for constructor in constructors:
|
||||
@@ -187,7 +197,7 @@ async def test_non_optimization_modes_never_optimize(monkeypatch, offline_ems, m
|
||||
@pytest.mark.parametrize("prefix", ["Genetic", "Genetic0"])
|
||||
async def test_missing_preparation_does_not_dispatch_controls(monkeypatch, offline_ems, prefix):
|
||||
constructor = Mock()
|
||||
monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
|
||||
_patch_optimizer(monkeypatch, prefix, constructor)
|
||||
prepare = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(getattr(ems_module, prefix + "OptimizationParameters"), "prepare", prepare)
|
||||
await ems_module.EnergyManagement.run(
|
||||
@@ -273,7 +283,7 @@ async def test_failed_legacy_http_run_never_reports_previous_solution(
|
||||
native.optimization_solution.side_effect = error
|
||||
else:
|
||||
native.energy_management_plan.side_effect = error
|
||||
monkeypatch.setattr(ems_module, "Genetic0Optimization", constructor)
|
||||
_patch_optimizer(monkeypatch, "Genetic0", constructor)
|
||||
offline_ems.run = MethodType(cls.run, offline_ems)
|
||||
offline_ems.genetic0_solution = cls.genetic0_solution
|
||||
monkeypatch.setattr(eos, "get_ems", lambda: offline_ems)
|
||||
@@ -311,7 +321,7 @@ async def test_conversion_failure_preserves_consistent_previous_results(
|
||||
solution.energy_management_plan.side_effect = RuntimeError("synthetic plan failure")
|
||||
constructor = Mock()
|
||||
constructor.return_value.optimize_ems.return_value = solution
|
||||
monkeypatch.setattr(ems_module, prefix + "Optimization", constructor)
|
||||
_patch_optimizer(monkeypatch, prefix, constructor)
|
||||
supplied_parameters: dict[str, Any] = {suffix + "_parameters": object()}
|
||||
result = await cls.run(
|
||||
offline_ems,
|
||||
|
||||
@@ -1308,8 +1308,8 @@ class TestCalculation:
|
||||
assert isinstance(result, pd.DataFrame)
|
||||
assert result.empty
|
||||
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.Location")
|
||||
@patch("pvlib.modelchain.ModelChain")
|
||||
@patch("pvlib.location.Location")
|
||||
def test_location_created(
|
||||
self,
|
||||
mock_location,
|
||||
@@ -1352,7 +1352,7 @@ class TestCalculation:
|
||||
|
||||
mock_location.assert_called_once()
|
||||
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
|
||||
@patch("pvlib.modelchain.ModelChain")
|
||||
def test_modelchain_run_called(
|
||||
self,
|
||||
mock_modelchain,
|
||||
@@ -1395,7 +1395,7 @@ class TestCalculation:
|
||||
|
||||
mc.run_model.assert_called_once_with(weather)
|
||||
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
|
||||
@patch("pvlib.modelchain.ModelChain")
|
||||
def test_negative_power_clipped(
|
||||
self,
|
||||
mock_modelchain,
|
||||
@@ -1444,7 +1444,7 @@ class TestCalculation:
|
||||
assert (result["pv_dc_power"] >= 0).all()
|
||||
assert (result["ac_power"] >= 0).all()
|
||||
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
|
||||
@patch("pvlib.modelchain.ModelChain")
|
||||
def test_dataframe_columns(
|
||||
self,
|
||||
mock_modelchain,
|
||||
@@ -1489,7 +1489,7 @@ class TestCalculation:
|
||||
assert "pv_dc_power" in result.columns
|
||||
assert "ac_power" in result.columns
|
||||
|
||||
@patch("akkudoktoreos.prediction.pvforecastpvlib.ModelChain")
|
||||
@patch("pvlib.modelchain.ModelChain")
|
||||
def test_result_index_preserved(
|
||||
self,
|
||||
mock_modelchain,
|
||||
|
||||
Reference in New Issue
Block a user