Mirrors/EOS
1
0
Fork 0
mirror of https://github.com/Akkudoktor-EOS/EOS.git synced 2025-08-25 23:11:14 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Improve caching. (#431)

Browse Source 
* Move the caching module to core.

Add an in memory cache that for caching function and method results
during an energy management run (optimization run). Two decorators
are provided for methods and functions.

* Improve the file cache store by load and save functions.

Make EOS load the cache file store on startup and save it on shutdown.
Add a cyclic task that cleans the cache file store from outdated cache files.

* Improve startup of EOSdash by EOS

Make EOS starting EOSdash adhere to path configuration given in EOS.
The whole environment from EOS is now passed to EOSdash.
Should also prevent test errors due to unwanted/ wrong config file creation.

Both servers now provide a health endpoint that can be used to detect whether
the server is running. This is also used for testing now.

* Improve startup of EOS

EOS now has got an energy management task that runs shortly after startup.
It tries to execute energy management runs with predictions newly fetched
or initialized from cached data on first run.

* Improve shutdown of EOS

EOS has now a shutdown task that shuts EOS down gracefully with some
time delay to allow REST API requests for shutdwon or restart to be fully serviced.

* Improve EMS

Add energy management task for repeated energy management controlled by
startup delay and interval configuration parameters.
Translate EnergieManagementSystem to english EnergyManagement.

* Add administration endpoints

  - endpoints to control caching from REST API.
  - endpoints to control server restart (will not work on Windows) and shutdown from REST API

* Improve doc generation

Use "\n" linenend convention also on Windows when generating doc files.
Replace Windows specific 127.0.0.1 address by standard 0.0.0.0.

* Improve test support (to be able to test caching)

  - Add system test option to pytest for running tests with "real" resources
  - Add new test fixture to start server for test class and test function
  - Make kill signal adapt to Windows/ Linux
  - Use consistently "\n" for lineends when writing text files in  doc test
  - Fix test_logging under Windows
  - Fix conftest config_default_dirs test fixture under Windows

From @Lasall

* Improve Windows support

 - Use 127.0.0.1 as default config host (model defaults) and
   addionally redirect 0.0.0.0 to localhost on Windows (because default
   config file still has 0.0.0.0).
 - Update install/startup instructions as package installation is
   required atm.

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
This commit is contained in:
Bobby Noelte
2025-02-12 21:35:51 +01:00
committed by GitHub
parent 1a2cb4d37d
commit 80bfe4d0f0
54 changed files with 3661 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
tests/conftest.py
View File

@@ -1,18 +1,26 @@
import json
import logging
import os
import signal
import subprocess
import sys
import tempfile
import time
from contextlib import contextmanager
from http import HTTPStatus
from pathlib import Path
from typing import Optional
from typing import Generator, Optional, Union
from unittest.mock import PropertyMock, patch
import pendulum
import psutil
import pytest
from xprocess import ProcessStarter
import requests
from xprocess import ProcessStarter, XProcess
from akkudoktoreos.config.config import ConfigEOS, get_config
from akkudoktoreos.core.logging import get_logger
from akkudoktoreos.server.server import get_default_host
logger = get_logger(__name__)
@@ -48,6 +56,12 @@ def pytest_addoption(parser):
default=False,
help="Verify that user config file is non-existent (will also fail if user config file exists before test run).",
)
parser.addoption(
"--system-test",
action="store_true",
default=False,
help="System test mode. Tests may access real resources, like prediction providers!",
)
@pytest.fixture
@@ -64,6 +78,18 @@ def config_mixin(config_eos):
yield config_mixin_patch
@pytest.fixture
def is_system_test(request):
yield bool(request.config.getoption("--system-test"))
@pytest.fixture
def prediction_eos():
from akkudoktoreos.prediction.prediction import get_prediction
return get_prediction()
@pytest.fixture
def devices_eos(config_mixin):
from akkudoktoreos.devices.devices import get_devices
@@ -87,13 +113,33 @@ def devices_mixin(devices_eos):
# Before activating, make sure that no user config file exists (e.g. ~/.config/net.akkudoktoreos.eos/EOS.config.json)
@pytest.fixture(autouse=True)
def cfg_non_existent(request):
yield
if bool(request.config.getoption("--check-config-side-effect")):
from platformdirs import user_config_dir
if not bool(request.config.getoption("--check-config-side-effect")):
yield
return
user_dir = user_config_dir(ConfigEOS.APP_NAME)
assert not Path(user_dir).joinpath(ConfigEOS.CONFIG_FILE_NAME).exists()
assert not Path.cwd().joinpath(ConfigEOS.CONFIG_FILE_NAME).exists()
# Before test
from platformdirs import user_config_dir
user_dir = user_config_dir(ConfigEOS.APP_NAME)
user_config_file = Path(user_dir).joinpath(ConfigEOS.CONFIG_FILE_NAME)
cwd_config_file = Path.cwd().joinpath(ConfigEOS.CONFIG_FILE_NAME)
assert (
not user_config_file.exists()
), f"Config file {user_config_file} exists, please delete before test!"
assert (
not cwd_config_file.exists()
), f"Config file {cwd_config_file} exists, please delete before test!"
# Yield to test
yield
# After test
assert (
not user_config_file.exists()
), f"Config file {user_config_file} created, please check test!"
assert (
not cwd_config_file.exists()
), f"Config file {cwd_config_file} created, please check test!"
@pytest.fixture(autouse=True)
@@ -149,52 +195,252 @@ def config_eos(
assert config_file.exists()
assert not config_file_cwd.exists()
assert config_default_dirs[-1] / "data" == config_eos.general.data_folder_path
assert config_default_dirs[-1] / "data/cache" == config_eos.general.data_cache_path
assert config_default_dirs[-1] / "data/cache" == config_eos.cache.path()
assert config_default_dirs[-1] / "data/output" == config_eos.general.data_output_path
return config_eos
@pytest.fixture
def config_default_dirs():
def config_default_dirs(tmpdir):
"""Fixture that provides a list of directories to be used as config dir."""
with tempfile.TemporaryDirectory() as tmp_user_home_dir:
# Default config directory from platform user config directory
config_default_dir_user = Path(tmp_user_home_dir) / "config"
tmp_user_home_dir = Path(tmpdir)
# Default config directory from current working directory
config_default_dir_cwd = Path(tmp_user_home_dir) / "cwd"
config_default_dir_cwd.mkdir()
# Default config directory from platform user config directory
config_default_dir_user = tmp_user_home_dir / "config"
# Default config directory from default config file
config_default_dir_default = Path(__file__).parent.parent.joinpath("src/akkudoktoreos/data")
# Default config directory from current working directory
config_default_dir_cwd = tmp_user_home_dir / "cwd"
config_default_dir_cwd.mkdir()
# Default data directory from platform user data directory
data_default_dir_user = Path(tmp_user_home_dir)
yield (
config_default_dir_user,
config_default_dir_cwd,
config_default_dir_default,
data_default_dir_user,
)
# Default config directory from default config file
config_default_dir_default = Path(__file__).parent.parent.joinpath("src/akkudoktoreos/data")
# Default data directory from platform user data directory
data_default_dir_user = tmp_user_home_dir
return (
config_default_dir_user,
config_default_dir_cwd,
config_default_dir_default,
data_default_dir_user,
)
@contextmanager
def server_base(xprocess: XProcess) -> Generator[dict[str, Union[str, int]], None, None]:
"""Fixture to start the server with temporary EOS_DIR and default config.
Args:
xprocess (XProcess): The pytest-xprocess fixture to manage the server process.
Yields:
dict[str, str]: A dictionary containing:
- "server" (str): URL of the server.
- "eos_dir" (str): Path to the temporary EOS_DIR.
"""
host = get_default_host()
port = 8503
eosdash_port = 8504
# Port of server may be still blocked by a server usage despite the other server already
# shut down. CLOSE_WAIT, TIME_WAIT may typically take up to 120 seconds.
server_timeout = 120
server = f"http://{host}:{port}"
eosdash_server = f"http://{host}:{eosdash_port}"
eos_tmp_dir = tempfile.TemporaryDirectory()
eos_dir = str(eos_tmp_dir.name)
class Starter(ProcessStarter):
# assure server to be installed
try:
project_dir = Path(__file__).parent.parent
subprocess.run(
[sys.executable, "-c", "import", "akkudoktoreos.server.eos"],
check=True,
env=os.environ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=project_dir,
)
except subprocess.CalledProcessError:
subprocess.run(
[sys.executable, "-m", "pip", "install", "-e", str(project_dir)],
env=os.environ,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=project_dir,
)
# Set environment for server run
env = os.environ.copy()
env["EOS_DIR"] = eos_dir
env["EOS_CONFIG_DIR"] = eos_dir
# command to start server process
args = [
sys.executable,
"-m",
"akkudoktoreos.server.eos",
"--host",
host,
"--port",
str(port),
]
# Will wait for 'server_timeout' seconds before timing out
timeout = server_timeout
# xprocess will now attempt to clean up upon interruptions
terminate_on_interrupt = True
# checks if our server is ready
def startup_check(self):
try:
result = requests.get(f"{server}/v1/health", timeout=2)
if result.status_code == 200:
return True
except:
pass
return False
def cleanup_eos_eosdash():
# Cleanup any EOS process left.
if os.name == "nt":
# Windows does not provide SIGKILL
sigkill = signal.SIGTERM
else:
sigkill = signal.SIGKILL
# - Use pid on EOS health endpoint
try:
result = requests.get(f"{server}/v1/health", timeout=2)
if result.status_code == HTTPStatus.OK:
pid = result.json()["pid"]
os.kill(pid, sigkill)
time.sleep(1)
result = requests.get(f"{server}/v1/health", timeout=2)
assert result.status_code != HTTPStatus.OK
except:
pass
# - Use pids from processes on EOS port
for retries in range(int(server_timeout / 3)):
pids: list[int] = []
for conn in psutil.net_connections(kind="inet"):
if conn.laddr.port == port:
if conn.pid not in pids:
# Get fresh process info
try:
process = psutil.Process(conn.pid)
process_info = process.as_dict(attrs=["pid", "cmdline"])
if "akkudoktoreos.server.eos" in process_info["cmdline"]:
pids.append(conn.pid)
except:
# PID may already be dead
pass
for pid in pids:
os.kill(pid, sigkill)
if len(pids) == 0:
break
time.sleep(3)
assert len(pids) == 0
# Cleanup any EOSdash processes left.
# - Use pid on EOSdash health endpoint
try:
result = requests.get(f"{eosdash_server}/eosdash/health", timeout=2)
if result.status_code == HTTPStatus.OK:
pid = result.json()["pid"]
os.kill(pid, sigkill)
time.sleep(1)
result = requests.get(f"{eosdash_server}/eosdash/health", timeout=2)
assert result.status_code != HTTPStatus.OK
except:
pass
# - Use pids from processes on EOSdash port
for retries in range(int(server_timeout / 3)):
pids = []
for conn in psutil.net_connections(kind="inet"):
if conn.laddr.port == eosdash_port:
if conn.pid not in pids:
# Get fresh process info
try:
process = psutil.Process(conn.pid)
process_info = process.as_dict(attrs=["pid", "cmdline"])
if "akkudoktoreos.server.eosdash" in process_info["cmdline"]:
pids.append(conn.pid)
except:
# PID may already be dead
pass
for pid in pids:
os.kill(pid, sigkill)
if len(pids) == 0:
break
time.sleep(3)
assert len(pids) == 0
# Kill all running eos and eosdash process - just to be sure
cleanup_eos_eosdash()
# Ensure there is an empty config file in the temporary EOS directory
config_file_path = Path(eos_dir).joinpath(ConfigEOS.CONFIG_FILE_NAME)
with config_file_path.open(mode="w", encoding="utf-8", newline="\n") as fd:
json.dump({}, fd)
# ensure process is running and return its logfile
pid, logfile = xprocess.ensure("eos", Starter)
logger.info(f"Started EOS ({pid}). This may take very long (up to {server_timeout} seconds).")
logger.info(f"View xprocess logfile at: {logfile}")
yield {
"server": server,
"eosdash_server": eosdash_server,
"eos_dir": eos_dir,
"timeout": server_timeout,
}
# clean up whole process tree afterwards
xprocess.getinfo("eos").terminate()
# Cleanup any EOS process left.
cleanup_eos_eosdash()
# Remove temporary EOS_DIR
eos_tmp_dir.cleanup()
@pytest.fixture(scope="class")
def server_setup_for_class(xprocess) -> Generator[dict[str, Union[str, int]], None, None]:
"""A fixture to start the server for a test class."""
with server_base(xprocess) as result:
yield result
@pytest.fixture(scope="function")
def server_setup_for_function(xprocess) -> Generator[dict[str, Union[str, int]], None, None]:
"""A fixture to start the server for a test function."""
with server_base(xprocess) as result:
yield result
@pytest.fixture
def server(xprocess, config_eos, config_default_dirs):
def server(xprocess, config_eos, config_default_dirs) -> Generator[str, None, None]:
"""Fixture to start the server.
Provides URL of the server.
"""
# create url/port info to the server
url = "http://0.0.0.0:8503"
class Starter(ProcessStarter):
# Set environment before any subprocess run, to keep custom config dir
env = os.environ.copy()
env["EOS_DIR"] = str(config_default_dirs[-1])
project_dir = config_eos.package_root_path
project_dir = config_eos.package_root_path.parent.parent
# assure server to be installed
try:
subprocess.run(
[sys.executable, "-c", "import akkudoktoreos.server.eos"],
[sys.executable, "-c", "import", "akkudoktoreos.server.eos"],
check=True,
env=env,
stdout=subprocess.PIPE,
@@ -203,7 +449,7 @@ def server(xprocess, config_eos, config_default_dirs):
)
except subprocess.CalledProcessError:
subprocess.run(
[sys.executable, "-m", "pip", "install", "-e", project_dir],
[sys.executable, "-m", "pip", "install", "-e", str(project_dir)],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
@@ -212,24 +458,26 @@ def server(xprocess, config_eos, config_default_dirs):
# command to start server process
args = [sys.executable, "-m", "akkudoktoreos.server.eos"]
# startup pattern
pattern = "Application startup complete."
# search this number of lines for the startup pattern, if not found
# a RuntimeError will be raised informing the user
max_read_lines = 30
# will wait for 30 seconds before timing out
timeout = 30
# will wait for xx seconds before timing out
timeout = 10
# xprocess will now attempt to clean up upon interruptions
terminate_on_interrupt = True
# checks if our server is ready
def startup_check(self):
try:
result = requests.get(f"{url}/v1/health")
if result.status_code == 200:
return True
except:
pass
return False
# ensure process is running and return its logfile
pid, logfile = xprocess.ensure("eos", Starter)
print(f"View xprocess logfile at: {logfile}")
# create url/port info to the server
url = "http://127.0.0.1:8503"
yield url
# clean up whole process tree afterwards