Mirrors/EOS
1
0
Fork 0
mirror of https://github.com/Akkudoktor-EOS/EOS.git synced 2026-07-21 17:28:11 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

fix: move data management to async (#1015)

Browse Source 
FAstAPI is an async framework. Data may be imported and exported, load and save, set and get
asynchronously. Prevent interleaving data operations to corrupt the data. In the previous design
sync and async data access was intermixed leading to data corruption.

The basic data classes DataSequence and DataContainer and the derived classes like Provider and
Measurement now are async. Data access is protected by several async locks.

To support the async design of the data classes the database interface became async.

The energy management is also adapted to the new async design. Optimization is still off-loaded
to another thread, but the prepration for the optimization and the post optimization actions now
follow the async design.

Adapter operations are now also protected by async locks.

Tests were adapted to the async design and new tests were created.

Besides this major fix several other improvements and fixes are included in this PR.

* fix: key_to_dict/list/array only regard data records with key value set.

  Before the exclusion of no value data records was only done if the dropna flag was set.

* fix: test for visual result pdf generation

  Due to updates in the library the generated charts text was a little bit different.
  Adapt the test to create the comaprison pdf in the test data durectory and
  update the reference pdf.

* chore: Remove MutableMapping from DataSequence and DataContainer.

  Mutable Mapping does not fit to the now async design.

* chore: Add NoDB database backend

  This backend implements the full database backend interface but performs
  no actual persistence. It is intended for configurations where database
  persistence is disabled (`provider=None`).

* chore: Improve measurement data import testing with real world scenarios.

  Added two new endpoints to support testing.

* chore: Add mermaid to supported documentation tools

* chore: Add documentation about async design

* chore: Add documentation about generic data handling

  Covers the basics of measurement and prediction time series data handling.

* chore: Add empty lines around markdown lists.

* chore: sync pre-commit config to updated package versions

Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
This commit is contained in:
Bobby Noelte
2026-07-15 16:38:53 +02:00
committed by GitHub
parent 38011780c5
commit eb9e966de9
99 changed files with 11971 additions and 5981 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/akkudoktoreos/adapter/adapterabc.py
View File

@@ -1,5 +1,6 @@
"""Abstract and base classes for adapters."""
import asyncio
from abc import abstractmethod
from typing import Any, Optional
@@ -53,12 +54,27 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin,
return self.provider_id() in self.config.adapter.provider
return False
@property
def _adapter_lock(self) -> asyncio.Lock:
"""Per-instance asyncio lock guarding adapter-level bulk operations.
The lock guards the full adapter state during bulk operations.
"""
try:
return object.__getattribute__(self, "_adapter_lock_instance")
except AttributeError:
lock = asyncio.Lock()
object.__setattr__(self, "_adapter_lock_instance", lock)
return lock
@abstractmethod
def _update_data(self) -> None:
async def _update_data(self) -> None:
"""Abstract method for custom adapter data update logic, to be implemented by derived classes.
Data update may be requested at different stages of energy management. The stage can be
detected by self.ems.stage().
This method is always called while `_adapter_lock` is held by the caller.
"""
pass
@@ -67,7 +83,7 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin,
return
super().__init__(*args, **kwargs)
def update_data(
async def update_data(
self,
force_enable: Optional[bool] = False,
) -> None:
@@ -81,8 +97,9 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin,
return
# Call the custom update logic
logger.debug(f"Update adapter provider: {self.provider_id()}")
self._update_data()
async with self._adapter_lock:
logger.debug(f"Update adapter provider: {self.provider_id()}")
await self._update_data()
class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel):
@@ -105,6 +122,19 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel):
)
return value
@property
def _container_lock(self) -> asyncio.Lock:
"""Coarse-grained lock for bulk operations across providers.
The lock guards cross-provider consistency during container operations.
"""
try:
return object.__getattribute__(self, "_container_lock_instance")
except AttributeError:
lock = asyncio.Lock()
object.__setattr__(self, "_container_lock_instance", lock)
return lock
@property
def enabled_providers(self) -> list[Any]:
"""List of providers that are currently enabled."""
@@ -145,7 +175,7 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel):
raise ValueError(error_msg)
return providers[provider_id]
def update_data(
async def update_data(
self,
force_enable: Optional[bool] = False,
) -> None:
@@ -154,7 +184,10 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel):
Args:
force_enable (bool, optional): If True, forces the update even if the provider is disabled.
"""
if len(self.providers) <= 0:
return
# Call the custom update logic
if len(self.providers) > 0:
async with self._container_lock:
for provider in self.providers:
provider.update_data(force_enable=force_enable)
await provider.update_data(force_enable=force_enable)

12
src/akkudoktoreos/adapter/homeassistant.py
View File

@@ -419,7 +419,7 @@ class HomeAssistantAdapter(AdapterProvider):
# Preserve original state for enums and free-text states
return raw_state
def _update_data(self) -> None:
async def _update_data(self) -> None:
stage = self.ems.stage()
if stage == EnergyManagementStage.DATA_ACQUISITION:
# Sync configuration
@@ -451,7 +451,7 @@ class HomeAssistantAdapter(AdapterProvider):
logger.debug(f"Entity {entity_id}: {state}")
if state:
measurement_value = float(state)
self.measurement.update_value(
await self.measurement.update_value(
self.ems_start_datetime, measurement_key, measurement_value
)
except Exception as e:
@@ -473,7 +473,7 @@ class HomeAssistantAdapter(AdapterProvider):
logger.debug(f"Entity {entity_id}: {state}")
if state:
measurement_value = float(state)
self.measurement.update_value(
await self.measurement.update_value(
self.ems_start_datetime, measurement_key, measurement_value
)
except Exception as e:
@@ -495,7 +495,7 @@ class HomeAssistantAdapter(AdapterProvider):
logger.debug(f"Entity {entity_id}: {state}")
if state:
measurement_value = float(state)
self.measurement.update_value(
await self.measurement.update_value(
self.ems_start_datetime, measurement_key, measurement_value
)
except Exception as e:
@@ -517,7 +517,7 @@ class HomeAssistantAdapter(AdapterProvider):
logger.debug(f"Entity {entity_id}: {state}")
if state:
measurement_value = float(state)
self.measurement.update_value(
await self.measurement.update_value(
self.ems_start_datetime, measurement_key, measurement_value
)
except Exception as e:
@@ -539,7 +539,7 @@ class HomeAssistantAdapter(AdapterProvider):
logger.debug(f"Entity {entity_id}: {state}")
if state:
measurement_value = float(state)
self.measurement.update_value(
await self.measurement.update_value(
self.ems_start_datetime, measurement_key, measurement_value
)
except Exception as e:

2
src/akkudoktoreos/adapter/nodered.py
View File

@@ -66,7 +66,7 @@ class NodeREDAdapter(AdapterProvider):
"""Return the unique identifier for the adapter provider."""
return "NodeRED"
def _update_data(self) -> None:
async def _update_data(self) -> None:
"""Custom adapter data update logic.
Data update may be requested at different stages of energy management. The stage can be

1036
src/akkudoktoreos/core/dataabc.py
View File

File diff suppressed because it is too large Load Diff

662
src/akkudoktoreos/core/database.py
View File

@@ -9,25 +9,34 @@ namespaces with a `namespace` column.
from __future__ import annotations
import asyncio
import shutil
import sqlite3
from pathlib import Path
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
from typing import (
Any,
AsyncIterator,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
)
import lmdb
from loguru import logger
from pydantic import Field, computed_field, field_validator
from akkudoktoreos.config.configabc import SettingsBaseModel
from akkudoktoreos.core.coreabc import SingletonMixin
from akkudoktoreos.core.coreabc import ConfigMixin, SingletonMixin
from akkudoktoreos.core.databaseabc import (
DATABASE_METADATA_KEY,
DatabaseABC,
DatabaseBackendABC,
)
# Valid database providers
database_providers: List[str] = ["LMDB", "SQLite"]
database_providers: List[str] = ["LMDB", "SQLite", "NoDB"]
class DatabaseCommonSettings(SettingsBaseModel):
@@ -40,6 +49,8 @@ class DatabaseCommonSettings(SettingsBaseModel):
batch_size: Batch size for batch operations.
"""
model_config = {"validate_assignment": True} # keeps field validation on assignment
provider: Optional[str] = Field(
default=None,
json_schema_extra={
@@ -177,12 +188,17 @@ class LMDBDatabase(DatabaseBackendABC):
"""Return the unique identifier for the database provider."""
return "LMDB"
def open(self, namespace: Optional[str] = None) -> None:
def open(self, *, namespace: Optional[str] = None) -> None:
"""Open LMDB environment and optionally ensure a namespace DBI.
Args:
namespace: Optional default namespace to open (DBI created on demand).
"""
if self.is_open:
if namespace is not None:
self._ensure_dbi(namespace=namespace)
return
self.storage_path.mkdir(parents=True, exist_ok=True)
self.env = lmdb.open(
@@ -201,7 +217,7 @@ class LMDBDatabase(DatabaseBackendABC):
self.default_namespace = namespace
if namespace is not None:
self._ensure_dbi(namespace)
self._ensure_dbi(namespace=namespace)
def close(self) -> None:
"""Close the LMDB environment and clear cached DBIs."""
@@ -214,7 +230,7 @@ class LMDBDatabase(DatabaseBackendABC):
self._dbis.clear()
logger.debug("Closed LMDB at %s", self.storage_path)
def flush(self, namespace: Optional[str] = None) -> None:
def flush(self, *, namespace: Optional[str] = None) -> None:
"""Sync LMDB environment (writes to disk)."""
if not isinstance(self.env, lmdb.Environment):
raise ValueError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
@@ -230,7 +246,7 @@ class LMDBDatabase(DatabaseBackendABC):
"""Return explicit namespace or default if None."""
return namespace if namespace is not None else self.default_namespace
def _ensure_dbi(self, namespace: Optional[str]) -> Optional[Any]:
def _ensure_dbi(self, *, namespace: Optional[str]) -> Optional[Any]:
"""Open and cache a DBI for the given namespace.
Args:
@@ -271,15 +287,15 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
with self.env.begin(write=True) as txn:
if metadata is None:
txn.delete(DATABASE_METADATA_KEY)
txn.delete(DATABASE_METADATA_KEY, db=dbi)
else:
txn.put(DATABASE_METADATA_KEY, metadata)
txn.put(DATABASE_METADATA_KEY, metadata, db=dbi)
def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]:
def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]:
"""Load metadata for a given namespace.
Returns None if no metadata exists.
@@ -293,10 +309,10 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
with self.env.begin(write=False) as txn:
return txn.get(DATABASE_METADATA_KEY)
return txn.get(DATABASE_METADATA_KEY, db=dbi)
# ------------------------------------------------------------------
# Bulk Write Operations
@@ -305,6 +321,7 @@ class LMDBDatabase(DatabaseBackendABC):
def save_records(
self,
records: Iterable[tuple[bytes, bytes]],
*,
namespace: Optional[str] = None,
) -> int:
"""Save multiple records into the specified namespace (or default).
@@ -324,7 +341,7 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
saved = 0
with self.lock:
@@ -338,6 +355,7 @@ class LMDBDatabase(DatabaseBackendABC):
def delete_records(
self,
keys: Iterable[bytes],
*,
namespace: Optional[str] = None,
) -> int:
"""Delete multiple records by key from the specified namespace.
@@ -352,7 +370,7 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError("Database not open")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
deleted = 0
with self.lock:
@@ -371,6 +389,7 @@ class LMDBDatabase(DatabaseBackendABC):
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
reverse: bool = False,
) -> Iterator[tuple[bytes, bytes]]:
@@ -391,11 +410,12 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong type `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
META = DATABASE_METADATA_KEY
results: list[tuple[bytes, bytes]] = []
cursor = None
txn = self.env.begin(write=False)
try:
cursor = txn.cursor(dbi)
@@ -454,7 +474,8 @@ class LMDBDatabase(DatabaseBackendABC):
finally:
# Ensure reader slot is always released
cursor.close()
if cursor is not None:
cursor.close()
txn.abort()
# Transaction is closed here — safe to yield
@@ -478,7 +499,7 @@ class LMDBDatabase(DatabaseBackendABC):
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
META = DATABASE_METADATA_KEY
count = 0
@@ -510,13 +531,14 @@ class LMDBDatabase(DatabaseBackendABC):
def get_key_range(
self,
*,
namespace: Optional[str] = None,
) -> tuple[Optional[bytes], Optional[bytes]]:
"""Return (min_key, max_key) in the given namespace or (None, None) if empty."""
if not isinstance(self.env, lmdb.Environment):
raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.")
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
with self.env.begin(write=False) as txn:
cursor = txn.cursor(db=dbi)
@@ -541,12 +563,12 @@ class LMDBDatabase(DatabaseBackendABC):
return min_key, max_key
def get_backend_stats(self, namespace: Optional[str] = None) -> dict[str, Any]:
def get_backend_stats(self, *, namespace: Optional[str] = None) -> dict[str, Any]:
"""Get LMDB backend-specific statistics."""
if not self.env:
return {}
dbi = self._ensure_dbi(namespace)
dbi = self._ensure_dbi(namespace=namespace)
with self.env.begin(write=False) as txn:
stat = txn.stat(db=dbi)
@@ -657,12 +679,15 @@ class SQLiteDatabase(DatabaseBackendABC):
"""Return the unique identifier for the database provider."""
return "SQLite"
def open(self, namespace: Optional[str] = None) -> None:
def open(self, *, namespace: Optional[str] = None) -> None:
"""Open SQLite connection and optionally set default namespace.
Args:
namespace: Optional default namespace to use when operations omit namespace.
"""
if self.is_open:
return
self.storage_path.mkdir(parents=True, exist_ok=True)
self.conn = sqlite3.connect(
@@ -700,7 +725,7 @@ class SQLiteDatabase(DatabaseBackendABC):
self._is_open = False
logger.debug("Closed SQLite at %s", self.db_file)
def flush(self, namespace: Optional[str] = None) -> None:
def flush(self, *, namespace: Optional[str] = None) -> None:
"""Commit any pending transactions to disk (no-op if autocommit)."""
if not isinstance(self.conn, sqlite3.Connection):
raise RuntimeError(f"SQLite connection is of wrong tpe `{type(self.conn)}`.")
@@ -745,7 +770,7 @@ class SQLiteDatabase(DatabaseBackendABC):
(ns, metadata),
)
def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]:
def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]:
"""Load metadata for a given namespace.
Returns None if no metadata exists.
@@ -777,6 +802,7 @@ class SQLiteDatabase(DatabaseBackendABC):
def save_records(
self,
records: Iterable[tuple[bytes, bytes]],
*,
namespace: Optional[str] = None,
) -> int:
"""Bulk insert or replace records.
@@ -794,18 +820,18 @@ class SQLiteDatabase(DatabaseBackendABC):
return 0
with self.lock:
self.conn.execute("BEGIN")
self.conn.executemany(
"INSERT OR REPLACE INTO records (namespace, key, value) VALUES (?, ?, ?)",
rows,
)
self.conn.execute("COMMIT")
with self.conn:
self.conn.executemany(
"INSERT OR REPLACE INTO records (namespace, key, value) VALUES (?, ?, ?)",
rows,
)
return len(rows)
def delete_records(
self,
keys: Iterable[bytes],
*,
namespace: Optional[str] = None,
) -> int:
"""Delete multiple records by key.
@@ -817,21 +843,25 @@ class SQLiteDatabase(DatabaseBackendABC):
ns = self._ns(namespace)
deleted: int = 0
with self.lock:
for key in keys:
cursor = self.conn.execute(
"DELETE FROM records WHERE namespace = ? AND key = ?",
(ns, key),
)
deleted += cursor.rowcount
rows = [(ns, key) for key in keys]
return deleted
if not rows:
return 0
with self.lock:
with self.conn:
cursor = self.conn.executemany(
"DELETE FROM records WHERE namespace = ? AND key = ?",
rows,
)
return cursor.rowcount
def iterate_records(
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
reverse: bool = False,
) -> Iterator[Tuple[bytes, bytes]]:
@@ -913,7 +943,7 @@ class SQLiteDatabase(DatabaseBackendABC):
return int(cursor.fetchone()[0])
def get_key_range(
self, namespace: Optional[str] = None
self, *, namespace: Optional[str] = None
) -> Tuple[Optional[bytes], Optional[bytes]]:
"""Return (min_key, max_key) for the namespace or (None, None) if empty."""
if not isinstance(self.conn, sqlite3.Connection):
@@ -928,7 +958,7 @@ class SQLiteDatabase(DatabaseBackendABC):
result = cursor.fetchone()
return result[0], result[1]
def get_backend_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]:
def get_backend_stats(self, *, namespace: Optional[str] = None) -> Dict[str, Any]:
"""Return SQLite-specific stats and namespace metrics."""
if not self.conn:
return {}
@@ -959,10 +989,216 @@ class SQLiteDatabase(DatabaseBackendABC):
logger.info("SQLite vacuum completed")
# ==================== Generic Database Implementation ====================
# ==================== NoDB Implementation ====================
class Database(DatabaseABC, SingletonMixin):
class NoDB(DatabaseBackendABC):
"""No-op database backend.
This backend implements the full database backend interface but performs
no actual persistence. It is intended for configurations where database
persistence is disabled (`provider=None`).
Characteristics:
- All write operations are ignored.
- All read operations return empty results.
- No files or external resources are created.
- Lifecycle operations are no-ops.
- Compression is disabled.
This backend allows the database wrapper to avoid special-case handling
for a missing database provider by always providing a valid backend
implementation.
"""
def __init__(self, **kwargs: Any) -> None:
"""Initialize the no-op backend."""
super().__init__()
self._is_open = True # Stateless
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
def provider_id(self) -> str:
"""Return the unique identifier for the database provider."""
return "NoDB"
def open(self, *, namespace: Optional[str] = None) -> None:
"""Mark backend as open.
Args:
namespace: Ignored.
"""
self.default_namespace = namespace
def close(self) -> None:
"""Mark backend as closed."""
return
def flush(self, *, namespace: Optional[str] = None) -> None:
"""Flush pending writes (no-op).
Args:
namespace: Ignored.
"""
return
# ------------------------------------------------------------------
# Effective backend configuration
# ------------------------------------------------------------------
@property
def is_open(self) -> bool:
"""Return dummy not open."""
return False
@property
def storage_path(self) -> Path:
"""Return dummy storage path."""
return Path("/dev/null")
@property
def compression_level(self) -> int:
"""Compression is disabled."""
return 0
@property
def compression(self) -> bool:
"""Compression is disabled."""
return False
# ------------------------------------------------------------------
# Metadata operations
# ------------------------------------------------------------------
def set_metadata(
self,
metadata: Optional[bytes],
*,
namespace: Optional[str] = None,
) -> None:
"""Store metadata (ignored).
Args:
metadata: Ignored.
namespace: Ignored.
"""
return
def get_metadata(
self,
*,
namespace: Optional[str] = None,
) -> Optional[bytes]:
"""Load metadata.
Always returns:
None
"""
return None
# ------------------------------------------------------------------
# Record operations
# ------------------------------------------------------------------
def save_records(
self,
records: Iterable[tuple[bytes, bytes]],
*,
namespace: Optional[str] = None,
) -> int:
"""Pretend to save records.
Args:
records: Ignored.
namespace: Ignored.
Returns:
Always 0.
"""
return 0
def delete_records(
self,
keys: Iterable[bytes],
*,
namespace: Optional[str] = None,
) -> int:
"""Pretend to delete records.
Args:
keys: Ignored.
namespace: Ignored.
Returns:
Always 0.
"""
return 0
def iterate_records(
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
reverse: bool = False,
) -> Iterator[tuple[bytes, bytes]]:
"""Iterate records.
Always yields:
Nothing
"""
return iter(())
def count_records(
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
) -> int:
"""Count records.
Returns:
Always 0.
"""
return 0
def get_key_range(
self,
*,
namespace: Optional[str] = None,
) -> tuple[Optional[bytes], Optional[bytes]]:
"""Return key range.
Returns:
Always (None, None).
"""
return None, None
def get_backend_stats(
self,
*,
namespace: Optional[str] = None,
) -> dict[str, Any]:
"""Return backend statistics.
Returns:
Minimal backend information.
"""
return {
"backend": "none",
"namespace": namespace,
"persistent": False,
"records": 0,
}
# ==================== Generic Database ====================
class Database(ConfigMixin, SingletonMixin):
"""Generic database.
All operations accept an optional `namespace` argument. Implementations should
@@ -971,16 +1207,15 @@ class Database(DatabaseABC, SingletonMixin):
a namespace column).
"""
_db: Optional[DatabaseBackendABC] = None
_db: DatabaseBackendABC = NoDB()
@classmethod
def reset_instance(cls) -> None:
"""Resets the singleton instance, forcing it to be recreated on next access."""
with cls._lock:
# Close current database backend
if cls._db:
cls._db.close()
cls._db = None
cls._db.close()
cls._db = NoDB()
# Remove current database instance
if cls in cls._instances:
del cls._instances[cls]
@@ -989,95 +1224,228 @@ class Database(DatabaseABC, SingletonMixin):
def __init__(self) -> None:
"""Initialize database."""
super().__init__()
self._db = None
self._db = NoDB()
def _setup_db(self) -> None:
"""Setup database."""
# Database helpers
@property
def _database_lock(self) -> asyncio.Lock:
"""Per-instance asyncio lock guarding database operations.
The lock guards the database state during async operations.
"""
# Lock must be a loop-local asyncio lock to provide singleton + pytest async compatibility.
loop = asyncio.get_running_loop()
try:
locks = object.__getattribute__(self, "_database_locks")
except AttributeError:
locks = {}
object.__setattr__(self, "_database_locks", locks)
lock = locks.get(loop)
if lock is None:
lock = asyncio.Lock()
locks[loop] = lock
return lock
async def _ensure_open(
self,
*,
namespace: Optional[str] = None,
) -> DatabaseBackendABC:
"""Ensure the configured backend exists, is open, and namespace is prepared.
Expects the database lock to already be held when called.
Args:
namespace: Optional namespace to prepare/open.
Returns:
The active and opened backend instance.
Raises:
RuntimeError: If no database provider is configured.
"""
provider_id = self.config.database.provider
database: Optional[DatabaseBackendABC] = None
if provider_id is None:
database = None
elif provider_id == "LMDB":
database = LMDBDatabase()
elif provider_id == "SQLite":
database = SQLiteDatabase()
else:
raise RuntimeError("Invalid database provider '{provider_id}'")
if self._db is not None:
self._db.close()
self._db = database
old_provider_id = self._db.provider_id()
def _database(self) -> DatabaseBackendABC:
"""Get database."""
provider_id = self.config.database.provider
if provider_id is None:
raise RuntimeError("Database not configured")
if self._db is None or self._db.provider_id() != provider_id:
if old_provider_id != provider_id:
# No database or configuration does not match
self._setup_db()
if self._db is None:
raise RuntimeError("Database not configured")
logger.debug(
f"Switching database provider from '{old_provider_id}' to '{provider_id}'."
)
old_db = self._db
database: DatabaseBackendABC
if provider_id is None or provider_id == "NoDB":
database = NoDB()
elif provider_id == "LMDB":
database = LMDBDatabase()
elif provider_id == "SQLite":
database = SQLiteDatabase()
else:
raise RuntimeError(f"Invalid database provider '{provider_id}'")
# Auto-close database that was used before
if old_db.is_open:
old_db.close()
self._db = database
if not self._db.is_open:
self._db.open()
await asyncio.to_thread(self._db.open, namespace=namespace)
elif namespace is not None:
# Allow backend to lazily prepare namespace resources
await asyncio.to_thread(self._db.open, namespace=namespace)
return self._db
async def _run_db(
self,
method_name: str,
*args: Any,
**kwargs: Any,
) -> Any:
"""Execute a synchronous database backend operation in a thread-safe, non-blocking way.
The backend is automatically initialized/opened before execution.
If a ``namespace`` keyword argument is present, the namespace is
prepared during backend initialization.
This helper ensures that all interactions with the underlying synchronous
database backend are:
- **Serialized** using an instance-level ``asyncio.Lock`` to protect backend state.
- **Non-blocking** by offloading execution to a worker thread via
``asyncio.to_thread``.
It should be used for all backend calls that may perform blocking I/O or
CPU-bound work.
Args:
method_name: The synchronous callable to execute (a backend method).
*args: Positional arguments forwarded to ``method``.
**kwargs: Keyword arguments forwarded to ``method``.
Returns:
The return value of ``method``.
Raises:
Any exception raised by ``func`` is propagated unchanged.
Notes:
- The callable is executed in a separate thread, so it must be thread-safe.
- The database lock is held for the duration of the call, ensuring that
no concurrent operations interfere with backend state.
- Avoid passing coroutines or async functions to this method; it is
intended strictly for synchronous callables.
"""
namespace = kwargs.get("namespace")
async with self._database_lock:
db = await self._ensure_open(namespace=namespace)
# Get actual method. _ensure_open may have changed the backend
method = getattr(db, method_name)
def backend_call() -> Any:
result = method(
*args,
**kwargs,
)
# Materialize iterators inside worker thread
if isinstance(result, Iterator):
return list(result)
return result
return await asyncio.to_thread(backend_call)
def provider_id(self) -> str:
"""Return the unique identifier for the database provider."""
try:
return self._database().provider_id()
except:
return "None"
return self._db.provider_id()
@property
def is_open(self) -> bool:
"""Return whether the database connection is open."""
try:
return self._database().is_open
except:
return False
return self._db.is_open
@property
def storage_path(self) -> Path:
"""Storage path for the database."""
return self._database().storage_path
"""Return effective storage path of active backend."""
return self._db.storage_path
@property
def compression_level(self) -> int:
"""Compression level for database record data."""
return self._database().compression_level
"""Return effective compression level of active backend."""
return self._db.compression_level
@property
def compression(self) -> bool:
"""Whether to compress stored values."""
return self._database().compression_level > 0
"""Return whether the active backend compresses stored values.
Returns:
True if compression is enabled for the active backend,
False if disabled,
or None if no backend is currently initialized.
"""
return self._db.compression_level > 0
# Lifecycle
def open(self, namespace: Optional[str] = None) -> None:
"""Open database connection and optionally set default namespace.
async def ensure_open(
self,
*,
namespace: Optional[str] = None,
) -> DatabaseBackendABC:
"""Ensure the configured backend exists, is open, and namespace is prepared.
Args:
namespace: Optional default namespace to prepare.
namespace: Optional namespace to prepare/open.
Returns:
The active and opened backend instance.
Raises:
RuntimeError: If the database cannot be opened.
RuntimeError: If no database provider is configured.
"""
self._database().open(namespace)
async with self._database_lock:
return await self._ensure_open(namespace=namespace)
def close(self) -> None:
async def open(self, *, namespace: Optional[str] = None) -> None:
"""Ensure the database backend is initialized and open.
If the backend is already open, this operation is a no-op except
that the backend may prepare the specified namespace.
Args:
namespace: Optional namespace to prepare/open.
Raises:
RuntimeError: If no database provider is configured or opening fails.
"""
async with self._database_lock:
await self._ensure_open(namespace=namespace)
async def close(self) -> None:
"""Close the database connection and cleanup resources."""
self._database().close()
async with self._database_lock:
if self._db is not None and self._db.is_open:
await asyncio.to_thread(self._db.close)
def flush(self, namespace: Optional[str] = None) -> None:
async def flush(self, *, namespace: Optional[str] = None) -> None:
"""Force synchronization of pending writes to storage (optional per-namespace)."""
return self._database().flush(namespace)
await self._run_db("flush", namespace=namespace)
# Metadata operations
def set_metadata(self, metadata: Optional[bytes], *, namespace: Optional[str] = None) -> None:
async def set_metadata(
self, metadata: Optional[bytes], *, namespace: Optional[str] = None
) -> None:
"""Save metadata for a given namespace.
Metadata is treated separately from data records and stored as a single object.
@@ -1086,9 +1454,13 @@ class Database(DatabaseABC, SingletonMixin):
metadata (bytes): Arbitrary metadata to save or None to delete metadata.
namespace (Optional[str]): Optional namespace under which to store metadata.
"""
self._database().set_metadata(metadata, namespace=namespace)
await self._run_db(
"set_metadata",
metadata,
namespace=namespace,
)
def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]:
async def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]:
"""Load metadata for a given namespace.
Returns None if no metadata exists.
@@ -1099,12 +1471,15 @@ class Database(DatabaseABC, SingletonMixin):
Returns:
Optional[bytes]: The loaded metadata, or None if not found.
"""
return self._database().get_metadata(namespace=namespace)
return await self._run_db(
"get_metadata",
namespace=namespace,
)
# Basic record operations
def save_records(
self, records: Iterable[tuple[bytes, bytes]], namespace: Optional[str] = None
async def save_records(
self, records: Iterable[tuple[bytes, bytes]], *, namespace: Optional[str] = None
) -> int:
"""Save multiple records into the specified namespace (or default).
@@ -1120,9 +1495,15 @@ class Database(DatabaseABC, SingletonMixin):
Raises:
RuntimeError: If DB not open or write failed.
"""
return self._database().save_records(records, namespace)
return await self._run_db(
"save_records",
records,
namespace=namespace,
)
def delete_records(self, keys: Iterable[bytes], namespace: Optional[str] = None) -> int:
async def delete_records(
self, keys: Iterable[bytes], *, namespace: Optional[str] = None
) -> int:
"""Delete multiple records by key from the specified namespace.
Args:
@@ -1132,15 +1513,20 @@ class Database(DatabaseABC, SingletonMixin):
Returns:
Number of records actually deleted.
"""
return self._database().delete_records(keys, namespace)
return await self._run_db(
"delete_records",
keys,
namespace=namespace,
)
def iterate_records(
async def iterate_records(
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
reverse: bool = False,
) -> Iterator[tuple[bytes, bytes]]:
) -> AsyncIterator[tuple[bytes, bytes]]:
"""Iterate over records for a namespace with optional bounds.
Args:
@@ -1152,9 +1538,18 @@ class Database(DatabaseABC, SingletonMixin):
Yields:
Tuples of (key, record).
"""
return self._database().iterate_records(start_key, end_key, namespace, reverse)
records: list[tuple[bytes, bytes]] = await self._run_db(
"iterate_records",
start_key,
end_key,
namespace=namespace,
reverse=reverse,
)
def count_records(
for item in records:
yield item
async def count_records(
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
@@ -1165,14 +1560,49 @@ class Database(DatabaseABC, SingletonMixin):
Excludes metadata records.
"""
return self._database().count_records(start_key, end_key, namespace=namespace)
return await self._run_db(
"count_records",
start_key,
end_key,
namespace=namespace,
)
def get_key_range(
self, namespace: Optional[str] = None
async def get_key_range(
self, *, namespace: Optional[str] = None
) -> Tuple[Optional[bytes], Optional[bytes]]:
"""Return (min_key, max_key) in the given namespace or (None, None) if empty."""
return self._database().get_key_range(namespace)
return await self._run_db(
"get_key_range",
namespace=namespace,
)
def get_backend_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]:
async def get_backend_stats(self, *, namespace: Optional[str] = None) -> Dict[str, Any]:
"""Get backend-specific statistics; implementations may return namespace-specific data."""
return self._database().get_backend_stats(namespace)
return await self._run_db(
"get_backend_stats",
namespace=namespace,
)
# Compression helpers
def serialize_data(self, data: bytes) -> bytes:
"""Optionally compress raw pickled data before storage.
Args:
data: Raw pickled bytes.
Returns:
Possibly compressed bytes.
"""
return self._db.serialize_data(data)
def deserialize_data(self, data: bytes) -> bytes:
"""Optionally decompress stored data.
Args:
data: Stored bytes.
Returns:
Raw pickled bytes (decompressed if needed).
"""
return self._db.deserialize_data(data)

343
src/akkudoktoreos/core/databaseabc.py
View File

@@ -12,6 +12,7 @@ from threading import Lock
from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Final,
Generic,
Iterable,
@@ -46,20 +47,38 @@ DATABASE_METADATA_KEY: bytes = b"__metadata__"
# ==================== Abstract Database Interface ====================
class DatabaseABC(ABC, ConfigMixin):
"""Abstract base class for database.
class DatabaseBackendABC(ABC, ConfigMixin, SingletonMixin):
"""Abstract base class for database backends.
All operations accept an optional `namespace` argument. Implementations should
treat None as the default/root namespace. Concrete implementations can map
namespace -> native namespace (LMDB DBI) or emulate namespaces (SQLite uses
a namespace column).
The database backend provides a synchronous interface. Asynchrounous access is
handled by the generic database class.
"""
connection: Any
lock: Lock
_is_open: bool
default_namespace: Optional[str]
def __init__(self, **kwargs: Any) -> None:
"""Initialize the DatabaseBackendABC base.
Args:
**kwargs: Backend-specific options (ignored by base).
"""
self.connection = None
self.lock = Lock()
self._is_open = False
self.default_namespace = None
@property
@abstractmethod
def is_open(self) -> bool:
"""Return whether the database connection is open."""
raise NotImplementedError
return self._is_open
@property
def storage_path(self) -> Path:
@@ -87,7 +106,7 @@ class DatabaseABC(ABC, ConfigMixin):
raise NotImplementedError
@abstractmethod
def open(self, namespace: Optional[str] = None) -> None:
def open(self, *, namespace: Optional[str] = None) -> None:
"""Open database connection and optionally set default namespace.
Args:
@@ -104,7 +123,7 @@ class DatabaseABC(ABC, ConfigMixin):
raise NotImplementedError
@abstractmethod
def flush(self, namespace: Optional[str] = None) -> None:
def flush(self, *, namespace: Optional[str] = None) -> None:
"""Force synchronization of pending writes to storage (optional per-namespace)."""
raise NotImplementedError
@@ -123,7 +142,7 @@ class DatabaseABC(ABC, ConfigMixin):
raise NotImplementedError
@abstractmethod
def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]:
def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]:
"""Load metadata for a given namespace.
Returns None if no metadata exists.
@@ -140,7 +159,7 @@ class DatabaseABC(ABC, ConfigMixin):
@abstractmethod
def save_records(
self, records: Iterable[tuple[bytes, bytes]], namespace: Optional[str] = None
self, records: Iterable[tuple[bytes, bytes]], *, namespace: Optional[str] = None
) -> int:
"""Save multiple records into the specified namespace (or default).
@@ -159,7 +178,7 @@ class DatabaseABC(ABC, ConfigMixin):
raise NotImplementedError
@abstractmethod
def delete_records(self, keys: Iterable[bytes], namespace: Optional[str] = None) -> int:
def delete_records(self, keys: Iterable[bytes], *, namespace: Optional[str] = None) -> int:
"""Delete multiple records by key from the specified namespace.
Args:
@@ -176,6 +195,7 @@ class DatabaseABC(ABC, ConfigMixin):
self,
start_key: Optional[bytes] = None,
end_key: Optional[bytes] = None,
*,
namespace: Optional[str] = None,
reverse: bool = False,
) -> Iterator[tuple[bytes, bytes]]:
@@ -208,13 +228,13 @@ class DatabaseABC(ABC, ConfigMixin):
@abstractmethod
def get_key_range(
self, namespace: Optional[str] = None
self, *, namespace: Optional[str] = None
) -> tuple[Optional[bytes], Optional[bytes]]:
"""Return (min_key, max_key) in the given namespace or (None, None) if empty."""
raise NotImplementedError
@abstractmethod
def get_backend_stats(self, namespace: Optional[str] = None) -> dict[str, Any]:
def get_backend_stats(self, *, namespace: Optional[str] = None) -> dict[str, Any]:
"""Get backend-specific statistics; implementations may return namespace-specific data."""
raise NotImplementedError
@@ -250,37 +270,6 @@ class DatabaseABC(ABC, ConfigMixin):
return data
class DatabaseBackendABC(DatabaseABC, SingletonMixin):
"""Abstract base class for database backends.
All operations accept an optional `namespace` argument. Implementations should
treat None as the default/root namespace. Concrete implementations can map
namespace -> native namespace (LMDB DBI) or emulate namespaces (SQLite uses
a namespace column).
"""
connection: Any
lock: Lock
_is_open: bool
default_namespace: Optional[str]
def __init__(self, **kwargs: Any) -> None:
"""Initialize the DatabaseBackendABC base.
Args:
**kwargs: Backend-specific options (ignored by base).
"""
self.connection = None
self.lock = Lock()
self._is_open = False
self.default_namespace = None
@property
def is_open(self) -> bool:
"""Return whether the database connection is open."""
return self._is_open
# ==================== Database Record Protocol Mixin ====================
@@ -442,7 +431,7 @@ class DatabaseRecordProtocol(Protocol, Generic[T_Record]):
@property
def db_enabled(self) -> bool: ...
def db_timestamp_range(self) -> tuple[DatabaseTimestampType, DatabaseTimestampType]: ...
async def db_timestamp_range(self) -> tuple[DatabaseTimestampType, DatabaseTimestampType]: ...
def db_generate_timestamps(
self,
@@ -451,52 +440,49 @@ class DatabaseRecordProtocol(Protocol, Generic[T_Record]):
interval: Optional[Duration] = None,
) -> Iterator[DatabaseTimestamp]: ...
def db_get_record(self, target_timestamp: DatabaseTimestamp) -> Optional[T_Record]: ...
async def db_get_record(self, target_timestamp: DatabaseTimestamp) -> Optional[T_Record]: ...
def db_insert_record(
async def db_insert_record(
self,
record: T_Record,
*,
mark_dirty: bool = True,
) -> None: ...
def db_iterate_records(
async def db_iterate_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
) -> Iterator[T_Record]: ...
) -> AsyncIterator[T_Record]: ...
def db_load_records(
async def db_load_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
) -> int: ...
def db_delete_records(
async def db_delete_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
) -> int: ...
# ---- dirty tracking ----
def db_mark_dirty_record(self, record: T_Record) -> None: ...
async def db_mark_dirty_record(self, record: T_Record) -> None: ...
def db_save_records(self) -> int: ...
# ---- autosave ----
def db_autosave(self) -> int: ...
async def db_save_records(self) -> int: ...
# ---- Remove old records from database to free space ----
def db_vacuum(
async def db_vacuum(
self,
keep_hours: Optional[int] = None,
keep_datetime: Optional[DatabaseTimestampType] = None,
) -> int: ...
# ---- statistics about database storage ----
def db_count_records(self) -> int: ...
async def db_count_records(self) -> int: ...
def db_get_stats(self) -> dict: ...
async def db_get_stats(self) -> dict: ...
T_DatabaseRecordProtocol = TypeVar("T_DatabaseRecordProtocol", bound="DatabaseRecordProtocol")
@@ -533,10 +519,12 @@ class DatabaseRecordProtocolMixin(
Completely manages in memory records and database storage.
Expects records with date_time (DatabaseTimestamp) property and the a record list
Expects records with date_time (DatabaseTimestamp) property and a record list
in self.records of the derived class.
DatabaseRecordProtocolMixin expects the derived classes to be singletons.
DatabaseRecordProtocolMixin expects the derived classes to be singletons and to have
the sequence guarded against asynchronous sequence state (_sequence_lock) and
asynchronous record state (_record_lock) changes.
"""
# Tell mypy these attributes exist (will be provided by subclasses)
@@ -549,7 +537,7 @@ class DatabaseRecordProtocolMixin(
@property
def record_keys_writable(self) -> list[str]: ...
def key_to_array(
async def key_to_array(
self,
key: str,
start_datetime: Optional[DateTime] = None,
@@ -583,7 +571,18 @@ class DatabaseRecordProtocolMixin(
# Initialization
# -----------------------------------------------------
def _db_ensure_initialized(self) -> None:
async def _db_init_metadata(self) -> None:
"""Initialize DB metadata."""
self._db_metadata: Optional[dict] = {
"version": self._db_version,
"created": to_datetime(as_string=True),
"provider_id": getattr(self, "provider_id", lambda: "unknown")(),
"compression": self.database.compression,
"backend": self.database.__class__.__name__,
}
await self._db_save_metadata(self._db_metadata)
async def _db_ensure_initialized(self) -> None:
"""Initialize DB runtime state.
Idempotent — safe to call multiple times.
@@ -613,25 +612,18 @@ class DatabaseRecordProtocolMixin(
self._db_version: int = 1
# Storage
self._db_metadata: Optional[dict] = None
self._db_metadata = None
self._db_storage_initialized: bool = False
self._db_initialized: bool = True
if not self._db_storage_initialized and self.db_enabled:
# Metadata
existing_metadata = self._db_load_metadata()
existing_metadata = await self._db_load_metadata()
if existing_metadata:
self._db_metadata = existing_metadata
else:
self._db_metadata = {
"version": self._db_version,
"created": to_datetime(as_string=True),
"provider_id": getattr(self, "provider_id", lambda: "unknown")(),
"compression": self.database.compression,
"backend": self.database.__class__.__name__,
}
self._db_save_metadata(self._db_metadata)
await self._db_init_metadata()
logger.info(
f"Initialized {self.database.__class__.__name__}:{self.db_namespace()} storage at "
@@ -641,12 +633,6 @@ class DatabaseRecordProtocolMixin(
self._db_storage_initialized = True
def model_post_init(self, __context: Any) -> None:
"""Initialize DB state attributes immediately after Pydantic construction."""
# Always call super() first — other mixins may also define model_post_init
super().model_post_init(__context) # type: ignore[misc]
self._db_ensure_initialized()
# -----------------------------------------------------
# Helpers
# -----------------------------------------------------
@@ -669,7 +655,7 @@ class DatabaseRecordProtocolMixin(
db_datetime_after = DatabaseTimestamp.from_datetime(target.add(seconds=1))
return db_datetime_after
def db_previous_timestamp(
async def db_previous_timestamp(
self,
timestamp: DatabaseTimestamp,
) -> Optional[DatabaseTimestamp]:
@@ -677,7 +663,7 @@ class DatabaseRecordProtocolMixin(
Search memory-first, then fallback to database if necessary.
"""
self._db_ensure_initialized()
await self._db_ensure_initialized()
# Step 1: Memory-first search
if self._db_sorted_timestamps:
@@ -689,7 +675,7 @@ class DatabaseRecordProtocolMixin(
if not self.db_enabled:
return None
db_min_key, _ = self.database.get_key_range(self.db_namespace())
db_min_key, _ = await self.database.get_key_range(namespace=self.db_namespace())
if db_min_key is None:
return None
@@ -710,7 +696,7 @@ class DatabaseRecordProtocolMixin(
start_key = self._db_key_from_timestamp(loaded_start)
previous_ts: Optional[DatabaseTimestamp] = None
for key, _ in self.database.iterate_records(
async for key, _ in self.database.iterate_records(
start_key=start_key,
end_key=end_key,
namespace=self.db_namespace(),
@@ -722,7 +708,7 @@ class DatabaseRecordProtocolMixin(
return previous_ts
def db_next_timestamp(
async def db_next_timestamp(
self,
timestamp: DatabaseTimestamp,
) -> Optional[DatabaseTimestamp]:
@@ -730,7 +716,7 @@ class DatabaseRecordProtocolMixin(
Search memory-first, then fallback to database if necessary.
"""
self._db_ensure_initialized()
await self._db_ensure_initialized()
# Step 1: Memory-first search
if self._db_sorted_timestamps:
@@ -742,7 +728,7 @@ class DatabaseRecordProtocolMixin(
if not self.db_enabled:
return None
_, db_max_key = self.database.get_key_range(self.db_namespace())
_, db_max_key = await self.database.get_key_range(namespace=self.db_namespace())
if db_max_key is None:
return None
@@ -762,7 +748,7 @@ class DatabaseRecordProtocolMixin(
if isinstance(loaded_end, DatabaseTimestamp) and timestamp < loaded_end:
start_key = self._db_key_from_timestamp(max(timestamp, loaded_end))
for key, _ in self.database.iterate_records(
async for key, _ in self.database.iterate_records(
start_key=start_key,
end_key=end_key,
namespace=self.db_namespace(),
@@ -796,22 +782,22 @@ class DatabaseRecordProtocolMixin(
record_data = pickle.loads(data) # noqa: S301
return self.record_class()(**record_data)
def _db_save_metadata(self, metadata: dict) -> None:
async def _db_save_metadata(self, metadata: dict) -> None:
"""Save metadata to database."""
if not self.db_enabled:
return
key = DATABASE_METADATA_KEY
value = pickle.dumps(metadata)
self.database.set_metadata(value, namespace=self.db_namespace())
await self.database.set_metadata(value, namespace=self.db_namespace())
def _db_load_metadata(self) -> Optional[dict]:
async def _db_load_metadata(self) -> Optional[dict]:
"""Load metadata from database."""
if not self.db_enabled:
return None
try:
value = self.database.get_metadata(namespace=self.db_namespace())
value = await self.database.get_metadata(namespace=self.db_namespace())
return pickle.loads(value) # noqa: S301
except Exception:
logger.debug("Can not load metadata.")
@@ -942,7 +928,7 @@ class DatabaseRecordProtocolMixin(
return loaded_start <= start_timestamp and end_timestamp <= loaded_end
def _db_load_initial_window(
async def _db_load_initial_window(
self,
center_timestamp: Optional[DatabaseTimestampType] = None,
) -> None:
@@ -1001,11 +987,11 @@ class DatabaseRecordProtocolMixin(
window = to_duration(window_h * 3600)
start, end = self._search_window(center_timestamp, window)
self.db_load_records(start, end)
await self.db_load_records(start, end)
self._db_load_phase = DatabaseRecordProtocolLoadPhase.INITIAL
def _db_load_full(self) -> int:
async def _db_load_full(self) -> int:
"""Load all remaining records from the database into memory.
This method performs a **full load** of the database, ensuring that all
@@ -1038,14 +1024,14 @@ class DatabaseRecordProtocolMixin(
# Perform full database load (memory is authoritative; skips duplicates)
# This also sets _db_loaded_range
loaded_count = self.db_load_records()
loaded_count = await self.db_load_records()
# Update state
self._db_load_phase = DatabaseRecordProtocolLoadPhase.FULL
return loaded_count
def _extend_boundaries(
async def _extend_boundaries(
self,
start_timestamp: DatabaseTimestampType,
end_timestamp: DatabaseTimestampType,
@@ -1069,7 +1055,7 @@ class DatabaseRecordProtocolMixin(
):
# There may be earlier DB records
# Reverse iterate to get nearest smaller key
for key, _ in self.database.iterate_records(
async for key, _ in self.database.iterate_records(
start_key=UNBOUND_START,
end_key=self._db_key_from_timestamp(start_timestamp),
namespace=self.db_namespace(),
@@ -1091,7 +1077,7 @@ class DatabaseRecordProtocolMixin(
and end_timestamp > self._db_sorted_timestamps[-1]
):
# There may be later DB records
for key, _ in self.database.iterate_records(
async for key, _ in self.database.iterate_records(
start_key=self._db_key_from_timestamp(end_timestamp),
end_key=UNBOUND_END,
namespace=self.db_namespace(),
@@ -1107,7 +1093,7 @@ class DatabaseRecordProtocolMixin(
return new_start, new_end
def _db_ensure_loaded(
async def _db_ensure_loaded(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
@@ -1172,11 +1158,11 @@ class DatabaseRecordProtocolMixin(
# Phase 0: NOTHING LOADED
if self._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE:
if start_timestamp is UNBOUND_START and end_timestamp is UNBOUND_END:
self._db_load_initial_window(center_timestamp)
await self._db_load_initial_window(center_timestamp)
# _db_load_initial_window sets _db_loaded_range and _db_load_phase
else:
# Load the records
loaded = self.db_load_records(start_timestamp, end_timestamp)
loaded = await self.db_load_records(start_timestamp, end_timestamp)
self._db_load_phase = DatabaseRecordProtocolLoadPhase.INITIAL
return
@@ -1196,7 +1182,7 @@ class DatabaseRecordProtocolMixin(
return # already have it
if start_timestamp == UNBOUND_START and end_timestamp == UNBOUND_END:
self._db_load_full()
await self._db_load_full()
return
current_start, current_end = self._db_loaded_range
@@ -1207,11 +1193,11 @@ class DatabaseRecordProtocolMixin(
# Left expansion
if start_timestamp < current_start:
self.db_load_records(start_timestamp, current_start)
await self.db_load_records(start_timestamp, current_start)
# Right expansion
if end_timestamp > current_end:
self.db_load_records(current_end, end_timestamp)
await self.db_load_records(current_end, end_timestamp)
return
@@ -1251,15 +1237,15 @@ class DatabaseRecordProtocolMixin(
def db_enabled(self) -> bool:
return self.database.is_open
def db_timestamp_range(
async def db_timestamp_range(
self,
) -> tuple[Optional[DatabaseTimestamp], Optional[DatabaseTimestamp]]:
"""Get the timestamp range of records in database.
Regards records in storage plus extra records in memory.
"""
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
if self._db_sorted_timestamps:
memory_min_timestamp: Optional[DatabaseTimestamp] = self._db_sorted_timestamps[0]
@@ -1271,7 +1257,7 @@ class DatabaseRecordProtocolMixin(
if not self.db_enabled:
return memory_min_timestamp, memory_max_timestamp
db_min_key, db_max_key = self.database.get_key_range(self.db_namespace())
db_min_key, db_max_key = await self.database.get_key_range(namespace=self.db_namespace())
if db_min_key is None or db_max_key is None:
return memory_min_timestamp, memory_max_timestamp
@@ -1331,7 +1317,7 @@ class DatabaseRecordProtocolMixin(
yield DatabaseTimestamp.from_datetime(current_utc)
current_utc = current_utc.add(seconds=step_seconds)
def db_get_record(
async def db_get_record(
self,
target_timestamp: DatabaseTimestamp,
*,
@@ -1353,11 +1339,11 @@ class DatabaseRecordProtocolMixin(
Returns:
Exact match, nearest record within the window, or None.
"""
self._db_ensure_initialized()
await self._db_ensure_initialized()
if time_window is None:
# Exact match only — load the minimal range containing this point
self._db_ensure_loaded(
await self._db_ensure_loaded(
target_timestamp,
self._db_timestamp_after(target_timestamp),
center_timestamp=target_timestamp,
@@ -1367,7 +1353,7 @@ class DatabaseRecordProtocolMixin(
# load the relevant range
# in case of unbounded escalates to FULL
search_start, search_end = self._search_window(target_timestamp, time_window)
self._db_ensure_loaded(search_start, search_end, center_timestamp=target_timestamp)
await self._db_ensure_loaded(search_start, search_end, center_timestamp=target_timestamp)
# Exact match first (works for all three cases once loaded)
record = self._db_record_index.get(target_timestamp, None)
@@ -1406,19 +1392,19 @@ class DatabaseRecordProtocolMixin(
return record
def db_insert_record(
async def db_insert_record(
self,
record: T_Record,
*,
mark_dirty: bool = True,
) -> None:
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
# Ensure normalized to UTC
db_record_date_time = DatabaseTimestamp.from_datetime(record.date_time)
self._db_ensure_loaded(
await self._db_ensure_loaded(
start_timestamp=db_record_date_time,
end_timestamp=db_record_date_time,
)
@@ -1446,7 +1432,7 @@ class DatabaseRecordProtocolMixin(
# Load (range)
# -----------------------------------------------------
def db_load_records(
async def db_load_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
@@ -1475,8 +1461,8 @@ class DatabaseRecordProtocolMixin(
Note:
record.date_time shall be DateTime or None
"""
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
if not self.db_enabled:
return 0
@@ -1488,7 +1474,7 @@ class DatabaseRecordProtocolMixin(
end_timestamp = UNBOUND_END
# Extend boundaries to include first record < start and first record >= end
query_start, query_end = self._extend_boundaries(start_timestamp, end_timestamp)
query_start, query_end = await self._extend_boundaries(start_timestamp, end_timestamp)
if isinstance(query_start, _DatabaseTimestampUnbound):
start_key = None
@@ -1504,7 +1490,7 @@ class DatabaseRecordProtocolMixin(
loaded_count = 0
# Iterate DB records (already sorted by key)
for db_key, value in self.database.iterate_records(
async for db_key, value in self.database.iterate_records(
start_key=start_key,
end_key=end_key,
namespace=namespace,
@@ -1551,16 +1537,16 @@ class DatabaseRecordProtocolMixin(
# Delete (range)
# -----------------------------------------------------
def db_delete_records(
async def db_delete_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
) -> int:
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
# Deletion is global — ensure we see everything
self._db_ensure_loaded(
await self._db_ensure_loaded(
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
)
@@ -1598,21 +1584,21 @@ class DatabaseRecordProtocolMixin(
# Iteration from DB (no duplicates)
# -----------------------------------------------------
def db_iterate_records(
async def db_iterate_records(
self,
start_timestamp: Optional[DatabaseTimestampType] = None,
end_timestamp: Optional[DatabaseTimestampType] = None,
) -> Iterator[T_Record]:
) -> AsyncIterator[T_Record]:
"""Iterate records in requested range.
Ensures storage is loaded into memory first,
then iterates over in-memory records only.
"""
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
# Ensure memory contains required range
self._db_ensure_loaded(
await self._db_ensure_loaded(
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
)
@@ -1635,9 +1621,9 @@ class DatabaseRecordProtocolMixin(
# Dirty tracking
# -----------------------------------------------------
def db_mark_dirty_record(self, record: T_Record) -> None:
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
async def db_mark_dirty_record(self, record: T_Record) -> None:
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)
self._db_dirty_timestamps.add(record_date_time_timestamp)
@@ -1646,9 +1632,9 @@ class DatabaseRecordProtocolMixin(
# Bulk save (flush dirty only)
# -----------------------------------------------------
def db_save_records(self) -> int:
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
async def db_save_records(self) -> int:
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
if not self.db_enabled:
return 0
@@ -1670,23 +1656,20 @@ class DatabaseRecordProtocolMixin(
save_items.append((key, value))
saved_count = len(save_items)
if saved_count:
self.database.save_records(save_items, namespace=namespace)
await self.database.save_records(save_items, namespace=namespace)
self._db_dirty_timestamps.clear()
self._db_new_timestamps.clear()
# --- handle deletions ---
if self._db_deleted_timestamps:
delete_keys = [self._db_key_from_timestamp(dt) for dt in self._db_deleted_timestamps]
self.database.delete_records(delete_keys, namespace=namespace)
await self.database.delete_records(delete_keys, namespace=namespace)
deleted_count = len(self._db_deleted_timestamps)
self._db_deleted_timestamps.clear()
return saved_count + deleted_count
def db_autosave(self) -> int:
return self.db_save_records()
def db_vacuum(
async def db_vacuum(
self,
keep_hours: Optional[int] = None,
keep_timestamp: Optional[DatabaseTimestampType] = None,
@@ -1708,8 +1691,8 @@ class DatabaseRecordProtocolMixin(
Returns:
Number of records deleted
"""
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
if keep_hours is None and keep_timestamp is None:
keep_duration = self.db_keep_duration()
@@ -1722,7 +1705,7 @@ class DatabaseRecordProtocolMixin(
keep_hours = keep_duration.hours
if keep_hours is not None:
_, db_max = self.db_timestamp_range()
_, db_max = await self.db_timestamp_range()
if db_max is None or isinstance(db_max, _DatabaseTimestampUnbound):
# No records
return 0 # nothing to delete
@@ -1740,9 +1723,9 @@ class DatabaseRecordProtocolMixin(
raise ValueError("Must specify either keep_hours or keep_timestamp")
# Delete records
deleted_count = self.db_delete_records(end_timestamp=db_cutoff_timestamp)
deleted_count = await self.db_delete_records(end_timestamp=db_cutoff_timestamp)
self.db_save_records()
await self.db_save_records()
logger.info(
f"Vacuumed {deleted_count} old records from database '{self.db_namespace()}' "
@@ -1750,14 +1733,14 @@ class DatabaseRecordProtocolMixin(
)
return deleted_count
def db_count_records(self) -> int:
async def db_count_records(self) -> int:
"""Return total logical number of records.
Memory is authoritative. If DB is enabled but not fully loaded,
we conservatively include storage-only records.
"""
# Defensive call - model_post_init() may not have initialized metadata
self._db_ensure_initialized()
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
if not self.db_enabled:
return len(self.records)
@@ -1766,13 +1749,13 @@ class DatabaseRecordProtocolMixin(
if self._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL:
return len(self.records)
storage_count = self.database.count_records(namespace=self.db_namespace())
storage_count = await self.database.count_records(namespace=self.db_namespace())
pending_deletes = len(self._db_deleted_timestamps)
new_count = len(self._db_new_timestamps)
return storage_count + new_count - pending_deletes
def db_get_stats(self) -> dict:
async def db_get_stats(self) -> dict:
"""Get comprehensive statistics about database storage.
Returns:
@@ -1783,6 +1766,8 @@ class DatabaseRecordProtocolMixin(
ns = self.db_namespace()
total_records = await self.database.count_records(namespace=ns)
stats = {
"enabled": True,
"backend": self.database.__class__.__name__,
@@ -1791,13 +1776,14 @@ class DatabaseRecordProtocolMixin(
"compression_enabled": self.database.compression,
"keep_duration_h": self.config.database.keep_duration_h,
"autosave_interval_sec": self.config.database.autosave_interval_sec,
"total_records": self.database.count_records(namespace=ns),
"total_records": total_records,
}
# Add backend-specific stats
stats.update(self.database.get_backend_stats(namespace=ns))
backend_stats = await self.database.get_backend_stats(namespace=ns)
stats.update(backend_stats)
min_timestamp, max_timestamp = self.db_timestamp_range()
min_timestamp, max_timestamp = await self.db_timestamp_range()
stats["timestamp_range"] = {
"min": str(min_timestamp),
"max": str(max_timestamp),
@@ -1866,7 +1852,7 @@ class DatabaseRecordProtocolMixin(
cutoff_str = self._db_metadata.get(key)
return DatabaseTimestamp(cutoff_str) if cutoff_str else None
def _db_set_compact_state(
async def _db_set_compact_state(
self,
tier_interval: Duration,
cutoff_ts: DatabaseTimestamp,
@@ -1881,13 +1867,13 @@ class DatabaseRecordProtocolMixin(
self._db_metadata = {}
key = f"last_compact_cutoff_{int(tier_interval.total_seconds())}"
self._db_metadata[key] = str(cutoff_ts)
self._db_save_metadata(self._db_metadata)
await self._db_save_metadata(self._db_metadata)
# ------------------------------------------------------------------
# Single-tier worker
# ------------------------------------------------------------------
def _db_compact_tier(
async def _db_compact_tier(
self,
age_threshold: Duration,
target_interval: Duration,
@@ -1923,14 +1909,14 @@ class DatabaseRecordProtocolMixin(
Number of original records deleted (before re-insertion of downsampled
records). Returns 0 if skipped.
"""
self._db_ensure_initialized()
await self._db_ensure_initialized()
interval_sec = int(target_interval.total_seconds())
if interval_sec <= 0:
return 0
# ---- Determine raw new cutoff ------------------------------------
_, db_max = self.db_timestamp_range()
_, db_max = await self.db_timestamp_range()
if db_max is None or isinstance(db_max, _DatabaseTimestampUnbound):
return 0
@@ -1955,7 +1941,7 @@ class DatabaseRecordProtocolMixin(
)
return 0
db_min, _ = self.db_timestamp_range()
db_min, _ = await self.db_timestamp_range()
if db_min is None or isinstance(db_min, _DatabaseTimestampUnbound):
return 0
@@ -1981,7 +1967,11 @@ class DatabaseRecordProtocolMixin(
window_end_ts = new_cutoff_ts
# ---- Sparse-data guard -------------------------------------------
existing_count = self.database.count_records(
# Ensure the window is loaded into memory so the sparse guard's
# records_in_window list comprehension sees actual data.
await self._db_ensure_loaded(window_start_ts, window_end_ts)
existing_count = await self.database.count_records(
start_key=self._db_key_from_timestamp(window_start_ts),
end_key=self._db_key_from_timestamp(window_end_ts),
namespace=self.db_namespace(),
@@ -1993,7 +1983,7 @@ class DatabaseRecordProtocolMixin(
if existing_count == 0:
# Nothing in window — just advance the cutoff
self._db_set_compact_state(target_interval, new_cutoff_ts)
await self._db_set_compact_state(target_interval, new_cutoff_ts)
return 0
if existing_count <= resampled_count:
@@ -2016,7 +2006,7 @@ class DatabaseRecordProtocolMixin(
f"and all timestamps already aligned "
f"(window={window_start_dt}..{window_end_dt})"
)
self._db_set_compact_state(target_interval, new_cutoff_ts)
await self._db_set_compact_state(target_interval, new_cutoff_ts)
return 0
# ---- Sparse but misaligned: full window rewrite -----------------
@@ -2049,7 +2039,7 @@ class DatabaseRecordProtocolMixin(
bucket[key] = val
# Delete entire window (aligned + misaligned)
deleted = self.db_delete_records(
deleted = await self.db_delete_records(
start_timestamp=window_start_ts,
end_timestamp=window_end_ts,
)
@@ -2060,10 +2050,10 @@ class DatabaseRecordProtocolMixin(
continue
snapped_dt = DateTime.fromtimestamp(snapped_epoch, tz="UTC")
record = self.record_class()(date_time=snapped_dt, **values)
self.db_insert_record(record, mark_dirty=True)
await self.db_insert_record(record, mark_dirty=True)
self.db_save_records()
self._db_set_compact_state(target_interval, new_cutoff_ts)
await self.db_save_records()
await self._db_set_compact_state(target_interval, new_cutoff_ts)
logger.info(
f"Rewrote sparse window in namespace '{self.db_namespace()}' "
f"tier {target_interval}: deleted={deleted}, "
@@ -2086,7 +2076,7 @@ class DatabaseRecordProtocolMixin(
if key == "date_time":
continue
try:
array = self.key_to_array(
array = await self.key_to_array(
key,
start_datetime=window_start_dt,
end_datetime=window_end_dt,
@@ -2095,7 +2085,9 @@ class DatabaseRecordProtocolMixin(
boundary="context",
align_to_interval=True,
)
except (KeyError, TypeError, ValueError):
logger.debug(f"key={key}, array_len={len(array)}")
except (KeyError, TypeError, ValueError) as e:
logger.error(f"key_to_array failed for {key}: {e}")
continue # non-numeric or missing key — skip silently
if len(array) == 0:
@@ -2125,11 +2117,11 @@ class DatabaseRecordProtocolMixin(
if not compacted_data or not compacted_timestamps:
# Nothing to write back — still advance cutoff
self._db_set_compact_state(target_interval, new_cutoff_ts)
await self._db_set_compact_state(target_interval, new_cutoff_ts)
return 0
# ---- Delete originals, re-insert downsampled records -------------
deleted = self.db_delete_records(
deleted = await self.db_delete_records(
start_timestamp=window_start_ts,
end_timestamp=window_end_ts,
)
@@ -2142,12 +2134,12 @@ class DatabaseRecordProtocolMixin(
}
if values:
record = self.record_class()(date_time=dt, **values)
self.db_insert_record(record, mark_dirty=True)
await self.db_insert_record(record, mark_dirty=True)
self.db_save_records()
await self.db_save_records()
# Persist the aligned new cutoff for this tier
self._db_set_compact_state(target_interval, new_cutoff_ts)
await self._db_set_compact_state(target_interval, new_cutoff_ts)
logger.info(
f"Compacted tier {target_interval}: deleted {deleted} records in "
@@ -2161,7 +2153,7 @@ class DatabaseRecordProtocolMixin(
# Public entry point
# ------------------------------------------------------------------
def db_compact(
async def db_compact(
self,
compact_tiers: Optional[list[tuple[Duration, Duration]]] = None,
) -> int:
@@ -2183,12 +2175,13 @@ class DatabaseRecordProtocolMixin(
compact_tiers = self.db_compact_tiers()
if not compact_tiers:
logger.debug(f"Compaction called but no compact_tiers '{compact_tiers}' given.")
return 0
total_deleted = 0
# Coarsest tier first (reversed) to avoid redundant work
for age_threshold, target_interval in reversed(compact_tiers):
total_deleted += self._db_compact_tier(age_threshold, target_interval)
total_deleted += await self._db_compact_tier(age_threshold, target_interval)
return total_deleted

386
src/akkudoktoreos/core/ems.py
View File

@@ -2,8 +2,7 @@ import traceback
from asyncio import Lock, get_running_loop
from concurrent.futures import ThreadPoolExecutor
from enum import StrEnum
from functools import partial
from typing import ClassVar, Optional
from typing import ClassVar, Optional, cast
from loguru import logger
from pydantic import computed_field
@@ -147,11 +146,11 @@ class EnergyManagement(
"""
return cls._genetic_solution
@classmethod
def _run(
cls,
start_datetime: DateTime,
mode: EnergyManagementMode,
async def run(
self,
start_datetime: Optional[DateTime] = None,
mode: Optional[EnergyManagementMode] = None,
algorithm: Optional[str] = None,
genetic_parameters: Optional[GeneticOptimizationParameters] = None,
genetic_individuals: Optional[int] = None,
genetic_seed: Optional[int] = None,
@@ -161,7 +160,6 @@ class EnergyManagement(
"""Run the energy management.
This method initializes the energy management run by setting its
start datetime, updating predictions, and optionally starting
optimization depending on the selected mode or configuration.
@@ -171,161 +169,20 @@ class EnergyManagement(
- "OPTIMIZATION": Runs the optimization process.
- "PREDICTION": Updates the forecast without optimization.
- "DISABLED": Does not run.
genetic_parameters (GeneticOptimizationParameters, optional): The
parameter set for the genetic algorithm. If not provided, it will
be constructed based on the current configuration and predictions.
genetic_individuals (int, optional): The number of individuals for the
genetic algorithm. Defaults to the algorithm's internal default (400)
if not specified.
genetic_seed (int, optional): The seed for the genetic algorithm. Defaults
to the algorithm's internal random seed if not specified.
force_enable (bool, optional): If True, bypasses any disabled state
to force the update process. This is mostly applicable to
prediction providers.
force_update (bool, optional): If True, forces data to be refreshed
even if a cached version is still valid.
Returns:
None
"""
# Ensure there is only one optimization/ energy management run at a time
if not mode in EnergyManagementMode._value2member_map_:
raise ValueError(f"Unknown energy management mode {mode}.")
if mode == EnergyManagementMode.DISABLED:
return
logger.info("Starting energy management run.")
cls._stage = EnergyManagementStage.DATA_ACQUISITION
# Remember/ set the start datetime of this energy management run.
# None leads
cls.set_start_datetime(start_datetime)
# Throw away any memory cached results of the last energy management run.
CacheEnergyManagementStore().clear()
# Do data aquisition by adapters
try:
cls.adapter.update_data(force_enable)
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
error_msg = f"Adapter update failed - phase {cls._stage}:\n{e}\n{trace}"
logger.error(error_msg)
cls._stage = EnergyManagementStage.FORECAST_RETRIEVAL
if mode == EnergyManagementMode.PREDICTION:
# Update the predictions
cls.prediction.update_data(force_enable=force_enable, force_update=force_update)
logger.info("Energy management run done (predictions updated)")
cls._stage = EnergyManagementStage.IDLE
return
# Prepare optimization parameters
# This also creates default configurations for missing values and updates the predictions
logger.info(
"Starting energy management prediction update and optimzation parameter preparation."
)
if genetic_parameters is None:
genetic_parameters = GeneticOptimizationParameters.prepare()
if not genetic_parameters:
logger.error(
"Energy management run canceled. Could not prepare optimisation parameters."
)
cls._stage = EnergyManagementStage.IDLE
return
cls._stage = EnergyManagementStage.OPTIMIZATION
logger.info("Starting energy management optimization.")
# Take values from config if not given
if genetic_individuals is None:
genetic_individuals = cls.config.optimization.genetic.individuals
if genetic_seed is None:
genetic_seed = cls.config.optimization.genetic.seed
if cls._start_datetime is None: # Make mypy happy - already set by us
raise RuntimeError("Start datetime not set.")
try:
optimization = GeneticOptimization(
verbose=bool(cls.config.server.verbose),
fixed_seed=genetic_seed,
)
solution = optimization.optimierung_ems(
start_hour=cls._start_datetime.hour,
parameters=genetic_parameters,
ngen=genetic_individuals,
)
except:
logger.exception("Energy management optimization failed.")
cls._stage = EnergyManagementStage.IDLE
return
cls._stage = EnergyManagementStage.CONTROL_DISPATCH
# Make genetic solution public
cls._genetic_solution = solution
# Make optimization solution public
cls._optimization_solution = solution.optimization_solution()
# Make plan public
cls._plan = solution.energy_management_plan()
logger.debug("Energy management genetic solution:\n{}", cls._genetic_solution)
logger.debug("Energy management optimization solution:\n{}", cls._optimization_solution)
logger.debug("Energy management plan:\n{}", cls._plan)
logger.info("Energy management run done (optimization updated)")
# Do control dispatch by adapters
try:
cls.adapter.update_data(force_enable)
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
error_msg = f"Adapter update failed - phase {cls._stage}:\n{e}\n{trace}"
logger.error(error_msg)
# Remember energy run datetime.
EnergyManagement._last_run_datetime = to_datetime()
# energy management run finished
cls._stage = EnergyManagementStage.IDLE
async def run(
self,
start_datetime: Optional[DateTime] = None,
mode: Optional[EnergyManagementMode] = None,
genetic_parameters: Optional[GeneticOptimizationParameters] = None,
genetic_individuals: Optional[int] = None,
genetic_seed: Optional[int] = None,
force_enable: Optional[bool] = False,
force_update: Optional[bool] = False,
) -> None:
"""Run the energy management.
This method initializes the energy management run by setting its
start datetime, updating predictions, and optionally starting
optimization depending on the selected mode or configuration.
Args:
start_datetime (DateTime, optional): The starting timestamp
of the energy management run. Defaults to the current datetime
if not provided.
mode (EnergyManagementMode, optional): The management mode to use. Must be one of:
- "OPTIMIZATION": Runs the optimization process.
- "PREDICTION": Updates the forecast without optimization.
Defaults to the mode defined in the current configuration.
algorithm (str, optional):
The algorithm to use. Must be one of:
- "GENETIC": Optimization uses the `GENETIC` optimization algorithm.
Defaults to the algorithm defined in the current configuration.
genetic_parameters (GeneticOptimizationParameters, optional): The
parameter set for the genetic algorithm. If not provided, it will
parameter set for the `GENETIC` algorithm. If not provided, it will
be constructed based on the current configuration and predictions.
genetic_individuals (int, optional): The number of individuals for the
genetic algorithm. Defaults to the algorithm's internal default (400)
`GENETIC` algorithm. Defaults to the algorithm's internal default (400)
if not specified.
genetic_seed (int, optional): The seed for the genetic algorithm. Defaults
genetic_seed (int, optional): The seed for the `GENETIC` algorithm. Defaults
to the algorithm's internal random seed if not specified.
force_enable (bool, optional): If True, bypasses any disabled state
to force the update process. This is mostly applicable to
@@ -336,22 +193,205 @@ class EnergyManagement(
Returns:
None
"""
async with self._run_lock:
loop = get_running_loop()
# Create a partial function with parameters "baked in"
if start_datetime is None:
start_datetime = to_datetime()
async with EnergyManagement._run_lock:
if mode is None:
mode = self.config.ems.mode
func = partial(
EnergyManagement._run,
start_datetime=start_datetime,
mode=mode,
genetic_parameters=genetic_parameters,
genetic_individuals=genetic_individuals,
genetic_seed=genetic_seed,
force_enable=force_enable,
force_update=force_update,
if mode not in EnergyManagementMode._value2member_map_:
raise ValueError(f"Unknown energy management mode {mode}.")
if mode == EnergyManagementMode.DISABLED:
logger.info("Energy management run disabled.")
return
logger.info("Starting energy management run.")
# --- Data Aquisition ---
EnergyManagement._stage = EnergyManagementStage.DATA_ACQUISITION
# Remember/ set the start datetime of this energy management run.
# None leads to current time as start datetime
self.set_start_datetime(start_datetime)
# Throw away any memory cached results of the last energy management run.
CacheEnergyManagementStore().clear()
# --- Adapter update ---
try:
await self.adapter.update_data(force_enable)
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
error_msg = (
f"Adapter update failed - phase {EnergyManagement._stage}:\n{e}\n{trace}"
)
logger.error(error_msg)
# --- Prediction ---
EnergyManagement._stage = EnergyManagementStage.FORECAST_RETRIEVAL
# Update the predictions
logger.info("Starting energy management prediction update.")
await self.prediction.update_data(force_enable=force_enable, force_update=force_update)
if mode == EnergyManagementMode.PREDICTION:
logger.info("Energy management run done (predictions updated)")
EnergyManagement._stage = EnergyManagementStage.IDLE
return
# --- Optimization ---
EnergyManagement._stage = EnergyManagementStage.OPTIMIZATION
optimization_start = to_datetime()
logger.info("Starting energy management optimization.")
if algorithm is None:
algorithm = self.config.optimization.algorithm
if algorithm == "GENETIC":
# Prepare optimization parameters
# This also creates default configurations for missing values and updates the predictions
logger.info("Starting optimzation parameter preparation.")
if genetic_parameters is None:
genetic_parameters = await GeneticOptimizationParameters.prepare()
if genetic_parameters is None:
logger.error(
"Energy management run canceled. Could not prepare optimisation parameters."
)
EnergyManagement._stage = EnergyManagementStage.IDLE
return
# Take values from config if not given
if genetic_individuals is None:
genetic_individuals = self.config.optimization.genetic.individuals
if genetic_seed is None:
genetic_seed = self.config.optimization.genetic.seed
if EnergyManagement._start_datetime is None: # Make mypy happy - already set by us
raise RuntimeError("Start datetime not set.")
# --- Optimization (CPU-bound → MUST offload) ---
try:
optimization = GeneticOptimization(
verbose=bool(self.config.server.verbose),
fixed_seed=genetic_seed,
)
loop = get_running_loop()
start_hour = EnergyManagement._start_datetime.hour
solution = await loop.run_in_executor(
None,
lambda: optimization.optimierung_ems(
start_hour=start_hour,
parameters=cast(
GeneticOptimizationParameters, genetic_parameters
), # cast for mypy
ngen=genetic_individuals,
),
)
except Exception:
logger.exception("Energy management optimization failed.")
EnergyManagement._stage = EnergyManagementStage.IDLE
return
else:
logger.error(f"Unknown optimization algorithm: '{algorithm}'. Skipping.")
EnergyManagement._stage = EnergyManagementStage.IDLE
return
optimization_duration = to_datetime() - optimization_start
logger.info(
"Energy management optimization ({}) completed in {:.1f} seconds.",
algorithm,
optimization_duration.total_seconds(),
)
# Run optimization in background thread to avoid blocking event loop
await loop.run_in_executor(executor, func)
logger.debug(
"Energy management optimization solution:\n{}",
EnergyManagement._optimization_solution,
)
logger.debug("Energy management plan:\n{}", EnergyManagement._plan)
# --- Control dispatch by adapters ---
EnergyManagement._stage = EnergyManagementStage.CONTROL_DISPATCH
# Make genetic solution public
EnergyManagement._genetic_solution = solution
# Make optimization solution public
EnergyManagement._optimization_solution = await solution.optimization_solution()
# Make plan public
EnergyManagement._plan = solution.energy_management_plan()
logger.debug(
"Energy management genetic solution:\n{}", EnergyManagement._genetic_solution
)
if genetic_parameters is None:
genetic_parameters = await GeneticOptimizationParameters.prepare()
if not genetic_parameters:
logger.error("Energy management run canceled. Could not prepare parameters.")
EnergyManagement._stage = EnergyManagementStage.IDLE
return
EnergyManagement._stage = EnergyManagementStage.OPTIMIZATION
if genetic_individuals is None:
genetic_individuals = self.config.optimization.genetic.individuals
if genetic_seed is None:
genetic_seed = self.config.optimization.genetic.seed
if EnergyManagement._start_datetime is None:
raise RuntimeError("Start datetime not set.")
# --- Optimization (CPU-bound → MUST offload) ---
try:
optimization = GeneticOptimization(
verbose=bool(self.config.server.verbose),
fixed_seed=genetic_seed,
)
loop = get_running_loop()
start_hour = EnergyManagement._start_datetime.hour
solution = await loop.run_in_executor(
None,
lambda: optimization.optimierung_ems(
start_hour=start_hour,
parameters=genetic_parameters,
ngen=genetic_individuals,
),
)
except Exception:
logger.exception("Energy management optimization failed.")
EnergyManagement._stage = EnergyManagementStage.IDLE
return
EnergyManagement._genetic_solution = solution
EnergyManagement._optimization_solution = await solution.optimization_solution()
EnergyManagement._plan = solution.energy_management_plan()
logger.debug("Genetic solution:\n{}", EnergyManagement._genetic_solution)
logger.debug("Optimization solution:\n{}", EnergyManagement._optimization_solution)
logger.debug("Plan:\n{}", EnergyManagement._plan)
logger.info("Energy management run done (optimization updated)")
# --- Dispatch control by adapters ---
EnergyManagement._stage = EnergyManagementStage.CONTROL_DISPATCH
# Dispatch (sync → optionally offload)
try:
await self.adapter.update_data(force_enable)
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
error_msg = (
f"Adapter update failed - phase {EnergyManagement._stage}:\n{e}\n{trace}"
)
logger.error(error_msg)
# --- Idle ---
# Remember energy run datetime.
EnergyManagement._last_run_datetime = to_datetime()
# energy management run finished
EnergyManagement._stage = EnergyManagementStage.IDLE

6
src/akkudoktoreos/core/pydantic.py
View File

@@ -543,10 +543,10 @@ class PydanticModelNestedValueMixin:
if not inspect.isclass(model):
raise TypeError(f"Model '{model}' is not of class type.")
if key not in model.model_fields:
if key not in model.model_fields: # type: ignore[attr-defined]
raise TypeError(f"Field '{key}' does not exist in model '{model.__name__}'.")
field_annotation = model.model_fields[key].annotation
field_annotation = model.model_fields[key].annotation # type: ignore[attr-defined]
if not field_annotation:
raise TypeError(
f"Missing type annotation for field '{key}' in model '{model.__name__}'."
@@ -692,7 +692,7 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel):
return super().model_dump(*args, **kwargs)
def to_dict(self) -> dict:
"""Convert this PredictionRecord instance to a dictionary representation.
"""Convert this pydantic model instance to a dictionary representation.
Returns:
dict: A dictionary where the keys are the field names of the PydanticBaseModel,

24
src/akkudoktoreos/measurement/measurement.py
View File

@@ -149,7 +149,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
# Return ceiling of division to include partial intervals
return int(np.ceil(diff_seconds / interval_seconds))
def _energy_from_meter_readings(
async def _energy_from_meter_readings(
self,
key: str,
start_datetime: DateTime,
@@ -170,7 +170,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
"""
size = self._interval_count(start_datetime, end_datetime, interval)
energy_mr_array = self.key_to_array(
energy_mr_array = await self.key_to_array(
key=key,
start_datetime=start_datetime,
end_datetime=end_datetime + interval,
@@ -203,7 +203,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
logger.debug(debug_msg)
return energy_array
def load_total_kwh(
async def load_total_kwh(
self,
start_datetime: Optional[DateTime] = None,
end_datetime: Optional[DateTime] = None,
@@ -232,9 +232,11 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
return np.zeros(size)
if start_datetime is None:
start_datetime = self.min_datetime
start_datetime = await self.min_datetime()
if end_datetime is None:
end_datetime = self.max_datetime.add(seconds=1)
end_datetime = await self.max_datetime()
if end_datetime:
end_datetime = end_datetime.add(seconds=1)
size = self._interval_count(start_datetime, end_datetime, interval)
load_total_kwh_array = np.zeros(size)
@@ -242,7 +244,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
if isinstance(self.config.measurement.load_emr_keys, list):
for key in self.config.measurement.load_emr_keys:
# Calculate load per interval
load_array = self._energy_from_meter_readings(
load_array = await self._energy_from_meter_readings(
key=key,
start_datetime=start_datetime,
end_datetime=end_datetime,
@@ -270,14 +272,14 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
"""
return to_datetime().subtract(hours=self.config.measurement.historic_hours)
def save(self) -> bool:
async def save(self) -> bool:
"""Save the measurements to persistent storage.
Returns:
True in case the measurements were saved, False otherwise.
"""
# Use db storage if available
saved_to_db = DataSequence.save(self)
saved_to_db = await DataSequence.save(self)
if not saved_to_db:
measurement_file_path = self._measurement_file_path()
if measurement_file_path is None:
@@ -292,14 +294,14 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
logger.exception("Cannot save measurements")
return True
def load(self) -> bool:
async def load(self) -> bool:
"""Load measurements from persistent storage.
Returns:
True in case the measurements were loaded, False otherwise.
"""
# Use db storage if available
loaded_from_db = DataSequence.load(self)
loaded_from_db = await DataSequence.load(self)
if not loaded_from_db:
measurement_file_path = self._measurement_file_path()
if measurement_file_path is None:
@@ -314,7 +316,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence):
# Explicitly add data records to the existing singleton
for record in loaded.records:
self.insert_by_datetime(record)
await self.insert_by_datetime(record)
except Exception as e:
logger.exception("Cannot load measurements")
return True

18
src/akkudoktoreos/optimization/genetic/geneticparams.py
View File

@@ -147,7 +147,7 @@ class GeneticOptimizationParameters(
return start_solution
@classmethod
def prepare(cls) -> "Optional[GeneticOptimizationParameters]":
async def prepare(cls) -> "Optional[GeneticOptimizationParameters]":
"""Prepare optimization parameters from config, forecast and measurement data.
Fills in values needed for optimization from available configuration, predictions and
@@ -231,11 +231,11 @@ class GeneticOptimizationParameters(
raise ValueError(error_msg)
# Assure predictions are uptodate
cls.prediction.update_data()
await cls.prediction.update_data()
try:
pvforecast_ac_power = (
cls.prediction.key_to_array(
await cls.prediction.key_to_array(
key="pvforecast_ac_power",
start_datetime=parameter_start_datetime,
end_datetime=parameter_end_datetime,
@@ -290,7 +290,7 @@ class GeneticOptimizationParameters(
# Retry
continue
try:
elecprice_marketprice_wh = cls.prediction.key_to_array(
elecprice_marketprice_wh = await cls.prediction.key_to_array(
key="elecprice_marketprice_wh",
start_datetime=parameter_start_datetime,
end_datetime=parameter_end_datetime,
@@ -306,7 +306,7 @@ class GeneticOptimizationParameters(
# Retry
continue
try:
loadforecast_power_w = cls.prediction.key_to_array(
loadforecast_power_w = await cls.prediction.key_to_array(
key="loadforecast_power_w",
start_datetime=parameter_start_datetime,
end_datetime=parameter_end_datetime,
@@ -331,7 +331,7 @@ class GeneticOptimizationParameters(
# Retry
continue
try:
feed_in_tariff_wh = cls.prediction.key_to_array(
feed_in_tariff_wh = await cls.prediction.key_to_array(
key="feed_in_tariff_wh",
start_datetime=parameter_start_datetime,
end_datetime=parameter_end_datetime,
@@ -358,7 +358,7 @@ class GeneticOptimizationParameters(
# Retry
continue
try:
weather_temp_air = cls.prediction.key_to_array(
weather_temp_air = await cls.prediction.key_to_array(
key="weather_temp_air",
start_datetime=parameter_start_datetime,
end_datetime=parameter_end_datetime,
@@ -418,7 +418,7 @@ class GeneticOptimizationParameters(
battery_lcos_kwh = battery_config.levelized_cost_of_storage_kwh
# Initial SOC
try:
initial_soc_factor = cls.measurement.key_to_value(
initial_soc_factor = await cls.measurement.key_to_value(
key=battery_config.measurement_key_soc_factor,
target_datetime=ems.start_datetime,
time_window=to_duration(to_duration("48 hours")),
@@ -490,7 +490,7 @@ class GeneticOptimizationParameters(
continue
# Initial SOC
try:
initial_soc_factor = cls.measurement.key_to_value(
initial_soc_factor = await cls.measurement.key_to_value(
key=electric_vehicle_config.measurement_key_soc_factor,
target_datetime=ems.start_datetime,
time_window=to_duration(to_duration("48 hours")),

4
src/akkudoktoreos/optimization/genetic/geneticsolution.py
View File

@@ -344,7 +344,7 @@ class GeneticSolution(ConfigMixin, GeneticParametersBaseModel):
return effective_ac, effective_dc, effective_dis
def optimization_solution(self) -> OptimizationSolution:
async def optimization_solution(self) -> OptimizationSolution:
"""Provide the genetic solution as a general optimization solution.
The battery modes are controlled by the grid control triggers:
@@ -612,7 +612,7 @@ class GeneticSolution(ConfigMixin, GeneticParametersBaseModel):
),
]:
if pred_key in pred.record_keys:
array = pred.key_to_array(
array = await pred.key_to_array(
key=pred_key,
start_datetime=start_datetime,
end_datetime=end_datetime,

10
src/akkudoktoreos/prediction/elecpriceakkudoktor.py
View File

@@ -135,7 +135,7 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
clean_history = self._cap_outliers(history)
return np.full(hours, np.median(clean_history))
def _update_data(
async def _update_data(
self, force_update: Optional[bool] = False
) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Update forecast data in the ElecPriceDataRecord format.
@@ -170,10 +170,10 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
series_data.at[orig_datetime] = price_wh
# Update values using key_from_series
self.key_from_series("elecprice_marketprice_wh", series_data)
await self.key_from_series("elecprice_marketprice_wh", series_data)
# Generate history array for prediction
history = self.key_to_array(
history = await self.key_to_array(
key="elecprice_marketprice_wh", end_datetime=highest_orig_datetime, fill_method="linear"
)
@@ -213,9 +213,9 @@ class ElecPriceAkkudoktor(ElecPriceProvider):
for i in range(len(prediction))
],
)
self.key_from_series("elecprice_marketprice_wh", prediction_series)
await self.key_from_series("elecprice_marketprice_wh", prediction_series)
# history2 = self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002
# history2 = await self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002
# return history, history2, prediction # for debug main

10
src/akkudoktoreos/prediction/elecpriceenergycharts.py
View File

@@ -195,7 +195,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
clean_history = self._cap_outliers(history)
return np.full(hours, np.median(clean_history))
def _update_data(
async def _update_data(
self, force_update: Optional[bool] = False
) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Update forecast data in the ElecPriceDataRecord format.
@@ -217,7 +217,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
# Determine if update is needed and how many days
past_days = 35
if self.highest_orig_datetime:
history_series = self.key_to_series(
history_series = await self.key_to_series(
key="elecprice_marketprice_wh", start_datetime=self.ems_start_datetime
)
# If history lower, then start_datetime
@@ -244,14 +244,14 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
# Parse and store data
series_data = self._parse_data(energy_charts_data)
self.highest_orig_datetime = series_data.index.max()
self.key_from_series("elecprice_marketprice_wh", series_data)
await self.key_from_series("elecprice_marketprice_wh", series_data)
else:
logger.info(
f"No Update ElecPriceEnergyCharts is needed, last in history: {self.highest_orig_datetime}"
)
# Generate history array for prediction
history = self.key_to_array(
history = await self.key_to_array(
key="elecprice_marketprice_wh",
end_datetime=self.highest_orig_datetime,
fill_method="linear",
@@ -293,4 +293,4 @@ class ElecPriceEnergyCharts(ElecPriceProvider):
for i in range(len(prediction))
],
)
self.key_from_series("elecprice_marketprice_wh", prediction_series)
await self.key_from_series("elecprice_marketprice_wh", prediction_series)

4
src/akkudoktoreos/prediction/elecpricefixed.py
View File

@@ -59,7 +59,7 @@ class ElecPriceFixed(ElecPriceProvider):
"""Return the unique identifier for the ElecPriceFixed provider."""
return "ElecPriceFixed"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update electricity price data from fixed schedule.
Generates electricity prices based on the configured time windows
@@ -106,6 +106,6 @@ class ElecPriceFixed(ElecPriceProvider):
# Convert kWh → Wh and store one entry per interval step.
for idx, price_kwh in enumerate(prices_kwh):
current_dt = start_datetime.add(seconds=idx * interval_seconds)
self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0)
await self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0)
logger.debug(f"Successfully generated {len(prices_kwh)} fixed electricity price entries")

8
src/akkudoktoreos/prediction/elecpriceimport.py
View File

@@ -63,14 +63,16 @@ class ElecPriceImport(ElecPriceProvider, PredictionImportProvider):
"""Return the unique identifier for the ElecPriceImport provider."""
return "ElecPriceImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.elecprice.elecpriceimport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.elecprice.elecpriceimport.import_file_path,
key_prefix="elecprice",
)
if self.config.elecprice.elecpriceimport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.elecprice.elecpriceimport.import_json,
key_prefix="elecprice",
)

4
src/akkudoktoreos/prediction/feedintarifffixed.py
View File

@@ -34,7 +34,7 @@ class FeedInTariffFixed(FeedInTariffProvider):
"""Return the unique identifier for the FeedInTariffFixed provider."""
return "FeedInTariffFixed"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
error_msg = "Feed in tariff not provided"
try:
feed_in_tariff = (
@@ -47,4 +47,4 @@ class FeedInTariffFixed(FeedInTariffProvider):
logger.error(error_msg)
raise ValueError(error_msg)
feed_in_tariff_wh = feed_in_tariff / 1000
self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh)
await self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh)

8
src/akkudoktoreos/prediction/feedintariffimport.py
View File

@@ -64,17 +64,19 @@ class FeedInTariffImport(FeedInTariffProvider, PredictionImportProvider):
"""Return the unique identifier for the FeedInTariffImport provider."""
return "FeedInTariffImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.feedintariff.provider_settings.FeedInTariffImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.feedintariff.provider_settings.FeedInTariffImport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.provider_settings.FeedInTariffImport.import_file_path,
key_prefix="feedintariff",
)
if self.config.feedintariff.provider_settings.FeedInTariffImport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.feedintariff.provider_settings.FeedInTariffImport.import_json,
key_prefix="feedintariff",
)

29
src/akkudoktoreos/prediction/loadakkudoktor.py
View File

@@ -68,7 +68,7 @@ class LoadAkkudoktor(LoadProvider):
raise ValueError(error_msg)
return data_year_energy
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Adds the load means and standard deviations."""
data_year_energy = self.load_data()
# We provide prediction starting at start of day, to be compatible to old system.
@@ -84,7 +84,7 @@ class LoadAkkudoktor(LoadProvider):
"loadakkudoktor_mean_power_w": hourly_stats[0],
"loadakkudoktor_std_power_w": hourly_stats[1],
}
self.update_value(date, values)
await self.update_value(date, values)
date += to_duration("1 hour")
# We are working on fresh data (no cache), report update time
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
@@ -98,7 +98,9 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
"""Return the unique identifier for the LoadAkkudoktor provider."""
return "LoadAkkudoktorAdjusted"
def _calculate_adjustment(self, data_year_energy: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
async def _calculate_adjustment(
self, data_year_energy: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
"""Calculate weekday and week end adjustment from total load measurement data.
Returns:
@@ -110,19 +112,22 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
weekend_adjust = np.zeros(24)
weekend_adjust_weight = np.zeros(24)
if self.measurement.max_datetime is None:
max_dt = await self.measurement.max_datetime()
if max_dt is None:
# No measurements - return 0 adjustment
return (weekday_adjust, weekday_adjust)
min_dt = await self.measurement.min_datetime()
# compare predictions with real measurement - try to use last 7 days
compare_start = self.measurement.max_datetime - to_duration("7 days")
if compare_datetimes(compare_start, self.measurement.min_datetime).lt:
compare_start = max_dt - to_duration("7 days")
if compare_datetimes(compare_start, min_dt).lt:
# Not enough measurements for 7 days - use what is available
compare_start = self.measurement.min_datetime
compare_end = self.measurement.max_datetime
compare_start = min_dt
compare_end = max_dt
compare_interval = to_duration("1 hour")
load_total_kwh_array = self.measurement.load_total_kwh(
load_total_kwh_array = await self.measurement.load_total_kwh(
start_datetime=compare_start,
end_datetime=compare_end,
interval=compare_interval,
@@ -159,10 +164,10 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
return (weekday_adjust, weekend_adjust)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Adds the load means and standard deviations."""
data_year_energy = self.load_data()
weekday_adjust, weekend_adjust = self._calculate_adjustment(data_year_energy)
weekday_adjust, weekend_adjust = await self._calculate_adjustment(data_year_energy)
# We provide prediction starting at start of day, to be compatible to old system.
# End date for prediction is prediction hours from now.
date = self.ems_start_datetime.start_of("day")
@@ -182,7 +187,7 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor):
# Saturday, Sunday (5, 6)
value_adjusted = hourly_stats[0] + weekend_adjust[date.hour]
values["loadforecast_power_w"] = max(0, value_adjusted)
self.update_value(date, values)
await self.update_value(date, values)
date += to_duration("1 hour")
# We are working on fresh data (no cache), report update time
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)

10
src/akkudoktoreos/prediction/loadimport.py
View File

@@ -62,8 +62,12 @@ class LoadImport(LoadProvider, PredictionImportProvider):
"""Return the unique identifier for the LoadImport provider."""
return "LoadImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.load.loadimport.import_file_path:
self.import_from_file(self.config.load.loadimport.import_file_path, key_prefix="load")
await self._import_from_file(
self.config.load.loadimport.import_file_path, key_prefix="load"
)
if self.config.load.loadimport.import_json:
self.import_from_json(self.config.load.loadimport.import_json, key_prefix="load")
await self._import_from_json(self.config.load.loadimport.import_json, key_prefix="load")

9
src/akkudoktoreos/prediction/loadvrm.py
View File

@@ -84,7 +84,7 @@ class LoadVrm(LoadProvider):
"""Convert UNIX ms timestamp to timezone-aware datetime."""
return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Fetch and store VRM load forecast as loadforecast_power_w and related values."""
if self.enabled is False:
logger.info("LoadVrm is disabled, skipping update.")
@@ -102,7 +102,7 @@ class LoadVrm(LoadProvider):
date = self._ts_to_datetime(timestamp)
rounded_value = round(value, 2)
self.update_value(
await self.update_value(
date,
{"loadforecast_power_w": rounded_value},
)
@@ -111,8 +111,3 @@ class LoadVrm(LoadProvider):
logger.debug(f"Updated loadforecast_power_w with {len(loadforecast_power_w_data)} entries.")
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
if __name__ == "__main__":
lv = LoadVrm()
lv._update_data()

2
src/akkudoktoreos/prediction/prediction.py
View File

@@ -14,7 +14,7 @@ Example:
# Create singleton prediction instance with prediction providers
from akkudoktoreos.prediction.prediction import prediction
prediction.update_data()
await prediction.update_data()
print("Prediction:", prediction)
Classes:

6
src/akkudoktoreos/prediction/predictionabc.py
View File

@@ -225,7 +225,7 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider):
hours = max(self.config.prediction.hours, self.config.prediction_historic_hours, 24)
return to_duration(hours * 3600)
def update_data(
async def update_data(
self,
force_enable: Optional[bool] = False,
force_update: Optional[bool] = False,
@@ -243,10 +243,10 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider):
return
# Delete outdated records before updating
self.delete_by_datetime(end_datetime=self.keep_datetime)
await self.delete_by_datetime(end_datetime=self.keep_datetime)
# Call the custom update logic
self._update_data(force_update=force_update)
await self._update_data(force_update=force_update)
class PredictionImportProvider(PredictionProvider, DataImportProvider):

20
src/akkudoktoreos/prediction/pvforecastakkudoktor.py
View File

@@ -52,10 +52,10 @@ Example:
forecast = PVForecastAkkudoktor(settings=config)
# Get an actual forecast
forecast.update_data()
await forecast.update_data()
# Update the AC power measurement for a specific date and time
forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0)
await forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0)
# Report the DC and AC power forecast along with AC measurements
print(forecast.report_ac_power_and_measurement())
@@ -286,7 +286,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
return akkudoktor_data
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the PVForecastAkkudoktorDataRecord format.
Retrieves data from Akkudoktor. The processed data is inserted into the sequence as
@@ -341,7 +341,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
"pvforecastakkudoktor_temp_air": forecast_values[0].temperature,
}
self.update_value(dt, data)
await self.update_value(dt, data)
if len(self) < self.config.prediction.hours:
raise ValueError(
@@ -385,7 +385,7 @@ class PVForecastAkkudoktor(PVForecastProvider):
# Example of how to use the PVForecastAkkudoktor class
if __name__ == "__main__":
async def main() -> None:
"""Main execution block to demonstrate the use of the PVForecastAkkudoktor class.
Sets up the forecast configuration fields, fetches PV power forecast data,
@@ -441,12 +441,18 @@ if __name__ == "__main__":
forecast = PVForecastAkkudoktor()
# Get an actual forecast
forecast.update_data()
await forecast.update_data()
# Update the AC power measurement for a specific date and time
forecast.update_value(
await forecast.update_value(
to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0
)
# Report the DC and AC power forecast along with AC measurements
print(forecast.report_ac_power_and_measurement())
if __name__ == "__main__":
import asyncio
asyncio.run(main())

8
src/akkudoktoreos/prediction/pvforecastimport.py
View File

@@ -64,17 +64,19 @@ class PVForecastImport(PVForecastProvider, PredictionImportProvider):
"""Return the unique identifier for the PVForecastImport provider."""
return "PVForecastImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.pvforecast.provider_settings.PVForecastImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.pvforecast.provider_settings.PVForecastImport.import_file_path is not None:
self.import_from_file(
await self._import_from_file(
self.config.pvforecast.provider_settings.PVForecastImport.import_file_path,
key_prefix="pvforecast",
)
if self.config.pvforecast.provider_settings.PVForecastImport.import_json is not None:
self.import_from_json(
await self._import_from_json(
self.config.pvforecast.provider_settings.PVForecastImport.import_json,
key_prefix="pvforecast",
)

10
src/akkudoktoreos/prediction/pvforecastvrm.py
View File

@@ -83,7 +83,7 @@ class PVForecastVrm(PVForecastProvider):
"""Convert UNIX ms timestamp to timezone-aware datetime."""
return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the PVForecastDataRecord format."""
if self.enabled is False:
logger.info("PVForecastVrm is disabled, skipping update.")
@@ -101,16 +101,10 @@ class PVForecastVrm(PVForecastProvider):
date = self._ts_to_datetime(timestamp)
dc_power = round(value, 2)
ac_power = round(dc_power * 0.96, 2)
self.update_value(
await self.update_value(
date, {"pvforecast_dc_power": dc_power, "pvforecast_ac_power": ac_power}
)
pv_forecast.append((date, dc_power))
logger.debug(f"Updated pvforecast_dc_power with {len(pv_forecast)} entries.")
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
# Example usage
if __name__ == "__main__":
pv = PVForecastVrm()
pv._update_data()

26
src/akkudoktoreos/prediction/weatherbrightsky.py
View File

@@ -111,7 +111,7 @@ class WeatherBrightSky(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return brightsky_data
def _description_to_series(self, description: str) -> pd.Series:
async def _description_to_series(self, description: str) -> pd.Series:
"""Retrieve a pandas Series corresponding to a weather data description.
This method fetches the key associated with the provided description
@@ -132,10 +132,10 @@ class WeatherBrightSky(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
series = self.key_to_series(key)
series = await self.key_to_series(key)
return series
def _description_from_series(self, description: str, data: pd.Series) -> None:
async def _description_from_series(self, description: str, data: pd.Series) -> None:
"""Update a weather data with a pandas Series based on its description.
This method fetches the key associated with the provided description
@@ -154,9 +154,9 @@ class WeatherBrightSky(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
self.key_from_series(key, data)
await self.key_from_series(key, data)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the WeatherDataRecord format.
Retrieves data from BrightSky, maps each BrightSky field to the corresponding
@@ -197,31 +197,31 @@ class WeatherBrightSky(WeatherProvider):
else:
value = value * corr_factor
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)
# Converting the cloud cover into Irradiance (GHI, DNI, DHI)
description = "Total Clouds (% Sky Obscured)"
cloud_cover = self._description_to_series(description)
cloud_cover = await self._description_to_series(description)
ghi, dni, dhi = self.estimate_irradiance_from_cloud_cover(
self.config.general.latitude, self.config.general.longitude, cloud_cover
)
description = "Global Horizontal Irradiance (W/m2)"
ghi = pd.Series(data=ghi, index=cloud_cover.index)
self._description_from_series(description, ghi)
await self._description_from_series(description, ghi)
description = "Direct Normal Irradiance (W/m2)"
dni = pd.Series(data=dni, index=cloud_cover.index)
self._description_from_series(description, dni)
await self._description_from_series(description, dni)
description = "Diffuse Horizontal Irradiance (W/m2)"
dhi = pd.Series(data=dhi, index=cloud_cover.index)
self._description_from_series(description, dhi)
await self._description_from_series(description, dhi)
# Add Preciptable Water (PWAT) with a PVLib method.
key = WeatherDataRecord.key_from_description("Temperature (°C)")
assert key # noqa: S101
temperature = self.key_to_array(
temperature = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -234,7 +234,7 @@ class WeatherBrightSky(WeatherProvider):
return
key = WeatherDataRecord.key_from_description("Relative Humidity (%)")
assert key # noqa: S101
humidity = self.key_to_array(
humidity = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -258,4 +258,4 @@ class WeatherBrightSky(WeatherProvider):
),
)
description = "Preciptable Water (cm)"
self._description_from_series(description, pwat)
await self._description_from_series(description, pwat)

6
src/akkudoktoreos/prediction/weatherclearoutside.py
View File

@@ -97,7 +97,7 @@ class WeatherClearOutside(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return response
def _update_data(self, force_update: Optional[bool] = None) -> None:
async def _update_data(self, force_update: Optional[bool] = None) -> None:
"""Scrape weather forecast data from ClearOutside's website.
This method requests weather forecast data from ClearOutside based on latitude
@@ -202,7 +202,7 @@ class WeatherClearOutside(WeatherProvider):
raise ValueError(error_msg)
# Delete all records that will be newly added
self.delete_by_datetime(start_datetime=forecast_start_datetime)
await self.delete_by_datetime(start_datetime=forecast_start_datetime)
# Collect weather data, loop over all days
for day, p_day in enumerate(p_days):
@@ -341,4 +341,4 @@ class WeatherClearOutside(WeatherProvider):
if corr_factor:
value = value * corr_factor
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)

8
src/akkudoktoreos/prediction/weatherimport.py
View File

@@ -64,17 +64,19 @@ class WeatherImport(WeatherProvider, PredictionImportProvider):
"""Return the unique identifier for the WeatherImport provider."""
return "WeatherImport"
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
# Both _sequence_lock and _record_lock are already held by the caller.
# Use internal sync methods only — never await public async counterparts.
if self.config.weather.provider_settings.WeatherImport is None:
logger.debug(f"{self.provider_id()} data update without provider settings.")
return
if self.config.weather.provider_settings.WeatherImport.import_file_path:
self.import_from_file(
await self._import_from_file(
self.config.weather.provider_settings.WeatherImport.import_file_path,
key_prefix="weather",
)
if self.config.weather.provider_settings.WeatherImport.import_json:
self.import_from_json(
await self._import_from_json(
self.config.weather.provider_settings.WeatherImport.import_json,
key_prefix="weather",
)

20
src/akkudoktoreos/prediction/weatheropenmeteo.py
View File

@@ -197,7 +197,7 @@ class WeatherOpenMeteo(WeatherProvider):
self.update_datetime = to_datetime(in_timezone=self.config.general.timezone)
return openmeteo_data
def _description_to_series(self, description: str) -> pd.Series:
async def _description_to_series(self, description: str) -> pd.Series:
"""Retrieve a pandas Series corresponding to a weather data description.
This method fetches the key associated with the provided description
@@ -218,10 +218,10 @@ class WeatherOpenMeteo(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
series = self.key_to_series(key)
series = await self.key_to_series(key)
return series
def _description_from_series(self, description: str, data: pd.Series) -> None:
async def _description_from_series(self, description: str, data: pd.Series) -> None:
"""Update a weather data with a pandas Series based on its description.
This method fetches the key associated with the provided description
@@ -240,9 +240,9 @@ class WeatherOpenMeteo(WeatherProvider):
error_msg = f"No WeatherDataRecord key for '{description}'"
logger.error(error_msg)
raise ValueError(error_msg)
self.key_from_series(key, data)
await self.key_from_series(key, data)
def _update_data(self, force_update: Optional[bool] = False) -> None:
async def _update_data(self, force_update: Optional[bool] = False) -> None:
"""Update forecast data in the WeatherDataRecord format.
Retrieves data from Open-Meteo, maps each Open-Meteo field to the corresponding
@@ -299,11 +299,11 @@ class WeatherOpenMeteo(WeatherProvider):
setattr(weather_record, key, value)
self.insert_by_datetime(weather_record)
await self.insert_by_datetime(weather_record)
# Check whether radiation values exist (for logging)
description_ghi = "Global Horizontal Irradiance (W/m2)"
ghi_series = self._description_to_series(description_ghi)
ghi_series = await self._description_to_series(description_ghi)
if ghi_series.isnull().all():
logger.warning("No GHI data received from Open-Meteo")
@@ -315,7 +315,7 @@ class WeatherOpenMeteo(WeatherProvider):
# Add Precipitable Water (PWAT) using PVLib method
key = WeatherDataRecord.key_from_description("Temperature (°C)")
assert key # noqa: S101
temperature = self.key_to_array(
temperature = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -329,7 +329,7 @@ class WeatherOpenMeteo(WeatherProvider):
key = WeatherDataRecord.key_from_description("Relative Humidity (%)")
assert key # noqa: S101
humidity = self.key_to_array(
humidity = await self.key_to_array(
key=key,
start_datetime=self.ems_start_datetime,
end_datetime=self.end_datetime,
@@ -354,4 +354,4 @@ class WeatherOpenMeteo(WeatherProvider):
),
)
description = "Precipitable Water (cm)"
self._description_from_series(description, pwat)
await self._description_from_series(description, pwat)

247
src/akkudoktoreos/server/eos.py
View File

@@ -73,20 +73,18 @@ from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration
# ----------------------
def save_eos_state() -> None:
async def save_eos_state() -> None:
"""Save EOS state."""
get_resource_registry().save()
get_prediction().save()
get_measurement().save()
await get_prediction().save()
await get_measurement().save()
cache_save() # keep last
def load_eos_state() -> None:
async def load_eos_state() -> None:
"""Load EOS state."""
cache_load() # keep first
get_measurement().load()
get_prediction().load()
get_resource_registry().load()
await get_measurement().load()
await get_prediction().load()
def terminate_eos() -> None:
@@ -100,18 +98,18 @@ def terminate_eos() -> None:
logger.info(f"🚀 EOS terminated, PID {pid}")
def save_eos_database() -> None:
async def save_eos_database() -> None:
"""Save EOS database."""
get_prediction().save()
get_measurement().save()
await get_prediction().save()
await get_measurement().save()
def compact_eos_database() -> None:
async def compact_eos_database() -> None:
"""Compact EOS database."""
get_prediction().db_compact()
get_measurement().db_compact()
get_prediction().db_vacuum()
get_measurement().db_vacuum()
await get_prediction().db_compact()
await get_measurement().db_compact()
await get_prediction().db_vacuum()
await get_measurement().db_vacuum()
def autosave_config() -> None:
@@ -134,7 +132,7 @@ async def server_shutdown_task() -> None:
Finally, logs a message indicating that the EOS server has been terminated.
"""
save_eos_state()
await save_eos_state()
# Give EOS time to finish some work
await asyncio.sleep(5)
@@ -162,7 +160,7 @@ def config_eos_ready() -> bool:
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Lifespan manager for the app."""
# On startup
load_eos_state()
await load_eos_state()
# Prepare the Manager and all task that are handled by the manager
manager = RetentionManager(
@@ -200,7 +198,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
await asyncio.gather(retention_manager_task, return_exceptions=True)
# On shutdown
save_eos_state()
await save_eos_state()
app = FastAPI(
@@ -297,7 +295,7 @@ def fastapi_admin_cache_get() -> dict:
@app.get("/v1/admin/database/stats", tags=["admin"])
def fastapi_admin_database_stats_get() -> dict:
async def fastapi_admin_database_stats_get() -> dict:
"""Get statistics from database.
Returns:
@@ -306,8 +304,8 @@ def fastapi_admin_database_stats_get() -> dict:
data = {}
try:
# Get the stats
data[get_measurement().db_namespace()] = get_measurement().db_get_stats()
data[get_prediction().__class__.__name__] = get_prediction().db_get_stats()
data[get_measurement().db_namespace()] = await get_measurement().db_get_stats()
data[get_prediction().__class__.__name__] = await get_prediction().db_get_stats()
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
raise HTTPException(
@@ -316,8 +314,28 @@ def fastapi_admin_database_stats_get() -> dict:
return data
@app.post("/v1/admin/database/save", tags=["admin"])
async def fastapi_admin_database_save_post() -> dict:
"""Save in memory data to database.
Returns:
data (dict): The database stats after saving the records.
"""
data = {}
try:
await get_measurement().save()
await get_prediction().save()
# Get the stats
data[get_measurement().db_namespace()] = await get_measurement().db_get_stats()
data[get_prediction().__class__.__name__] = await get_prediction().db_get_stats()
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
raise HTTPException(status_code=400, detail=f"Error on database save: {e}\n{trace}")
return data
@app.post("/v1/admin/database/vacuum", tags=["admin"])
def fastapi_admin_database_vacuum_post() -> dict:
async def fastapi_admin_database_vacuum_post() -> dict:
"""Remove old records from database.
Returns:
@@ -325,11 +343,13 @@ def fastapi_admin_database_vacuum_post() -> dict:
"""
data = {}
try:
get_measurement().db_vacuum()
get_prediction().db_vacuum()
await get_measurement().db_vacuum()
await get_prediction().db_vacuum()
# Get the stats
data[get_measurement().db_namespace()] = get_measurement().db_get_stats()
data[get_prediction().__class__.__name__] = get_prediction().db_get_stats()
measuremet_stats = await get_measurement().db_get_stats()
data[get_measurement().db_namespace()] = measuremet_stats
prediction_stats = await get_prediction().db_get_stats()
data[get_prediction().__class__.__name__] = prediction_stats
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
raise HTTPException(status_code=400, detail=f"Error on database vacuum: {e}\n{trace}")
@@ -342,7 +362,7 @@ async def fastapi_admin_server_restart_post() -> dict:
Restart EOS properly by starting a new instance before exiting the old one.
"""
save_eos_state()
await save_eos_state()
# Start a new EOS (Uvicorn) process
logger.info("🔄 Restarting EOS...")
@@ -525,7 +545,11 @@ def fastapi_config_put(settings: SettingsEOS) -> ConfigEOS:
get_config().merge_settings(settings)
return get_config()
except Exception as e:
raise HTTPException(status_code=400, detail=f"Error on update of configuration: {e}")
trace = "".join(traceback.TracebackException.from_exception(e).format())
raise HTTPException(
status_code=400,
detail=f"Error on update of configuration '{settings}':\n{e}\n{trace}",
)
@app.put("/v1/config/{path:path}", tags=["config"])
@@ -687,14 +711,14 @@ def fastapi_measurement_keys_get() -> list[str]:
@app.get("/v1/measurement/series", tags=["measurement"])
def fastapi_measurement_series_get(
async def fastapi_measurement_series_get(
key: Annotated[str, Query(description="Measurement key.")],
) -> PydanticDateTimeSeries:
"""Get the measurements of given key as series."""
try:
if key not in get_measurement().record_keys:
raise HTTPException(status_code=404, detail=f"Key '{key}' is not available.")
pdseries = get_measurement().key_to_series(key=key)
pdseries = await get_measurement().key_to_series(key=key)
return PydanticDateTimeSeries.from_series(pdseries)
except HTTPException:
# Re-raise HTTP exceptions
@@ -710,7 +734,7 @@ def fastapi_measurement_series_get(
@app.put("/v1/measurement/value", tags=["measurement"])
def fastapi_measurement_value_put(
async def fastapi_measurement_value_put(
datetime: Annotated[str, Query(description="Datetime.")],
key: Annotated[str, Query(description="Measurement key.")],
value: Union[float | str],
@@ -740,8 +764,8 @@ def fastapi_measurement_value_put(
detail=f"Invalid datetime '{datetime}': {e}",
)
get_measurement().update_value(dt, key, value)
pdseries = get_measurement().key_to_series(key=key)
await get_measurement().update_value(dt, key, value)
pdseries = await get_measurement().key_to_series(key=key)
return PydanticDateTimeSeries.from_series(pdseries)
except HTTPException:
raise
@@ -755,7 +779,7 @@ def fastapi_measurement_value_put(
@app.put("/v1/measurement/series", tags=["measurement"])
def fastapi_measurement_series_put(
async def fastapi_measurement_series_put(
key: Annotated[str, Query(description="Measurement key.")], series: PydanticDateTimeSeries
) -> PydanticDateTimeSeries:
"""Merge measurement given as series into given key."""
@@ -763,8 +787,8 @@ def fastapi_measurement_series_put(
if key not in get_measurement().record_keys:
raise HTTPException(status_code=404, detail=f"Key '{key}' is not available.")
pdseries = series.to_series() # make pandas series from PydanticDateTimeSeries
get_measurement().key_from_series(key=key, series=pdseries)
pdseries = get_measurement().key_to_series(key=key)
await get_measurement().key_from_series(key=key, series=pdseries)
pdseries = await get_measurement().key_to_series(key=key)
return PydanticDateTimeSeries.from_series(pdseries)
except HTTPException:
# Re-raise HTTP exceptions
@@ -780,11 +804,11 @@ def fastapi_measurement_series_put(
@app.put("/v1/measurement/dataframe", tags=["measurement"])
def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None:
async def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None:
"""Merge the measurement data given as dataframe into EOS measurements."""
try:
dataframe = data.to_dataframe()
get_measurement().import_from_dataframe(dataframe)
await get_measurement().import_from_dataframe(dataframe)
except Exception as e:
# Log unexpected errors
trace = "".join(traceback.TracebackException.from_exception(e).format())
@@ -796,11 +820,11 @@ def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None:
@app.put("/v1/measurement/data", tags=["measurement"])
def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None:
async def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None:
"""Merge the measurement data given as datetime data into EOS measurements."""
try:
datetimedata = data.to_dict()
get_measurement().import_from_dict(datetimedata)
await get_measurement().import_from_dict(datetimedata)
except Exception as e:
# Log unexpected errors
trace = "".join(traceback.TracebackException.from_exception(e).format())
@@ -811,6 +835,49 @@ def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None:
)
@app.delete("/v1/measurement/range", tags=["measurement"])
async def fastapi_measurement_range_delete(
key: Annotated[str, Query(description="Measurement key.")],
start_datetime: Annotated[Optional[str], Query(description="Start datetime.")] = None,
end_datetime: Annotated[Optional[str], Query(description="End datetime.")] = None,
) -> PydanticDateTimeSeries:
"""Delete measurement values for a key within a datetime range."""
try:
if key not in get_measurement().record_keys:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Key '{key}' not found in measurements",
)
try:
start_dt = to_datetime(start_datetime) if start_datetime else None
end_dt = to_datetime(end_datetime) if end_datetime else None
except Exception as e:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail=f"Invalid datetime: {e}",
)
await get_measurement().key_delete_by_datetime(
key=key,
start_datetime=start_dt,
end_datetime=end_dt,
)
pdseries = await get_measurement().key_to_series(key=key)
return PydanticDateTimeSeries.from_series(pdseries)
except HTTPException:
raise
except Exception as e:
trace = "".join(traceback.TracebackException.from_exception(e).format())
logger.exception(f"Unexpected error deleting measurement range: {key}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Internal server error:\n{e}\n{trace}",
)
@app.get("/v1/prediction/providers", tags=["prediction"])
def fastapi_prediction_providers_get(enabled: Optional[bool] = None) -> list[str]:
"""Get a list of available prediction providers.
@@ -838,7 +905,7 @@ def fastapi_prediction_keys_get() -> list[str]:
@app.get("/v1/prediction/series", tags=["prediction"])
def fastapi_prediction_series_get(
async def fastapi_prediction_series_get(
key: Annotated[str, Query(description="Prediction key.")],
start_datetime: Annotated[
Optional[str],
@@ -868,14 +935,14 @@ def fastapi_prediction_series_get(
end_datetime = get_prediction().end_datetime
else:
end_datetime = to_datetime(end_datetime)
pdseries = get_prediction().key_to_series(
pdseries = await get_prediction().key_to_series(
key=key, start_datetime=start_datetime, end_datetime=end_datetime
)
return PydanticDateTimeSeries.from_series(pdseries)
@app.get("/v1/prediction/dataframe", tags=["prediction"])
def fastapi_prediction_dataframe_get(
async def fastapi_prediction_dataframe_get(
keys: Annotated[list[str], Query(description="Prediction keys.")],
start_datetime: Annotated[
Optional[str],
@@ -911,14 +978,14 @@ def fastapi_prediction_dataframe_get(
end_datetime = get_prediction().end_datetime
else:
end_datetime = to_datetime(end_datetime)
df = get_prediction().keys_to_dataframe(
df = await get_prediction().keys_to_dataframe(
keys=keys, start_datetime=start_datetime, end_datetime=end_datetime, interval=interval
)
return PydanticDateTimeDataFrame.from_dataframe(df, tz=get_config().general.timezone)
@app.get("/v1/prediction/list", tags=["prediction"])
def fastapi_prediction_list_get(
async def fastapi_prediction_list_get(
key: Annotated[str, Query(description="Prediction key.")],
start_datetime: Annotated[
Optional[str],
@@ -958,16 +1025,13 @@ def fastapi_prediction_list_get(
interval = to_duration("1 hour")
else:
interval = to_duration(interval)
prediction_list = (
get_prediction()
.key_to_array(
key=key,
start_datetime=start_datetime,
end_datetime=end_datetime,
interval=interval,
)
.tolist()
prediction_array = await get_prediction().key_to_array(
key=key,
start_datetime=start_datetime,
end_datetime=end_datetime,
interval=interval,
)
prediction_list = prediction_array.tolist()
return prediction_list
@@ -1130,22 +1194,19 @@ async def fastapi_strompreis() -> list[float]:
start_datetime = to_datetime().start_of("day")
end_datetime = start_datetime.add(days=2)
try:
elecprice = (
get_prediction()
.key_to_array(
key="elecprice_marketprice_wh",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
.tolist()
elecprice_array = await get_prediction().key_to_array(
key="elecprice_marketprice_wh",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
elecprice_list = elecprice_array.tolist()
except Exception as e:
raise HTTPException(
status_code=404,
detail=f"Can not get the electricity price forecast: {e}.\nDid you configure the electricity price forecast provider?",
)
return elecprice
return elecprice_list
class GesamtlastRequest(PydanticBaseModel):
@@ -1191,7 +1252,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]:
# Insert measured data into EOS measurement
# Convert from energy per interval to dummy energy meter readings
measurement_key = "gesamtlast_emr"
get_measurement().key_delete_by_datetime(
await get_measurement().key_delete_by_datetime(
key=measurement_key
) # delete all gesamtlast_emr measurements
energy = {}
@@ -1218,7 +1279,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]:
energy_mr_values.append(0.0)
energy_mr_dates.append(dt)
energy_mr_values.append(energy_mr)
get_measurement().key_from_lists(measurement_key, energy_mr_dates, energy_mr_values)
await get_measurement().key_from_lists(measurement_key, energy_mr_dates, energy_mr_values)
# Ensure there is only one optimization/ energy management run at a time
try:
@@ -1236,15 +1297,12 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]:
start_datetime = to_datetime().start_of("day")
end_datetime = start_datetime.add(days=2)
try:
prediction_list = (
get_prediction()
.key_to_array(
key="loadforecast_power_w",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
.tolist()
prediction_array = await get_prediction().key_to_array(
key="loadforecast_power_w",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
prediction_list = prediction_array.tolist()
except Exception as e:
raise HTTPException(
status_code=404,
@@ -1299,15 +1357,12 @@ async def fastapi_gesamtlast_simple(year_energy: float) -> list[float]:
start_datetime = to_datetime().start_of("day")
end_datetime = start_datetime.add(days=2)
try:
prediction_list = (
get_prediction()
.key_to_array(
key="loadforecast_power_w",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
.tolist()
prediction_array = await get_prediction().key_to_array(
key="loadforecast_power_w",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
prediction_list = prediction_array.tolist()
except Exception as e:
raise HTTPException(
status_code=404,
@@ -1358,24 +1413,18 @@ async def fastapi_pvforecast() -> ForecastResponse:
start_datetime = to_datetime().start_of("day")
end_datetime = start_datetime.add(days=2)
try:
ac_power = (
get_prediction()
.key_to_array(
key="pvforecast_ac_power",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
.tolist()
ac_power_array = await get_prediction().key_to_array(
key="pvforecast_ac_power",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
temp_air = (
get_prediction()
.key_to_array(
key="pvforecastakkudoktor_temp_air",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
.tolist()
ac_power_list = ac_power_array.tolist()
temp_air_array = await get_prediction().key_to_array(
key="pvforecastakkudoktor_temp_air",
start_datetime=start_datetime,
end_datetime=end_datetime,
)
temp_air_list = temp_air_array.tolist()
except Exception as e:
raise HTTPException(
status_code=404,
@@ -1383,7 +1432,7 @@ async def fastapi_pvforecast() -> ForecastResponse:
)
# Return both forecasts as a JSON response
return ForecastResponse(temperature=temp_air, pvpower=ac_power)
return ForecastResponse(temperature=temp_air_list, pvpower=ac_power_list)
@app.post("/optimize", tags=["optimize"])