Mirrors/EOS
1
0
Fork 0
mirror of https://github.com/Akkudoktor-EOS/EOS.git synced 2026-10-17 11:31:56 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix: unify mypy environments for local checks and CI (#1291)

Browse Source 
The isolated pre-commit mypy hook previously omitted runtime type information that
make mypy used, hiding errors involving dependencies such as Pydantic and Pendulum.
Makefile, pre-commit and CI now run the same full-project typing policy in the
development environment defined by uv.lock.

- Use uv run --locked --exact --extra dev and the same mypy arguments for Makefile
  and the local hook. Check all of src and tests, including on configuration-only
  changes.
- Pin Python 3.13 for local development and the pre-commit CI job, and install the
  locked pre-commit version in CI.
- Disable incremental analysis because existing Pendulum cache state changes mypy 2.3.1
  diagnostics. Document the policy, the performance tradeoff and the existing typing debt.
- Add a regression test that exercises Makefile, the hook and the CI command in a
  temporary project, accepting valid dependency types and detecting deliberate
  Pydantic/Pendulum assignment errors.

Resolve the newly detected mypy diagnostics.

- Enable the numpydantic and Pydantic mypy plugins, retaining strict Pydantic
  constructor typing with init_typed = true. Validate raw/coercible payloads through model_validate.
- Propagate concrete record, provider and time-window types through generic collections,
  factories and lookup methods. Preserve runtime field inspection and generated time-window
  documentation.
- Align Pendulum annotations with actual factory/arithmetic results while retaining Pydantic
  validation adapters at runtime. Correct optional values, array boundaries, REST handlers
  and plotting interfaces.
- Add pinned scipy-stubs and types-psutil, update uv.lock, and supply the plugins' dependencies.
- Add runtime regression coverage for validated path defaults, normalized time-series metadata,
  generic field inspection, invalid timestamps and unsupported provider imports.

Runtime and compatibility details:

- Validate path defaults as Path objects while retaining raw string defaults needed by
  migration serialization with exclude_defaults.
- Normalize feed-in tariff lists and default charge rates to NumPy arrays; reject missing
  timestamps/uninitialized values explicitly. Importing into a provider without import support
  returns HTTP 400.
- Public JSON schemas and OpenAPI structure match main (excluding the generated version).

Signed-off-by: dr-dimitry

Signed-off-by: dr-dimitry
Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
Co-authored-by: dr-dimitri <87113560+dr-dimitri@users.noreply.github.com>
Co-authored-by: Normann <github@koldrack.com>
This commit is contained in:
Bobby Noelte
2026-09-10 23:20:35 +02:00
committed by GitHub
co-authored by dr-dimitri Normann
parent 5b584cbb57
commit 1abdd345c4
114 changed files with 2217 additions and 1322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/akkudoktoreos/core/cache.py
+24 -17
View File
@@ -25,6 +25,7 @@ from typing import (
Optional,
ParamSpec,
TypeVar,
cast,
)
import cachebox
@@ -46,7 +47,8 @@ from akkudoktoreos.utils.datetimeutil import (
# ---------------------------------
# Define a type variable for methods and functions
TCallable = TypeVar("TCallable", bound=Callable[..., Any])
Param = ParamSpec("Param")
RetType = TypeVar("RetType")
def cache_energy_management_store_callback(event: int, key: Any, value: Any) -> None:
@@ -195,7 +197,9 @@ class CacheEnergyManagementStore(SingletonMixin):
raise AttributeError(f"'{self.cache.__class__.__name__}' object has no method 'clear'")
def cache_energy_management(callable: TCallable) -> TCallable:
def cache_energy_management(
func: Callable[Param, RetType],
) -> Callable[Param, RetType]:
"""Decorator for in memory caching the result of a callable.
This decorator caches the method or function's result in `CacheEnergyManagementStore`,
@@ -203,7 +207,7 @@ def cache_energy_management(callable: TCallable) -> TCallable:
next energy management start.
Args:
callable (Callable): The function or method to be decorated.
func (Callable): The function or method to be decorated.
Returns:
Callable: The wrapped function with caching functionality.
@@ -218,24 +222,22 @@ def cache_energy_management(callable: TCallable) -> TCallable:
"""
@cachebox.cached(
cache=CacheEnergyManagementStore().cache, callback=cache_energy_management_store_callback
)
@functools.wraps(callable)
def wrapper(*args: Any, **kwargs: Any) -> Any:
result = callable(*args, **kwargs)
return result
@functools.wraps(func)
def wrapper(*args: Param.args, **kwargs: Param.kwargs) -> RetType:
return func(*args, **kwargs)
return wrapper
cached_wrapper = cachebox.cached(
cache=CacheEnergyManagementStore().cache,
callback=cache_energy_management_store_callback,
)(wrapper)
return cast(Callable[Param, RetType], cached_wrapper)
# ---------------------------------
# Cache File Management
# ---------------------------------
Param = ParamSpec("Param")
RetType = TypeVar("RetType")
def cache_clear(clear_all: Optional[bool] = None) -> None:
"""Cleanup expired cache files."""
@@ -742,6 +744,9 @@ class CacheFileStore(ConfigMixin, SingletonMixin):
if clear_all:
clear_file = True
else:
# Initialized above when clear_all is false; should never raise.
if before_datetime is None:
raise AssertionError("Cache expiry threshold is not initialized")
clear_file = compare_datetimes(cache_item.until_datetime, before_datetime).lt
if clear_file:
@@ -782,9 +787,11 @@ class CacheFileStore(ConfigMixin, SingletonMixin):
with self._store_lock:
store_current = {}
for key, record in self._store.items():
ttl_duration = record.ttl_duration
if ttl_duration:
ttl_duration = ttl_duration.total_seconds()
ttl_duration = (
record.ttl_duration.total_seconds()
if record.ttl_duration
else record.ttl_duration
)
store_current[key] = {
# Convert file-like objects to file paths for serialization
"cache_file": self._get_file_path(record.cache_file),
src/akkudoktoreos/core/cachesettings.py
+2
View File
@@ -14,8 +14,10 @@ from akkudoktoreos.config.configabc import SettingsBaseModel
class CacheCommonSettings(SettingsBaseModel):
"""Cache Configuration."""
# Retain the raw serialized default for exclude_defaults compatibility.
subpath: Optional[Path] = Field(
default="cache",
validate_default=True,
json_schema_extra={"description": "Sub-path for the EOS cache data directory."},
)
src/akkudoktoreos/core/dataabc.py
+54 -31
View File
@@ -20,11 +20,14 @@ from typing import (
TYPE_CHECKING,
Any,
Dict,
Generic,
Iterator,
Optional,
Tuple,
Type,
TypeVar,
Union,
cast,
get_args,
overload,
)
@@ -289,7 +292,8 @@ class DataRecord(DataABC, MutableMapping):
except AttributeError:
raise KeyError(f"'{key}' is not a recognized field.")
def __iter__(self) -> Iterator[str]:
# EOS collections iterate over their elements instead of BaseModel field/value pairs.
def __iter__(self) -> Iterator[str]: # type: ignore[override]
"""Iterate over the field names in the data record.
Returns:
@@ -441,7 +445,10 @@ class DataRecord(DataABC, MutableMapping):
# ==================== DataSequence ====================
class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
DataRecordT = TypeVar("DataRecordT", bound=DataRecord)
class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecordT], Generic[DataRecordT]):
"""A managed sequence of DataRecord instances with time series behavior.
The DataSequence class provides an ordered, mutable collection of DataRecord
@@ -488,7 +495,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
"""
# To be overloaded by derived classes.
records: list[DataRecord] = Field(
records: list[DataRecordT] = Field(
default_factory=list, json_schema_extra={"description": "List of data records"}
)
@@ -553,7 +560,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
f"Key '{key}' is not in writable record keys: {self.record_keys_writable}"
)
def _validate_record(self, value: DataRecord) -> None:
def _validate_record(self, value: DataRecordT) -> None:
"""Check if the provided value is a valid DataRecord with compatible keys.
Args:
@@ -629,7 +636,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
return self.record_class().record_keys_writable()
@classmethod
def record_class(cls) -> Type:
def record_class(cls) -> Type[DataRecordT]:
"""Get the class of the data record handled by this data sequence.
This method determines the class of the data record type associated with
@@ -646,6 +653,8 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
field_info = cls.model_fields["records"]
# Get the list element type from the 'type_' attribute
list_element_type = get_args(field_info.annotation)[0]
if isinstance(list_element_type, TypeVar):
list_element_type = list_element_type.__bound__
if not isinstance(list_element_type(), DataRecord):
raise ValueError(
f"Data record must be an instance of DataRecord: '{list_element_type}'."
@@ -736,17 +745,18 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
# Sequence methods
def __iter__(self) -> Iterator[DataRecord]:
# EOS collections iterate over their elements instead of BaseModel field/value pairs.
def __iter__(self) -> Iterator[DataRecordT]: # type: ignore[override]
"""Create an iterator for accessing DataRecords sequentially (memory only).
Returns:
Iterator[DataRecord]: An iterator for the records.
Iterator[DataRecordT]: An iterator for the records.
"""
return iter(self.records)
async def get_by_datetime(
self, target_datetime: DateTime, *, time_window: Optional[Duration] = None
) -> Optional[DataRecord]:
) -> Optional[DataRecordT]:
"""Get the record at the specified datetime, with an optional fallback search window.
Args:
@@ -770,7 +780,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
async def get_nearest_by_datetime(
self, target_datetime: DateTime, time_window: Optional[Duration] = None
) -> Optional[DataRecord]:
) -> Optional[DataRecordT]:
"""Get the record nearest to the specified datetime within an optional time window.
Args:
@@ -800,7 +810,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
# sync rw write access to data sequence, needs locking in case of use in async.
async def _insert_by_datetime(self, record: DataRecord) -> None:
async def _insert_by_datetime(self, record: DataRecordT) -> None:
"""Insert or merge a DataRecord into the sequence based on its datetime.
Internal implementation of `insert_by_datetime`. Callers must
@@ -822,8 +832,10 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
"""
self._validate_record(record)
# Ensure datetime objects are normalized
record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)
# _validate_record normalizes the timestamp, including a missing value.
record_date_time_timestamp = DatabaseTimestamp.from_datetime(
self._db_require_date_time(record)
)
avail_record = await self.db_get_record(record_date_time_timestamp)
if avail_record:
@@ -914,7 +926,9 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
avail_record = await self.db_get_record(db_target)
if avail_record is None:
# Create a new DataRecord if none exists
new_record = self.record_class()(date_time=date_time, **{key: values[i]})
new_record = self.record_class().model_validate(
{"date_time": date_time, key: values[i]}
)
await self.db_insert_record(new_record)
else:
# Update existing record's specified key
@@ -949,7 +963,9 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
avail_record = await self.db_get_record(db_target)
if avail_record is None:
# Create a new DataRecord if none exists
new_record = self.record_class()(date_time=date_time, **{key: value})
new_record = self.record_class().model_validate(
{"date_time": date_time, key: value}
)
await self.db_insert_record(new_record)
else:
# Update existing record's specified key
@@ -958,7 +974,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
# data sequence access usable also for async access
async def insert_by_datetime(self, record: DataRecord) -> None:
async def insert_by_datetime(self, record: DataRecordT) -> None:
"""Insert or merge a DataRecord into the sequence based on its date.
If a record with the same date exists, merges new data fields with the existing record.
@@ -1010,7 +1026,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
start_datetime: Optional[DateTime] = None,
end_datetime: Optional[DateTime] = None,
dropna: bool = True,
) -> Dict[DateTime, Any]:
) -> Dict[str, Any]:
"""Extract a dictionary indexed by the date_time field of the DataRecords.
The dictionary will contain values extracted from the specified key attribute of each DataRecord,
@@ -1130,7 +1146,8 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
end_timestamp is None or record_date_time_timestamp < end_timestamp
):
filtered_records.append(record)
dates = [record.date_time for record in filtered_records]
# The filter above already excludes records without timestamps.
dates = cast(list[DateTime], [record.date_time for record in filtered_records])
values = [getattr(record, key, None) for record in filtered_records]
return dates, values
@@ -1310,7 +1327,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
query_start = DatabaseTimestamp.to_datetime(query_start_timestamp)
if end_datetime is not None:
# We have a end datetime - look for next entry
end_timestamp = DatabaseTimestamp.from_datetime(query_end)
end_timestamp = DatabaseTimestamp.from_datetime(end_datetime)
query_end_timestamp = await self.db_next_timestamp(end_timestamp)
if query_end_timestamp is None:
# Ensure at least end_datetime is included (excluded by definition)
@@ -1382,13 +1399,12 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
floored_epoch, unit="s", tz="UTC"
)
else:
resample_origin = resample_start
resample_origin = pd.Timestamp(resample_start)
else:
# Preserve original behaviour: buckets start at the resample start.
resample_origin = resample_start
if resample_origin is None:
# We have no resample origin - take start of day as default
resample_origin = "start_day"
resample_origin = (
pd.Timestamp(resample_start) if resample_start is not None else "start_day"
)
# Check for numeric values
numeric_series = pd.to_numeric(series, errors="coerce") # ensures float64, not object dtype
@@ -1746,7 +1762,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]):
# ==================== DataProvider ====================
class DataProvider(SingletonMixin, DataSequence):
class DataProvider(SingletonMixin, DataSequence[DataRecordT], Generic[DataRecordT]):
"""Abstract base class for data providers with singleton thread-safety and configurable data parameters.
This class serves as a base for managing generic data, providing an interface for derived
@@ -2013,6 +2029,8 @@ class DataImportMixin(StartMixin):
# Generate value_datetime_mapping once if not using datetime index
if not has_datetime_index:
# Create values datetime list
if start_datetime is None:
raise ValueError("Timezone-aware datetime required")
start_timestamp = DatabaseTimestamp.from_datetime(start_datetime)
value_db_datetimes = list(
self.db_generate_timestamps(start_timestamp, values_count, interval) # type: ignore[attr-defined]
@@ -2094,6 +2112,7 @@ class DataImportMixin(StartMixin):
json_str = json_str.strip() # strip remaining white space at start and end
# Try pandas dataframe with orient="split"
import_data: PydanticDateTimeDataFrame | PydanticDateTimeData | dict[str, Any]
try:
import_data = PydanticDateTimeDataFrame.model_validate_json(json_str)
await self._import_from_dataframe(import_data.to_dataframe())
@@ -2123,7 +2142,7 @@ class DataImportMixin(StartMixin):
# Use simple dict format
try:
import_data = json.loads(json_str)
import_data = cast(dict[str, Any], json.loads(json_str))
await self._import_from_dict(
import_data, key_prefix=key_prefix, start_datetime=start_datetime, interval=interval
)
@@ -2332,7 +2351,7 @@ class DataImportMixin(StartMixin):
# ==================== DataImportProvider ====================
class DataImportProvider(DataImportMixin, DataProvider):
class DataImportProvider(DataImportMixin, DataProvider[DataRecordT], Generic[DataRecordT]):
"""Abstract base class for data providers that import generic data.
This class is designed to handle generic data provided in the form of a key-value dictionary.
@@ -2350,7 +2369,10 @@ class DataImportProvider(DataImportMixin, DataProvider):
# ==================== DataContainer ====================
class DataContainer(SingletonMixin, DataABC):
DataProviderT = TypeVar("DataProviderT", bound=DataProvider)
class DataContainer(SingletonMixin, DataABC, Generic[DataProviderT]):
"""A container for managing multiple DataProvider instances.
This class enables access to data from multiple data providers, supporting retrieval and
@@ -2363,7 +2385,7 @@ class DataContainer(SingletonMixin, DataABC):
"""
# To be overloaded by derived classes.
providers: list[DataProvider] = Field(
providers: list[DataProviderT] = Field(
default_factory=list, json_schema_extra={"description": "List of data providers"}
)
@@ -2381,7 +2403,7 @@ class DataContainer(SingletonMixin, DataABC):
return lock
@field_validator("providers", mode="after")
def check_providers(cls, value: list[DataProvider]) -> list[DataProvider]:
def check_providers(cls, value: list[DataProviderT]) -> list[DataProviderT]:
# Check each item in the list
for item in value:
if not isinstance(item, DataProvider):
@@ -2422,7 +2444,8 @@ class DataContainer(SingletonMixin, DataABC):
return
super().__init__(*args, **kwargs)
def __iter__(self) -> Iterator[str]:
# EOS collections iterate over their elements instead of BaseModel field/value pairs.
def __iter__(self) -> Iterator[str]: # type: ignore[override]
"""Return an iterator over all unique keys available across providers.
Returns:
@@ -2894,7 +2917,7 @@ class DataContainer(SingletonMixin, DataABC):
if key_error:
raise KeyError(f"key `{key}` is not in predictions")
def provider_by_id(self, provider_id: str) -> DataProvider:
def provider_by_id(self, provider_id: str) -> DataProviderT:
"""Retrieves a data provider by its unique identifier.
This method searches through the list of all available providers and
src/akkudoktoreos/core/databaseabc.py
+67 -24
View File
@@ -23,6 +23,7 @@ from typing import (
Type,
TypeVar,
Union,
cast,
)
from loguru import logger
@@ -39,6 +40,7 @@ from akkudoktoreos.core.types import (
ResampleMethod,
)
from akkudoktoreos.utils.datetimeutil import (
UTC,
DateTime,
Duration,
to_datetime,
@@ -278,9 +280,10 @@ class DatabaseBackendABC(ABC, ConfigMixin, SingletonMixin):
class DataRecordProtocol(Protocol):
date_time: DateTime
# Records may be incomplete in memory; database entry points require a timestamp.
date_time: Optional[DateTime]
def __init__(self, date_time: Any) -> None: ...
def __init__(self, date_time: Optional[DateTime]) -> None: ...
def __getitem__(self, key: str) -> Any: ...
@@ -303,15 +306,13 @@ class DatabaseTimestamp(str):
@classmethod
def from_datetime(cls, dt: DateTime) -> "DatabaseTimestamp":
if dt.tz is None:
if dt is None or dt.tz is None:
raise ValueError("Timezone-aware datetime required")
return cls(dt.in_timezone("UTC").format("YYYYMMDDTHHmmss[Z]"))
def to_datetime(self) -> DateTime:
from pendulum import parse
return parse(self)
return to_datetime(self, in_timezone="UTC")
class _DatabaseTimestampUnbound(str):
@@ -801,6 +802,19 @@ class DatabaseRecordProtocolMixin(
return None
def _db_require_date_time(self, record: T_Record) -> DateTime:
"""Validate a record's timestamp before it enters the database index."""
date_time = record.date_time
if date_time is None:
try:
namespace = self.db_namespace()
except NotImplementedError:
namespace = self.__class__.__name__
raise ValueError(
f"Database records require a datetime (namespace='{namespace}', got {record!r})"
)
return date_time
def _db_serialize_record(self, record: T_Record) -> bytes:
"""Serialize a DataRecord to bytes."""
if self.database is None:
@@ -1404,10 +1418,14 @@ class DatabaseRecordProtocolMixin(
if not candidates:
return None
# Indexed records have timestamps, validated when inserted or loaded.
# We validate again to be safe for future refactoring/ changes.
record = min(
candidates,
key=lambda r: abs(
(r.date_time - DatabaseTimestamp.to_datetime(target_timestamp)).total_seconds()
(
self._db_require_date_time(r) - DatabaseTimestamp.to_datetime(target_timestamp)
).total_seconds()
),
)
@@ -1417,7 +1435,8 @@ class DatabaseRecordProtocolMixin(
if (
abs(
(
record.date_time - DatabaseTimestamp.to_datetime(target_timestamp)
self._db_require_date_time(record)
- DatabaseTimestamp.to_datetime(target_timestamp)
).total_seconds()
)
> half_seconds
@@ -1436,7 +1455,7 @@ class DatabaseRecordProtocolMixin(
await self._db_ensure_initialized()
# Ensure normalized to UTC
db_record_date_time = DatabaseTimestamp.from_datetime(record.date_time)
db_record_date_time = DatabaseTimestamp.from_datetime(self._db_require_date_time(record))
await self._db_ensure_loaded(
start_timestamp=db_record_date_time,
@@ -1533,7 +1552,9 @@ class DatabaseRecordProtocolMixin(
continue
record = self._db_deserialize_record(value)
db_record_date_time = DatabaseTimestamp.from_datetime(record.date_time)
db_record_date_time = DatabaseTimestamp.from_datetime(
self._db_require_date_time(record)
)
# Do not resurrect explicitly deleted records
if db_record_date_time in self._db_deleted_timestamps:
@@ -1645,7 +1666,11 @@ class DatabaseRecordProtocolMixin(
start_idx = bisect.bisect_left(self._db_sorted_timestamps, start_timestamp)
for record in self.records[start_idx:]:
record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)
# Indexed records were validated on insertion or loading.
# We validate againg to be safe for future refactoring/ changes.
record_date_time_timestamp = DatabaseTimestamp.from_datetime(
self._db_require_date_time(record)
)
if start_timestamp and record_date_time_timestamp < start_timestamp:
continue
@@ -1666,7 +1691,9 @@ class DatabaseRecordProtocolMixin(
# Ensure db in memory data and metadata is initialized
await self._db_ensure_initialized()
record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time)
record_date_time_timestamp = DatabaseTimestamp.from_datetime(
self._db_require_date_time(record)
)
self._db_dirty_timestamps.add(record_date_time_timestamp)
# -----------------------------------------------------
@@ -1691,10 +1718,20 @@ class DatabaseRecordProtocolMixin(
save_items = []
for dt in self._db_dirty_timestamps:
record = self._db_record_index.get(dt)
if record:
key = self._db_key_from_timestamp(dt)
value = self._db_serialize_record(record)
save_items.append((key, value))
if record is None:
continue
# Do sanity checks on the record - date_time set and no shift since insertion
current_ts = DatabaseTimestamp.from_datetime(self._db_require_date_time(record))
if current_ts != dt:
raise RuntimeError(
f"Record date_time was mutated after insertion "
f"(index key {dt!r} != current {current_ts!r}); "
"use delete_by_datetime()+insert_by_datetime() to re-time a record."
)
# Add to save
key = self._db_key_from_timestamp(dt)
value = self._db_serialize_record(record)
save_items.append((key, value))
saved_count = len(save_items)
if saved_count:
await self.database.save_records(save_items, namespace=namespace)
@@ -1969,7 +2006,7 @@ class DatabaseRecordProtocolMixin(
# run — they are inside the age window but straddle an incomplete bucket.
raw_cutoff_epoch = int(raw_cutoff_dt.timestamp())
floored_cutoff_epoch = (raw_cutoff_epoch // interval_sec) * interval_sec
new_cutoff_dt = DateTime.fromtimestamp(floored_cutoff_epoch, tz="UTC")
new_cutoff_dt = DateTime.fromtimestamp(floored_cutoff_epoch, tz=UTC)
new_cutoff_ts = DatabaseTimestamp.from_datetime(new_cutoff_dt)
# ---- Determine window start (incremental) ------------------------
@@ -2001,7 +2038,7 @@ class DatabaseRecordProtocolMixin(
# overwritten with the same values).
raw_start_epoch = int(raw_window_start_dt.timestamp())
floored_start_epoch = (raw_start_epoch // interval_sec) * interval_sec
window_start_dt = DateTime.fromtimestamp(floored_start_epoch, tz="UTC")
window_start_dt = DateTime.fromtimestamp(floored_start_epoch, tz=UTC)
window_start_ts = DatabaseTimestamp.from_datetime(window_start_dt)
window_end_dt = new_cutoff_dt # exclusive upper bound, already aligned
@@ -2031,13 +2068,19 @@ class DatabaseRecordProtocolMixin(
# Data is already sparse — check whether timestamps are aligned.
# If every record already sits on an interval boundary, nothing to do.
# If any are misaligned, snap them in place without resampling.
# Indexed records have timestamps, validated when inserted or loaded.
# We validate again to be safe for future refactoring/ changes.
records_in_window = [
r
for r in self.records
if r.date_time is not None and window_start_dt <= r.date_time < window_end_dt
if window_start_dt <= self._db_require_date_time(r) < window_end_dt
]
# The window filter above raises an exception for records without timestamps.
misaligned = [
r for r in records_in_window if int(r.date_time.timestamp()) % interval_sec != 0
r
for r in records_in_window
if int(cast(DateTime, r.date_time).timestamp()) % interval_sec != 0
]
if not misaligned:
logger.debug(
@@ -2065,8 +2108,8 @@ class DatabaseRecordProtocolMixin(
# Process chronologically so the earliest record's values win when
# multiple records floor to the same bucket.
snapped_bucket: dict[int, dict[str, Any]] = {}
for r in sorted(records_in_window, key=lambda x: x.date_time):
ts_epoch = int(r.date_time.timestamp())
for r in sorted(records_in_window, key=lambda r: cast(DateTime, r.date_time)):
ts_epoch = int(cast(DateTime, r.date_time).timestamp())
snapped_epoch = (ts_epoch // interval_sec) * interval_sec
bucket = snapped_bucket.setdefault(snapped_epoch, {})
for key in self.record_keys_writable:
@@ -2089,7 +2132,7 @@ class DatabaseRecordProtocolMixin(
for snapped_epoch, values in snapped_bucket.items():
if not values:
continue
snapped_dt = DateTime.fromtimestamp(snapped_epoch, tz="UTC")
snapped_dt = DateTime.fromtimestamp(snapped_epoch, tz=UTC)
record = self.record_class()(date_time=snapped_dt, **values)
await self.db_insert_record(record, mark_dirty=True)
@@ -2148,7 +2191,7 @@ class DatabaseRecordProtocolMixin(
while first_bucket_epoch < int(window_start_dt.timestamp()):
first_bucket_epoch += interval_sec
compacted_timestamps = [
DateTime.fromtimestamp(first_bucket_epoch + i * interval_sec, tz="UTC")
DateTime.fromtimestamp(first_bucket_epoch + i * interval_sec, tz=UTC)
for i in range(len(array))
]
src/akkudoktoreos/core/ems.py
+1 -1
View File
@@ -92,7 +92,7 @@ class EnergyManagement(
def start_datetime(self) -> DateTime:
"""The starting datetime of the current or latest energy management."""
if EnergyManagement._start_datetime is None:
EnergyManagement.set_start_datetime()
return EnergyManagement.set_start_datetime()
return EnergyManagement._start_datetime
@computed_field # type: ignore[prop-decorator]
src/akkudoktoreos/core/logging.py
+4 -4
View File
@@ -7,7 +7,7 @@ import re
import sys
from pathlib import Path
from types import FrameType
from typing import Any, List, Optional
from typing import Any, List, Optional, cast
import pendulum
from loguru import logger
@@ -176,8 +176,8 @@ def read_file_log(
raise FileNotFoundError("Log file not found")
try:
from_dt = pendulum.parse(from_time) if from_time else None
to_dt = pendulum.parse(to_time) if to_time else None
from_dt = cast(pendulum.DateTime, pendulum.parse(from_time)) if from_time else None
to_dt = cast(pendulum.DateTime, pendulum.parse(to_time)) if to_time else None
except Exception as e:
raise ValueError(f"Invalid date/time format: {e}")
@@ -192,7 +192,7 @@ def read_file_log(
return False
if from_dt or to_dt:
try:
log_time = pendulum.parse(log["time"])
log_time = cast(pendulum.DateTime, pendulum.parse(log["time"]))
except Exception:
return False
if from_dt and log_time < from_dt:
src/akkudoktoreos/core/logsettings.py
+3 -3
View File
@@ -19,7 +19,7 @@ class LoggingCommonSettings(SettingsBaseModel):
default=None,
json_schema_extra={
"description": "Logging level for API response.",
"examples": LOGGING_LEVELS,
"examples": [*LOGGING_LEVELS],
},
)
@@ -27,7 +27,7 @@ class LoggingCommonSettings(SettingsBaseModel):
default=None,
json_schema_extra={
"description": "Logging level for logging to console.",
"examples": LOGGING_LEVELS,
"examples": [*LOGGING_LEVELS],
},
)
@@ -35,7 +35,7 @@ class LoggingCommonSettings(SettingsBaseModel):
default=None,
json_schema_extra={
"description": "Logging level for logging to file.",
"examples": LOGGING_LEVELS,
"examples": [*LOGGING_LEVELS],
},
)
src/akkudoktoreos/core/pydantic.py
+30 -15
View File
@@ -20,13 +20,17 @@ import uuid
import weakref
from copy import deepcopy
from typing import (
Annotated,
Any,
Callable,
Dict,
List,
Optional,
Self,
Type,
TypeVar,
Union,
cast,
get_args,
get_origin,
)
@@ -39,6 +43,7 @@ from pydantic import (
BaseModel,
ConfigDict,
Field,
GetPydanticSchema,
PrivateAttr,
RootModel,
ValidationError,
@@ -49,6 +54,7 @@ from pydantic.fields import ComputedFieldInfo, FieldInfo
from akkudoktoreos.utils.datetimeutil import (
DateTime,
Duration,
to_datetime,
to_duration,
to_timezone,
@@ -415,7 +421,7 @@ class PydanticModelNestedValueMixin:
# If this is the final key, set the value
if is_final_key:
try:
model.validate_and_set(key, value)
getattr(model, "validate_and_set")(key, value)
except Exception as e:
raise ValueError(f"Error updating model: {e}") from e
return
@@ -549,10 +555,10 @@ class PydanticModelNestedValueMixin:
if not inspect.isclass(model):
raise TypeError(f"Model '{model}' is not of class type.")
if key not in model.model_fields: # type: ignore[attr-defined]
if key not in model.model_fields:
raise TypeError(f"Field '{key}' does not exist in model '{model.__name__}'.")
field_annotation = model.model_fields[key].annotation # type: ignore[attr-defined]
field_annotation = model.model_fields[key].annotation
if not field_annotation:
raise TypeError(
f"Missing type annotation for field '{key}' in model '{model.__name__}'."
@@ -563,6 +569,8 @@ class PydanticModelNestedValueMixin:
while queue:
annotation = queue.pop(0)
if isinstance(annotation, TypeVar):
annotation = annotation.__bound__ or Any
origin = get_origin(annotation)
args = get_args(annotation)
@@ -679,7 +687,7 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel):
"""Resets the fields to their default values."""
for field_name, field_info in self.__class__.model_fields.items():
if field_info.default_factory is not None: # Handle fields with default_factory
default_value = field_info.default_factory()
default_value = field_info.get_default(call_default_factory=True)
else:
default_value = field_info.default
try:
@@ -707,7 +715,7 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel):
return self.model_dump()
@classmethod
def from_dict(cls: Type["PydanticBaseModel"], data: dict) -> "PydanticBaseModel":
def from_dict(cls, data: dict) -> Self:
"""Create a PydanticBaseModel instance from a dictionary.
Args:
@@ -735,7 +743,7 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel):
return self.model_dump_json()
@classmethod
def from_json(cls: Type["PydanticBaseModel"], json_str: str) -> "PydanticBaseModel":
def from_json(cls, json_str: str) -> Self:
"""Create an instance of the PydanticBaseModel class or its subclass from a JSON string.
Args:
@@ -926,6 +934,10 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel):
return None
DateTimeDataInput = dict[str, str | list[float | int | str | None]]
DateTimeDataValues = dict[str, str | DateTime | Duration | list[float | int | str | None]]
class PydanticDateTimeData(RootModel):
"""Pydantic model for time series data with consistent value lengths.
@@ -948,13 +960,16 @@ class PydanticDateTimeData(RootModel):
"""
root: Dict[str, Union[str, List[Union[float, int, str, None]]]]
# The wire format contains strings; validate_root normalizes the two
# indexing values to Pendulum objects. Keep the existing input schema.
root: Annotated[
DateTimeDataValues,
GetPydanticSchema(lambda source_type, handler: handler(DateTimeDataInput)),
]
@field_validator("root", mode="after")
@classmethod
def validate_root(
cls, value: Dict[str, Union[str, List[Union[float, int, str, None]]]]
) -> Dict[str, Union[str, List[Union[float, int, str, None]]]]:
def validate_root(cls, value: dict[str, Any]) -> DateTimeDataValues:
# Validate that all keys are strings
if not all(isinstance(k, str) for k in value.keys()):
raise ValueError("All keys in the dictionary must be strings.")
@@ -977,7 +992,7 @@ class PydanticDateTimeData(RootModel):
return value
def to_dict(self) -> Dict[str, Union[str, List[Union[float, int, str, None]]]]:
def to_dict(self) -> DateTimeDataValues:
"""Convert the model to a plain dictionary.
Returns:
@@ -1176,8 +1191,8 @@ class PydanticDateTimeDataFrame(PydanticBaseModel):
df[col] = df[col].dt.tz_convert(resolved_tz)
return cls(
data=df.to_dict(orient="index"),
dtypes={col: str(dtype) for col, dtype in df.dtypes.items()},
data=cast(dict[str, dict[str, Any]], df.to_dict(orient="index")),
dtypes=cast(dict[str, str], {col: str(dtype) for col, dtype in df.dtypes.items()}),
tz=resolved_tz,
datetime_columns=datetime_columns,
)
@@ -1401,10 +1416,10 @@ class PydanticDateTimeSeries(PydanticBaseModel):
series.index = index
if len(index) > 0:
tz = to_datetime(series.index[0]).timezone.name
tz = to_datetime(series.index[0]).timezone_name
return cls(
data=series.to_dict(),
data=cast(dict[str, Any], series.to_dict()),
dtype=str(series.dtype),
tz=tz,
)