From eb9e966de965f288553f41842feb5880b21500b0 Mon Sep 17 00:00:00 2001 From: Bobby Noelte Date: Wed, 15 Jul 2026 16:38:53 +0200 Subject: [PATCH] fix: move data management to async (#1015) FAstAPI is an async framework. Data may be imported and exported, load and save, set and get asynchronously. Prevent interleaving data operations to corrupt the data. In the previous design sync and async data access was intermixed leading to data corruption. The basic data classes DataSequence and DataContainer and the derived classes like Provider and Measurement now are async. Data access is protected by several async locks. To support the async design of the data classes the database interface became async. The energy management is also adapted to the new async design. Optimization is still off-loaded to another thread, but the prepration for the optimization and the post optimization actions now follow the async design. Adapter operations are now also protected by async locks. Tests were adapted to the async design and new tests were created. Besides this major fix several other improvements and fixes are included in this PR. * fix: key_to_dict/list/array only regard data records with key value set. Before the exclusion of no value data records was only done if the dropna flag was set. * fix: test for visual result pdf generation Due to updates in the library the generated charts text was a little bit different. Adapt the test to create the comaprison pdf in the test data durectory and update the reference pdf. * chore: Remove MutableMapping from DataSequence and DataContainer. Mutable Mapping does not fit to the now async design. * chore: Add NoDB database backend This backend implements the full database backend interface but performs no actual persistence. It is intended for configurations where database persistence is disabled (`provider=None`). * chore: Improve measurement data import testing with real world scenarios. Added two new endpoints to support testing. * chore: Add mermaid to supported documentation tools * chore: Add documentation about async design * chore: Add documentation about generic data handling Covers the basics of measurement and prediction time series data handling. * chore: Add empty lines around markdown lists. * chore: sync pre-commit config to updated package versions Signed-off-by: Bobby Noelte --- .env | 2 +- .pre-commit-config.yaml | 16 +- Makefile | 2 +- config.yaml | 2 +- docs/_generated/configdatabase.md | 3 +- docs/_generated/openapi.md | 59 +- docs/akkudoktoreos/adapter/adapternodered.md | 3 + docs/akkudoktoreos/database.md | 4 + docs/akkudoktoreos/optimpost.md | 1 + docs/conf.py | 4 + docs/develop/async.md | 184 ++ docs/develop/dataabc.md | 568 ++++ docs/index.md | 1 + openapi.json | 107 +- pyproject.toml | 1 + src/akkudoktoreos/adapter/adapterabc.py | 47 +- src/akkudoktoreos/adapter/homeassistant.py | 12 +- src/akkudoktoreos/adapter/nodered.py | 2 +- src/akkudoktoreos/core/dataabc.py | 1036 ++++--- src/akkudoktoreos/core/database.py | 662 ++++- src/akkudoktoreos/core/databaseabc.py | 343 ++- src/akkudoktoreos/core/ems.py | 386 +-- src/akkudoktoreos/core/pydantic.py | 6 +- src/akkudoktoreos/measurement/measurement.py | 24 +- .../optimization/genetic/geneticparams.py | 18 +- .../optimization/genetic/geneticsolution.py | 4 +- .../prediction/elecpriceakkudoktor.py | 10 +- .../prediction/elecpriceenergycharts.py | 10 +- .../prediction/elecpricefixed.py | 4 +- .../prediction/elecpriceimport.py | 8 +- .../prediction/feedintarifffixed.py | 4 +- .../prediction/feedintariffimport.py | 8 +- .../prediction/loadakkudoktor.py | 29 +- src/akkudoktoreos/prediction/loadimport.py | 10 +- src/akkudoktoreos/prediction/loadvrm.py | 9 +- src/akkudoktoreos/prediction/prediction.py | 2 +- src/akkudoktoreos/prediction/predictionabc.py | 6 +- .../prediction/pvforecastakkudoktor.py | 20 +- .../prediction/pvforecastimport.py | 8 +- src/akkudoktoreos/prediction/pvforecastvrm.py | 10 +- .../prediction/weatherbrightsky.py | 26 +- .../prediction/weatherclearoutside.py | 6 +- src/akkudoktoreos/prediction/weatherimport.py | 8 +- .../prediction/weatheropenmeteo.py | 20 +- src/akkudoktoreos/server/eos.py | 247 +- tests/single_test_optimization.py | 16 +- tests/single_test_prediction.py | 7 +- tests/test_adapternodered.py | 25 +- tests/test_dataabc.py | 1627 ----------- tests/test_dataabccompact.py | 544 ++-- tests/test_dataabccontainer.py | 308 +++ tests/test_dataabcprovider.py | 332 +++ tests/test_dataabcrecord.py | 312 +++ tests/test_dataabcsequence.py | 979 +++++++ tests/test_dataabcsequencedb.py | 701 +++++ tests/test_database.py | 1604 +++++------ tests/test_databaseabc.py | 615 ++-- tests/test_docsphinx.py | 69 +- tests/test_elecpriceakkudoktor.py | 293 +- tests/test_elecpriceenergycharts.py | 328 +-- tests/test_elecpricefixed.py | 36 +- tests/test_elecpriceimport.py | 129 +- tests/test_geneticoptimize.py | 1 + tests/test_inverter_efficiency.py | 6 +- tests/test_loadakkudoktor.py | 315 +-- tests/test_loadvrm.py | 99 +- tests/test_measurement.py | 62 +- tests/test_prediction.py | 5 +- tests/test_predictionabc.py | 96 +- tests/test_pvforecastakkudoktor.py | 36 +- tests/test_pvforecastimport.py | 10 +- tests/test_pvforecastvrm.py | 10 +- tests/test_system.py | 281 ++ tests/test_visualize.py | 8 +- tests/test_weatherbrightsky.py | 10 +- tests/test_weatherclearoutside.py | 17 +- tests/test_weatherimport.py | 9 +- tests/test_weatheropenmeteo.py | 21 +- tests/testdata/docs/_generated/config.md | 25 + .../testdata/docs/_generated/configadapter.md | 238 ++ tests/testdata/docs/_generated/configcache.md | 28 + .../docs/_generated/configdatabase.md | 73 + .../testdata/docs/_generated/configdevices.md | 549 ++++ .../docs/_generated/configelecprice.md | 275 ++ tests/testdata/docs/_generated/configems.md | 30 + .../testdata/docs/_generated/configexample.md | 260 ++ .../docs/_generated/configfeedintariff.md | 149 + .../testdata/docs/_generated/configgeneral.md | 67 + tests/testdata/docs/_generated/configload.md | 163 ++ .../testdata/docs/_generated/configlogging.md | 45 + .../docs/_generated/configmeasurement.md | 75 + .../docs/_generated/configoptimization.md | 104 + .../docs/_generated/configprediction.md | 28 + .../docs/_generated/configpvforecast.md | 346 +++ .../testdata/docs/_generated/configserver.md | 42 + tests/testdata/docs/_generated/configutils.md | 23 + .../testdata/docs/_generated/configweather.md | 116 + tests/testdata/test_example_report.pdf | Bin 90937 -> 97354 bytes uv.lock | 2463 +++++++++-------- 99 files changed, 11971 insertions(+), 5981 deletions(-) create mode 100644 docs/develop/async.md create mode 100644 docs/develop/dataabc.md delete mode 100644 tests/test_dataabc.py create mode 100644 tests/test_dataabccontainer.py create mode 100644 tests/test_dataabcprovider.py create mode 100644 tests/test_dataabcrecord.py create mode 100644 tests/test_dataabcsequence.py create mode 100644 tests/test_dataabcsequencedb.py create mode 100644 tests/testdata/docs/_generated/config.md create mode 100644 tests/testdata/docs/_generated/configadapter.md create mode 100644 tests/testdata/docs/_generated/configcache.md create mode 100644 tests/testdata/docs/_generated/configdatabase.md create mode 100644 tests/testdata/docs/_generated/configdevices.md create mode 100644 tests/testdata/docs/_generated/configelecprice.md create mode 100644 tests/testdata/docs/_generated/configems.md create mode 100644 tests/testdata/docs/_generated/configexample.md create mode 100644 tests/testdata/docs/_generated/configfeedintariff.md create mode 100644 tests/testdata/docs/_generated/configgeneral.md create mode 100644 tests/testdata/docs/_generated/configload.md create mode 100644 tests/testdata/docs/_generated/configlogging.md create mode 100644 tests/testdata/docs/_generated/configmeasurement.md create mode 100644 tests/testdata/docs/_generated/configoptimization.md create mode 100644 tests/testdata/docs/_generated/configprediction.md create mode 100644 tests/testdata/docs/_generated/configpvforecast.md create mode 100644 tests/testdata/docs/_generated/configserver.md create mode 100644 tests/testdata/docs/_generated/configutils.md create mode 100644 tests/testdata/docs/_generated/configweather.md diff --git a/.env b/.env index c73a9e18..7264f758 100644 --- a/.env +++ b/.env @@ -11,7 +11,7 @@ DOCKER_COMPOSE_DATA_DIR=${HOME}/.local/share/net.akkudoktor.eos # ----------------------------------------------------------------------------- # Image / build # ----------------------------------------------------------------------------- -VERSION=0.3.0.dev2604141105859917 +VERSION=0.3.0.dev2607150960221650 PYTHON_VERSION=3.13.9 # ----------------------------------------------------------------------------- diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 272be86b..131abcd8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,13 +15,13 @@ repos: # --- Import sorting --- - repo: https://github.com/PyCQA/isort - rev: 7.0.0 + rev: 8.0.1 hooks: - id: isort # --- Linting + Formatting via Ruff --- - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.7 + rev: v0.15.21 hooks: # Run the linter and fix simple isssues automatically - id: ruff @@ -31,20 +31,20 @@ repos: # --- Static type checking --- - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.19.1 + rev: v2.3.0 hooks: - id: mypy additional_dependencies: - - types-requests==2.33.0.20260408 - - pandas-stubs==3.0.0.260204 + - types-requests==2.33.0.20260712 + - pandas-stubs==3.0.3.260530 - tokenize-rt==6.2.0 - - types-docutils==0.22.3.20260322 - - types-PyYaml==6.0.12.20260408 + - types-docutils==0.22.3.20260518 + - types-PyYaml==6.0.12.20260518 pass_filenames: false # --- Markdown linter --- - repo: https://github.com/jackdewinter/pymarkdown - rev: v0.9.36 + rev: v0.9.39 hooks: - id: pymarkdown files: ^docs/ diff --git a/Makefile b/Makefile index 0b9178a4..0bde7b03 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ install: version-txt # Target to rebuild the virtual environment. update-env: @echo "Rebuilding virtual environment to match pyproject.toml..." - uv rebuild + $(UV) sync --upgrade --extra dev @echo "Environment rebuilt." # Target to create a distribution. diff --git a/config.yaml b/config.yaml index d41cf054..605c554c 100644 --- a/config.yaml +++ b/config.yaml @@ -6,7 +6,7 @@ # the root directory (no add-on folder as usual). name: "Akkudoktor-EOS" -version: "0.3.0.dev2604141105859917" +version: "0.3.0.dev2607150960221650" slug: "eos" description: "Akkudoktor-EOS add-on" url: "https://github.com/Akkudoktor-EOS/EOS" diff --git a/docs/_generated/configdatabase.md b/docs/_generated/configdatabase.md index 01a7a708..92c6ec49 100644 --- a/docs/_generated/configdatabase.md +++ b/docs/_generated/configdatabase.md @@ -64,7 +64,8 @@ None indicates forever. Database namespaces may have diverging definitions. | "batch_size": 100, "providers": [ "LMDB", - "SQLite" + "SQLite", + "NoDB" ] } } diff --git a/docs/_generated/openapi.md b/docs/_generated/openapi.md index e60ed7f5..f0af6c6f 100644 --- a/docs/_generated/openapi.md +++ b/docs/_generated/openapi.md @@ -1,6 +1,6 @@ # Akkudoktor-EOS -**Version**: `v0.3.0.dev2604141105859917` +**Version**: `v0.3.0.dev2607150960221650` **Description**: This project provides a comprehensive solution for simulating and optimizing an energy system based on renewable energy sources. With a focus on photovoltaic (PV) systems, battery storage (batteries), load management (consumer requirements), heat pumps, electric vehicles, and consideration of electricity price data, this system enables forecasting and optimization of energy flow and costs over a specified period. @@ -338,6 +338,31 @@ Returns: --- +## POST /v1/admin/database/save + + +**Links**: [local](http://localhost:8503/docs#/default/fastapi_admin_database_save_post_v1_admin_database_save_post), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/fastapi_admin_database_save_post_v1_admin_database_save_post) + + +Fastapi Admin Database Save Post + + +```python +""" +Save in memory data to database. + +Returns: + data (dict): The database stats after saving the records. +""" +``` + + +**Responses**: + +- **200**: Successful Response + +--- + ## GET /v1/admin/database/stats @@ -882,6 +907,38 @@ Fastapi Measurement Keys Get --- +## DELETE /v1/measurement/range + + +**Links**: [local](http://localhost:8503/docs#/default/fastapi_measurement_range_delete_v1_measurement_range_delete), [eos](https://petstore3.swagger.io/?url=https://raw.githubusercontent.com/Akkudoktor-EOS/EOS/refs/heads/main/openapi.json#/default/fastapi_measurement_range_delete_v1_measurement_range_delete) + + +Fastapi Measurement Range Delete + + +```python +""" +Delete measurement values for a key within a datetime range. +""" +``` + + +**Parameters**: + +- `key` (query, required): Measurement key. + +- `start_datetime` (query, optional): Start datetime. + +- `end_datetime` (query, optional): End datetime. + +**Responses**: + +- **200**: Successful Response + +- **422**: Validation Error + +--- + ## GET /v1/measurement/series diff --git a/docs/akkudoktoreos/adapter/adapternodered.md b/docs/akkudoktoreos/adapter/adapternodered.md index 40e21c53..ca6b95ac 100644 --- a/docs/akkudoktoreos/adapter/adapternodered.md +++ b/docs/akkudoktoreos/adapter/adapternodered.md @@ -38,6 +38,7 @@ Typical use cases with Node-RED: #### 1. Enable and configure the Node-RED adapter EOS must be configured with access to the Node-RED instance in Config->adapter. + * prerequisite is an already installed and running Node-RED instance * adapter.nodered.host: 192.168.1.109 (example IP of your Node-RED instance) * adapter.nodered.port: 1880 (default) @@ -46,10 +47,12 @@ EOS must be configured with access to the Node-RED instance in Config->adapter. #### 2. Run energy optimisation Before the run, EOS receives: + * EOS receives measurement values via HTTP-IN "eos_data_acquisition" before optimisation. (The HTTP-IN Node "eos_data_acquisition" is NOT yet functional) After the run, EOS provides: + * The device instruction and solution entities for the current time slot via HTTP-IN "Control Dispatch". ### Configuration steps in NodeRED diff --git a/docs/akkudoktoreos/database.md b/docs/akkudoktoreos/database.md index e8b0d362..77fab0af 100644 --- a/docs/akkudoktoreos/database.md +++ b/docs/akkudoktoreos/database.md @@ -284,6 +284,7 @@ DatabaseTimestamp.from_datetime(dt: DateTime) -> "20241027T123456[Z]" ``` **Properties:** + - Always stored in UTC (timezone-aware required) - Lexicographically sortable - Bijective conversion to/from `pendulum.DateTime` @@ -330,6 +331,7 @@ The system uses a progressive loading model to minimize memory footprint: ### Boundary Extension When loading a range `[start, end)`, the system automatically extends boundaries to include: + - **First record before** `start` (for interpolation/context) - **First record at or after** `end` (for closing boundary) @@ -348,6 +350,7 @@ SELECT * FROM records WHERE namespace='measurement' ``` **Default Namespace:** + - Can be set during `open(namespace="default")` - Operations with `namespace=None` use the default - Each record class typically defines its own namespace via `db_namespace()` @@ -537,6 +540,7 @@ db_vacuum(keep_timestamp=cutoff) # Keep from cutoff onward ``` **Strategy:** + - Computes cutoff relative to `max_timestamp - keep_hours` - Deletes all records before cutoff - Immediately persists changes via `db_save_records()` diff --git a/docs/akkudoktoreos/optimpost.md b/docs/akkudoktoreos/optimpost.md index 5451550e..50fa7c19 100644 --- a/docs/akkudoktoreos/optimpost.md +++ b/docs/akkudoktoreos/optimpost.md @@ -223,6 +223,7 @@ penalty = ac_wh_charged × (break_even_price − best_uncovered_price) × factor ``` where: + - `break_even_price = charge_price / η_round_trip` - `best_uncovered_price` = highest future price not already covered by free PV battery energy - `factor` = `optimization.genetic.penalties.ac_charge_break_even` (default `1.0`) diff --git a/docs/conf.py b/docs/conf.py index a1f50820..602930f0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,6 +33,7 @@ extensions = [ "sphinx_rtd_theme", "myst_parser", "sphinx_tabs.tabs", + "sphinxcontrib.mermaid", ] templates_path = ["_templates"] @@ -141,3 +142,6 @@ napoleon_use_rtype = True napoleon_preprocess_types = False napoleon_type_aliases = None napoleon_attr_annotations = True + +# -- Options for mermaid ------------------------------------------------- +mermaid_output_format = "html" # no mmdc to be installed diff --git a/docs/develop/async.md b/docs/develop/async.md new file mode 100644 index 00000000..fb2534d3 --- /dev/null +++ b/docs/develop/async.md @@ -0,0 +1,184 @@ +# Asynchronous Design of Akkudoktor‑EOS + +The Akkudoktor‑EOS server is built on **FastAPI** and is transitioning to a fully asynchronous +design to improve scalability, responsiveness, and resource utilisation, especially for +long‑running or I/O‑bound operations. +The server manages a variety of background tasks through a central **Retention Manager**, which +schedules and supervises all periodic and asynchronous work. + +## Core Asynchronous Components + +- **FastAPI REST Interface** + All HTTP endpoints are defined as `async def` handlers (or synchronous where no I/O waiting is + required), allowing the server to handle many concurrent connections without blocking the event + loop. + +- **Retention Manager** + A dedicated component responsible for orchestrating all background tasks. It runs an asynchronous + tick loop that executes registered functions at configurable intervals. The manager also provides + graceful shutdown handling, waiting for in‑flight jobs to finish. + + +- **Managed Asynchronous Tasks** + The following tasks are registered with the Retention Manager and run periodically: + + | Task Name | Function | Interval Configuration | Description | + |-----------|----------|------------------------|-------------| + | `supervise_eosdash` | `supervise_eosdash` | `server/eosdash_supervise_interval_sec` | Monitors and restarts the EOSdash UI process if needed. | + | `autosave_config` | `autosave_config` | `general/config_save_interval_sec` | Saves the current configuration to disk automatically. | + | `cache_clear` | `cache_clear` | `cache/cleanup_interval` | Removes expired entries from the cache. | + | `save_eos_database` | `save_eos_database` | `database/autosave_interval_sec` | Persists in‑memory measurement and prediction data to the database. | + | `compact_eos_database` | `compact_eos_database` | `database/compaction_interval_sec` | Compacts and vacuums the database to reclaim space and improve performance. | + | `manage_energy` | `ems_manage_energy` | `ems/interval` | Core energy management loop: triggers predictions, optimisation, and device control. | + + The `manage_energy` task is the central orchestrator that itself calls asynchronous prediction + updates, adapter scheduling and optimisation runs. It uses the `EnergyManagementSystem` + (`get_ems()`) which internally manages concurrency to ensure only one energy management run + happens at a time. + + +- **On‑Demand Asynchronous Endpoints** + Several REST endpoints are asynchronous and delegate heavy work to the `EnergyManagementSystem`. + Examples include: + - `POST /v1/prediction/update` – updates all prediction providers asynchronously. + - `POST /optimize` – runs a genetic optimisation (deprecated, but still async). + - `POST /v1/admin/server/restart` – spawns a new process and schedules a shutdown task. + +## Asynchronous Workflow + +The diagram below shows how periodic tasks are registered and executed by the Retention Manager, +and how a client request can trigger an asynchronous update. + +```mermaid + +sequenceDiagram + participant Client + participant FastAPI as FastAPI (Async) + participant Retention as Retention Manager (Async) + participant EMS as EnergyManagementSystem (Async) + participant Adapter as Adapter (Async) + participant Prediction as Prediction (Async) + participant Measurement as Measurement (Async) + participant DatabaseRecords as DatabaseRecords (Async) + participant Database as Database (Sync) + Note over Retention: On startup (lifespan) + FastAPI->>Retention: register tasks with intervals + Retention-->>FastAPI: tasks registered + Retention->>Retention: start tick loop + + loop every EMS interval + Retention->>EMS: run(mode=PREDICTION+OPTIMIZATION) + EMS->>Adapter: update_data, DATA_AQUISITION + Adapter->>Measurement: update_value + Measurement-->>Adapter: done + Adapter-->>EMS: done + EMS->>Prediction: update_data, FORECAST_RETRIEVAL + Prediction-->>EMS: done + EMS->>Adapter: update_data, CONTROL_DISPATCH + Adapter-->>EMS: done + EMS-->>Retention: done + end + + loop every DatabaseRecords autosave interval + Retention->>Prediction: save + Prediction->>DatabaseRecords: db_save_records + DatabaseRecords-->>Prediction: done + Prediction-->>Retention: done + Retention->>Measurement: save + Measurement->>DatabaseRecords: db_save_records + DatabaseRecords-->>Measurement: done + Measurement-->>Retention: done + end + + loop every DatabaseRecords compaction interval + Retention->>Prediction: db_compact + Prediction->>DatabaseRecords: db_delete_records + DatabaseRecords-->>Prediction: done + Prediction->>DatabaseRecords: db_save_records + DatabaseRecords-->>Prediction: done + Prediction-->>Retention: done + Retention->>Measurement: db_compact + Measurement->>DatabaseRecords: db_delete_records + DatabaseRecords-->>Measurement: done + Measurement->>DatabaseRecords: db_save_records + DatabaseRecords-->>Measurement: done + Measurement-->>Retention: done + end + + Client->>FastAPI: POST /v1/prediction/update + FastAPI->>EMS: await run(mode=PREDICTION) + EMS-->>FastAPI: predictions updated + FastAPI-->>Client: 200 OK +``` + +For server shutdown or restart, the Retention Manager’s task is cancelled, and the server waits +for in‑flight jobs to finish (shutdown timeout = 10 seconds). The state is saved via +`save_eos_state()`. + +## Asynchronous vs. Synchronous + +- **REST endpoints** that perform I/O or heavy computation are `async def`. +- **Retention Manager** uses `asyncio.create_task()` to run the tick loop and individual task + executions. +- **Database** that synchronizes database access is `async def`, but the database backends + **DataBaseBackendABC** are synchronous. +- **DatabaseRecordProtocolMixin** provides asynchronous access to in memory data records and + database storage. +- **Prediction** and **Measurement** provide asynchronous access to the undelying database using + the **DatabaseRecordProtocolMixin**. +- **Energy management runs** are serialised using an internal lock (via `EMS.run()`) to avoid + overlapping optimisation cycles. +- **Process management** for shutdown/restart uses `asyncio.create_task(server_shutdown_task())` + to gracefully terminate after a delay. + +## Benefits of Full Asynchrony + +- **Higher throughput** – FastAPI’s event loop can handle thousands of idle keep‑alive connections. +- **Lower latency** – Long‑running tasks (database compaction, prediction updates) do not block HTTP + responses. +- **Easier maintenance** – Uniform async patterns replace mixed sync/async code. +- **Better resource usage** – The Retention Manager can throttle, skip, or prioritise tasks based on + configuration and system load. +- **Graceful shutdown** – All background tasks are cancelled cooperatively, and state is saved + before exit. + +## Additional Asynchronous Patterns + +Beyond the Retention Manager, the server uses: + +- **Asynchronous shutdown/restart** – When a restart is requested, a new process is spawned, and the + current process schedules a delayed termination (`server_shutdown_task`). This ensures zero + downtime if the new process starts before the old one exits. +- **Concurrent request handling** – Multiple clients can call prediction update endpoints + simultaneously; the `EMS.run()` method serialises them internally, preventing race conditions. +- **Non‑blocking logging** – Log entries are written asynchronously (via loguru’s async sinks when + configured). + +The combination of FastAPI, the Retention Manager, and asynchronous I/O enables Akkudoktor‑EOS to +run efficiently on resource‑constrained devices (e.g., Raspberry Pi) while maintaining responsive +REST APIs and reliable background data maintenance. + +## Asynchronous Detailed Design + +### Database + +The database backends are synchronous. The selected backend is wrapped in the asynchronous +thread-safe database singleton defined by the Database class. + +### DatabaseRecordProtocol, DataRecordProtocol + +The DatabaseRecordProtocol completely manages in memory records and database storage. It acesses +the database backend by the database singleton and is therefor asynchronous. The +DatabaseRecordProtocol has a minimum expectation for data records defined by the DataRecordProtocol. +Data records are expected to be synchronous. + +### Data Records + +Data records are synchronous. + +### Data Sequence, Data Provide, Data Container + +Data sequences, the derived data provider, and the data provider aggregation data containers are +asynchronous. Data sequences' data can be backed by the database using the +DatabaseRecordProtocol to access the data. That is the reason why all the classes are +asynchronous. diff --git a/docs/develop/dataabc.md b/docs/develop/dataabc.md new file mode 100644 index 00000000..30654789 --- /dev/null +++ b/docs/develop/dataabc.md @@ -0,0 +1,568 @@ +# `dataabc` — Generic Data Handling + +## Overview + +The `dataabc` module provides the foundational abstractions for managing time-series data +in EOS. It defines a layered class hierarchy that covers individual data records, ordered +sequences of records, singleton data providers, and multi-provider containers. + +All classes in this module are designed for use in predictive modelling workflows and share +three cross-cutting concerns: + +- **Configuration access** via `ConfigMixin`, exposing the global EOS configuration as + `self.config`. +- **Database persistence** via `DatabaseRecordProtocolMixin`, providing optional storage + in a time-series database alongside in-memory records. +- **Async safety** via a three-level locking scheme described in detail in the + [Concurrency and Locking](#concurrency-and-locking) section. + +## Class Hierarchy + +```text +PydanticBaseModel +└── DataABC (ConfigMixin, StartMixin) + ├── DataRecord (MutableMapping) + └── DataSequence (DatabaseRecordProtocolMixin) + └── DataProvider (SingletonMixin) + └── DataImportProvider (DataImportMixin) + +DataContainer (SingletonMixin, MutableMapping) +DataImportMixin (StartMixin) +``` + +## Classes + +### `DataABC` + +Base class for all data-handling objects. Inherits from `ConfigMixin` and `StartMixin`, +making the global EOS configuration available as `self.config` on every derived instance. +Not intended to be instantiated directly. + +### `DataRecord` + +A single measurement or forecast point at a specific datetime, implemented as a +`MutableMapping` so that field values can be accessed and mutated both as dictionary +entries (`record["field"]`) and as attributes (`record.field`). + +#### Key concepts + +**Static fields** are declared as Pydantic model fields in the class body. They are always +present and validated on assignment. + +**Configured fields** are dynamic: their names are returned by the classmethod +`configured_data_keys()`, which derived classes override to pull key names from the EOS +configuration. These keys are stored in the internal `configured_data` dict but appear +transparent to callers — they show up in `dir()`, `iter()`, and attribute access just like +static fields. + +```python +class MeasurementDataRecord(DataRecord): + @classmethod + def configured_data_keys(cls) -> Optional[list[str]]: + return cls.config.measurement.keys +``` + +#### Important methods + +| Method | Description | +|---|---| +| `record_keys()` | All field names, including configured keys. | +| `record_keys_writable()` | Subset of `record_keys()` that can be written. | +| `key_from_description(desc)` | Fuzzy-matches a description string to a field name. | +| `keys_from_descriptions(descs)` | Batch version of `key_from_description`. | + +--- + +### `DataSequence` + +An ordered, mutable collection of `DataRecord` instances with time-series behaviour. +Records are always kept sorted in ascending `date_time` order. `DataSequence` is also the +level at which **async safety is enforced** — see +[Concurrency and Locking](#concurrency-and-locking). + +Derived classes must redeclare the `records` field with the concrete record type: + +```python +class Measurement(DataSequence): + records: list[MeasurementDataRecord] = Field(default_factory=list) +``` + +#### Public async interface of `DataSequence` + +All methods that mutate sequence state are `async`. Callers in an async context (e.g. +FastAPI endpoint handlers) must `await` them. + +| Method | Description | +|---|---| +| `await insert_by_datetime(record)` | Insert or merge a record by its datetime. | +| `await update_value(date, key, value)` | Insert or update a single field value at a datetime. | +| `await key_from_lists(key, dates, values)` | Populate a field from parallel date/value lists. | +| `await key_from_series(key, series)` | Populate a field from a `pd.Series`. | +| `await save()` | Persist all records to the configured storage backend. | +| `await load()` | Load records from the configured storage backend. | +| `await import_from_dict(data)` | Import records from a key-value dictionary. | +| `await import_from_dataframe(df)` | Import records from a `pd.DataFrame`. | +| `await import_from_json(json_str)` | Import records from a JSON string. | +| `await import_from_file(path)` | Import records from a JSON file. | + +#### Internal sync interface of `DataSequence` + +Each public async method has a private sync counterpart prefixed with `_`. These are +intended **only** for callers that already hold the appropriate locks, or that are running +in a purely sequential context (startup, `_load()` internals, tests). See +[Choosing the right call site](#choosing-the-right-call-site). + +| Internal method | Corresponding public method | +|---|---| +| `_insert_by_datetime(record)` | `.insert_by_datetime()` | +| `_update_value(date, ...)` | `.update_value()` | +| `_key_from_lists(key, dates, values)` | `.key_from_lists()` | +| `_key_from_series(key, series)` | `.key_from_series()` | +| `_save()` | `.save()` | +| `_load()` | `.load()` | +| `_import_from_dict(...)` | `.import_from_dict()` | +| `_import_from_dataframe(...)` | `.import_from_dataframe()` | +| `_import_from_json(...)` | `.import_from_json()` | +| `_import_from_file(...)` | `.import_from_file()` | + +#### Read-only methods (always sync) of `DataSequence` + +These methods do not modify sequence state and require no locking: + +| Method | Description | +|---|---| +| `get_by_datetime(dt)` | Exact or nearest record lookup. | +| `get_nearest_by_datetime(dt)` | Nearest record within an optional time window. | +| `key_to_dict(key, ...)` | Extract a `{datetime: value}` dict for a key. | +| `key_to_lists(key, ...)` | Extract parallel date and value lists. | +| `key_to_series(key, ...)` | Extract a `pd.Series` indexed by datetime. | +| `key_to_array(key, ...)` | Extract a resampled `np.ndarray` at a fixed interval. | +| `key_to_value(key, dt)` | Scalar lookup nearest to a datetime. | +| `to_dataframe(...)` | Convert all records to a `pd.DataFrame`. | +| `delete_by_datetime(...)` | Delete records within a datetime range. | +| `key_delete_by_datetime(key, ...)` | Set a field to `None` across a datetime range. | + +#### Computed properties of `DataSequence` + +| Property | Description | +|---|---| +| `min_datetime` | Earliest datetime in the sequence. | +| `max_datetime` | Latest datetime in the sequence. | +| `record_keys` | All field names for this sequence's record type. | +| `record_keys_writable` | Writable subset of `record_keys`. | + +### `DataProvider` + +Abstract singleton base class for objects that own and update a `DataSequence`. Each +concrete provider represents one data source (e.g. weather forecast, load measurement, +grid price). `DataProvider` is a specialisation of `DataSequence` and inherits its full +async interface and locking infrastructure — `_record_lock` and `_sequence_lock` — without +adding any new locks of its own. + +Derived classes must implement: + +| Abstract method | Description | +|---|---| +| `provider_id() -> str` | Unique string identifier for this provider. | +| `enabled() -> bool` | Whether this provider is active per configuration. | +| `_update_data(force_update)` | Custom data fetch/update logic. | + +#### `_update_data` contract of `DataProvider` + +`_update_data` is always called while the provider's own `_sequence_lock` and +`_record_lock` are both held by `update_data`. Implementations must therefore observe +the following constraints to avoid deadlock: + +- Use only the internal sync methods (`_insert_by_datetime`, `_update_value`, + `_key_from_lists`, `_key_from_series`). Never call their public `async` counterparts, + which would attempt to re-acquire `_record_lock`. +- Do not call `save()`, `load()`, `_save()`, or `_load()`. Both locks are already held; + attempting to re-acquire either will deadlock. +- Network or I/O calls are permitted but should be kept short. Offload long-running I/O + to a thread via `asyncio.to_thread` in the caller before entering the lock scope. + +```python +# Correct — both _sequence_lock and _record_lock are already held by the caller +def _update_data(self, force_update=False): + for dt, value in self._fetch_from_api(): + self._update_value(dt, "temperature_c", value) + +# Wrong — would deadlock: _record_lock is not reentrant +def _update_data(self, force_update=False): + for dt, value in self._fetch_from_api(): + await self.update_value(dt, "temperature_c", value) +``` + +#### Public async interface of `DataProvider` + + +| Method | Description | +|---|---| +| `await update_data(force_enable, force_update)` | Call `_update_data` if enabled or forced, holding both `_sequence_lock` and `_record_lock` for the duration. | + + +### `DataImportMixin` + +Mixin that adds bulk import capability to any class that also provides `update_value` and +`record_keys_writable`. Provides `import_from_dict`, `import_from_dataframe`, +`import_from_json`, and `import_from_file`. + +The mixin expects values to be lists aligned to a fixed time interval starting from a +`start_datetime`. Two special dictionary keys are handled automatically: + +- `start_datetime` — overrides the start of the import window. +- `interval` — overrides the fixed time step between values. + +### `DataImportProvider` + +Convenience base class combining `DataImportMixin` and `DataProvider`. Derive from this +when a provider's data arrives via JSON, file, or dict import rather than a live API. + +### `DataContainer` + +A singleton `MutableMapping` that aggregates multiple `DataProvider` instances and +presents their combined data through a single interface. Providers are tried in order; +the first one that contains the requested key wins. + +`DataContainer` carries its own pair of locks (`_record_lock` and `_container_lock`) +that are independent of the locks on each provider. See +[Concurrency and Locking](#concurrency-and-locking) for the full acquisition matrix. + +#### Public async interface of `DataContainer` + +| Method | Description | +|---|---| +| `await __setitem__(key, series)` | Write a `pd.Series` into the appropriate provider. | +| `await __delitem__(key)` | Clear a field across all providers. | +| `await update_data(force_enable, force_update)` | Update all providers. | +| `await save()` | Save all providers to persistent storage. | +| `await load()` | Load all providers from persistent storage. | +| `await db_vacuum()` | Remove old records from all provider databases. | +| `await db_compact()` | Apply tiered compaction to all provider databases. | +| `await keys_to_dataframe(keys, ...)` | Consistent cross-provider snapshot as a `pd.DataFrame`. | + +#### Read-only methods (always sync) of `DataContainer` + +| Method | Description | +|---|---| +| `__getitem__(key)` | Return a `pd.Series` for a key from the first matching provider. | +| `__iter__()` | Iterate over all unique keys across enabled providers. | +| `__len__()` | Total number of unique keys. | +| `key_to_series(key, ...)` | Extract a series from the first matching provider. | +| `key_to_array(key, ...)` | Extract a resampled array from the first matching provider. | +| `provider_by_id(provider_id)` | Look up a provider by its string identifier. | +| `enabled_providers` | List of currently active providers. | +| `record_keys` | Union of all record keys across enabled providers. | +| `record_keys_writable` | Union of all writable record keys across enabled providers. | + +## Concurrency and Locking + +### Problem + +EOS runs under FastAPI with an async event loop. Multiple HTTP requests are handled +concurrently as coroutines on a single thread. Because coroutines yield at `await` points, +two coroutines can interleave between a read and a subsequent write, producing a +**check-then-act race condition**: + +```text +Coroutine A: db_get_record(ts) → None # record does not exist yet +Coroutine B: db_get_record(ts) → None # same — A has not inserted yet +Coroutine A: db_insert_record(new_rec) # inserts successfully +Coroutine B: db_insert_record(new_rec) # raises: duplicate timestamp +``` + +This is the root cause of the `ValueError: Duplicate timestamp` errors seen in production. + +### Solution: three-level locking + +The solution uses three distinct `asyncio.Lock` objects, each protecting a different scope +of state. Each name answers the question "what is being protected": + + +| Lock | Defined on | Attribute | Protects | +|---|---|---|---| +| Record lock | `DataSequence` | `_record_lock` | A single check-then-act on one record. | +| Sequence lock | `DataSequence` | `_sequence_lock` | The full sequence state during operations that touch many records at once. | +| Container lock | `DataContainer` | `_container_lock` | Cross-provider consistency during container-level bulk operations. | + + +`DataProvider` inherits `_record_lock` and `_sequence_lock` from `DataSequence` without +adding any new locks of its own. It is a specialisation of `DataSequence`, not a new +locking scope. + +The fixed acquisition order across all levels is: + +```text +_container_lock → _record_lock → provider _sequence_lock → provider _record_lock +``` + +No code path may acquire a finer-grained lock and then wait for a coarser one at the +same level. This invariant prevents deadlock. + +### Lock creation + +All locks are created lazily on first access via `_get_or_create_lock()`, which bypasses +Pydantic's `__setattr__` using `object.__setattr__` directly. This is necessary because +Pydantic v2 rejects attributes that are not declared model fields, and `asyncio.Lock` +cannot be safely created outside a running event loop, making `__init__`-time creation +unsafe. `cached_property` is not used for the same reason — Pydantic v2's tight +`__dict__` control makes it unreliable on model instances. + +```python +def _get_or_create_lock(self, attr: str) -> asyncio.Lock: + try: + return object.__getattribute__(self, attr) + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, attr, lock) + return lock + +@property +def _record_lock(self) -> asyncio.Lock: + return self._get_or_create_lock("_record_lock_instance") + +@property +def _sequence_lock(self) -> asyncio.Lock: # DataSequence / DataProvider + return self._get_or_create_lock("_sequence_lock_instance") + +@property +def _container_lock(self) -> asyncio.Lock: # DataContainer only + return self._get_or_create_lock("_container_lock_instance") +``` + +### Lock acquisition rules + +**Individual record writes** acquire only `_record_lock`, held for the minimum time +needed to make the check-then-act atomic: + +```python +async def update_value(self, date, *args, **kwargs): + async with self._record_lock: + self._update_value(date, *args, **kwargs) +``` + +**Sequence-level bulk operations** (`save`, `load`, `import_from_*`) acquire +`_sequence_lock` first, then `_record_lock`. This blocks both concurrent bulk operations +and concurrent individual writes for the full duration: + +```python +async def load(self): + async with self._sequence_lock: + async with self._record_lock: + self._load() +``` + +**`update_data`** on a provider acquires both `_sequence_lock` and `_record_lock`, +protecting the provider's full sequence state against concurrent saves, loads, imports, +or individual record writes for the entire duration of `_update_data`: + +```python +async def update_data(self, force_enable=False, force_update=False): + if not force_enable and not self.enabled(): + return + async with self._sequence_lock: + async with self._record_lock: + self._update_data(force_update=force_update) +``` + +**Container-level bulk operations** acquire `_container_lock` and `_record_lock` on the +container, then delegate to each provider (which independently acquires its own +`_sequence_lock` and `_record_lock`): + +```python +async def save(self): # DataContainer + async with self._container_lock: + async with self._record_lock: + for provider in self.providers: + await provider.save() # provider acquires _sequence_lock + _record_lock +``` + +**Cross-provider consistent reads** acquire only the container's `_record_lock`, which +prevents concurrent writes from producing an inconsistent snapshot without blocking +other read operations: + +```python +async def keys_to_dataframe(self, keys, ...): + async with self._record_lock: + ... # read from multiple providers atomically +``` + +### Lock acquisition matrix + +#### `DataSequence` / `DataProvider` lock + +| Operation | `_sequence_lock` | `_record_lock` | +|---|---|---| +| `insert_by_datetime` | | ✓ | +| `update_value` | | ✓ | +| `key_from_lists` | | ✓ | +| `key_from_series` | | ✓ | +| `update_data` | ✓ | ✓ | +| `save` | ✓ | ✓ | +| `load` | ✓ | ✓ | +| `import_from_dict` | ✓ | ✓ | +| `import_from_dataframe` | ✓ | ✓ | +| `import_from_json` | ✓ | ✓ | +| `import_from_file` | ✓ | ✓ | +| Read-only methods | | | + +#### `DataContainer` lock + + +| Operation | `_container_lock` | `_record_lock` | Provider `_sequence_lock` | Provider `_record_lock` | +|---|---|---|---|---| +| `update_data` | ✓ | ✓ | ✓ | ✓ | +| `__setitem__` | | ✓ | | | +| `__delitem__` | | ✓ | | | +| `keys_to_dataframe` | | ✓ | | | +| `save` | ✓ | ✓ | ✓ | ✓ | +| `load` | ✓ | ✓ | ✓ | ✓ | +| `db_vacuum` | ✓ | ✓ | | | +| `db_compact` | ✓ | ✓ | | | +| Read-only methods | | | | | + + +### Why `asyncio.Lock` and not `threading.Lock` + +FastAPI's default async workers run all coroutines on a **single OS thread**. Coroutines +interleave at `await` points, not at thread boundaries. `threading.Lock` would not protect +against this interleaving and would risk deadlock if the same coroutine attempts to +re-acquire it. `asyncio.Lock` yields control correctly at `await` points while keeping +other coroutines blocked. + +### Why `asyncio.Lock` is not reentrant + +Python's `asyncio.Lock` is intentionally non-reentrant. If a coroutine that already holds +`_record_lock` calls a public async method (which also tries to acquire `_record_lock`), +it will deadlock. This is why `_update_data()` implementations and all other internal +callers must use the private `_method()` variants rather than `await self.method()`. + +### Choosing the right call site + + +| Caller context | Correct call | +|---|---| +| FastAPI endpoint or any `async def` | `await sequence.insert_by_datetime(record)` | +| `_update_data()` implementation | `self._insert_by_datetime(record)` | +| Startup, `_load()` internals, tests | `sequence._insert_by_datetime(record)` | +| `DataContainer` delegating to a provider | `await provider.save()` — container holds its own locks; provider independently holds its own | + + +## Usage Examples + +### Reading data (sync, no locking required) + +```python +measurement = get_measurement() + +# Scalar lookup +value = measurement.key_to_value("grid_import_w", target_datetime=now) + +# Resampled array for the next 24 hours +array = measurement.key_to_array( + key="grid_import_w", + start_datetime=now, + end_datetime=now.add(hours=24), + interval=to_duration("1 hour"), +) + +# Pandas Series +series = measurement.key_to_series("grid_import_w", start_datetime=now) +``` + +### Writing a single value from a FastAPI endpoint + +```python +@router.put("/measurement/value") +async def put_measurement_value(datetime: str, key: str, value: float): + dt = to_datetime(datetime) + await get_measurement().update_value(dt, key, value) +``` + +### Bulk import from a FastAPI endpoint + +```python +@router.put("/measurement/import") +async def put_measurement_import(data: dict): + await get_measurement().import_from_dict(data) +``` + +### Writing from a sync context (startup / tests) + +```python +def load_initial_data(measurement: Measurement, records: list[MeasurementDataRecord]): + # Sequential — no concurrency, use internal sync methods directly + for record in records: + measurement._insert_by_datetime(record) +``` + +### Implementing a custom `DataProvider` + +```python +class MyProvider(DataProvider): + records: list[MyRecord] = Field(default_factory=list) + + def provider_id(self) -> str: + return "MyProvider" + + def enabled(self) -> bool: + return self.config.my_provider.enabled + + def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. + for dt, reading in fetch_from_hardware(): + self._update_value(dt, "sensor_w", reading) + + def db_namespace(self) -> str: + return "MyProvider" + + def db_keep_datetime(self) -> Optional[DateTime]: + return to_datetime().subtract(hours=48) +``` + +--- + +## Design Notes + +### Singleton providers and containers + +`DataProvider` and `DataContainer` both inherit from `SingletonMixin`. A single instance +is shared across all coroutines handling concurrent requests. This is precisely why +locking is necessary — every concurrent request operates on the same in-memory object. + +### Lock naming rationale + +The three lock names were chosen to reflect *what is being protected*, not *how coarse +the operation is*: + +- `_record_lock` — makes it immediately clear that a single record's check-then-act is + being made atomic. The same name is used at both `DataSequence` and `DataContainer` + level because it plays the same role in both: serialising individual write operations. +- `_sequence_lock` — defined on `DataSequence` and inherited unchanged by `DataProvider`. + The name reflects that the entire sequence state is held exclusively, not just one + record. `DataProvider` is a specialisation of `DataSequence`, so the name remains + accurate at both levels without needing a separate `_provider_lock`. +- `_container_lock` — exclusive to `DataContainer`, reflecting that cross-provider + container-level state is held exclusively during bulk operations. + +This naming also makes lock misuse visible in code review: a method that acquires +`_sequence_lock` without also acquiring `_record_lock` (or acquires them in the wrong +order) stands out immediately against the documented fixed acquisition order. + +### Database vs in-memory storage + +`DataSequence` uses `DatabaseRecordProtocolMixin` to optionally back its records with a +persistent time-series database. When a database is configured, `save()` and `load()` +delegate to it; otherwise they fall back to a JSON file. The in-memory `records` list acts +as a write-through cache. The locking scheme covers both paths — the database operations +are included inside the lock scope so that in-memory state and database state remain +consistent. + +### Pydantic v2 compatibility + +The `asyncio.Lock` instances are stored on the instance using `object.__setattr__`, +bypassing Pydantic's field validation entirely. This is intentional: `asyncio.Lock` is not +serialisable and must not appear in `model_dump()` or `model_dump_json()` output. The +locks are therefore invisible to Pydantic serialisation and are reconstructed fresh on +each process start, which is correct behaviour for a lock. diff --git a/docs/index.md b/docs/index.md index e9ff0540..a3a86d17 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,6 +63,7 @@ akkudoktoreos/api.rst develop/develop.md develop/release.md +develop/async.md develop/CHANGELOG.md ``` diff --git a/openapi.json b/openapi.json index 02d54274..7e2f35ec 100644 --- a/openapi.json +++ b/openapi.json @@ -8,7 +8,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.html" }, - "version": "v0.3.0.dev2604141105859917" + "version": "v0.3.0.dev2607150960221650" }, "paths": { "/v1/admin/cache/clear": { @@ -155,6 +155,30 @@ } } }, + "/v1/admin/database/save": { + "post": { + "tags": [ + "admin" + ], + "summary": "Fastapi Admin Database Save Post", + "description": "Save in memory data to database.\n\nReturns:\n data (dict): The database stats after saving the records.", + "operationId": "fastapi_admin_database_save_post_v1_admin_database_save_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Fastapi Admin Database Save Post V1 Admin Database Save Post" + } + } + } + } + } + } + }, "/v1/admin/database/vacuum": { "post": { "tags": [ @@ -1198,6 +1222,87 @@ } } }, + "/v1/measurement/range": { + "delete": { + "tags": [ + "measurement" + ], + "summary": "Fastapi Measurement Range Delete", + "description": "Delete measurement values for a key within a datetime range.", + "operationId": "fastapi_measurement_range_delete_v1_measurement_range_delete", + "parameters": [ + { + "name": "key", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Measurement key.", + "title": "Key" + }, + "description": "Measurement key." + }, + { + "name": "start_datetime", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Start datetime.", + "title": "Start Datetime" + }, + "description": "Start datetime." + }, + { + "name": "end_datetime", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "End datetime.", + "title": "End Datetime" + }, + "description": "End datetime." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PydanticDateTimeSeries" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/prediction/providers": { "get": { "tags": [ diff --git a/pyproject.toml b/pyproject.toml index f3836b3c..e0a09341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ dev = [ "GitPython==3.1.51", "myst-parser==5.1.0", "docutils==0.21.2", + "sphinxcontrib-mermaid==2.0.3 ", # Pytest "pytest==9.1.1", diff --git a/src/akkudoktoreos/adapter/adapterabc.py b/src/akkudoktoreos/adapter/adapterabc.py index 819b01d6..22b67f9b 100644 --- a/src/akkudoktoreos/adapter/adapterabc.py +++ b/src/akkudoktoreos/adapter/adapterabc.py @@ -1,5 +1,6 @@ """Abstract and base classes for adapters.""" +import asyncio from abc import abstractmethod from typing import Any, Optional @@ -53,12 +54,27 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin, return self.provider_id() in self.config.adapter.provider return False + @property + def _adapter_lock(self) -> asyncio.Lock: + """Per-instance asyncio lock guarding adapter-level bulk operations. + + The lock guards the full adapter state during bulk operations. + """ + try: + return object.__getattribute__(self, "_adapter_lock_instance") + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, "_adapter_lock_instance", lock) + return lock + @abstractmethod - def _update_data(self) -> None: + async def _update_data(self) -> None: """Abstract method for custom adapter data update logic, to be implemented by derived classes. Data update may be requested at different stages of energy management. The stage can be detected by self.ems.stage(). + + This method is always called while `_adapter_lock` is held by the caller. """ pass @@ -67,7 +83,7 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin, return super().__init__(*args, **kwargs) - def update_data( + async def update_data( self, force_enable: Optional[bool] = False, ) -> None: @@ -81,8 +97,9 @@ class AdapterProvider(SingletonMixin, ConfigMixin, MeasurementMixin, StartMixin, return # Call the custom update logic - logger.debug(f"Update adapter provider: {self.provider_id()}") - self._update_data() + async with self._adapter_lock: + logger.debug(f"Update adapter provider: {self.provider_id()}") + await self._update_data() class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel): @@ -105,6 +122,19 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel): ) return value + @property + def _container_lock(self) -> asyncio.Lock: + """Coarse-grained lock for bulk operations across providers. + + The lock guards cross-provider consistency during container operations. + """ + try: + return object.__getattribute__(self, "_container_lock_instance") + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, "_container_lock_instance", lock) + return lock + @property def enabled_providers(self) -> list[Any]: """List of providers that are currently enabled.""" @@ -145,7 +175,7 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel): raise ValueError(error_msg) return providers[provider_id] - def update_data( + async def update_data( self, force_enable: Optional[bool] = False, ) -> None: @@ -154,7 +184,10 @@ class AdapterContainer(SingletonMixin, ConfigMixin, PydanticBaseModel): Args: force_enable (bool, optional): If True, forces the update even if the provider is disabled. """ + if len(self.providers) <= 0: + return + # Call the custom update logic - if len(self.providers) > 0: + async with self._container_lock: for provider in self.providers: - provider.update_data(force_enable=force_enable) + await provider.update_data(force_enable=force_enable) diff --git a/src/akkudoktoreos/adapter/homeassistant.py b/src/akkudoktoreos/adapter/homeassistant.py index f7b2fb23..3544d08c 100644 --- a/src/akkudoktoreos/adapter/homeassistant.py +++ b/src/akkudoktoreos/adapter/homeassistant.py @@ -419,7 +419,7 @@ class HomeAssistantAdapter(AdapterProvider): # Preserve original state for enums and free-text states return raw_state - def _update_data(self) -> None: + async def _update_data(self) -> None: stage = self.ems.stage() if stage == EnergyManagementStage.DATA_ACQUISITION: # Sync configuration @@ -451,7 +451,7 @@ class HomeAssistantAdapter(AdapterProvider): logger.debug(f"Entity {entity_id}: {state}") if state: measurement_value = float(state) - self.measurement.update_value( + await self.measurement.update_value( self.ems_start_datetime, measurement_key, measurement_value ) except Exception as e: @@ -473,7 +473,7 @@ class HomeAssistantAdapter(AdapterProvider): logger.debug(f"Entity {entity_id}: {state}") if state: measurement_value = float(state) - self.measurement.update_value( + await self.measurement.update_value( self.ems_start_datetime, measurement_key, measurement_value ) except Exception as e: @@ -495,7 +495,7 @@ class HomeAssistantAdapter(AdapterProvider): logger.debug(f"Entity {entity_id}: {state}") if state: measurement_value = float(state) - self.measurement.update_value( + await self.measurement.update_value( self.ems_start_datetime, measurement_key, measurement_value ) except Exception as e: @@ -517,7 +517,7 @@ class HomeAssistantAdapter(AdapterProvider): logger.debug(f"Entity {entity_id}: {state}") if state: measurement_value = float(state) - self.measurement.update_value( + await self.measurement.update_value( self.ems_start_datetime, measurement_key, measurement_value ) except Exception as e: @@ -539,7 +539,7 @@ class HomeAssistantAdapter(AdapterProvider): logger.debug(f"Entity {entity_id}: {state}") if state: measurement_value = float(state) - self.measurement.update_value( + await self.measurement.update_value( self.ems_start_datetime, measurement_key, measurement_value ) except Exception as e: diff --git a/src/akkudoktoreos/adapter/nodered.py b/src/akkudoktoreos/adapter/nodered.py index b7e94eb5..6009c57a 100644 --- a/src/akkudoktoreos/adapter/nodered.py +++ b/src/akkudoktoreos/adapter/nodered.py @@ -66,7 +66,7 @@ class NodeREDAdapter(AdapterProvider): """Return the unique identifier for the adapter provider.""" return "NodeRED" - def _update_data(self) -> None: + async def _update_data(self) -> None: """Custom adapter data update logic. Data update may be requested at different stages of energy management. The stage can be diff --git a/src/akkudoktoreos/core/dataabc.py b/src/akkudoktoreos/core/dataabc.py index b95aa423..a2eee923 100644 --- a/src/akkudoktoreos/core/dataabc.py +++ b/src/akkudoktoreos/core/dataabc.py @@ -8,6 +8,7 @@ This module is designed for use in predictive modeling workflows, facilitating t and manipulation of configuration and generic data in a clear, scalable, and structured manner. """ +import asyncio import difflib import json import traceback @@ -16,6 +17,7 @@ from collections.abc import KeysView, MutableMapping from itertools import chain from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Dict, Iterator, @@ -53,6 +55,7 @@ from akkudoktoreos.core.databaseabc import ( DatabaseTimestamp, DatabaseTimeWindowType, ) +from akkudoktoreos.core.decorators import classproperty from akkudoktoreos.core.pydantic import ( PydanticBaseModel, PydanticDateTimeData, @@ -433,7 +436,7 @@ class DataRecord(DataABC, MutableMapping): class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): - """A managed sequence of DataRecord instances with ltime series behavior. + """A managed sequence of DataRecord instances with time series behavior. The DataSequence class provides an ordered, mutable collection of DataRecord instances. @@ -466,15 +469,15 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): records: list[DerivedDataRecord] = Field(default_factory=list, json_schema_extra={ "description": "List of data records" }) seq = DerivedSequence() - seq.insert(DerivedDataRecord(date_time=datetime.now(), temperature=72)) - seq.insert(DerivedDataRecord(date_time=datetime.now(), temperature=75)) + await seq.insert(DerivedDataRecord(date_time=datetime.now(), temperature=72)) + await seq.insert(DerivedDataRecord(date_time=datetime.now(), temperature=75)) # Convert to JSON and back - json_data = seq.to_json() - new_seq = DerivedSequence.from_json(json_data) + json_data = await seq.to_json_async() + new_seq = await DerivedSequence.from_json_async(json_data) # Convert to Pandas Series - series = seq.key_to_series('temperature') + series = await seq.key_to_series('temperature') """ @@ -485,6 +488,39 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Sequence helpers + @property + def _record_lock(self) -> asyncio.Lock: + """Per-instance asyncio lock guarding per record check-then-act write paths. + + The lock guards a single check-then-act on one record. + + The Lock is created lazily on first access (so construction outside an event loop is safe) + and is cached on the instance so all coroutines sharing the same DataSequence share the + same lock. + """ + try: + return object.__getattribute__(self, "_record_lock_instance") + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, "_record_lock_instance", lock) + return lock + + @property + def _sequence_lock(self) -> asyncio.Lock: + """Per-instance asyncio lock guarding sequence-level bulk operations. + + The lock guards the full sequence state during bulk operations (save, load, import, export). + + While held, all individual writes are also blocked because bulk + operations acquire both locks in order. + """ + try: + return object.__getattribute__(self, "_sequence_lock_instance") + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, "_sequence_lock_instance", lock) + return lock + def _validate_key(self, key: str) -> None: """Verify that a specified key exists in the current record keys. @@ -530,10 +566,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Sequence state - # Derived fields (computed) - @computed_field # type: ignore[prop-decorator] - @property - def min_datetime(self) -> Optional[DateTime]: + async def min_datetime(self) -> Optional[DateTime]: """Minimum (earliest) datetime in the time series sequence of data records. This property computes the earliest datetime from the sequence of data records. @@ -543,16 +576,14 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): Optional[DateTime]: The earliest datetime in the sequence, or `None` if no data records exist. """ - min_timestamp, _ = self.db_timestamp_range() + min_timestamp, _ = await self.db_timestamp_range() if min_timestamp is None: return None # Timestamps are in UTC - convert to timezone utc_datetime = DatabaseTimestamp.to_datetime(min_timestamp) return utc_datetime.in_timezone(self.config.general.timezone) - @computed_field # type: ignore[prop-decorator] - @property - def max_datetime(self) -> Optional[DateTime]: + async def max_datetime(self) -> Optional[DateTime]: """Maximum (latest) datetime in the time series sequence of data records. This property computes the latest datetime from the sequence of data records. @@ -562,13 +593,15 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): Optional[DateTime]: The latest datetime in the sequence, or `None` if no data records exist. """ - _, max_timestamp = self.db_timestamp_range() + _, max_timestamp = await self.db_timestamp_range() if max_timestamp is None: return None # Timestamps are in UTC - convert to timezone utc_datetime = DatabaseTimestamp.to_datetime(max_timestamp) return utc_datetime.in_timezone(self.config.general.timezone) + # Derived fields (computed) + @computed_field # type: ignore[prop-decorator] @property def record_keys(self) -> list[str]: @@ -613,8 +646,23 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): ) return list_element_type + async def to_dict_async(self) -> dict: + """Convert the sequence to a dictionary representation. + + Returns: + dict: A dictionary where the keys are the field names of the PydanticBaseModel, + and the values are the corresponding field values. + """ + if not self.records: + return {"records": []} # Return empty records dict + + # Convert records to a dictionary list + data = [record.model_dump() async for record in self.db_iterate_records()] + + return {"records": data} + @classmethod - def from_dict(cls, data: dict) -> "DataSequence": + async def from_dict_async(cls, data: dict) -> "DataSequence": """Reconstruct a sequence from its serialized dictionary form. Fully subclass-safe and invariant-safe. @@ -639,13 +687,38 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): record = record_model(**record_dict) # Important: use insert_by_datetime to rebuild invariants - sequence.insert_by_datetime(record) + await sequence.insert_by_datetime(record) return sequence + async def to_json_async(self) -> str: + """Convert the sequence instance to a JSON string. + + Returns: + str: The JSON representation of the instance. + """ + seq_dict: dict = await self.to_dict_async() + return json.dumps(seq_dict, default=str) + + @classmethod + async def from_json_async(cls, json_str: str) -> "DataSequence": + """Create an instance of the sequence from a JSON string. + + Args: + json_str (str): JSON string to parse and convert into a sequence instance. + + Returns: + DataSequence: A new instance of the class, populated with data from the JSON string. + + Notes: + Works with derived classes by ensuring the `cls` argument is used to instantiate the object. + """ + seq_dict: dict = json.loads(json_str) + return await cls.from_dict_async(seq_dict) + def __len__(self) -> int: - """Get total number of DataRecords in sequence (DB + memory-only).""" - return self.db_count_records() + """Get total number of DataRecords in sequence (memory-only).""" + return len(self.records) def __repr__(self) -> str: """Provide a string representation of the DataSequence. @@ -658,14 +731,14 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Sequence methods def __iter__(self) -> Iterator[DataRecord]: - """Create an iterator for accessing DataRecords sequentially. + """Create an iterator for accessing DataRecords sequentially (memory only). Returns: Iterator[DataRecord]: An iterator for the records. """ return iter(self.records) - def get_by_datetime( + async def get_by_datetime( self, target_datetime: DateTime, *, time_window: Optional[Duration] = None ) -> Optional[DataRecord]: """Get the record at the specified datetime, with an optional fallback search window. @@ -687,9 +760,9 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Ensure datetime objects are normalized db_target = DatabaseTimestamp.from_datetime(target_datetime) - return self.db_get_record(db_target, time_window=time_window) + return await self.db_get_record(db_target, time_window=time_window) - def get_nearest_by_datetime( + async def get_nearest_by_datetime( self, target_datetime: DateTime, time_window: Optional[Duration] = None ) -> Optional[DataRecord]: """Get the record nearest to the specified datetime within an optional time window. @@ -717,62 +790,63 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): twin: DatabaseTimeWindowType = UNBOUND_WINDOW else: twin = time_window - return self.db_get_record(db_target, time_window=twin) + return await self.db_get_record(db_target, time_window=twin) - def insert_by_datetime(self, record: DataRecord) -> None: - """Insert or merge a DataRecord into the sequence based on its date. + # sync rw write access to data sequence, needs locking in case of use in async. - If a record with the same date exists, merges new data fields with the existing record. - Otherwise, appends the record and maintains chronological order. + async def _insert_by_datetime(self, record: DataRecord) -> None: + """Insert or merge a DataRecord into the sequence based on its datetime. + + Internal implementation of `insert_by_datetime`. Callers must + acquire ``self._record_lock`` before calling this method. + + If a record with the same datetime exists, merges non-None fields from + the incoming record into the existing one. Otherwise inserts the record + and maintains chronological order. Args: - record (DataRecord): The record to add or merge. + record: The record to insert or merge. ``record.date_time`` must + be a ``DateTime`` or ``None``. - Note: - record.date_time shall be a DateTime or None + Raises: + ValueError: If ``record`` is not an instance of the expected record + class, or if ``record.date_time`` cannot be converted to a + ``DateTime``. + KeyError: If a field in ``record`` is not in the writable record keys. """ self._validate_record(record) # Ensure datetime objects are normalized record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time) - avail_record = self.db_get_record(record_date_time_timestamp) + avail_record = await self.db_get_record(record_date_time_timestamp) if avail_record: # Merge values, only updating fields where data record has a non-None value for field, val in record.model_dump(exclude_unset=True).items(): if field in record.record_keys_writable(): setattr(avail_record, field, val) - self.db_mark_dirty_record(record) + await self.db_mark_dirty_record(avail_record) else: - self.db_insert_record(record) + await self.db_insert_record(record) - @overload - def update_value(self, date: DateTime, key: str, value: Any) -> None: ... + async def _update_value(self, date: DateTime, *args: Any, **kwargs: Any) -> None: + """Update or insert field values for a record at the given datetime. - @overload - def update_value(self, date: DateTime, values: Dict[str, Any]) -> None: ... + Internal implementation of `update_value`. Callers must + acquire ``self._record_lock`` before calling this method. - def update_value(self, date: DateTime, *args: Any, **kwargs: Any) -> None: - """Updates specific values in the data record for a given date. - - If a record for the date exists, updates the specified attributes with the new values. - Otherwise, appends a new record with the given values and maintains chronological order. + If a record for ``date`` already exists, updates the specified fields. + Otherwise creates and inserts a new record with those values. Args: - date (datetime): The date for which the values are to be added or updated. - key (str), value (Any): Single key-value pair to update - OR - values (Dict[str, Any]): Dictionary of key-value pairs to update - OR - **kwargs: Key-value pairs as keyword arguments - - Examples: - .. code-block:: python - - update_value(date, 'temperature', 25.5) - update_value(date, {'temperature': 25.5, 'humidity': 80}) - update_value(date, temperature=25.5, humidity=80) + date: The datetime of the record to update or create. + *args: Either a single ``(key, value)`` pair, or a single + ``dict`` of key-value pairs. + **kwargs: Additional key-value pairs to update. + Raises: + ValueError: If the argument combination is invalid. + KeyError: If any key is not in the writable record keys. """ # Process input arguments into a dictionary values: Dict[str, Any] = {} @@ -796,18 +870,135 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): db_target = DatabaseTimestamp.from_datetime(date) # Check if a record with the given date already exists - record = self.db_get_record(db_target) + record = await self.db_get_record(db_target) if record is None: # Create a new record and append to the list new_record = self.record_class()(date_time=date, **values) - self.db_insert_record(new_record) + await self.db_insert_record(new_record) else: # Update the DataRecord with all new values for key, value in values.items(): setattr(record, key, value) - self.db_mark_dirty_record(record) + await self.db_mark_dirty_record(record) - def key_to_dict( + async def _key_from_lists(self, key: str, dates: list[DateTime], values: list[float]) -> None: + """Update the sequence from parallel lists of datetimes and values. + + Internal implementation of `key_from_lists`. Callers must + acquire ``self._record_lock`` before calling this method. + + For each datetime, updates the existing record's ``key`` field if one + exists, otherwise inserts a new record. The lists must be ordered from + oldest to newest datetime. + + Args: + key: Field name in the data record to update. + dates: Ordered list of datetimes, one per value. + values: Values corresponding to each datetime in ``dates``. + + Raises: + KeyError: If ``key`` is not in the writable record keys. + """ + self._validate_key_writable(key) + + for i, date_time in enumerate(dates): + # Ensure datetime objects are normalized + db_target = DatabaseTimestamp.from_datetime(date_time) + # Check if there's an existing record for this date_time + avail_record = await self.db_get_record(db_target) + if avail_record is None: + # Create a new DataRecord if none exists + new_record = self.record_class()(date_time=date_time, **{key: values[i]}) + await self.db_insert_record(new_record) + else: + # Update existing record's specified key + setattr(avail_record, key, values[i]) + await self.db_mark_dirty_record(avail_record) + + async def _key_from_series(self, key: str, series: pd.Series) -> None: + """Update the sequence from a Pandas Series. + + Internal implementation of `key_from_series`. Callers must + acquire ``self._record_lock`` before calling this method. + + The series index must contain datetime values representing the + ``date_time`` of each record. For each index entry, updates the + existing record's ``key`` field if one exists, otherwise inserts + a new record. + + Args: + key: Field name in the data record to update. + series: Series whose index is datetime values and whose values + correspond to ``key`` in each record. + + Raises: + KeyError: If ``key`` is not in the writable record keys. + """ + self._validate_key_writable(key) + + for date_time, value in series.items(): + # Ensure datetime objects are normalized + db_target = DatabaseTimestamp.from_datetime(to_datetime(date_time)) + # Check if there's an existing record for this date_time + avail_record = await self.db_get_record(db_target) + if avail_record is None: + # Create a new DataRecord if none exists + new_record = self.record_class()(date_time=date_time, **{key: value}) + await self.db_insert_record(new_record) + else: + # Update existing record's specified key + setattr(avail_record, key, value) + await self.db_mark_dirty_record(avail_record) + + # data sequence access usable also for async access + + async def insert_by_datetime(self, record: DataRecord) -> None: + """Insert or merge a DataRecord into the sequence based on its date. + + If a record with the same date exists, merges new data fields with the existing record. + Otherwise, appends the record and maintains chronological order. + + Args: + record (DataRecord): The record to add or merge. + + Note: + record.date_time shall be a DateTime or None + """ + async with self._record_lock: + await self._insert_by_datetime(record) + + @overload + async def update_value(self, date: DateTime, key: str, value: Any) -> None: ... + + @overload + async def update_value(self, date: DateTime, values: Dict[str, Any]) -> None: ... + + async def update_value(self, date: DateTime, *args: Any, **kwargs: Any) -> None: + """Updates specific values in the data record for a given date. + + If a record for the date exists, updates the specified attributes with the new values. + Otherwise, appends a new record with the given values and maintains chronological order. + + Args: + date (datetime): The date for which the values are to be added or updated. + key (str), value (Any): Single key-value pair to update + OR + values (Dict[str, Any]): Dictionary of key-value pairs to update + OR + **kwargs: Key-value pairs as keyword arguments + + Examples: + .. code-block:: python + + await update_value(date, 'temperature', 25.5) + await update_value(date, {'temperature': 25.5, 'humidity': 80}) + await update_value(date, temperature=25.5, humidity=80) + + """ + async with self._record_lock: + await self._update_value(date, *args, **kwargs) + + async def key_to_dict( self, key: str, start_datetime: Optional[DateTime] = None, @@ -844,7 +1035,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): if dropna is None: dropna = True filtered_data = {} - for record in self.db_iterate_records(start_timestamp, end_timestamp): + async for record in self.db_iterate_records(start_timestamp, end_timestamp): if ( record.date_time is None or (dropna and getattr(record, key, None) is None) @@ -861,7 +1052,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): return filtered_data - def key_to_value( + async def key_to_value( self, key: str, target_datetime: DateTime, time_window: Optional[Duration] = None ) -> Optional[float]: """Returns the value corresponding to the specified key that is nearest to the given datetime. @@ -884,11 +1075,11 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Ensure datetime objects are normalized db_target = DatabaseTimestamp.from_datetime(to_datetime(target_datetime)) - record = self.db_get_record(db_target, time_window=time_window) + record = await self.db_get_record(db_target, time_window=time_window) return getattr(record, key, None) - def key_to_lists( + async def key_to_lists( self, key: str, start_datetime: Optional[DateTime] = None, @@ -925,10 +1116,10 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): if dropna is None: dropna = True filtered_records = [] - for record in self.db_iterate_records(start_timestamp, end_timestamp): + async for record in self.db_iterate_records(start_timestamp, end_timestamp): if ( record.date_time is None - or (dropna and getattr(record, key, None) is None) + or (getattr(record, key, None) is None) # key is not in record or (dropna and getattr(record, key, None) == float("nan")) ): continue @@ -942,7 +1133,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): return dates, values - def key_from_lists(self, key: str, dates: list[DateTime], values: list[float]) -> None: + async def key_from_lists(self, key: str, dates: list[DateTime], values: list[float]) -> None: """Update the DataSequence from lists of datetime and value elements. The dates list should represent the date_time of each DataRecord, and the values list @@ -955,23 +1146,10 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): dates: List of datetime elements. values: List of values corresponding to the specified key in the data records. """ - self._validate_key_writable(key) + async with self._record_lock: + await self._key_from_lists(key, dates, values) - for i, date_time in enumerate(dates): - # Ensure datetime objects are normalized - db_target = DatabaseTimestamp.from_datetime(date_time) - # Check if there's an existing record for this date_time - avail_record = self.db_get_record(db_target) - if avail_record is None: - # Create a new DataRecord if none exists - new_record = self.record_class()(date_time=date_time, **{key: values[i]}) - self.db_insert_record(new_record) - else: - # Update existing record's specified key - setattr(avail_record, key, values[i]) - self.db_mark_dirty_record(avail_record) - - def key_to_series( + async def key_to_series( self, key: str, start_datetime: Optional[DateTime] = None, @@ -993,13 +1171,13 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): Raises: KeyError: If the specified key is not found in any of the DataRecords. """ - dates, values = self.key_to_lists( + dates, values = await self.key_to_lists( key=key, start_datetime=start_datetime, end_datetime=end_datetime, dropna=dropna ) series = pd.Series(data=values, index=pd.DatetimeIndex(dates), name=key) return series - def key_from_series(self, key: str, series: pd.Series) -> None: + async def key_from_series(self, key: str, series: pd.Series) -> None: """Update the DataSequence from a Pandas Series. The series index should represent the date_time of each DataRecord, and the series values @@ -1009,23 +1187,10 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): series (pd.Series): A Pandas Series containing data to update the DataSequence. key (str): The field name in the DataRecord that corresponds to the values in the Series. """ - self._validate_key_writable(key) + async with self._record_lock: + await self._key_from_series(key, series) - for date_time, value in series.items(): - # Ensure datetime objects are normalized - db_target = DatabaseTimestamp.from_datetime(to_datetime(date_time)) - # Check if there's an existing record for this date_time - avail_record = self.db_get_record(db_target) - if avail_record is None: - # Create a new DataRecord if none exists - new_record = self.record_class()(date_time=date_time, **{key: value}) - self.db_insert_record(new_record) - else: - # Update existing record's specified key - setattr(avail_record, key, value) - self.db_mark_dirty_record(avail_record) - - def key_to_array( + async def key_to_array( self, key: str, start_datetime: Optional[DateTime] = None, @@ -1108,13 +1273,13 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): if query_start is not None: # We have a start datetime - look for previous entry start_timestamp = DatabaseTimestamp.from_datetime(query_start) - query_start_timestamp = self.db_previous_timestamp(start_timestamp) + query_start_timestamp = await self.db_previous_timestamp(start_timestamp) if query_start_timestamp: query_start = DatabaseTimestamp.to_datetime(query_start_timestamp) if end_datetime is not None: # We have a end datetime - look for next entry end_timestamp = DatabaseTimestamp.from_datetime(query_end) - query_end_timestamp = self.db_next_timestamp(end_timestamp) + query_end_timestamp = await self.db_next_timestamp(end_timestamp) if query_end_timestamp is None: # Ensure at least end_datetime is included (excluded by definition) query_end = end_datetime.add(seconds=1) @@ -1122,7 +1287,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): query_end = DatabaseTimestamp.to_datetime(query_end_timestamp).add(seconds=1) # Load raw lists (already sorted & filtered) - dates, values = self.key_to_lists( + dates, values = await self.key_to_lists( key=key, start_datetime=query_start, end_datetime=query_end, dropna=dropna ) values_len = len(values) @@ -1255,7 +1420,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): return array - def to_dataframe( + async def to_dataframe( self, start_datetime: Optional[DateTime] = None, end_datetime: Optional[DateTime] = None, @@ -1283,7 +1448,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): # Convert filtered records to a dictionary list data = [ record.model_dump() - for record in self.db_iterate_records( + async for record in self.db_iterate_records( start_timestamp=start_timestamp, end_timestamp=end_timestamp ) ] @@ -1301,7 +1466,7 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): df.index = pd.DatetimeIndex(df["date_time"]) return df - def delete_by_datetime( + async def delete_by_datetime( self, start_datetime: Optional[DateTime] = None, end_datetime: Optional[DateTime] = None, @@ -1324,9 +1489,11 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): ) end_timestamp = DatabaseTimestamp.from_datetime(end_datetime) if end_datetime else None - return self.db_delete_records(start_timestamp=start_timestamp, end_timestamp=end_timestamp) + return await self.db_delete_records( + start_timestamp=start_timestamp, end_timestamp=end_timestamp + ) - def key_delete_by_datetime( + async def key_delete_by_datetime( self, key: str, start_datetime: Optional[DateTime] = None, @@ -1357,34 +1524,66 @@ class DataSequence(DataABC, DatabaseRecordProtocolMixin[DataRecord]): ) end_timestamp = DatabaseTimestamp.from_datetime(end_datetime) if end_datetime else None - for record in self.db_iterate_records(start_timestamp, end_timestamp): + async for record in self.db_iterate_records(start_timestamp, end_timestamp): del record[key] - self.db_mark_dirty_record(record) + await self.db_mark_dirty_record(record) - def save(self) -> bool: + async def _save(self) -> bool: """Save data records to persistent storage. + Internal implementation of `save`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + Returns: True in case the data records were saved, False otherwise. """ if not self.db_enabled: return False - saved = self.db_save_records() + saved = await self.db_save_records() return saved > 0 - def load(self) -> bool: + async def _load(self) -> bool: """Load data records from from persistent storage. + Internal implementation of `load`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + Returns: True in case the data records were loaded, False otherwise. """ if not self.db_enabled: return False - loaded = self.db_load_records() + loaded = await self.db_load_records() return loaded > 0 + async def save(self) -> bool: + """Save data records to persistent storage. + + Acquires both the bulk and write locks for the duration, ensuring no + concurrent writes or other bulk operations can interleave. + + Returns: + True if the records were saved, False otherwise. + """ + async with self._sequence_lock: + async with self._record_lock: + return await self._save() + + async def load(self) -> bool: + """Load data records from persistent storage. + + Acquires both the bulk and write locks for the duration, ensuring no + concurrent writes or other bulk operations can interleave. + + Returns: + True if the records were loaded, False otherwise. + """ + async with self._sequence_lock: + async with self._record_lock: + return await self._load() + # ----------------------- DataSequence Database Protocol --------------------- # Required interface propagated to derived class. @@ -1427,11 +1626,16 @@ class DataProvider(SingletonMixin, DataSequence): raise NotImplementedError() @abstractmethod - def _update_data(self, force_update: Optional[bool] = False) -> None: - """Abstract method for custom data update logic, to be implemented by derived classes. + async def _update_data(self, force_update: Optional[bool] = False) -> None: + """Custom data update logic to be implemented by derived classes. + + This method is always called while `_sequence_lock` and ``_record_lock`` is held by the + caller. Implementations must therefore use the internal ``_insert_by_datetime()``, + ``_update_value()``, ``_key_from_lists()``, and ``_key_from_series()`` + methods rather than their public async counterparts, to avoid deadlock. Args: - force_update (bool, optional): If True, forces the provider to update the data even if still cached. + force_update: If True, forces update even if data is still cached. """ pass @@ -1444,7 +1648,7 @@ class DataProvider(SingletonMixin, DataSequence): """Namespace of database.""" return self.provider_id() - def update_data( + async def update_data( self, force_enable: Optional[bool] = False, force_update: Optional[bool] = False, @@ -1460,7 +1664,9 @@ class DataProvider(SingletonMixin, DataSequence): return # Call the custom update logic - self._update_data(force_update=force_update) + async with self._sequence_lock: + async with self._record_lock: + await self._update_data(force_update=force_update) # ==================== DataImportMixin ==================== @@ -1484,12 +1690,22 @@ class DataImportMixin(StartMixin): """ - # Attributes required but defined elsehere. - # - start_datetime - # - record_keys_writable - # - update_value + # Tell mypy these attributes exist (will be provided by subclasses or other mixins) + if TYPE_CHECKING: - def import_from_dict( + @property + def _sequence_lock(self) -> asyncio.Lock: ... + + @property + def _record_lock(self) -> asyncio.Lock: ... + + @classproperty + def ems_start_datetime(cls) -> Optional[DateTime]: ... + + @property + def record_keys_writable(self) -> list[str]: ... + + async def _import_from_dict( self, import_data: dict, key_prefix: str = "", @@ -1502,6 +1718,9 @@ class DataImportMixin(StartMixin): record keys and the provided `key_prefix`, and updates the data values sequentially. All value lists must have the same length. + Internal implementation of `import_from_dict`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + Args: import_data (dict): Dictionary containing the generic data with optional 'start_datetime' and 'interval' keys. @@ -1542,7 +1761,7 @@ class DataImportMixin(StartMixin): key for key in import_data.keys() if key.startswith(key_prefix) - and key in self.record_keys_writable # type: ignore + and key in self.record_keys_writable and key not in ("start_datetime", "interval") ] @@ -1578,12 +1797,12 @@ class DataImportMixin(StartMixin): value = values[value_index] value_datetime = DatabaseTimestamp.to_datetime(value_db_datetime) if value is not None and not pd.isna(value): - self.update_value(value_datetime, key, value) # type: ignore + await self._update_value(value_datetime, key, value) # type: ignore except (IndexError, TypeError) as e: raise ValueError(f"Error processing values for key '{key}': {e}") - def import_from_dataframe( + async def _import_from_dataframe( self, df: pd.DataFrame, key_prefix: str = "", @@ -1596,6 +1815,9 @@ class DataImportMixin(StartMixin): record keys and the provided `key_prefix`, and updates the data values using the DataFrame's index as timestamps. + Internal implementation of `import_from_dataframe`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + Args: df (pd.DataFrame): DataFrame containing the generic data with datetime index or sequential values. @@ -1627,7 +1849,7 @@ class DataImportMixin(StartMixin): valid_columns = [ col for col in df.columns - if col.startswith(key_prefix) and col in self.record_keys_writable # type: ignore + if col.startswith(key_prefix) and col in self.record_keys_writable ] if not valid_columns: @@ -1653,7 +1875,7 @@ class DataImportMixin(StartMixin): # Use the DataFrame's datetime index for dt, value in zip(index_datetimes, values): if value is not None and not pd.isna(value): - self.update_value(dt, column, value) # type: ignore + await self._update_value(dt, column, value) # type: ignore else: # Use the pre-generated datetime index for value_index in range(values_count): @@ -1662,12 +1884,209 @@ class DataImportMixin(StartMixin): value_db_datetimes[value_index] ) if value is not None and not pd.isna(value): - self.update_value(value_datetime, column, value) # type: ignore + await self._update_value(value_datetime, column, value) # type: ignore except Exception as e: raise ValueError(f"Error processing column '{column}': {e}") - def import_from_json( + async def _import_from_json( + self, + json_str: str, + key_prefix: str = "", + start_datetime: Optional[DateTime] = None, + interval: Optional[Duration] = None, + ) -> None: + """Updates generic data by importing it from a JSON string. + + This method reads generic data from a JSON string, matches keys based on the + record keys and the provided `key_prefix`, and updates the data values sequentially, + starting from the `start_datetime`. + + If start_datetime and or interval is given in the JSON dict it will be used. Otherwise + the given parameters are used. If None is given start_datetime defaults to + 'self.ems_start_datetime' and interval defaults to 1 hour. + + Internal implementation of `import_from_json`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + + Args: + json_str (str): The JSON string containing the generic data. + key_prefix (str, optional): A prefix to filter relevant keys from the generic data. + Only keys starting with this prefix will be considered. Defaults to an empty string. + start_datetime (DateTime, optional): Start datetime of values. + interval (duration, optional): The fixed time interval. Defaults to 1 hour. + + Raises: + JSONDecodeError: If the file content is not valid JSON. + + Example: + Given a JSON string with the following content and `key_prefix = "load"`, only the + "loadforecast_power_w" key will be processed even though both keys are in the record. + + .. code-block:: json + + { + "start_datetime": "2024-11-10 00:00:00", + "interval": "30 minutes", + "loadforecast_power_w": [20.5, 21.0, 22.1], + "other_xyz: [10.5, 11.0, 12.1] + } + + """ + # Strip quotes if provided - does not effect unquoted string + json_str = json_str.strip() # strip white space at start and end + if (json_str.startswith("'") and json_str.endswith("'")) or ( + json_str.startswith('"') and json_str.endswith('"') + ): + json_str = json_str[1:-1] # strip outer quotes + json_str = json_str.strip() # strip remaining white space at start and end + + # Try pandas dataframe with orient="split" + try: + import_data = PydanticDateTimeDataFrame.model_validate_json(json_str) + await self._import_from_dataframe(import_data.to_dataframe()) + return + except ValidationError as e: + error_msg = "" + for error in e.errors(): + field = " -> ".join(str(x) for x in error["loc"]) + message = error["msg"] + error_type = error["type"] + error_msg += f"Field: {field}\nError: {message}\nType: {error_type}\n" + logger.debug(f"PydanticDateTimeDataFrame import: {error_msg}") + + # Try dictionary with special keys start_datetime and interval + try: + import_data = PydanticDateTimeData.model_validate_json(json_str) + await self._import_from_dict(import_data.to_dict()) + return + except ValidationError as e: + error_msg = "" + for error in e.errors(): + field = " -> ".join(str(x) for x in error["loc"]) + message = error["msg"] + error_type = error["type"] + error_msg += f"Field: {field}\nError: {message}\nType: {error_type}\n" + logger.debug(f"PydanticDateTimeData import: {error_msg}") + + # Use simple dict format + try: + import_data = json.loads(json_str) + await self._import_from_dict( + import_data, key_prefix=key_prefix, start_datetime=start_datetime, interval=interval + ) + except Exception as e: + error_msg = f"Invalid JSON string '{json_str}': {e}" + logger.debug(error_msg) + raise ValueError(error_msg) from e + + async def _import_from_file( + self, + import_file_path: Path, + key_prefix: str = "", + start_datetime: Optional[DateTime] = None, + interval: Optional[Duration] = None, + ) -> None: + """Updates generic data by importing it from a file. + + This method reads generic data from a JSON file, matches keys based on the + record keys and the provided `key_prefix`, and updates the data values sequentially, + starting from the `start_datetime`. Each data value is associated with an hourly + interval. + + If start_datetime and or interval is given in the JSON dict it will be used. Otherwise + the given parameters are used. If None is given start_datetime defaults to + 'self.ems_start_datetime' and interval defaults to 1 hour. + + Internal implementation of `import_from_file`. Callers must + acquire ``self._sequence_lock`` and "self._record_lock" before calling this method. + + Args: + import_file_path (Path): The path to the JSON file containing the generic data. + key_prefix (str, optional): A prefix to filter relevant keys from the generic data. + Only keys starting with this prefix will be considered. Defaults to an empty string. + start_datetime (DateTime, optional): Start datetime of values. + interval (duration, optional): The fixed time interval. Defaults to 1 hour. + + Raises: + FileNotFoundError: If the specified file does not exist. + JSONDecodeError: If the file content is not valid JSON. + + Example: + Given a JSON file with the following content and `key_prefix = "load"`, only the + "loadforecast_power_w" key will be processed even though both keys are in the record. + + .. code-block:: json + + { + "loadforecast_power_w": [20.5, 21.0, 22.1], + "other_xyz: [10.5, 11.0, 12.1], + } + + """ + with import_file_path.open("r", encoding="utf-8", newline=None) as import_file: + import_str = import_file.read() + await self._import_from_json( + import_str, key_prefix=key_prefix, start_datetime=start_datetime, interval=interval + ) + + async def import_from_dict( + self, + import_data: dict, + key_prefix: str = "", + start_datetime: Optional[DateTime] = None, + interval: Optional[Duration] = None, + ) -> None: + """Updates generic data by importing it from a dictionary. + + This method reads generic data from a dictionary, matches keys based on the + record keys and the provided `key_prefix`, and updates the data values sequentially. + All value lists must have the same length. + + Args: + import_data (dict): Dictionary containing the generic data with optional + 'start_datetime' and 'interval' keys. + key_prefix (str, optional): A prefix to filter relevant keys from the generic data. + Only keys starting with this prefix will be considered. Defaults to an empty string. + start_datetime (DateTime, optional): Start datetime of values if not in dict. + interval (Duration, optional): The fixed time interval if not in dict. + + Raises: + ValueError: If value lists have different lengths or if datetime conversion fails. + """ + async with self._sequence_lock: + async with self._record_lock: + await self._import_from_dict(import_data, key_prefix, start_datetime, interval) + + async def import_from_dataframe( + self, + df: pd.DataFrame, + key_prefix: str = "", + start_datetime: Optional[DateTime] = None, + interval: Optional[Duration] = None, + ) -> None: + """Updates generic data by importing it from a pandas DataFrame. + + This method reads generic data from a DataFrame, matches columns based on the + record keys and the provided `key_prefix`, and updates the data values using + the DataFrame's index as timestamps. + + Args: + df (pd.DataFrame): DataFrame containing the generic data with datetime index + or sequential values. + key_prefix (str, optional): A prefix to filter relevant columns from the DataFrame. + Only columns starting with this prefix will be considered. Defaults to an empty string. + start_datetime (DateTime, optional): Start datetime if DataFrame doesn't have datetime index. + interval (Duration, optional): The fixed time interval if DataFrame doesn't have datetime index. + + Raises: + ValueError: If DataFrame structure is invalid or datetime conversion fails. + """ + async with self._sequence_lock: + async with self._record_lock: + await self._import_from_dataframe(df, key_prefix, start_datetime, interval) + + async def import_from_json( self, json_str: str, key_prefix: str = "", @@ -1708,54 +2127,11 @@ class DataImportMixin(StartMixin): } """ - # Strip quotes if provided - does not effect unquoted string - json_str = json_str.strip() # strip white space at start and end - if (json_str.startswith("'") and json_str.endswith("'")) or ( - json_str.startswith('"') and json_str.endswith('"') - ): - json_str = json_str[1:-1] # strip outer quotes - json_str = json_str.strip() # strip remaining white space at start and end + async with self._sequence_lock: + async with self._record_lock: + await self._import_from_json(json_str, key_prefix, start_datetime, interval) - # Try pandas dataframe with orient="split" - try: - import_data = PydanticDateTimeDataFrame.model_validate_json(json_str) - self.import_from_dataframe(import_data.to_dataframe()) - return - except ValidationError as e: - error_msg = "" - for error in e.errors(): - field = " -> ".join(str(x) for x in error["loc"]) - message = error["msg"] - error_type = error["type"] - error_msg += f"Field: {field}\nError: {message}\nType: {error_type}\n" - logger.debug(f"PydanticDateTimeDataFrame import: {error_msg}") - - # Try dictionary with special keys start_datetime and interval - try: - import_data = PydanticDateTimeData.model_validate_json(json_str) - self.import_from_dict(import_data.to_dict()) - return - except ValidationError as e: - error_msg = "" - for error in e.errors(): - field = " -> ".join(str(x) for x in error["loc"]) - message = error["msg"] - error_type = error["type"] - error_msg += f"Field: {field}\nError: {message}\nType: {error_type}\n" - logger.debug(f"PydanticDateTimeData import: {error_msg}") - - # Use simple dict format - try: - import_data = json.loads(json_str) - self.import_from_dict( - import_data, key_prefix=key_prefix, start_datetime=start_datetime, interval=interval - ) - except Exception as e: - error_msg = f"Invalid JSON string '{json_str}': {e}" - logger.debug(error_msg) - raise ValueError(error_msg) from e - - def import_from_file( + async def import_from_file( self, import_file_path: Path, key_prefix: str = "", @@ -1796,11 +2172,9 @@ class DataImportMixin(StartMixin): } """ - with import_file_path.open("r", encoding="utf-8", newline=None) as import_file: - import_str = import_file.read() - self.import_from_json( - import_str, key_prefix=key_prefix, start_datetime=start_datetime, interval=interval - ) + async with self._sequence_lock: + async with self._record_lock: + await self._import_from_file(import_file_path, key_prefix, start_datetime, interval) # ==================== DataImportProvider ==================== @@ -1824,7 +2198,7 @@ class DataImportProvider(DataImportMixin, DataProvider): # ==================== DataContainer ==================== -class DataContainer(SingletonMixin, DataABC, MutableMapping): +class DataContainer(SingletonMixin, DataABC): """A container for managing multiple DataProvider instances. This class enables access to data from multiple data providers, supporting retrieval and @@ -1841,6 +2215,19 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): default_factory=list, json_schema_extra={"description": "List of data providers"} ) + @property + def _container_lock(self) -> asyncio.Lock: + """Coarse-grained lock for bulk operations across providers. + + The lock guards cross-provider consistency during container operations. + """ + try: + return object.__getattribute__(self, "_container_lock_instance") + except AttributeError: + lock = asyncio.Lock() + object.__setattr__(self, "_container_lock_instance", lock) + return lock + @field_validator("providers", mode="after") def check_providers(cls, value: list[DataProvider]) -> list[DataProvider]: # Check each item in the list @@ -1883,77 +2270,6 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): return super().__init__(*args, **kwargs) - def __getitem__(self, key: str) -> pd.Series: - """Retrieve a Pandas Series for a specified key from the data in each DataProvider. - - Iterates through providers to find and return the first available Series for the specified key. - - Args: - key (str): The field name to retrieve, representing a data attribute in DataRecords. - - Returns: - pd.Series: A Pandas Series containing aggregated data for the specified key. - - Raises: - KeyError: If no provider contains data for the specified key. - """ - series = None - for provider in self.enabled_providers: - try: - series = provider.key_to_series(key) - break - except KeyError: - continue - - if series is None: - raise KeyError(f"No data found for key '{key}'.") - - return series - - def __setitem__(self, key: str, value: pd.Series) -> None: - """Add or merge a Pandas Series for a specified key into the records of an appropriate provider. - - Attempts to update or insert the provided Series data in each provider. If no provider supports - the specified key, an error is raised. - - Args: - key (str): The field name to update, representing a data attribute in DataRecords. - value (pd.Series): A Pandas Series containing data for the specified key. - - Raises: - ValueError: If `value` is not an instance of `pd.Series`. - KeyError: If no provider supports the specified key. - """ - if not isinstance(value, pd.Series): - raise ValueError("Value must be an instance of pd.Series.") - - for provider in self.enabled_providers: - try: - provider.key_from_series(key, value) - break - except KeyError: - continue - else: - raise KeyError(f"Key '{key}' not found in any provider.") - - def __delitem__(self, key: str) -> None: - """Set the value of the specified key in the data records of each provider to None. - - Args: - key (str): The field name in DataRecords to clear. - - Raises: - KeyError: If the key is not found in any provider. - """ - for provider in self.enabled_providers: - try: - provider.key_delete_by_datetime(key) - break - except KeyError: - continue - else: - raise KeyError(f"Key '{key}' not found in any provider.") - def __iter__(self) -> Iterator[str]: """Return an iterator over all unique keys available across providers. @@ -1981,39 +2297,47 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): def keys(self) -> KeysView[str]: return dict.fromkeys(self.record_keys).keys() - def update_data( + async def update_data( self, force_enable: Optional[bool] = False, force_update: Optional[bool] = False, ) -> None: - """Update data. + """Update data from all providers. + + Acquires both the bulk and write locks for the duration, ensuring no + concurrent writes or other bulk operations can interleave with the + provider updates. Args: - force_enable (bool, optional): If True, forces the update even if a provider is disabled. - force_update (bool, optional): If True, forces the providers to update the data even if still cached. - """ - for provider in self.providers: - try: - provider.update_data(force_enable=force_enable, force_update=force_update) - except Exception as e: - trace = "".join(traceback.TracebackException.from_exception(e).format()) - error = ( - f"Provider {provider.provider_id()} fails on update - " - f"enabled={provider.enabled()}, " - f"force_enable={force_enable}, " - f"force_update={force_update}" - f":\n{e}\n{trace}" - ) - if provider.enabled(): - # The active provider failed — this is a real error worth propagating. - logger.error(error) - raise RuntimeError(error) - else: - # A non-active provider failed (e.g. missing config while force_enable=True). - # Log as warning and continue so the remaining providers still run. - logger.warning(error) + force_enable: If True, forces the update even if a provider is disabled. + force_update: If True, forces providers to update even if data is cached. - def key_to_series( + Raises: + RuntimeError: If an enabled provider fails during update. + """ + async with self._container_lock: + for provider in self.providers: + try: + await provider.update_data(force_enable=force_enable, force_update=force_update) + except Exception as e: + trace = "".join(traceback.TracebackException.from_exception(e).format()) + error = ( + f"Provider {provider.provider_id()} fails on update - " + f"enabled={provider.enabled()}, " + f"force_enable={force_enable}, " + f"force_update={force_update}" + f":\n{e}\n{trace}" + ) + if provider.enabled(): + # The active provider failed — this is a real error worth propagating. + logger.error(error) + raise RuntimeError(error) + else: + # A non-active provider failed (e.g. missing config while force_enable=True). + # Log as warning and continue so the remaining providers still run. + logger.warning(error) + + async def key_to_series( self, key: str, start_datetime: Optional[DateTime] = None, @@ -2040,7 +2364,7 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): series = None for provider in self.enabled_providers: try: - series = provider.key_to_series( + series = await provider.key_to_series( key, start_datetime=start_datetime, end_datetime=end_datetime, @@ -2055,7 +2379,7 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): return series - def key_to_array( + async def key_to_array( self, key: str, start_datetime: Optional[DateTime] = None, @@ -2091,7 +2415,7 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): array = None for provider in self.enabled_providers: try: - array = provider.key_to_array( + array = await provider.key_to_array( key, start_datetime=start_datetime, end_datetime=end_datetime, @@ -2108,7 +2432,7 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): return array - def keys_to_dataframe( + async def keys_to_dataframe( self, keys: list[str], start_datetime: Optional[DateTime] = None, @@ -2146,25 +2470,22 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): if start_datetime is None: # Take earliest datetime of all providers that are enabled for provider in self.enabled_providers: + min_dt = await provider.min_datetime() if start_datetime is None: - start_datetime = provider.min_datetime - elif ( - provider.min_datetime - and compare_datetimes(provider.min_datetime, start_datetime).lt - ): - start_datetime = provider.min_datetime + start_datetime = min_dt + elif min_dt and compare_datetimes(min_dt, start_datetime).lt: + start_datetime = min_dt if end_datetime is None: # Take latest datetime of all providers that are enabled for provider in self.enabled_providers: + max_dt = await provider.max_datetime() if end_datetime is None: - end_datetime = provider.max_datetime - elif ( - provider.max_datetime - and compare_datetimes(provider.max_datetime, end_datetime).gt - ): - end_datetime = provider.min_datetime + end_datetime = max_dt + elif max_dt and compare_datetimes(max_dt, end_datetime).gt: + min_dt = await provider.min_datetime() + end_datetime = max_dt if end_datetime: - end_datetime.add(seconds=1) + end_datetime = end_datetime.add(seconds=1) # Create a DatetimeIndex based on start, end, and interval if start_datetime is None or end_datetime is None: @@ -2181,7 +2502,9 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): data = {} for key in keys: try: - array = self.key_to_array(key, start_datetime, end_datetime, interval, fill_method) + array = await self.key_to_array( + key, start_datetime, end_datetime, interval, fill_method + ) if len(array) != len(reference_index): raise ValueError( @@ -2225,47 +2548,84 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): # ----------------------- DataContainer Database Protocol --------------------- - def save(self) -> None: - """Save data records to persistent storage.""" - for provider in self.providers: - try: - provider.save() - except Exception as ex: - error = f"Provider {provider.provider_id()} fails on save: {ex}" - logger.error(error) - raise RuntimeError(error) + async def save(self) -> bool: + """Save data records of all providers to persistent storage. - def load(self) -> None: - """Load data records from from persistent storage.""" - for provider in self.providers: - try: - provider.load() - except Exception as ex: - error = f"Provider {provider.provider_id()} fails on load: {ex}" - logger.error(error) - raise RuntimeError(error) + Returns: + True if all providers saved successfully, False if any provider saved nothing. - def db_vacuum(self) -> None: - """Remove old records of all providers from database to free space.""" - for provider in self.providers: - try: - provider.db_vacuum() - except Exception as ex: - error = f"Provider {provider.provider_id()} fails on db vacuum: {ex}" - logger.error(error) - raise RuntimeError(error) + Raises: + RuntimeError: If any provider fails to save. + """ + result = True + async with self._container_lock: + for provider in self.providers: + try: + saved = await provider.save() + if not saved: + result = False + except Exception as ex: + error = f"Provider {provider.provider_id()} fails on save: {ex}" + logger.error(error) + raise RuntimeError(error) + return result - def db_compact(self) -> None: - """Apply tiered compaction to all providers to reduce storage while retaining coverage.""" - for provider in self.providers: - try: - provider.db_compact() - except Exception as ex: - error = f"Provider {provider.provider_id()} fails on db_compact: {ex}" - logger.error(error) - raise RuntimeError(error) + async def load(self) -> bool: + """Load data records of all providers from persistent storage. - def db_get_stats(self) -> dict: + Shall never be called from any _sequence_lock-held context (DataSequence and derived + classes), it will deadlock because asyncio.Lock is not re-entrant. + + Returns: + True if all providers loaded successfully, False if any provider loaded nothing. + + Raises: + RuntimeError: If any provider fails to load. + """ + result = True + async with self._container_lock: + for provider in self.providers: + try: + loaded = await provider.load() + if not loaded: + result = False + except Exception as ex: + error = f"Provider {provider.provider_id()} fails on load: {ex}" + logger.error(error) + raise RuntimeError(error) + return result + + async def db_vacuum(self) -> None: + """Remove old records of all providers from the database. + + Raises: + RuntimeError: If any provider fails during vacuum. + """ + async with self._container_lock: + for provider in self.providers: + try: + await provider.db_vacuum() + except Exception as ex: + error = f"Provider {provider.provider_id()} fails on db vacuum: {ex}" + logger.error(error) + raise RuntimeError(error) + + async def db_compact(self) -> None: + """Apply tiered compaction to all providers. + + Raises: + RuntimeError: If any provider fails during compaction. + """ + async with self._container_lock: + for provider in self.providers: + try: + await provider.db_compact() + except Exception as ex: + error = f"Provider {provider.provider_id()} fails on db_compact: {ex}" + logger.error(error) + raise RuntimeError(error) + + async def db_get_stats(self) -> dict: """Get comprehensive statistics about database storage for all providers. Returns: @@ -2274,7 +2634,7 @@ class DataContainer(SingletonMixin, DataABC, MutableMapping): db_stats = {} for provider in self.providers: try: - db_stats[provider.db_namespace()] = provider.db_get_stats() + db_stats[provider.db_namespace()] = await provider.db_get_stats() except Exception as ex: error = f"Provider {provider.provider_id()} fails on db vacuum: {ex}" logger.error(error) diff --git a/src/akkudoktoreos/core/database.py b/src/akkudoktoreos/core/database.py index b806d330..69b74b6a 100644 --- a/src/akkudoktoreos/core/database.py +++ b/src/akkudoktoreos/core/database.py @@ -9,25 +9,34 @@ namespaces with a `namespace` column. from __future__ import annotations +import asyncio import shutil import sqlite3 from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple +from typing import ( + Any, + AsyncIterator, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, +) import lmdb from loguru import logger from pydantic import Field, computed_field, field_validator from akkudoktoreos.config.configabc import SettingsBaseModel -from akkudoktoreos.core.coreabc import SingletonMixin +from akkudoktoreos.core.coreabc import ConfigMixin, SingletonMixin from akkudoktoreos.core.databaseabc import ( DATABASE_METADATA_KEY, - DatabaseABC, DatabaseBackendABC, ) # Valid database providers -database_providers: List[str] = ["LMDB", "SQLite"] +database_providers: List[str] = ["LMDB", "SQLite", "NoDB"] class DatabaseCommonSettings(SettingsBaseModel): @@ -40,6 +49,8 @@ class DatabaseCommonSettings(SettingsBaseModel): batch_size: Batch size for batch operations. """ + model_config = {"validate_assignment": True} # keeps field validation on assignment + provider: Optional[str] = Field( default=None, json_schema_extra={ @@ -177,12 +188,17 @@ class LMDBDatabase(DatabaseBackendABC): """Return the unique identifier for the database provider.""" return "LMDB" - def open(self, namespace: Optional[str] = None) -> None: + def open(self, *, namespace: Optional[str] = None) -> None: """Open LMDB environment and optionally ensure a namespace DBI. Args: namespace: Optional default namespace to open (DBI created on demand). """ + if self.is_open: + if namespace is not None: + self._ensure_dbi(namespace=namespace) + return + self.storage_path.mkdir(parents=True, exist_ok=True) self.env = lmdb.open( @@ -201,7 +217,7 @@ class LMDBDatabase(DatabaseBackendABC): self.default_namespace = namespace if namespace is not None: - self._ensure_dbi(namespace) + self._ensure_dbi(namespace=namespace) def close(self) -> None: """Close the LMDB environment and clear cached DBIs.""" @@ -214,7 +230,7 @@ class LMDBDatabase(DatabaseBackendABC): self._dbis.clear() logger.debug("Closed LMDB at %s", self.storage_path) - def flush(self, namespace: Optional[str] = None) -> None: + def flush(self, *, namespace: Optional[str] = None) -> None: """Sync LMDB environment (writes to disk).""" if not isinstance(self.env, lmdb.Environment): raise ValueError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") @@ -230,7 +246,7 @@ class LMDBDatabase(DatabaseBackendABC): """Return explicit namespace or default if None.""" return namespace if namespace is not None else self.default_namespace - def _ensure_dbi(self, namespace: Optional[str]) -> Optional[Any]: + def _ensure_dbi(self, *, namespace: Optional[str]) -> Optional[Any]: """Open and cache a DBI for the given namespace. Args: @@ -271,15 +287,15 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) with self.env.begin(write=True) as txn: if metadata is None: - txn.delete(DATABASE_METADATA_KEY) + txn.delete(DATABASE_METADATA_KEY, db=dbi) else: - txn.put(DATABASE_METADATA_KEY, metadata) + txn.put(DATABASE_METADATA_KEY, metadata, db=dbi) - def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]: + def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]: """Load metadata for a given namespace. Returns None if no metadata exists. @@ -293,10 +309,10 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) with self.env.begin(write=False) as txn: - return txn.get(DATABASE_METADATA_KEY) + return txn.get(DATABASE_METADATA_KEY, db=dbi) # ------------------------------------------------------------------ # Bulk Write Operations @@ -305,6 +321,7 @@ class LMDBDatabase(DatabaseBackendABC): def save_records( self, records: Iterable[tuple[bytes, bytes]], + *, namespace: Optional[str] = None, ) -> int: """Save multiple records into the specified namespace (or default). @@ -324,7 +341,7 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) saved = 0 with self.lock: @@ -338,6 +355,7 @@ class LMDBDatabase(DatabaseBackendABC): def delete_records( self, keys: Iterable[bytes], + *, namespace: Optional[str] = None, ) -> int: """Delete multiple records by key from the specified namespace. @@ -352,7 +370,7 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError("Database not open") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) deleted = 0 with self.lock: @@ -371,6 +389,7 @@ class LMDBDatabase(DatabaseBackendABC): self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, + *, namespace: Optional[str] = None, reverse: bool = False, ) -> Iterator[tuple[bytes, bytes]]: @@ -391,11 +410,12 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong type `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) META = DATABASE_METADATA_KEY results: list[tuple[bytes, bytes]] = [] + cursor = None txn = self.env.begin(write=False) try: cursor = txn.cursor(dbi) @@ -454,7 +474,8 @@ class LMDBDatabase(DatabaseBackendABC): finally: # Ensure reader slot is always released - cursor.close() + if cursor is not None: + cursor.close() txn.abort() # Transaction is closed here — safe to yield @@ -478,7 +499,7 @@ class LMDBDatabase(DatabaseBackendABC): if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) META = DATABASE_METADATA_KEY count = 0 @@ -510,13 +531,14 @@ class LMDBDatabase(DatabaseBackendABC): def get_key_range( self, + *, namespace: Optional[str] = None, ) -> tuple[Optional[bytes], Optional[bytes]]: """Return (min_key, max_key) in the given namespace or (None, None) if empty.""" if not isinstance(self.env, lmdb.Environment): raise RuntimeError(f"LMDB Environment is of wrong tpe `{type(self.env)}`.") - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) with self.env.begin(write=False) as txn: cursor = txn.cursor(db=dbi) @@ -541,12 +563,12 @@ class LMDBDatabase(DatabaseBackendABC): return min_key, max_key - def get_backend_stats(self, namespace: Optional[str] = None) -> dict[str, Any]: + def get_backend_stats(self, *, namespace: Optional[str] = None) -> dict[str, Any]: """Get LMDB backend-specific statistics.""" if not self.env: return {} - dbi = self._ensure_dbi(namespace) + dbi = self._ensure_dbi(namespace=namespace) with self.env.begin(write=False) as txn: stat = txn.stat(db=dbi) @@ -657,12 +679,15 @@ class SQLiteDatabase(DatabaseBackendABC): """Return the unique identifier for the database provider.""" return "SQLite" - def open(self, namespace: Optional[str] = None) -> None: + def open(self, *, namespace: Optional[str] = None) -> None: """Open SQLite connection and optionally set default namespace. Args: namespace: Optional default namespace to use when operations omit namespace. """ + if self.is_open: + return + self.storage_path.mkdir(parents=True, exist_ok=True) self.conn = sqlite3.connect( @@ -700,7 +725,7 @@ class SQLiteDatabase(DatabaseBackendABC): self._is_open = False logger.debug("Closed SQLite at %s", self.db_file) - def flush(self, namespace: Optional[str] = None) -> None: + def flush(self, *, namespace: Optional[str] = None) -> None: """Commit any pending transactions to disk (no-op if autocommit).""" if not isinstance(self.conn, sqlite3.Connection): raise RuntimeError(f"SQLite connection is of wrong tpe `{type(self.conn)}`.") @@ -745,7 +770,7 @@ class SQLiteDatabase(DatabaseBackendABC): (ns, metadata), ) - def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]: + def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]: """Load metadata for a given namespace. Returns None if no metadata exists. @@ -777,6 +802,7 @@ class SQLiteDatabase(DatabaseBackendABC): def save_records( self, records: Iterable[tuple[bytes, bytes]], + *, namespace: Optional[str] = None, ) -> int: """Bulk insert or replace records. @@ -794,18 +820,18 @@ class SQLiteDatabase(DatabaseBackendABC): return 0 with self.lock: - self.conn.execute("BEGIN") - self.conn.executemany( - "INSERT OR REPLACE INTO records (namespace, key, value) VALUES (?, ?, ?)", - rows, - ) - self.conn.execute("COMMIT") + with self.conn: + self.conn.executemany( + "INSERT OR REPLACE INTO records (namespace, key, value) VALUES (?, ?, ?)", + rows, + ) return len(rows) def delete_records( self, keys: Iterable[bytes], + *, namespace: Optional[str] = None, ) -> int: """Delete multiple records by key. @@ -817,21 +843,25 @@ class SQLiteDatabase(DatabaseBackendABC): ns = self._ns(namespace) - deleted: int = 0 - with self.lock: - for key in keys: - cursor = self.conn.execute( - "DELETE FROM records WHERE namespace = ? AND key = ?", - (ns, key), - ) - deleted += cursor.rowcount + rows = [(ns, key) for key in keys] - return deleted + if not rows: + return 0 + + with self.lock: + with self.conn: + cursor = self.conn.executemany( + "DELETE FROM records WHERE namespace = ? AND key = ?", + rows, + ) + + return cursor.rowcount def iterate_records( self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, + *, namespace: Optional[str] = None, reverse: bool = False, ) -> Iterator[Tuple[bytes, bytes]]: @@ -913,7 +943,7 @@ class SQLiteDatabase(DatabaseBackendABC): return int(cursor.fetchone()[0]) def get_key_range( - self, namespace: Optional[str] = None + self, *, namespace: Optional[str] = None ) -> Tuple[Optional[bytes], Optional[bytes]]: """Return (min_key, max_key) for the namespace or (None, None) if empty.""" if not isinstance(self.conn, sqlite3.Connection): @@ -928,7 +958,7 @@ class SQLiteDatabase(DatabaseBackendABC): result = cursor.fetchone() return result[0], result[1] - def get_backend_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]: + def get_backend_stats(self, *, namespace: Optional[str] = None) -> Dict[str, Any]: """Return SQLite-specific stats and namespace metrics.""" if not self.conn: return {} @@ -959,10 +989,216 @@ class SQLiteDatabase(DatabaseBackendABC): logger.info("SQLite vacuum completed") -# ==================== Generic Database Implementation ==================== +# ==================== NoDB Implementation ==================== -class Database(DatabaseABC, SingletonMixin): +class NoDB(DatabaseBackendABC): + """No-op database backend. + + This backend implements the full database backend interface but performs + no actual persistence. It is intended for configurations where database + persistence is disabled (`provider=None`). + + Characteristics: + - All write operations are ignored. + - All read operations return empty results. + - No files or external resources are created. + - Lifecycle operations are no-ops. + - Compression is disabled. + + This backend allows the database wrapper to avoid special-case handling + for a missing database provider by always providing a valid backend + implementation. + """ + + def __init__(self, **kwargs: Any) -> None: + """Initialize the no-op backend.""" + super().__init__() + self._is_open = True # Stateless + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def provider_id(self) -> str: + """Return the unique identifier for the database provider.""" + return "NoDB" + + def open(self, *, namespace: Optional[str] = None) -> None: + """Mark backend as open. + + Args: + namespace: Ignored. + """ + self.default_namespace = namespace + + def close(self) -> None: + """Mark backend as closed.""" + return + + def flush(self, *, namespace: Optional[str] = None) -> None: + """Flush pending writes (no-op). + + Args: + namespace: Ignored. + """ + return + + # ------------------------------------------------------------------ + # Effective backend configuration + # ------------------------------------------------------------------ + + @property + def is_open(self) -> bool: + """Return dummy not open.""" + return False + + @property + def storage_path(self) -> Path: + """Return dummy storage path.""" + return Path("/dev/null") + + @property + def compression_level(self) -> int: + """Compression is disabled.""" + return 0 + + @property + def compression(self) -> bool: + """Compression is disabled.""" + return False + + # ------------------------------------------------------------------ + # Metadata operations + # ------------------------------------------------------------------ + + def set_metadata( + self, + metadata: Optional[bytes], + *, + namespace: Optional[str] = None, + ) -> None: + """Store metadata (ignored). + + Args: + metadata: Ignored. + namespace: Ignored. + """ + return + + def get_metadata( + self, + *, + namespace: Optional[str] = None, + ) -> Optional[bytes]: + """Load metadata. + + Always returns: + None + """ + return None + + # ------------------------------------------------------------------ + # Record operations + # ------------------------------------------------------------------ + + def save_records( + self, + records: Iterable[tuple[bytes, bytes]], + *, + namespace: Optional[str] = None, + ) -> int: + """Pretend to save records. + + Args: + records: Ignored. + namespace: Ignored. + + Returns: + Always 0. + """ + return 0 + + def delete_records( + self, + keys: Iterable[bytes], + *, + namespace: Optional[str] = None, + ) -> int: + """Pretend to delete records. + + Args: + keys: Ignored. + namespace: Ignored. + + Returns: + Always 0. + """ + return 0 + + def iterate_records( + self, + start_key: Optional[bytes] = None, + end_key: Optional[bytes] = None, + *, + namespace: Optional[str] = None, + reverse: bool = False, + ) -> Iterator[tuple[bytes, bytes]]: + """Iterate records. + + Always yields: + Nothing + """ + return iter(()) + + def count_records( + self, + start_key: Optional[bytes] = None, + end_key: Optional[bytes] = None, + *, + namespace: Optional[str] = None, + ) -> int: + """Count records. + + Returns: + Always 0. + """ + return 0 + + def get_key_range( + self, + *, + namespace: Optional[str] = None, + ) -> tuple[Optional[bytes], Optional[bytes]]: + """Return key range. + + Returns: + Always (None, None). + """ + return None, None + + def get_backend_stats( + self, + *, + namespace: Optional[str] = None, + ) -> dict[str, Any]: + """Return backend statistics. + + Returns: + Minimal backend information. + """ + return { + "backend": "none", + "namespace": namespace, + "persistent": False, + "records": 0, + } + + +# ==================== Generic Database ==================== + + +class Database(ConfigMixin, SingletonMixin): """Generic database. All operations accept an optional `namespace` argument. Implementations should @@ -971,16 +1207,15 @@ class Database(DatabaseABC, SingletonMixin): a namespace column). """ - _db: Optional[DatabaseBackendABC] = None + _db: DatabaseBackendABC = NoDB() @classmethod def reset_instance(cls) -> None: """Resets the singleton instance, forcing it to be recreated on next access.""" with cls._lock: # Close current database backend - if cls._db: - cls._db.close() - cls._db = None + cls._db.close() + cls._db = NoDB() # Remove current database instance if cls in cls._instances: del cls._instances[cls] @@ -989,95 +1224,228 @@ class Database(DatabaseABC, SingletonMixin): def __init__(self) -> None: """Initialize database.""" super().__init__() - self._db = None + self._db = NoDB() - def _setup_db(self) -> None: - """Setup database.""" + # Database helpers + + @property + def _database_lock(self) -> asyncio.Lock: + """Per-instance asyncio lock guarding database operations. + + The lock guards the database state during async operations. + """ + # Lock must be a loop-local asyncio lock to provide singleton + pytest async compatibility. + loop = asyncio.get_running_loop() + + try: + locks = object.__getattribute__(self, "_database_locks") + except AttributeError: + locks = {} + object.__setattr__(self, "_database_locks", locks) + + lock = locks.get(loop) + + if lock is None: + lock = asyncio.Lock() + locks[loop] = lock + + return lock + + async def _ensure_open( + self, + *, + namespace: Optional[str] = None, + ) -> DatabaseBackendABC: + """Ensure the configured backend exists, is open, and namespace is prepared. + + Expects the database lock to already be held when called. + + Args: + namespace: Optional namespace to prepare/open. + + Returns: + The active and opened backend instance. + + Raises: + RuntimeError: If no database provider is configured. + """ provider_id = self.config.database.provider - database: Optional[DatabaseBackendABC] = None - if provider_id is None: - database = None - elif provider_id == "LMDB": - database = LMDBDatabase() - elif provider_id == "SQLite": - database = SQLiteDatabase() - else: - raise RuntimeError("Invalid database provider '{provider_id}'") - if self._db is not None: - self._db.close() - self._db = database + old_provider_id = self._db.provider_id() - def _database(self) -> DatabaseBackendABC: - """Get database.""" - provider_id = self.config.database.provider - if provider_id is None: - raise RuntimeError("Database not configured") - - if self._db is None or self._db.provider_id() != provider_id: + if old_provider_id != provider_id: # No database or configuration does not match - self._setup_db() - if self._db is None: - raise RuntimeError("Database not configured") + logger.debug( + f"Switching database provider from '{old_provider_id}' to '{provider_id}'." + ) + old_db = self._db + + database: DatabaseBackendABC + if provider_id is None or provider_id == "NoDB": + database = NoDB() + elif provider_id == "LMDB": + database = LMDBDatabase() + elif provider_id == "SQLite": + database = SQLiteDatabase() + else: + raise RuntimeError(f"Invalid database provider '{provider_id}'") + + # Auto-close database that was used before + if old_db.is_open: + old_db.close() + + self._db = database if not self._db.is_open: - self._db.open() + await asyncio.to_thread(self._db.open, namespace=namespace) + elif namespace is not None: + # Allow backend to lazily prepare namespace resources + await asyncio.to_thread(self._db.open, namespace=namespace) return self._db + async def _run_db( + self, + method_name: str, + *args: Any, + **kwargs: Any, + ) -> Any: + """Execute a synchronous database backend operation in a thread-safe, non-blocking way. + + The backend is automatically initialized/opened before execution. + If a ``namespace`` keyword argument is present, the namespace is + prepared during backend initialization. + + This helper ensures that all interactions with the underlying synchronous + database backend are: + + - **Serialized** using an instance-level ``asyncio.Lock`` to protect backend state. + - **Non-blocking** by offloading execution to a worker thread via + ``asyncio.to_thread``. + + It should be used for all backend calls that may perform blocking I/O or + CPU-bound work. + + Args: + method_name: The synchronous callable to execute (a backend method). + *args: Positional arguments forwarded to ``method``. + **kwargs: Keyword arguments forwarded to ``method``. + + Returns: + The return value of ``method``. + + Raises: + Any exception raised by ``func`` is propagated unchanged. + + Notes: + - The callable is executed in a separate thread, so it must be thread-safe. + - The database lock is held for the duration of the call, ensuring that + no concurrent operations interfere with backend state. + - Avoid passing coroutines or async functions to this method; it is + intended strictly for synchronous callables. + """ + namespace = kwargs.get("namespace") + + async with self._database_lock: + db = await self._ensure_open(namespace=namespace) + + # Get actual method. _ensure_open may have changed the backend + method = getattr(db, method_name) + + def backend_call() -> Any: + result = method( + *args, + **kwargs, + ) + + # Materialize iterators inside worker thread + if isinstance(result, Iterator): + return list(result) + + return result + + return await asyncio.to_thread(backend_call) + def provider_id(self) -> str: """Return the unique identifier for the database provider.""" - try: - return self._database().provider_id() - except: - return "None" + return self._db.provider_id() @property def is_open(self) -> bool: """Return whether the database connection is open.""" - try: - return self._database().is_open - except: - return False + return self._db.is_open @property def storage_path(self) -> Path: - """Storage path for the database.""" - return self._database().storage_path + """Return effective storage path of active backend.""" + return self._db.storage_path @property def compression_level(self) -> int: - """Compression level for database record data.""" - return self._database().compression_level + """Return effective compression level of active backend.""" + return self._db.compression_level @property def compression(self) -> bool: - """Whether to compress stored values.""" - return self._database().compression_level > 0 + """Return whether the active backend compresses stored values. + + Returns: + True if compression is enabled for the active backend, + False if disabled, + or None if no backend is currently initialized. + """ + return self._db.compression_level > 0 # Lifecycle - def open(self, namespace: Optional[str] = None) -> None: - """Open database connection and optionally set default namespace. + async def ensure_open( + self, + *, + namespace: Optional[str] = None, + ) -> DatabaseBackendABC: + """Ensure the configured backend exists, is open, and namespace is prepared. Args: - namespace: Optional default namespace to prepare. + namespace: Optional namespace to prepare/open. + + Returns: + The active and opened backend instance. Raises: - RuntimeError: If the database cannot be opened. + RuntimeError: If no database provider is configured. """ - self._database().open(namespace) + async with self._database_lock: + return await self._ensure_open(namespace=namespace) - def close(self) -> None: + async def open(self, *, namespace: Optional[str] = None) -> None: + """Ensure the database backend is initialized and open. + + If the backend is already open, this operation is a no-op except + that the backend may prepare the specified namespace. + + Args: + namespace: Optional namespace to prepare/open. + + Raises: + RuntimeError: If no database provider is configured or opening fails. + """ + async with self._database_lock: + await self._ensure_open(namespace=namespace) + + async def close(self) -> None: """Close the database connection and cleanup resources.""" - self._database().close() + async with self._database_lock: + if self._db is not None and self._db.is_open: + await asyncio.to_thread(self._db.close) - def flush(self, namespace: Optional[str] = None) -> None: + async def flush(self, *, namespace: Optional[str] = None) -> None: """Force synchronization of pending writes to storage (optional per-namespace).""" - return self._database().flush(namespace) + await self._run_db("flush", namespace=namespace) # Metadata operations - def set_metadata(self, metadata: Optional[bytes], *, namespace: Optional[str] = None) -> None: + async def set_metadata( + self, metadata: Optional[bytes], *, namespace: Optional[str] = None + ) -> None: """Save metadata for a given namespace. Metadata is treated separately from data records and stored as a single object. @@ -1086,9 +1454,13 @@ class Database(DatabaseABC, SingletonMixin): metadata (bytes): Arbitrary metadata to save or None to delete metadata. namespace (Optional[str]): Optional namespace under which to store metadata. """ - self._database().set_metadata(metadata, namespace=namespace) + await self._run_db( + "set_metadata", + metadata, + namespace=namespace, + ) - def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]: + async def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]: """Load metadata for a given namespace. Returns None if no metadata exists. @@ -1099,12 +1471,15 @@ class Database(DatabaseABC, SingletonMixin): Returns: Optional[bytes]: The loaded metadata, or None if not found. """ - return self._database().get_metadata(namespace=namespace) + return await self._run_db( + "get_metadata", + namespace=namespace, + ) # Basic record operations - def save_records( - self, records: Iterable[tuple[bytes, bytes]], namespace: Optional[str] = None + async def save_records( + self, records: Iterable[tuple[bytes, bytes]], *, namespace: Optional[str] = None ) -> int: """Save multiple records into the specified namespace (or default). @@ -1120,9 +1495,15 @@ class Database(DatabaseABC, SingletonMixin): Raises: RuntimeError: If DB not open or write failed. """ - return self._database().save_records(records, namespace) + return await self._run_db( + "save_records", + records, + namespace=namespace, + ) - def delete_records(self, keys: Iterable[bytes], namespace: Optional[str] = None) -> int: + async def delete_records( + self, keys: Iterable[bytes], *, namespace: Optional[str] = None + ) -> int: """Delete multiple records by key from the specified namespace. Args: @@ -1132,15 +1513,20 @@ class Database(DatabaseABC, SingletonMixin): Returns: Number of records actually deleted. """ - return self._database().delete_records(keys, namespace) + return await self._run_db( + "delete_records", + keys, + namespace=namespace, + ) - def iterate_records( + async def iterate_records( self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, + *, namespace: Optional[str] = None, reverse: bool = False, - ) -> Iterator[tuple[bytes, bytes]]: + ) -> AsyncIterator[tuple[bytes, bytes]]: """Iterate over records for a namespace with optional bounds. Args: @@ -1152,9 +1538,18 @@ class Database(DatabaseABC, SingletonMixin): Yields: Tuples of (key, record). """ - return self._database().iterate_records(start_key, end_key, namespace, reverse) + records: list[tuple[bytes, bytes]] = await self._run_db( + "iterate_records", + start_key, + end_key, + namespace=namespace, + reverse=reverse, + ) - def count_records( + for item in records: + yield item + + async def count_records( self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, @@ -1165,14 +1560,49 @@ class Database(DatabaseABC, SingletonMixin): Excludes metadata records. """ - return self._database().count_records(start_key, end_key, namespace=namespace) + return await self._run_db( + "count_records", + start_key, + end_key, + namespace=namespace, + ) - def get_key_range( - self, namespace: Optional[str] = None + async def get_key_range( + self, *, namespace: Optional[str] = None ) -> Tuple[Optional[bytes], Optional[bytes]]: """Return (min_key, max_key) in the given namespace or (None, None) if empty.""" - return self._database().get_key_range(namespace) + return await self._run_db( + "get_key_range", + namespace=namespace, + ) - def get_backend_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]: + async def get_backend_stats(self, *, namespace: Optional[str] = None) -> Dict[str, Any]: """Get backend-specific statistics; implementations may return namespace-specific data.""" - return self._database().get_backend_stats(namespace) + return await self._run_db( + "get_backend_stats", + namespace=namespace, + ) + + # Compression helpers + + def serialize_data(self, data: bytes) -> bytes: + """Optionally compress raw pickled data before storage. + + Args: + data: Raw pickled bytes. + + Returns: + Possibly compressed bytes. + """ + return self._db.serialize_data(data) + + def deserialize_data(self, data: bytes) -> bytes: + """Optionally decompress stored data. + + Args: + data: Stored bytes. + + Returns: + Raw pickled bytes (decompressed if needed). + """ + return self._db.deserialize_data(data) diff --git a/src/akkudoktoreos/core/databaseabc.py b/src/akkudoktoreos/core/databaseabc.py index b80aac79..1e50844f 100644 --- a/src/akkudoktoreos/core/databaseabc.py +++ b/src/akkudoktoreos/core/databaseabc.py @@ -12,6 +12,7 @@ from threading import Lock from typing import ( TYPE_CHECKING, Any, + AsyncIterator, Final, Generic, Iterable, @@ -46,20 +47,38 @@ DATABASE_METADATA_KEY: bytes = b"__metadata__" # ==================== Abstract Database Interface ==================== -class DatabaseABC(ABC, ConfigMixin): - """Abstract base class for database. +class DatabaseBackendABC(ABC, ConfigMixin, SingletonMixin): + """Abstract base class for database backends. All operations accept an optional `namespace` argument. Implementations should treat None as the default/root namespace. Concrete implementations can map namespace -> native namespace (LMDB DBI) or emulate namespaces (SQLite uses a namespace column). + + The database backend provides a synchronous interface. Asynchrounous access is + handled by the generic database class. """ + connection: Any + lock: Lock + _is_open: bool + default_namespace: Optional[str] + + def __init__(self, **kwargs: Any) -> None: + """Initialize the DatabaseBackendABC base. + + Args: + **kwargs: Backend-specific options (ignored by base). + """ + self.connection = None + self.lock = Lock() + self._is_open = False + self.default_namespace = None + @property - @abstractmethod def is_open(self) -> bool: """Return whether the database connection is open.""" - raise NotImplementedError + return self._is_open @property def storage_path(self) -> Path: @@ -87,7 +106,7 @@ class DatabaseABC(ABC, ConfigMixin): raise NotImplementedError @abstractmethod - def open(self, namespace: Optional[str] = None) -> None: + def open(self, *, namespace: Optional[str] = None) -> None: """Open database connection and optionally set default namespace. Args: @@ -104,7 +123,7 @@ class DatabaseABC(ABC, ConfigMixin): raise NotImplementedError @abstractmethod - def flush(self, namespace: Optional[str] = None) -> None: + def flush(self, *, namespace: Optional[str] = None) -> None: """Force synchronization of pending writes to storage (optional per-namespace).""" raise NotImplementedError @@ -123,7 +142,7 @@ class DatabaseABC(ABC, ConfigMixin): raise NotImplementedError @abstractmethod - def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]: + def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]: """Load metadata for a given namespace. Returns None if no metadata exists. @@ -140,7 +159,7 @@ class DatabaseABC(ABC, ConfigMixin): @abstractmethod def save_records( - self, records: Iterable[tuple[bytes, bytes]], namespace: Optional[str] = None + self, records: Iterable[tuple[bytes, bytes]], *, namespace: Optional[str] = None ) -> int: """Save multiple records into the specified namespace (or default). @@ -159,7 +178,7 @@ class DatabaseABC(ABC, ConfigMixin): raise NotImplementedError @abstractmethod - def delete_records(self, keys: Iterable[bytes], namespace: Optional[str] = None) -> int: + def delete_records(self, keys: Iterable[bytes], *, namespace: Optional[str] = None) -> int: """Delete multiple records by key from the specified namespace. Args: @@ -176,6 +195,7 @@ class DatabaseABC(ABC, ConfigMixin): self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, + *, namespace: Optional[str] = None, reverse: bool = False, ) -> Iterator[tuple[bytes, bytes]]: @@ -208,13 +228,13 @@ class DatabaseABC(ABC, ConfigMixin): @abstractmethod def get_key_range( - self, namespace: Optional[str] = None + self, *, namespace: Optional[str] = None ) -> tuple[Optional[bytes], Optional[bytes]]: """Return (min_key, max_key) in the given namespace or (None, None) if empty.""" raise NotImplementedError @abstractmethod - def get_backend_stats(self, namespace: Optional[str] = None) -> dict[str, Any]: + def get_backend_stats(self, *, namespace: Optional[str] = None) -> dict[str, Any]: """Get backend-specific statistics; implementations may return namespace-specific data.""" raise NotImplementedError @@ -250,37 +270,6 @@ class DatabaseABC(ABC, ConfigMixin): return data -class DatabaseBackendABC(DatabaseABC, SingletonMixin): - """Abstract base class for database backends. - - All operations accept an optional `namespace` argument. Implementations should - treat None as the default/root namespace. Concrete implementations can map - namespace -> native namespace (LMDB DBI) or emulate namespaces (SQLite uses - a namespace column). - """ - - connection: Any - lock: Lock - _is_open: bool - default_namespace: Optional[str] - - def __init__(self, **kwargs: Any) -> None: - """Initialize the DatabaseBackendABC base. - - Args: - **kwargs: Backend-specific options (ignored by base). - """ - self.connection = None - self.lock = Lock() - self._is_open = False - self.default_namespace = None - - @property - def is_open(self) -> bool: - """Return whether the database connection is open.""" - return self._is_open - - # ==================== Database Record Protocol Mixin ==================== @@ -442,7 +431,7 @@ class DatabaseRecordProtocol(Protocol, Generic[T_Record]): @property def db_enabled(self) -> bool: ... - def db_timestamp_range(self) -> tuple[DatabaseTimestampType, DatabaseTimestampType]: ... + async def db_timestamp_range(self) -> tuple[DatabaseTimestampType, DatabaseTimestampType]: ... def db_generate_timestamps( self, @@ -451,52 +440,49 @@ class DatabaseRecordProtocol(Protocol, Generic[T_Record]): interval: Optional[Duration] = None, ) -> Iterator[DatabaseTimestamp]: ... - def db_get_record(self, target_timestamp: DatabaseTimestamp) -> Optional[T_Record]: ... + async def db_get_record(self, target_timestamp: DatabaseTimestamp) -> Optional[T_Record]: ... - def db_insert_record( + async def db_insert_record( self, record: T_Record, *, mark_dirty: bool = True, ) -> None: ... - def db_iterate_records( + async def db_iterate_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, - ) -> Iterator[T_Record]: ... + ) -> AsyncIterator[T_Record]: ... - def db_load_records( + async def db_load_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, ) -> int: ... - def db_delete_records( + async def db_delete_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, ) -> int: ... # ---- dirty tracking ---- - def db_mark_dirty_record(self, record: T_Record) -> None: ... + async def db_mark_dirty_record(self, record: T_Record) -> None: ... - def db_save_records(self) -> int: ... - - # ---- autosave ---- - def db_autosave(self) -> int: ... + async def db_save_records(self) -> int: ... # ---- Remove old records from database to free space ---- - def db_vacuum( + async def db_vacuum( self, keep_hours: Optional[int] = None, keep_datetime: Optional[DatabaseTimestampType] = None, ) -> int: ... # ---- statistics about database storage ---- - def db_count_records(self) -> int: ... + async def db_count_records(self) -> int: ... - def db_get_stats(self) -> dict: ... + async def db_get_stats(self) -> dict: ... T_DatabaseRecordProtocol = TypeVar("T_DatabaseRecordProtocol", bound="DatabaseRecordProtocol") @@ -533,10 +519,12 @@ class DatabaseRecordProtocolMixin( Completely manages in memory records and database storage. - Expects records with date_time (DatabaseTimestamp) property and the a record list + Expects records with date_time (DatabaseTimestamp) property and a record list in self.records of the derived class. - DatabaseRecordProtocolMixin expects the derived classes to be singletons. + DatabaseRecordProtocolMixin expects the derived classes to be singletons and to have + the sequence guarded against asynchronous sequence state (_sequence_lock) and + asynchronous record state (_record_lock) changes. """ # Tell mypy these attributes exist (will be provided by subclasses) @@ -549,7 +537,7 @@ class DatabaseRecordProtocolMixin( @property def record_keys_writable(self) -> list[str]: ... - def key_to_array( + async def key_to_array( self, key: str, start_datetime: Optional[DateTime] = None, @@ -583,7 +571,18 @@ class DatabaseRecordProtocolMixin( # Initialization # ----------------------------------------------------- - def _db_ensure_initialized(self) -> None: + async def _db_init_metadata(self) -> None: + """Initialize DB metadata.""" + self._db_metadata: Optional[dict] = { + "version": self._db_version, + "created": to_datetime(as_string=True), + "provider_id": getattr(self, "provider_id", lambda: "unknown")(), + "compression": self.database.compression, + "backend": self.database.__class__.__name__, + } + await self._db_save_metadata(self._db_metadata) + + async def _db_ensure_initialized(self) -> None: """Initialize DB runtime state. Idempotent — safe to call multiple times. @@ -613,25 +612,18 @@ class DatabaseRecordProtocolMixin( self._db_version: int = 1 # Storage - self._db_metadata: Optional[dict] = None + self._db_metadata = None self._db_storage_initialized: bool = False self._db_initialized: bool = True if not self._db_storage_initialized and self.db_enabled: # Metadata - existing_metadata = self._db_load_metadata() + existing_metadata = await self._db_load_metadata() if existing_metadata: self._db_metadata = existing_metadata else: - self._db_metadata = { - "version": self._db_version, - "created": to_datetime(as_string=True), - "provider_id": getattr(self, "provider_id", lambda: "unknown")(), - "compression": self.database.compression, - "backend": self.database.__class__.__name__, - } - self._db_save_metadata(self._db_metadata) + await self._db_init_metadata() logger.info( f"Initialized {self.database.__class__.__name__}:{self.db_namespace()} storage at " @@ -641,12 +633,6 @@ class DatabaseRecordProtocolMixin( self._db_storage_initialized = True - def model_post_init(self, __context: Any) -> None: - """Initialize DB state attributes immediately after Pydantic construction.""" - # Always call super() first — other mixins may also define model_post_init - super().model_post_init(__context) # type: ignore[misc] - self._db_ensure_initialized() - # ----------------------------------------------------- # Helpers # ----------------------------------------------------- @@ -669,7 +655,7 @@ class DatabaseRecordProtocolMixin( db_datetime_after = DatabaseTimestamp.from_datetime(target.add(seconds=1)) return db_datetime_after - def db_previous_timestamp( + async def db_previous_timestamp( self, timestamp: DatabaseTimestamp, ) -> Optional[DatabaseTimestamp]: @@ -677,7 +663,7 @@ class DatabaseRecordProtocolMixin( Search memory-first, then fallback to database if necessary. """ - self._db_ensure_initialized() + await self._db_ensure_initialized() # Step 1: Memory-first search if self._db_sorted_timestamps: @@ -689,7 +675,7 @@ class DatabaseRecordProtocolMixin( if not self.db_enabled: return None - db_min_key, _ = self.database.get_key_range(self.db_namespace()) + db_min_key, _ = await self.database.get_key_range(namespace=self.db_namespace()) if db_min_key is None: return None @@ -710,7 +696,7 @@ class DatabaseRecordProtocolMixin( start_key = self._db_key_from_timestamp(loaded_start) previous_ts: Optional[DatabaseTimestamp] = None - for key, _ in self.database.iterate_records( + async for key, _ in self.database.iterate_records( start_key=start_key, end_key=end_key, namespace=self.db_namespace(), @@ -722,7 +708,7 @@ class DatabaseRecordProtocolMixin( return previous_ts - def db_next_timestamp( + async def db_next_timestamp( self, timestamp: DatabaseTimestamp, ) -> Optional[DatabaseTimestamp]: @@ -730,7 +716,7 @@ class DatabaseRecordProtocolMixin( Search memory-first, then fallback to database if necessary. """ - self._db_ensure_initialized() + await self._db_ensure_initialized() # Step 1: Memory-first search if self._db_sorted_timestamps: @@ -742,7 +728,7 @@ class DatabaseRecordProtocolMixin( if not self.db_enabled: return None - _, db_max_key = self.database.get_key_range(self.db_namespace()) + _, db_max_key = await self.database.get_key_range(namespace=self.db_namespace()) if db_max_key is None: return None @@ -762,7 +748,7 @@ class DatabaseRecordProtocolMixin( if isinstance(loaded_end, DatabaseTimestamp) and timestamp < loaded_end: start_key = self._db_key_from_timestamp(max(timestamp, loaded_end)) - for key, _ in self.database.iterate_records( + async for key, _ in self.database.iterate_records( start_key=start_key, end_key=end_key, namespace=self.db_namespace(), @@ -796,22 +782,22 @@ class DatabaseRecordProtocolMixin( record_data = pickle.loads(data) # noqa: S301 return self.record_class()(**record_data) - def _db_save_metadata(self, metadata: dict) -> None: + async def _db_save_metadata(self, metadata: dict) -> None: """Save metadata to database.""" if not self.db_enabled: return key = DATABASE_METADATA_KEY value = pickle.dumps(metadata) - self.database.set_metadata(value, namespace=self.db_namespace()) + await self.database.set_metadata(value, namespace=self.db_namespace()) - def _db_load_metadata(self) -> Optional[dict]: + async def _db_load_metadata(self) -> Optional[dict]: """Load metadata from database.""" if not self.db_enabled: return None try: - value = self.database.get_metadata(namespace=self.db_namespace()) + value = await self.database.get_metadata(namespace=self.db_namespace()) return pickle.loads(value) # noqa: S301 except Exception: logger.debug("Can not load metadata.") @@ -942,7 +928,7 @@ class DatabaseRecordProtocolMixin( return loaded_start <= start_timestamp and end_timestamp <= loaded_end - def _db_load_initial_window( + async def _db_load_initial_window( self, center_timestamp: Optional[DatabaseTimestampType] = None, ) -> None: @@ -1001,11 +987,11 @@ class DatabaseRecordProtocolMixin( window = to_duration(window_h * 3600) start, end = self._search_window(center_timestamp, window) - self.db_load_records(start, end) + await self.db_load_records(start, end) self._db_load_phase = DatabaseRecordProtocolLoadPhase.INITIAL - def _db_load_full(self) -> int: + async def _db_load_full(self) -> int: """Load all remaining records from the database into memory. This method performs a **full load** of the database, ensuring that all @@ -1038,14 +1024,14 @@ class DatabaseRecordProtocolMixin( # Perform full database load (memory is authoritative; skips duplicates) # This also sets _db_loaded_range - loaded_count = self.db_load_records() + loaded_count = await self.db_load_records() # Update state self._db_load_phase = DatabaseRecordProtocolLoadPhase.FULL return loaded_count - def _extend_boundaries( + async def _extend_boundaries( self, start_timestamp: DatabaseTimestampType, end_timestamp: DatabaseTimestampType, @@ -1069,7 +1055,7 @@ class DatabaseRecordProtocolMixin( ): # There may be earlier DB records # Reverse iterate to get nearest smaller key - for key, _ in self.database.iterate_records( + async for key, _ in self.database.iterate_records( start_key=UNBOUND_START, end_key=self._db_key_from_timestamp(start_timestamp), namespace=self.db_namespace(), @@ -1091,7 +1077,7 @@ class DatabaseRecordProtocolMixin( and end_timestamp > self._db_sorted_timestamps[-1] ): # There may be later DB records - for key, _ in self.database.iterate_records( + async for key, _ in self.database.iterate_records( start_key=self._db_key_from_timestamp(end_timestamp), end_key=UNBOUND_END, namespace=self.db_namespace(), @@ -1107,7 +1093,7 @@ class DatabaseRecordProtocolMixin( return new_start, new_end - def _db_ensure_loaded( + async def _db_ensure_loaded( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, @@ -1172,11 +1158,11 @@ class DatabaseRecordProtocolMixin( # Phase 0: NOTHING LOADED if self._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE: if start_timestamp is UNBOUND_START and end_timestamp is UNBOUND_END: - self._db_load_initial_window(center_timestamp) + await self._db_load_initial_window(center_timestamp) # _db_load_initial_window sets _db_loaded_range and _db_load_phase else: # Load the records - loaded = self.db_load_records(start_timestamp, end_timestamp) + loaded = await self.db_load_records(start_timestamp, end_timestamp) self._db_load_phase = DatabaseRecordProtocolLoadPhase.INITIAL return @@ -1196,7 +1182,7 @@ class DatabaseRecordProtocolMixin( return # already have it if start_timestamp == UNBOUND_START and end_timestamp == UNBOUND_END: - self._db_load_full() + await self._db_load_full() return current_start, current_end = self._db_loaded_range @@ -1207,11 +1193,11 @@ class DatabaseRecordProtocolMixin( # Left expansion if start_timestamp < current_start: - self.db_load_records(start_timestamp, current_start) + await self.db_load_records(start_timestamp, current_start) # Right expansion if end_timestamp > current_end: - self.db_load_records(current_end, end_timestamp) + await self.db_load_records(current_end, end_timestamp) return @@ -1251,15 +1237,15 @@ class DatabaseRecordProtocolMixin( def db_enabled(self) -> bool: return self.database.is_open - def db_timestamp_range( + async def db_timestamp_range( self, ) -> tuple[Optional[DatabaseTimestamp], Optional[DatabaseTimestamp]]: """Get the timestamp range of records in database. Regards records in storage plus extra records in memory. """ - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() if self._db_sorted_timestamps: memory_min_timestamp: Optional[DatabaseTimestamp] = self._db_sorted_timestamps[0] @@ -1271,7 +1257,7 @@ class DatabaseRecordProtocolMixin( if not self.db_enabled: return memory_min_timestamp, memory_max_timestamp - db_min_key, db_max_key = self.database.get_key_range(self.db_namespace()) + db_min_key, db_max_key = await self.database.get_key_range(namespace=self.db_namespace()) if db_min_key is None or db_max_key is None: return memory_min_timestamp, memory_max_timestamp @@ -1331,7 +1317,7 @@ class DatabaseRecordProtocolMixin( yield DatabaseTimestamp.from_datetime(current_utc) current_utc = current_utc.add(seconds=step_seconds) - def db_get_record( + async def db_get_record( self, target_timestamp: DatabaseTimestamp, *, @@ -1353,11 +1339,11 @@ class DatabaseRecordProtocolMixin( Returns: Exact match, nearest record within the window, or None. """ - self._db_ensure_initialized() + await self._db_ensure_initialized() if time_window is None: # Exact match only — load the minimal range containing this point - self._db_ensure_loaded( + await self._db_ensure_loaded( target_timestamp, self._db_timestamp_after(target_timestamp), center_timestamp=target_timestamp, @@ -1367,7 +1353,7 @@ class DatabaseRecordProtocolMixin( # load the relevant range # in case of unbounded escalates to FULL search_start, search_end = self._search_window(target_timestamp, time_window) - self._db_ensure_loaded(search_start, search_end, center_timestamp=target_timestamp) + await self._db_ensure_loaded(search_start, search_end, center_timestamp=target_timestamp) # Exact match first (works for all three cases once loaded) record = self._db_record_index.get(target_timestamp, None) @@ -1406,19 +1392,19 @@ class DatabaseRecordProtocolMixin( return record - def db_insert_record( + async def db_insert_record( self, record: T_Record, *, mark_dirty: bool = True, ) -> None: - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() # Ensure normalized to UTC db_record_date_time = DatabaseTimestamp.from_datetime(record.date_time) - self._db_ensure_loaded( + await self._db_ensure_loaded( start_timestamp=db_record_date_time, end_timestamp=db_record_date_time, ) @@ -1446,7 +1432,7 @@ class DatabaseRecordProtocolMixin( # Load (range) # ----------------------------------------------------- - def db_load_records( + async def db_load_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, @@ -1475,8 +1461,8 @@ class DatabaseRecordProtocolMixin( Note: record.date_time shall be DateTime or None """ - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() if not self.db_enabled: return 0 @@ -1488,7 +1474,7 @@ class DatabaseRecordProtocolMixin( end_timestamp = UNBOUND_END # Extend boundaries to include first record < start and first record >= end - query_start, query_end = self._extend_boundaries(start_timestamp, end_timestamp) + query_start, query_end = await self._extend_boundaries(start_timestamp, end_timestamp) if isinstance(query_start, _DatabaseTimestampUnbound): start_key = None @@ -1504,7 +1490,7 @@ class DatabaseRecordProtocolMixin( loaded_count = 0 # Iterate DB records (already sorted by key) - for db_key, value in self.database.iterate_records( + async for db_key, value in self.database.iterate_records( start_key=start_key, end_key=end_key, namespace=namespace, @@ -1551,16 +1537,16 @@ class DatabaseRecordProtocolMixin( # Delete (range) # ----------------------------------------------------- - def db_delete_records( + async def db_delete_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, ) -> int: - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() # Deletion is global — ensure we see everything - self._db_ensure_loaded( + await self._db_ensure_loaded( start_timestamp=start_timestamp, end_timestamp=end_timestamp, ) @@ -1598,21 +1584,21 @@ class DatabaseRecordProtocolMixin( # Iteration from DB (no duplicates) # ----------------------------------------------------- - def db_iterate_records( + async def db_iterate_records( self, start_timestamp: Optional[DatabaseTimestampType] = None, end_timestamp: Optional[DatabaseTimestampType] = None, - ) -> Iterator[T_Record]: + ) -> AsyncIterator[T_Record]: """Iterate records in requested range. Ensures storage is loaded into memory first, then iterates over in-memory records only. """ - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() # Ensure memory contains required range - self._db_ensure_loaded( + await self._db_ensure_loaded( start_timestamp=start_timestamp, end_timestamp=end_timestamp, ) @@ -1635,9 +1621,9 @@ class DatabaseRecordProtocolMixin( # Dirty tracking # ----------------------------------------------------- - def db_mark_dirty_record(self, record: T_Record) -> None: - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + async def db_mark_dirty_record(self, record: T_Record) -> None: + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time) self._db_dirty_timestamps.add(record_date_time_timestamp) @@ -1646,9 +1632,9 @@ class DatabaseRecordProtocolMixin( # Bulk save (flush dirty only) # ----------------------------------------------------- - def db_save_records(self) -> int: - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + async def db_save_records(self) -> int: + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() if not self.db_enabled: return 0 @@ -1670,23 +1656,20 @@ class DatabaseRecordProtocolMixin( save_items.append((key, value)) saved_count = len(save_items) if saved_count: - self.database.save_records(save_items, namespace=namespace) + await self.database.save_records(save_items, namespace=namespace) self._db_dirty_timestamps.clear() self._db_new_timestamps.clear() # --- handle deletions --- if self._db_deleted_timestamps: delete_keys = [self._db_key_from_timestamp(dt) for dt in self._db_deleted_timestamps] - self.database.delete_records(delete_keys, namespace=namespace) + await self.database.delete_records(delete_keys, namespace=namespace) deleted_count = len(self._db_deleted_timestamps) self._db_deleted_timestamps.clear() return saved_count + deleted_count - def db_autosave(self) -> int: - return self.db_save_records() - - def db_vacuum( + async def db_vacuum( self, keep_hours: Optional[int] = None, keep_timestamp: Optional[DatabaseTimestampType] = None, @@ -1708,8 +1691,8 @@ class DatabaseRecordProtocolMixin( Returns: Number of records deleted """ - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() if keep_hours is None and keep_timestamp is None: keep_duration = self.db_keep_duration() @@ -1722,7 +1705,7 @@ class DatabaseRecordProtocolMixin( keep_hours = keep_duration.hours if keep_hours is not None: - _, db_max = self.db_timestamp_range() + _, db_max = await self.db_timestamp_range() if db_max is None or isinstance(db_max, _DatabaseTimestampUnbound): # No records return 0 # nothing to delete @@ -1740,9 +1723,9 @@ class DatabaseRecordProtocolMixin( raise ValueError("Must specify either keep_hours or keep_timestamp") # Delete records - deleted_count = self.db_delete_records(end_timestamp=db_cutoff_timestamp) + deleted_count = await self.db_delete_records(end_timestamp=db_cutoff_timestamp) - self.db_save_records() + await self.db_save_records() logger.info( f"Vacuumed {deleted_count} old records from database '{self.db_namespace()}' " @@ -1750,14 +1733,14 @@ class DatabaseRecordProtocolMixin( ) return deleted_count - def db_count_records(self) -> int: + async def db_count_records(self) -> int: """Return total logical number of records. Memory is authoritative. If DB is enabled but not fully loaded, we conservatively include storage-only records. """ - # Defensive call - model_post_init() may not have initialized metadata - self._db_ensure_initialized() + # Ensure db in memory data and metadata is initialized + await self._db_ensure_initialized() if not self.db_enabled: return len(self.records) @@ -1766,13 +1749,13 @@ class DatabaseRecordProtocolMixin( if self._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL: return len(self.records) - storage_count = self.database.count_records(namespace=self.db_namespace()) + storage_count = await self.database.count_records(namespace=self.db_namespace()) pending_deletes = len(self._db_deleted_timestamps) new_count = len(self._db_new_timestamps) return storage_count + new_count - pending_deletes - def db_get_stats(self) -> dict: + async def db_get_stats(self) -> dict: """Get comprehensive statistics about database storage. Returns: @@ -1783,6 +1766,8 @@ class DatabaseRecordProtocolMixin( ns = self.db_namespace() + total_records = await self.database.count_records(namespace=ns) + stats = { "enabled": True, "backend": self.database.__class__.__name__, @@ -1791,13 +1776,14 @@ class DatabaseRecordProtocolMixin( "compression_enabled": self.database.compression, "keep_duration_h": self.config.database.keep_duration_h, "autosave_interval_sec": self.config.database.autosave_interval_sec, - "total_records": self.database.count_records(namespace=ns), + "total_records": total_records, } # Add backend-specific stats - stats.update(self.database.get_backend_stats(namespace=ns)) + backend_stats = await self.database.get_backend_stats(namespace=ns) + stats.update(backend_stats) - min_timestamp, max_timestamp = self.db_timestamp_range() + min_timestamp, max_timestamp = await self.db_timestamp_range() stats["timestamp_range"] = { "min": str(min_timestamp), "max": str(max_timestamp), @@ -1866,7 +1852,7 @@ class DatabaseRecordProtocolMixin( cutoff_str = self._db_metadata.get(key) return DatabaseTimestamp(cutoff_str) if cutoff_str else None - def _db_set_compact_state( + async def _db_set_compact_state( self, tier_interval: Duration, cutoff_ts: DatabaseTimestamp, @@ -1881,13 +1867,13 @@ class DatabaseRecordProtocolMixin( self._db_metadata = {} key = f"last_compact_cutoff_{int(tier_interval.total_seconds())}" self._db_metadata[key] = str(cutoff_ts) - self._db_save_metadata(self._db_metadata) + await self._db_save_metadata(self._db_metadata) # ------------------------------------------------------------------ # Single-tier worker # ------------------------------------------------------------------ - def _db_compact_tier( + async def _db_compact_tier( self, age_threshold: Duration, target_interval: Duration, @@ -1923,14 +1909,14 @@ class DatabaseRecordProtocolMixin( Number of original records deleted (before re-insertion of downsampled records). Returns 0 if skipped. """ - self._db_ensure_initialized() + await self._db_ensure_initialized() interval_sec = int(target_interval.total_seconds()) if interval_sec <= 0: return 0 # ---- Determine raw new cutoff ------------------------------------ - _, db_max = self.db_timestamp_range() + _, db_max = await self.db_timestamp_range() if db_max is None or isinstance(db_max, _DatabaseTimestampUnbound): return 0 @@ -1955,7 +1941,7 @@ class DatabaseRecordProtocolMixin( ) return 0 - db_min, _ = self.db_timestamp_range() + db_min, _ = await self.db_timestamp_range() if db_min is None or isinstance(db_min, _DatabaseTimestampUnbound): return 0 @@ -1981,7 +1967,11 @@ class DatabaseRecordProtocolMixin( window_end_ts = new_cutoff_ts # ---- Sparse-data guard ------------------------------------------- - existing_count = self.database.count_records( + # Ensure the window is loaded into memory so the sparse guard's + # records_in_window list comprehension sees actual data. + await self._db_ensure_loaded(window_start_ts, window_end_ts) + + existing_count = await self.database.count_records( start_key=self._db_key_from_timestamp(window_start_ts), end_key=self._db_key_from_timestamp(window_end_ts), namespace=self.db_namespace(), @@ -1993,7 +1983,7 @@ class DatabaseRecordProtocolMixin( if existing_count == 0: # Nothing in window — just advance the cutoff - self._db_set_compact_state(target_interval, new_cutoff_ts) + await self._db_set_compact_state(target_interval, new_cutoff_ts) return 0 if existing_count <= resampled_count: @@ -2016,7 +2006,7 @@ class DatabaseRecordProtocolMixin( f"and all timestamps already aligned " f"(window={window_start_dt}..{window_end_dt})" ) - self._db_set_compact_state(target_interval, new_cutoff_ts) + await self._db_set_compact_state(target_interval, new_cutoff_ts) return 0 # ---- Sparse but misaligned: full window rewrite ----------------- @@ -2049,7 +2039,7 @@ class DatabaseRecordProtocolMixin( bucket[key] = val # Delete entire window (aligned + misaligned) - deleted = self.db_delete_records( + deleted = await self.db_delete_records( start_timestamp=window_start_ts, end_timestamp=window_end_ts, ) @@ -2060,10 +2050,10 @@ class DatabaseRecordProtocolMixin( continue snapped_dt = DateTime.fromtimestamp(snapped_epoch, tz="UTC") record = self.record_class()(date_time=snapped_dt, **values) - self.db_insert_record(record, mark_dirty=True) + await self.db_insert_record(record, mark_dirty=True) - self.db_save_records() - self._db_set_compact_state(target_interval, new_cutoff_ts) + await self.db_save_records() + await self._db_set_compact_state(target_interval, new_cutoff_ts) logger.info( f"Rewrote sparse window in namespace '{self.db_namespace()}' " f"tier {target_interval}: deleted={deleted}, " @@ -2086,7 +2076,7 @@ class DatabaseRecordProtocolMixin( if key == "date_time": continue try: - array = self.key_to_array( + array = await self.key_to_array( key, start_datetime=window_start_dt, end_datetime=window_end_dt, @@ -2095,7 +2085,9 @@ class DatabaseRecordProtocolMixin( boundary="context", align_to_interval=True, ) - except (KeyError, TypeError, ValueError): + logger.debug(f"key={key}, array_len={len(array)}") + except (KeyError, TypeError, ValueError) as e: + logger.error(f"key_to_array failed for {key}: {e}") continue # non-numeric or missing key — skip silently if len(array) == 0: @@ -2125,11 +2117,11 @@ class DatabaseRecordProtocolMixin( if not compacted_data or not compacted_timestamps: # Nothing to write back — still advance cutoff - self._db_set_compact_state(target_interval, new_cutoff_ts) + await self._db_set_compact_state(target_interval, new_cutoff_ts) return 0 # ---- Delete originals, re-insert downsampled records ------------- - deleted = self.db_delete_records( + deleted = await self.db_delete_records( start_timestamp=window_start_ts, end_timestamp=window_end_ts, ) @@ -2142,12 +2134,12 @@ class DatabaseRecordProtocolMixin( } if values: record = self.record_class()(date_time=dt, **values) - self.db_insert_record(record, mark_dirty=True) + await self.db_insert_record(record, mark_dirty=True) - self.db_save_records() + await self.db_save_records() # Persist the aligned new cutoff for this tier - self._db_set_compact_state(target_interval, new_cutoff_ts) + await self._db_set_compact_state(target_interval, new_cutoff_ts) logger.info( f"Compacted tier {target_interval}: deleted {deleted} records in " @@ -2161,7 +2153,7 @@ class DatabaseRecordProtocolMixin( # Public entry point # ------------------------------------------------------------------ - def db_compact( + async def db_compact( self, compact_tiers: Optional[list[tuple[Duration, Duration]]] = None, ) -> int: @@ -2183,12 +2175,13 @@ class DatabaseRecordProtocolMixin( compact_tiers = self.db_compact_tiers() if not compact_tiers: + logger.debug(f"Compaction called but no compact_tiers '{compact_tiers}' given.") return 0 total_deleted = 0 # Coarsest tier first (reversed) to avoid redundant work for age_threshold, target_interval in reversed(compact_tiers): - total_deleted += self._db_compact_tier(age_threshold, target_interval) + total_deleted += await self._db_compact_tier(age_threshold, target_interval) return total_deleted diff --git a/src/akkudoktoreos/core/ems.py b/src/akkudoktoreos/core/ems.py index d2aafd6e..1b34a46e 100644 --- a/src/akkudoktoreos/core/ems.py +++ b/src/akkudoktoreos/core/ems.py @@ -2,8 +2,7 @@ import traceback from asyncio import Lock, get_running_loop from concurrent.futures import ThreadPoolExecutor from enum import StrEnum -from functools import partial -from typing import ClassVar, Optional +from typing import ClassVar, Optional, cast from loguru import logger from pydantic import computed_field @@ -147,11 +146,11 @@ class EnergyManagement( """ return cls._genetic_solution - @classmethod - def _run( - cls, - start_datetime: DateTime, - mode: EnergyManagementMode, + async def run( + self, + start_datetime: Optional[DateTime] = None, + mode: Optional[EnergyManagementMode] = None, + algorithm: Optional[str] = None, genetic_parameters: Optional[GeneticOptimizationParameters] = None, genetic_individuals: Optional[int] = None, genetic_seed: Optional[int] = None, @@ -161,7 +160,6 @@ class EnergyManagement( """Run the energy management. This method initializes the energy management run by setting its - start datetime, updating predictions, and optionally starting optimization depending on the selected mode or configuration. @@ -171,161 +169,20 @@ class EnergyManagement( - "OPTIMIZATION": Runs the optimization process. - "PREDICTION": Updates the forecast without optimization. - "DISABLED": Does not run. - genetic_parameters (GeneticOptimizationParameters, optional): The - parameter set for the genetic algorithm. If not provided, it will - be constructed based on the current configuration and predictions. - genetic_individuals (int, optional): The number of individuals for the - genetic algorithm. Defaults to the algorithm's internal default (400) - if not specified. - genetic_seed (int, optional): The seed for the genetic algorithm. Defaults - to the algorithm's internal random seed if not specified. - force_enable (bool, optional): If True, bypasses any disabled state - to force the update process. This is mostly applicable to - prediction providers. - force_update (bool, optional): If True, forces data to be refreshed - even if a cached version is still valid. - - Returns: - None - """ - # Ensure there is only one optimization/ energy management run at a time - if not mode in EnergyManagementMode._value2member_map_: - raise ValueError(f"Unknown energy management mode {mode}.") - if mode == EnergyManagementMode.DISABLED: - return - - logger.info("Starting energy management run.") - - cls._stage = EnergyManagementStage.DATA_ACQUISITION - - # Remember/ set the start datetime of this energy management run. - # None leads - cls.set_start_datetime(start_datetime) - - # Throw away any memory cached results of the last energy management run. - CacheEnergyManagementStore().clear() - - # Do data aquisition by adapters - try: - cls.adapter.update_data(force_enable) - except Exception as e: - trace = "".join(traceback.TracebackException.from_exception(e).format()) - error_msg = f"Adapter update failed - phase {cls._stage}:\n{e}\n{trace}" - logger.error(error_msg) - - cls._stage = EnergyManagementStage.FORECAST_RETRIEVAL - - if mode == EnergyManagementMode.PREDICTION: - # Update the predictions - cls.prediction.update_data(force_enable=force_enable, force_update=force_update) - logger.info("Energy management run done (predictions updated)") - cls._stage = EnergyManagementStage.IDLE - return - - # Prepare optimization parameters - # This also creates default configurations for missing values and updates the predictions - logger.info( - "Starting energy management prediction update and optimzation parameter preparation." - ) - if genetic_parameters is None: - genetic_parameters = GeneticOptimizationParameters.prepare() - - if not genetic_parameters: - logger.error( - "Energy management run canceled. Could not prepare optimisation parameters." - ) - cls._stage = EnergyManagementStage.IDLE - return - - cls._stage = EnergyManagementStage.OPTIMIZATION - logger.info("Starting energy management optimization.") - - # Take values from config if not given - if genetic_individuals is None: - genetic_individuals = cls.config.optimization.genetic.individuals - if genetic_seed is None: - genetic_seed = cls.config.optimization.genetic.seed - - if cls._start_datetime is None: # Make mypy happy - already set by us - raise RuntimeError("Start datetime not set.") - - try: - optimization = GeneticOptimization( - verbose=bool(cls.config.server.verbose), - fixed_seed=genetic_seed, - ) - solution = optimization.optimierung_ems( - start_hour=cls._start_datetime.hour, - parameters=genetic_parameters, - ngen=genetic_individuals, - ) - except: - logger.exception("Energy management optimization failed.") - cls._stage = EnergyManagementStage.IDLE - return - - cls._stage = EnergyManagementStage.CONTROL_DISPATCH - - # Make genetic solution public - cls._genetic_solution = solution - - # Make optimization solution public - cls._optimization_solution = solution.optimization_solution() - - # Make plan public - cls._plan = solution.energy_management_plan() - - logger.debug("Energy management genetic solution:\n{}", cls._genetic_solution) - logger.debug("Energy management optimization solution:\n{}", cls._optimization_solution) - logger.debug("Energy management plan:\n{}", cls._plan) - logger.info("Energy management run done (optimization updated)") - - # Do control dispatch by adapters - try: - cls.adapter.update_data(force_enable) - except Exception as e: - trace = "".join(traceback.TracebackException.from_exception(e).format()) - error_msg = f"Adapter update failed - phase {cls._stage}:\n{e}\n{trace}" - logger.error(error_msg) - - # Remember energy run datetime. - EnergyManagement._last_run_datetime = to_datetime() - - # energy management run finished - cls._stage = EnergyManagementStage.IDLE - - async def run( - self, - start_datetime: Optional[DateTime] = None, - mode: Optional[EnergyManagementMode] = None, - genetic_parameters: Optional[GeneticOptimizationParameters] = None, - genetic_individuals: Optional[int] = None, - genetic_seed: Optional[int] = None, - force_enable: Optional[bool] = False, - force_update: Optional[bool] = False, - ) -> None: - """Run the energy management. - - This method initializes the energy management run by setting its - start datetime, updating predictions, and optionally starting - optimization depending on the selected mode or configuration. - - Args: - start_datetime (DateTime, optional): The starting timestamp - of the energy management run. Defaults to the current datetime - if not provided. - mode (EnergyManagementMode, optional): The management mode to use. Must be one of: - - "OPTIMIZATION": Runs the optimization process. - - "PREDICTION": Updates the forecast without optimization. Defaults to the mode defined in the current configuration. + algorithm (str, optional): + The algorithm to use. Must be one of: + - "GENETIC": Optimization uses the `GENETIC` optimization algorithm. + + Defaults to the algorithm defined in the current configuration. genetic_parameters (GeneticOptimizationParameters, optional): The - parameter set for the genetic algorithm. If not provided, it will + parameter set for the `GENETIC` algorithm. If not provided, it will be constructed based on the current configuration and predictions. genetic_individuals (int, optional): The number of individuals for the - genetic algorithm. Defaults to the algorithm's internal default (400) + `GENETIC` algorithm. Defaults to the algorithm's internal default (400) if not specified. - genetic_seed (int, optional): The seed for the genetic algorithm. Defaults + genetic_seed (int, optional): The seed for the `GENETIC` algorithm. Defaults to the algorithm's internal random seed if not specified. force_enable (bool, optional): If True, bypasses any disabled state to force the update process. This is mostly applicable to @@ -336,22 +193,205 @@ class EnergyManagement( Returns: None """ - async with self._run_lock: - loop = get_running_loop() - # Create a partial function with parameters "baked in" - if start_datetime is None: - start_datetime = to_datetime() + async with EnergyManagement._run_lock: if mode is None: mode = self.config.ems.mode - func = partial( - EnergyManagement._run, - start_datetime=start_datetime, - mode=mode, - genetic_parameters=genetic_parameters, - genetic_individuals=genetic_individuals, - genetic_seed=genetic_seed, - force_enable=force_enable, - force_update=force_update, + + if mode not in EnergyManagementMode._value2member_map_: + raise ValueError(f"Unknown energy management mode {mode}.") + if mode == EnergyManagementMode.DISABLED: + logger.info("Energy management run disabled.") + return + + logger.info("Starting energy management run.") + + # --- Data Aquisition --- + EnergyManagement._stage = EnergyManagementStage.DATA_ACQUISITION + + # Remember/ set the start datetime of this energy management run. + # None leads to current time as start datetime + self.set_start_datetime(start_datetime) + + # Throw away any memory cached results of the last energy management run. + CacheEnergyManagementStore().clear() + + # --- Adapter update --- + try: + await self.adapter.update_data(force_enable) + except Exception as e: + trace = "".join(traceback.TracebackException.from_exception(e).format()) + error_msg = ( + f"Adapter update failed - phase {EnergyManagement._stage}:\n{e}\n{trace}" + ) + logger.error(error_msg) + + # --- Prediction --- + EnergyManagement._stage = EnergyManagementStage.FORECAST_RETRIEVAL + + # Update the predictions + logger.info("Starting energy management prediction update.") + await self.prediction.update_data(force_enable=force_enable, force_update=force_update) + + if mode == EnergyManagementMode.PREDICTION: + logger.info("Energy management run done (predictions updated)") + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + # --- Optimization --- + EnergyManagement._stage = EnergyManagementStage.OPTIMIZATION + optimization_start = to_datetime() + logger.info("Starting energy management optimization.") + + if algorithm is None: + algorithm = self.config.optimization.algorithm + + if algorithm == "GENETIC": + # Prepare optimization parameters + # This also creates default configurations for missing values and updates the predictions + logger.info("Starting optimzation parameter preparation.") + if genetic_parameters is None: + genetic_parameters = await GeneticOptimizationParameters.prepare() + if genetic_parameters is None: + logger.error( + "Energy management run canceled. Could not prepare optimisation parameters." + ) + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + # Take values from config if not given + if genetic_individuals is None: + genetic_individuals = self.config.optimization.genetic.individuals + if genetic_seed is None: + genetic_seed = self.config.optimization.genetic.seed + + if EnergyManagement._start_datetime is None: # Make mypy happy - already set by us + raise RuntimeError("Start datetime not set.") + + # --- Optimization (CPU-bound → MUST offload) --- + try: + optimization = GeneticOptimization( + verbose=bool(self.config.server.verbose), + fixed_seed=genetic_seed, + ) + + loop = get_running_loop() + start_hour = EnergyManagement._start_datetime.hour + solution = await loop.run_in_executor( + None, + lambda: optimization.optimierung_ems( + start_hour=start_hour, + parameters=cast( + GeneticOptimizationParameters, genetic_parameters + ), # cast for mypy + ngen=genetic_individuals, + ), + ) + + except Exception: + logger.exception("Energy management optimization failed.") + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + else: + logger.error(f"Unknown optimization algorithm: '{algorithm}'. Skipping.") + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + optimization_duration = to_datetime() - optimization_start + logger.info( + "Energy management optimization ({}) completed in {:.1f} seconds.", + algorithm, + optimization_duration.total_seconds(), ) - # Run optimization in background thread to avoid blocking event loop - await loop.run_in_executor(executor, func) + + logger.debug( + "Energy management optimization solution:\n{}", + EnergyManagement._optimization_solution, + ) + logger.debug("Energy management plan:\n{}", EnergyManagement._plan) + + # --- Control dispatch by adapters --- + EnergyManagement._stage = EnergyManagementStage.CONTROL_DISPATCH + + # Make genetic solution public + EnergyManagement._genetic_solution = solution + + # Make optimization solution public + EnergyManagement._optimization_solution = await solution.optimization_solution() + + # Make plan public + EnergyManagement._plan = solution.energy_management_plan() + + logger.debug( + "Energy management genetic solution:\n{}", EnergyManagement._genetic_solution + ) + + if genetic_parameters is None: + genetic_parameters = await GeneticOptimizationParameters.prepare() + + if not genetic_parameters: + logger.error("Energy management run canceled. Could not prepare parameters.") + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + EnergyManagement._stage = EnergyManagementStage.OPTIMIZATION + + if genetic_individuals is None: + genetic_individuals = self.config.optimization.genetic.individuals + if genetic_seed is None: + genetic_seed = self.config.optimization.genetic.seed + + if EnergyManagement._start_datetime is None: + raise RuntimeError("Start datetime not set.") + + # --- Optimization (CPU-bound → MUST offload) --- + try: + optimization = GeneticOptimization( + verbose=bool(self.config.server.verbose), + fixed_seed=genetic_seed, + ) + + loop = get_running_loop() + start_hour = EnergyManagement._start_datetime.hour + solution = await loop.run_in_executor( + None, + lambda: optimization.optimierung_ems( + start_hour=start_hour, + parameters=genetic_parameters, + ngen=genetic_individuals, + ), + ) + + except Exception: + logger.exception("Energy management optimization failed.") + EnergyManagement._stage = EnergyManagementStage.IDLE + return + + EnergyManagement._genetic_solution = solution + EnergyManagement._optimization_solution = await solution.optimization_solution() + EnergyManagement._plan = solution.energy_management_plan() + + logger.debug("Genetic solution:\n{}", EnergyManagement._genetic_solution) + logger.debug("Optimization solution:\n{}", EnergyManagement._optimization_solution) + logger.debug("Plan:\n{}", EnergyManagement._plan) + logger.info("Energy management run done (optimization updated)") + + # --- Dispatch control by adapters --- + EnergyManagement._stage = EnergyManagementStage.CONTROL_DISPATCH + + # Dispatch (sync → optionally offload) + try: + await self.adapter.update_data(force_enable) + except Exception as e: + trace = "".join(traceback.TracebackException.from_exception(e).format()) + error_msg = ( + f"Adapter update failed - phase {EnergyManagement._stage}:\n{e}\n{trace}" + ) + logger.error(error_msg) + + # --- Idle --- + # Remember energy run datetime. + EnergyManagement._last_run_datetime = to_datetime() + + # energy management run finished + EnergyManagement._stage = EnergyManagementStage.IDLE diff --git a/src/akkudoktoreos/core/pydantic.py b/src/akkudoktoreos/core/pydantic.py index 18a6a070..b29466e4 100644 --- a/src/akkudoktoreos/core/pydantic.py +++ b/src/akkudoktoreos/core/pydantic.py @@ -543,10 +543,10 @@ class PydanticModelNestedValueMixin: if not inspect.isclass(model): raise TypeError(f"Model '{model}' is not of class type.") - if key not in model.model_fields: + if key not in model.model_fields: # type: ignore[attr-defined] raise TypeError(f"Field '{key}' does not exist in model '{model.__name__}'.") - field_annotation = model.model_fields[key].annotation + field_annotation = model.model_fields[key].annotation # type: ignore[attr-defined] if not field_annotation: raise TypeError( f"Missing type annotation for field '{key}' in model '{model.__name__}'." @@ -692,7 +692,7 @@ class PydanticBaseModel(PydanticModelNestedValueMixin, BaseModel): return super().model_dump(*args, **kwargs) def to_dict(self) -> dict: - """Convert this PredictionRecord instance to a dictionary representation. + """Convert this pydantic model instance to a dictionary representation. Returns: dict: A dictionary where the keys are the field names of the PydanticBaseModel, diff --git a/src/akkudoktoreos/measurement/measurement.py b/src/akkudoktoreos/measurement/measurement.py index dbbdca41..be3a5731 100644 --- a/src/akkudoktoreos/measurement/measurement.py +++ b/src/akkudoktoreos/measurement/measurement.py @@ -149,7 +149,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): # Return ceiling of division to include partial intervals return int(np.ceil(diff_seconds / interval_seconds)) - def _energy_from_meter_readings( + async def _energy_from_meter_readings( self, key: str, start_datetime: DateTime, @@ -170,7 +170,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): """ size = self._interval_count(start_datetime, end_datetime, interval) - energy_mr_array = self.key_to_array( + energy_mr_array = await self.key_to_array( key=key, start_datetime=start_datetime, end_datetime=end_datetime + interval, @@ -203,7 +203,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): logger.debug(debug_msg) return energy_array - def load_total_kwh( + async def load_total_kwh( self, start_datetime: Optional[DateTime] = None, end_datetime: Optional[DateTime] = None, @@ -232,9 +232,11 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): return np.zeros(size) if start_datetime is None: - start_datetime = self.min_datetime + start_datetime = await self.min_datetime() if end_datetime is None: - end_datetime = self.max_datetime.add(seconds=1) + end_datetime = await self.max_datetime() + if end_datetime: + end_datetime = end_datetime.add(seconds=1) size = self._interval_count(start_datetime, end_datetime, interval) load_total_kwh_array = np.zeros(size) @@ -242,7 +244,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): if isinstance(self.config.measurement.load_emr_keys, list): for key in self.config.measurement.load_emr_keys: # Calculate load per interval - load_array = self._energy_from_meter_readings( + load_array = await self._energy_from_meter_readings( key=key, start_datetime=start_datetime, end_datetime=end_datetime, @@ -270,14 +272,14 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): """ return to_datetime().subtract(hours=self.config.measurement.historic_hours) - def save(self) -> bool: + async def save(self) -> bool: """Save the measurements to persistent storage. Returns: True in case the measurements were saved, False otherwise. """ # Use db storage if available - saved_to_db = DataSequence.save(self) + saved_to_db = await DataSequence.save(self) if not saved_to_db: measurement_file_path = self._measurement_file_path() if measurement_file_path is None: @@ -292,14 +294,14 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): logger.exception("Cannot save measurements") return True - def load(self) -> bool: + async def load(self) -> bool: """Load measurements from persistent storage. Returns: True in case the measurements were loaded, False otherwise. """ # Use db storage if available - loaded_from_db = DataSequence.load(self) + loaded_from_db = await DataSequence.load(self) if not loaded_from_db: measurement_file_path = self._measurement_file_path() if measurement_file_path is None: @@ -314,7 +316,7 @@ class Measurement(SingletonMixin, DataImportMixin, DataSequence): # Explicitly add data records to the existing singleton for record in loaded.records: - self.insert_by_datetime(record) + await self.insert_by_datetime(record) except Exception as e: logger.exception("Cannot load measurements") return True diff --git a/src/akkudoktoreos/optimization/genetic/geneticparams.py b/src/akkudoktoreos/optimization/genetic/geneticparams.py index ec2d32bf..1c40ded4 100644 --- a/src/akkudoktoreos/optimization/genetic/geneticparams.py +++ b/src/akkudoktoreos/optimization/genetic/geneticparams.py @@ -147,7 +147,7 @@ class GeneticOptimizationParameters( return start_solution @classmethod - def prepare(cls) -> "Optional[GeneticOptimizationParameters]": + async def prepare(cls) -> "Optional[GeneticOptimizationParameters]": """Prepare optimization parameters from config, forecast and measurement data. Fills in values needed for optimization from available configuration, predictions and @@ -231,11 +231,11 @@ class GeneticOptimizationParameters( raise ValueError(error_msg) # Assure predictions are uptodate - cls.prediction.update_data() + await cls.prediction.update_data() try: pvforecast_ac_power = ( - cls.prediction.key_to_array( + await cls.prediction.key_to_array( key="pvforecast_ac_power", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, @@ -290,7 +290,7 @@ class GeneticOptimizationParameters( # Retry continue try: - elecprice_marketprice_wh = cls.prediction.key_to_array( + elecprice_marketprice_wh = await cls.prediction.key_to_array( key="elecprice_marketprice_wh", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, @@ -306,7 +306,7 @@ class GeneticOptimizationParameters( # Retry continue try: - loadforecast_power_w = cls.prediction.key_to_array( + loadforecast_power_w = await cls.prediction.key_to_array( key="loadforecast_power_w", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, @@ -331,7 +331,7 @@ class GeneticOptimizationParameters( # Retry continue try: - feed_in_tariff_wh = cls.prediction.key_to_array( + feed_in_tariff_wh = await cls.prediction.key_to_array( key="feed_in_tariff_wh", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, @@ -358,7 +358,7 @@ class GeneticOptimizationParameters( # Retry continue try: - weather_temp_air = cls.prediction.key_to_array( + weather_temp_air = await cls.prediction.key_to_array( key="weather_temp_air", start_datetime=parameter_start_datetime, end_datetime=parameter_end_datetime, @@ -418,7 +418,7 @@ class GeneticOptimizationParameters( battery_lcos_kwh = battery_config.levelized_cost_of_storage_kwh # Initial SOC try: - initial_soc_factor = cls.measurement.key_to_value( + initial_soc_factor = await cls.measurement.key_to_value( key=battery_config.measurement_key_soc_factor, target_datetime=ems.start_datetime, time_window=to_duration(to_duration("48 hours")), @@ -490,7 +490,7 @@ class GeneticOptimizationParameters( continue # Initial SOC try: - initial_soc_factor = cls.measurement.key_to_value( + initial_soc_factor = await cls.measurement.key_to_value( key=electric_vehicle_config.measurement_key_soc_factor, target_datetime=ems.start_datetime, time_window=to_duration(to_duration("48 hours")), diff --git a/src/akkudoktoreos/optimization/genetic/geneticsolution.py b/src/akkudoktoreos/optimization/genetic/geneticsolution.py index b03227a1..7a317086 100644 --- a/src/akkudoktoreos/optimization/genetic/geneticsolution.py +++ b/src/akkudoktoreos/optimization/genetic/geneticsolution.py @@ -344,7 +344,7 @@ class GeneticSolution(ConfigMixin, GeneticParametersBaseModel): return effective_ac, effective_dc, effective_dis - def optimization_solution(self) -> OptimizationSolution: + async def optimization_solution(self) -> OptimizationSolution: """Provide the genetic solution as a general optimization solution. The battery modes are controlled by the grid control triggers: @@ -612,7 +612,7 @@ class GeneticSolution(ConfigMixin, GeneticParametersBaseModel): ), ]: if pred_key in pred.record_keys: - array = pred.key_to_array( + array = await pred.key_to_array( key=pred_key, start_datetime=start_datetime, end_datetime=end_datetime, diff --git a/src/akkudoktoreos/prediction/elecpriceakkudoktor.py b/src/akkudoktoreos/prediction/elecpriceakkudoktor.py index 15711fc4..2e57d39d 100644 --- a/src/akkudoktoreos/prediction/elecpriceakkudoktor.py +++ b/src/akkudoktoreos/prediction/elecpriceakkudoktor.py @@ -135,7 +135,7 @@ class ElecPriceAkkudoktor(ElecPriceProvider): clean_history = self._cap_outliers(history) return np.full(hours, np.median(clean_history)) - def _update_data( + async def _update_data( self, force_update: Optional[bool] = False ) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]: """Update forecast data in the ElecPriceDataRecord format. @@ -170,10 +170,10 @@ class ElecPriceAkkudoktor(ElecPriceProvider): series_data.at[orig_datetime] = price_wh # Update values using key_from_series - self.key_from_series("elecprice_marketprice_wh", series_data) + await self.key_from_series("elecprice_marketprice_wh", series_data) # Generate history array for prediction - history = self.key_to_array( + history = await self.key_to_array( key="elecprice_marketprice_wh", end_datetime=highest_orig_datetime, fill_method="linear" ) @@ -213,9 +213,9 @@ class ElecPriceAkkudoktor(ElecPriceProvider): for i in range(len(prediction)) ], ) - self.key_from_series("elecprice_marketprice_wh", prediction_series) + await self.key_from_series("elecprice_marketprice_wh", prediction_series) - # history2 = self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002 + # history2 = await self.key_to_array(key="elecprice_marketprice_wh", fill_method="linear") + 0.0002 # return history, history2, prediction # for debug main diff --git a/src/akkudoktoreos/prediction/elecpriceenergycharts.py b/src/akkudoktoreos/prediction/elecpriceenergycharts.py index e33f2835..4334ad3e 100644 --- a/src/akkudoktoreos/prediction/elecpriceenergycharts.py +++ b/src/akkudoktoreos/prediction/elecpriceenergycharts.py @@ -195,7 +195,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider): clean_history = self._cap_outliers(history) return np.full(hours, np.median(clean_history)) - def _update_data( + async def _update_data( self, force_update: Optional[bool] = False ) -> None: # tuple[np.ndarray, np.ndarray, np.ndarray]: """Update forecast data in the ElecPriceDataRecord format. @@ -217,7 +217,7 @@ class ElecPriceEnergyCharts(ElecPriceProvider): # Determine if update is needed and how many days past_days = 35 if self.highest_orig_datetime: - history_series = self.key_to_series( + history_series = await self.key_to_series( key="elecprice_marketprice_wh", start_datetime=self.ems_start_datetime ) # If history lower, then start_datetime @@ -244,14 +244,14 @@ class ElecPriceEnergyCharts(ElecPriceProvider): # Parse and store data series_data = self._parse_data(energy_charts_data) self.highest_orig_datetime = series_data.index.max() - self.key_from_series("elecprice_marketprice_wh", series_data) + await self.key_from_series("elecprice_marketprice_wh", series_data) else: logger.info( f"No Update ElecPriceEnergyCharts is needed, last in history: {self.highest_orig_datetime}" ) # Generate history array for prediction - history = self.key_to_array( + history = await self.key_to_array( key="elecprice_marketprice_wh", end_datetime=self.highest_orig_datetime, fill_method="linear", @@ -293,4 +293,4 @@ class ElecPriceEnergyCharts(ElecPriceProvider): for i in range(len(prediction)) ], ) - self.key_from_series("elecprice_marketprice_wh", prediction_series) + await self.key_from_series("elecprice_marketprice_wh", prediction_series) diff --git a/src/akkudoktoreos/prediction/elecpricefixed.py b/src/akkudoktoreos/prediction/elecpricefixed.py index 5547e09c..8b40c90b 100644 --- a/src/akkudoktoreos/prediction/elecpricefixed.py +++ b/src/akkudoktoreos/prediction/elecpricefixed.py @@ -59,7 +59,7 @@ class ElecPriceFixed(ElecPriceProvider): """Return the unique identifier for the ElecPriceFixed provider.""" return "ElecPriceFixed" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update electricity price data from fixed schedule. Generates electricity prices based on the configured time windows @@ -106,6 +106,6 @@ class ElecPriceFixed(ElecPriceProvider): # Convert kWh → Wh and store one entry per interval step. for idx, price_kwh in enumerate(prices_kwh): current_dt = start_datetime.add(seconds=idx * interval_seconds) - self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0) + await self.update_value(current_dt, "elecprice_marketprice_wh", price_kwh / 1000.0) logger.debug(f"Successfully generated {len(prices_kwh)} fixed electricity price entries") diff --git a/src/akkudoktoreos/prediction/elecpriceimport.py b/src/akkudoktoreos/prediction/elecpriceimport.py index 5ac2687c..fe415328 100644 --- a/src/akkudoktoreos/prediction/elecpriceimport.py +++ b/src/akkudoktoreos/prediction/elecpriceimport.py @@ -63,14 +63,16 @@ class ElecPriceImport(ElecPriceProvider, PredictionImportProvider): """Return the unique identifier for the ElecPriceImport provider.""" return "ElecPriceImport" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. if self.config.elecprice.elecpriceimport.import_file_path: - self.import_from_file( + await self._import_from_file( self.config.elecprice.elecpriceimport.import_file_path, key_prefix="elecprice", ) if self.config.elecprice.elecpriceimport.import_json: - self.import_from_json( + await self._import_from_json( self.config.elecprice.elecpriceimport.import_json, key_prefix="elecprice", ) diff --git a/src/akkudoktoreos/prediction/feedintarifffixed.py b/src/akkudoktoreos/prediction/feedintarifffixed.py index e7f401c2..2d7bd88a 100644 --- a/src/akkudoktoreos/prediction/feedintarifffixed.py +++ b/src/akkudoktoreos/prediction/feedintarifffixed.py @@ -34,7 +34,7 @@ class FeedInTariffFixed(FeedInTariffProvider): """Return the unique identifier for the FeedInTariffFixed provider.""" return "FeedInTariffFixed" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: error_msg = "Feed in tariff not provided" try: feed_in_tariff = ( @@ -47,4 +47,4 @@ class FeedInTariffFixed(FeedInTariffProvider): logger.error(error_msg) raise ValueError(error_msg) feed_in_tariff_wh = feed_in_tariff / 1000 - self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh) + await self.update_value(to_datetime(), "feed_in_tariff_wh", feed_in_tariff_wh) diff --git a/src/akkudoktoreos/prediction/feedintariffimport.py b/src/akkudoktoreos/prediction/feedintariffimport.py index 3ad8c069..10e51ca4 100644 --- a/src/akkudoktoreos/prediction/feedintariffimport.py +++ b/src/akkudoktoreos/prediction/feedintariffimport.py @@ -64,17 +64,19 @@ class FeedInTariffImport(FeedInTariffProvider, PredictionImportProvider): """Return the unique identifier for the FeedInTariffImport provider.""" return "FeedInTariffImport" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. if self.config.feedintariff.provider_settings.FeedInTariffImport is None: logger.debug(f"{self.provider_id()} data update without provider settings.") return if self.config.feedintariff.provider_settings.FeedInTariffImport.import_file_path: - self.import_from_file( + await self._import_from_file( self.config.provider_settings.FeedInTariffImport.import_file_path, key_prefix="feedintariff", ) if self.config.feedintariff.provider_settings.FeedInTariffImport.import_json: - self.import_from_json( + await self._import_from_json( self.config.feedintariff.provider_settings.FeedInTariffImport.import_json, key_prefix="feedintariff", ) diff --git a/src/akkudoktoreos/prediction/loadakkudoktor.py b/src/akkudoktoreos/prediction/loadakkudoktor.py index 83aa8aba..83065709 100644 --- a/src/akkudoktoreos/prediction/loadakkudoktor.py +++ b/src/akkudoktoreos/prediction/loadakkudoktor.py @@ -68,7 +68,7 @@ class LoadAkkudoktor(LoadProvider): raise ValueError(error_msg) return data_year_energy - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Adds the load means and standard deviations.""" data_year_energy = self.load_data() # We provide prediction starting at start of day, to be compatible to old system. @@ -84,7 +84,7 @@ class LoadAkkudoktor(LoadProvider): "loadakkudoktor_mean_power_w": hourly_stats[0], "loadakkudoktor_std_power_w": hourly_stats[1], } - self.update_value(date, values) + await self.update_value(date, values) date += to_duration("1 hour") # We are working on fresh data (no cache), report update time self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) @@ -98,7 +98,9 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor): """Return the unique identifier for the LoadAkkudoktor provider.""" return "LoadAkkudoktorAdjusted" - def _calculate_adjustment(self, data_year_energy: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + async def _calculate_adjustment( + self, data_year_energy: np.ndarray + ) -> tuple[np.ndarray, np.ndarray]: """Calculate weekday and week end adjustment from total load measurement data. Returns: @@ -110,19 +112,22 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor): weekend_adjust = np.zeros(24) weekend_adjust_weight = np.zeros(24) - if self.measurement.max_datetime is None: + max_dt = await self.measurement.max_datetime() + if max_dt is None: # No measurements - return 0 adjustment return (weekday_adjust, weekday_adjust) + min_dt = await self.measurement.min_datetime() + # compare predictions with real measurement - try to use last 7 days - compare_start = self.measurement.max_datetime - to_duration("7 days") - if compare_datetimes(compare_start, self.measurement.min_datetime).lt: + compare_start = max_dt - to_duration("7 days") + if compare_datetimes(compare_start, min_dt).lt: # Not enough measurements for 7 days - use what is available - compare_start = self.measurement.min_datetime - compare_end = self.measurement.max_datetime + compare_start = min_dt + compare_end = max_dt compare_interval = to_duration("1 hour") - load_total_kwh_array = self.measurement.load_total_kwh( + load_total_kwh_array = await self.measurement.load_total_kwh( start_datetime=compare_start, end_datetime=compare_end, interval=compare_interval, @@ -159,10 +164,10 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor): return (weekday_adjust, weekend_adjust) - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Adds the load means and standard deviations.""" data_year_energy = self.load_data() - weekday_adjust, weekend_adjust = self._calculate_adjustment(data_year_energy) + weekday_adjust, weekend_adjust = await self._calculate_adjustment(data_year_energy) # We provide prediction starting at start of day, to be compatible to old system. # End date for prediction is prediction hours from now. date = self.ems_start_datetime.start_of("day") @@ -182,7 +187,7 @@ class LoadAkkudoktorAdjusted(LoadAkkudoktor): # Saturday, Sunday (5, 6) value_adjusted = hourly_stats[0] + weekend_adjust[date.hour] values["loadforecast_power_w"] = max(0, value_adjusted) - self.update_value(date, values) + await self.update_value(date, values) date += to_duration("1 hour") # We are working on fresh data (no cache), report update time self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) diff --git a/src/akkudoktoreos/prediction/loadimport.py b/src/akkudoktoreos/prediction/loadimport.py index fe98ead9..2d5533c0 100644 --- a/src/akkudoktoreos/prediction/loadimport.py +++ b/src/akkudoktoreos/prediction/loadimport.py @@ -62,8 +62,12 @@ class LoadImport(LoadProvider, PredictionImportProvider): """Return the unique identifier for the LoadImport provider.""" return "LoadImport" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. if self.config.load.loadimport.import_file_path: - self.import_from_file(self.config.load.loadimport.import_file_path, key_prefix="load") + await self._import_from_file( + self.config.load.loadimport.import_file_path, key_prefix="load" + ) if self.config.load.loadimport.import_json: - self.import_from_json(self.config.load.loadimport.import_json, key_prefix="load") + await self._import_from_json(self.config.load.loadimport.import_json, key_prefix="load") diff --git a/src/akkudoktoreos/prediction/loadvrm.py b/src/akkudoktoreos/prediction/loadvrm.py index b3958ad4..9e0ee598 100644 --- a/src/akkudoktoreos/prediction/loadvrm.py +++ b/src/akkudoktoreos/prediction/loadvrm.py @@ -84,7 +84,7 @@ class LoadVrm(LoadProvider): """Convert UNIX ms timestamp to timezone-aware datetime.""" return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone) - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Fetch and store VRM load forecast as loadforecast_power_w and related values.""" if self.enabled is False: logger.info("LoadVrm is disabled, skipping update.") @@ -102,7 +102,7 @@ class LoadVrm(LoadProvider): date = self._ts_to_datetime(timestamp) rounded_value = round(value, 2) - self.update_value( + await self.update_value( date, {"loadforecast_power_w": rounded_value}, ) @@ -111,8 +111,3 @@ class LoadVrm(LoadProvider): logger.debug(f"Updated loadforecast_power_w with {len(loadforecast_power_w_data)} entries.") self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) - - -if __name__ == "__main__": - lv = LoadVrm() - lv._update_data() diff --git a/src/akkudoktoreos/prediction/prediction.py b/src/akkudoktoreos/prediction/prediction.py index 9fe60539..4c099629 100644 --- a/src/akkudoktoreos/prediction/prediction.py +++ b/src/akkudoktoreos/prediction/prediction.py @@ -14,7 +14,7 @@ Example: # Create singleton prediction instance with prediction providers from akkudoktoreos.prediction.prediction import prediction - prediction.update_data() + await prediction.update_data() print("Prediction:", prediction) Classes: diff --git a/src/akkudoktoreos/prediction/predictionabc.py b/src/akkudoktoreos/prediction/predictionabc.py index e99aa518..c2e3b564 100644 --- a/src/akkudoktoreos/prediction/predictionabc.py +++ b/src/akkudoktoreos/prediction/predictionabc.py @@ -225,7 +225,7 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider): hours = max(self.config.prediction.hours, self.config.prediction_historic_hours, 24) return to_duration(hours * 3600) - def update_data( + async def update_data( self, force_enable: Optional[bool] = False, force_update: Optional[bool] = False, @@ -243,10 +243,10 @@ class PredictionProvider(PredictionStartEndKeepMixin, DataProvider): return # Delete outdated records before updating - self.delete_by_datetime(end_datetime=self.keep_datetime) + await self.delete_by_datetime(end_datetime=self.keep_datetime) # Call the custom update logic - self._update_data(force_update=force_update) + await self._update_data(force_update=force_update) class PredictionImportProvider(PredictionProvider, DataImportProvider): diff --git a/src/akkudoktoreos/prediction/pvforecastakkudoktor.py b/src/akkudoktoreos/prediction/pvforecastakkudoktor.py index 8632069d..1a891905 100644 --- a/src/akkudoktoreos/prediction/pvforecastakkudoktor.py +++ b/src/akkudoktoreos/prediction/pvforecastakkudoktor.py @@ -52,10 +52,10 @@ Example: forecast = PVForecastAkkudoktor(settings=config) # Get an actual forecast - forecast.update_data() + await forecast.update_data() # Update the AC power measurement for a specific date and time - forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0) + await forecast.update_value(to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0) # Report the DC and AC power forecast along with AC measurements print(forecast.report_ac_power_and_measurement()) @@ -286,7 +286,7 @@ class PVForecastAkkudoktor(PVForecastProvider): return akkudoktor_data - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update forecast data in the PVForecastAkkudoktorDataRecord format. Retrieves data from Akkudoktor. The processed data is inserted into the sequence as @@ -341,7 +341,7 @@ class PVForecastAkkudoktor(PVForecastProvider): "pvforecastakkudoktor_temp_air": forecast_values[0].temperature, } - self.update_value(dt, data) + await self.update_value(dt, data) if len(self) < self.config.prediction.hours: raise ValueError( @@ -385,7 +385,7 @@ class PVForecastAkkudoktor(PVForecastProvider): # Example of how to use the PVForecastAkkudoktor class -if __name__ == "__main__": +async def main() -> None: """Main execution block to demonstrate the use of the PVForecastAkkudoktor class. Sets up the forecast configuration fields, fetches PV power forecast data, @@ -441,12 +441,18 @@ if __name__ == "__main__": forecast = PVForecastAkkudoktor() # Get an actual forecast - forecast.update_data() + await forecast.update_data() # Update the AC power measurement for a specific date and time - forecast.update_value( + await forecast.update_value( to_datetime(None, to_maxtime=False), "pvforecastakkudoktor_ac_power_measured", 1000.0 ) # Report the DC and AC power forecast along with AC measurements print(forecast.report_ac_power_and_measurement()) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/src/akkudoktoreos/prediction/pvforecastimport.py b/src/akkudoktoreos/prediction/pvforecastimport.py index 78df2d9f..70b46675 100644 --- a/src/akkudoktoreos/prediction/pvforecastimport.py +++ b/src/akkudoktoreos/prediction/pvforecastimport.py @@ -64,17 +64,19 @@ class PVForecastImport(PVForecastProvider, PredictionImportProvider): """Return the unique identifier for the PVForecastImport provider.""" return "PVForecastImport" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. if self.config.pvforecast.provider_settings.PVForecastImport is None: logger.debug(f"{self.provider_id()} data update without provider settings.") return if self.config.pvforecast.provider_settings.PVForecastImport.import_file_path is not None: - self.import_from_file( + await self._import_from_file( self.config.pvforecast.provider_settings.PVForecastImport.import_file_path, key_prefix="pvforecast", ) if self.config.pvforecast.provider_settings.PVForecastImport.import_json is not None: - self.import_from_json( + await self._import_from_json( self.config.pvforecast.provider_settings.PVForecastImport.import_json, key_prefix="pvforecast", ) diff --git a/src/akkudoktoreos/prediction/pvforecastvrm.py b/src/akkudoktoreos/prediction/pvforecastvrm.py index 4a1db340..6943b5db 100644 --- a/src/akkudoktoreos/prediction/pvforecastvrm.py +++ b/src/akkudoktoreos/prediction/pvforecastvrm.py @@ -83,7 +83,7 @@ class PVForecastVrm(PVForecastProvider): """Convert UNIX ms timestamp to timezone-aware datetime.""" return to_datetime(timestamp / 1000, in_timezone=self.config.general.timezone) - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update forecast data in the PVForecastDataRecord format.""" if self.enabled is False: logger.info("PVForecastVrm is disabled, skipping update.") @@ -101,16 +101,10 @@ class PVForecastVrm(PVForecastProvider): date = self._ts_to_datetime(timestamp) dc_power = round(value, 2) ac_power = round(dc_power * 0.96, 2) - self.update_value( + await self.update_value( date, {"pvforecast_dc_power": dc_power, "pvforecast_ac_power": ac_power} ) pv_forecast.append((date, dc_power)) logger.debug(f"Updated pvforecast_dc_power with {len(pv_forecast)} entries.") self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) - - -# Example usage -if __name__ == "__main__": - pv = PVForecastVrm() - pv._update_data() diff --git a/src/akkudoktoreos/prediction/weatherbrightsky.py b/src/akkudoktoreos/prediction/weatherbrightsky.py index 93a17f90..b62a4c7c 100644 --- a/src/akkudoktoreos/prediction/weatherbrightsky.py +++ b/src/akkudoktoreos/prediction/weatherbrightsky.py @@ -111,7 +111,7 @@ class WeatherBrightSky(WeatherProvider): self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) return brightsky_data - def _description_to_series(self, description: str) -> pd.Series: + async def _description_to_series(self, description: str) -> pd.Series: """Retrieve a pandas Series corresponding to a weather data description. This method fetches the key associated with the provided description @@ -132,10 +132,10 @@ class WeatherBrightSky(WeatherProvider): error_msg = f"No WeatherDataRecord key for '{description}'" logger.error(error_msg) raise ValueError(error_msg) - series = self.key_to_series(key) + series = await self.key_to_series(key) return series - def _description_from_series(self, description: str, data: pd.Series) -> None: + async def _description_from_series(self, description: str, data: pd.Series) -> None: """Update a weather data with a pandas Series based on its description. This method fetches the key associated with the provided description @@ -154,9 +154,9 @@ class WeatherBrightSky(WeatherProvider): error_msg = f"No WeatherDataRecord key for '{description}'" logger.error(error_msg) raise ValueError(error_msg) - self.key_from_series(key, data) + await self.key_from_series(key, data) - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update forecast data in the WeatherDataRecord format. Retrieves data from BrightSky, maps each BrightSky field to the corresponding @@ -197,31 +197,31 @@ class WeatherBrightSky(WeatherProvider): else: value = value * corr_factor setattr(weather_record, key, value) - self.insert_by_datetime(weather_record) + await self.insert_by_datetime(weather_record) # Converting the cloud cover into Irradiance (GHI, DNI, DHI) description = "Total Clouds (% Sky Obscured)" - cloud_cover = self._description_to_series(description) + cloud_cover = await self._description_to_series(description) ghi, dni, dhi = self.estimate_irradiance_from_cloud_cover( self.config.general.latitude, self.config.general.longitude, cloud_cover ) description = "Global Horizontal Irradiance (W/m2)" ghi = pd.Series(data=ghi, index=cloud_cover.index) - self._description_from_series(description, ghi) + await self._description_from_series(description, ghi) description = "Direct Normal Irradiance (W/m2)" dni = pd.Series(data=dni, index=cloud_cover.index) - self._description_from_series(description, dni) + await self._description_from_series(description, dni) description = "Diffuse Horizontal Irradiance (W/m2)" dhi = pd.Series(data=dhi, index=cloud_cover.index) - self._description_from_series(description, dhi) + await self._description_from_series(description, dhi) # Add Preciptable Water (PWAT) with a PVLib method. key = WeatherDataRecord.key_from_description("Temperature (°C)") assert key # noqa: S101 - temperature = self.key_to_array( + temperature = await self.key_to_array( key=key, start_datetime=self.ems_start_datetime, end_datetime=self.end_datetime, @@ -234,7 +234,7 @@ class WeatherBrightSky(WeatherProvider): return key = WeatherDataRecord.key_from_description("Relative Humidity (%)") assert key # noqa: S101 - humidity = self.key_to_array( + humidity = await self.key_to_array( key=key, start_datetime=self.ems_start_datetime, end_datetime=self.end_datetime, @@ -258,4 +258,4 @@ class WeatherBrightSky(WeatherProvider): ), ) description = "Preciptable Water (cm)" - self._description_from_series(description, pwat) + await self._description_from_series(description, pwat) diff --git a/src/akkudoktoreos/prediction/weatherclearoutside.py b/src/akkudoktoreos/prediction/weatherclearoutside.py index 38f533b9..5427a13f 100644 --- a/src/akkudoktoreos/prediction/weatherclearoutside.py +++ b/src/akkudoktoreos/prediction/weatherclearoutside.py @@ -97,7 +97,7 @@ class WeatherClearOutside(WeatherProvider): self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) return response - def _update_data(self, force_update: Optional[bool] = None) -> None: + async def _update_data(self, force_update: Optional[bool] = None) -> None: """Scrape weather forecast data from ClearOutside's website. This method requests weather forecast data from ClearOutside based on latitude @@ -202,7 +202,7 @@ class WeatherClearOutside(WeatherProvider): raise ValueError(error_msg) # Delete all records that will be newly added - self.delete_by_datetime(start_datetime=forecast_start_datetime) + await self.delete_by_datetime(start_datetime=forecast_start_datetime) # Collect weather data, loop over all days for day, p_day in enumerate(p_days): @@ -341,4 +341,4 @@ class WeatherClearOutside(WeatherProvider): if corr_factor: value = value * corr_factor setattr(weather_record, key, value) - self.insert_by_datetime(weather_record) + await self.insert_by_datetime(weather_record) diff --git a/src/akkudoktoreos/prediction/weatherimport.py b/src/akkudoktoreos/prediction/weatherimport.py index 4842fe19..5d0d6de3 100644 --- a/src/akkudoktoreos/prediction/weatherimport.py +++ b/src/akkudoktoreos/prediction/weatherimport.py @@ -64,17 +64,19 @@ class WeatherImport(WeatherProvider, PredictionImportProvider): """Return the unique identifier for the WeatherImport provider.""" return "WeatherImport" - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Both _sequence_lock and _record_lock are already held by the caller. + # Use internal sync methods only — never await public async counterparts. if self.config.weather.provider_settings.WeatherImport is None: logger.debug(f"{self.provider_id()} data update without provider settings.") return if self.config.weather.provider_settings.WeatherImport.import_file_path: - self.import_from_file( + await self._import_from_file( self.config.weather.provider_settings.WeatherImport.import_file_path, key_prefix="weather", ) if self.config.weather.provider_settings.WeatherImport.import_json: - self.import_from_json( + await self._import_from_json( self.config.weather.provider_settings.WeatherImport.import_json, key_prefix="weather", ) diff --git a/src/akkudoktoreos/prediction/weatheropenmeteo.py b/src/akkudoktoreos/prediction/weatheropenmeteo.py index e6957da3..3b29d1f7 100644 --- a/src/akkudoktoreos/prediction/weatheropenmeteo.py +++ b/src/akkudoktoreos/prediction/weatheropenmeteo.py @@ -197,7 +197,7 @@ class WeatherOpenMeteo(WeatherProvider): self.update_datetime = to_datetime(in_timezone=self.config.general.timezone) return openmeteo_data - def _description_to_series(self, description: str) -> pd.Series: + async def _description_to_series(self, description: str) -> pd.Series: """Retrieve a pandas Series corresponding to a weather data description. This method fetches the key associated with the provided description @@ -218,10 +218,10 @@ class WeatherOpenMeteo(WeatherProvider): error_msg = f"No WeatherDataRecord key for '{description}'" logger.error(error_msg) raise ValueError(error_msg) - series = self.key_to_series(key) + series = await self.key_to_series(key) return series - def _description_from_series(self, description: str, data: pd.Series) -> None: + async def _description_from_series(self, description: str, data: pd.Series) -> None: """Update a weather data with a pandas Series based on its description. This method fetches the key associated with the provided description @@ -240,9 +240,9 @@ class WeatherOpenMeteo(WeatherProvider): error_msg = f"No WeatherDataRecord key for '{description}'" logger.error(error_msg) raise ValueError(error_msg) - self.key_from_series(key, data) + await self.key_from_series(key, data) - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: """Update forecast data in the WeatherDataRecord format. Retrieves data from Open-Meteo, maps each Open-Meteo field to the corresponding @@ -299,11 +299,11 @@ class WeatherOpenMeteo(WeatherProvider): setattr(weather_record, key, value) - self.insert_by_datetime(weather_record) + await self.insert_by_datetime(weather_record) # Check whether radiation values exist (for logging) description_ghi = "Global Horizontal Irradiance (W/m2)" - ghi_series = self._description_to_series(description_ghi) + ghi_series = await self._description_to_series(description_ghi) if ghi_series.isnull().all(): logger.warning("No GHI data received from Open-Meteo") @@ -315,7 +315,7 @@ class WeatherOpenMeteo(WeatherProvider): # Add Precipitable Water (PWAT) using PVLib method key = WeatherDataRecord.key_from_description("Temperature (°C)") assert key # noqa: S101 - temperature = self.key_to_array( + temperature = await self.key_to_array( key=key, start_datetime=self.ems_start_datetime, end_datetime=self.end_datetime, @@ -329,7 +329,7 @@ class WeatherOpenMeteo(WeatherProvider): key = WeatherDataRecord.key_from_description("Relative Humidity (%)") assert key # noqa: S101 - humidity = self.key_to_array( + humidity = await self.key_to_array( key=key, start_datetime=self.ems_start_datetime, end_datetime=self.end_datetime, @@ -354,4 +354,4 @@ class WeatherOpenMeteo(WeatherProvider): ), ) description = "Precipitable Water (cm)" - self._description_from_series(description, pwat) + await self._description_from_series(description, pwat) diff --git a/src/akkudoktoreos/server/eos.py b/src/akkudoktoreos/server/eos.py index 27cb2309..06620b5f 100755 --- a/src/akkudoktoreos/server/eos.py +++ b/src/akkudoktoreos/server/eos.py @@ -73,20 +73,18 @@ from akkudoktoreos.utils.datetimeutil import to_datetime, to_duration # ---------------------- -def save_eos_state() -> None: +async def save_eos_state() -> None: """Save EOS state.""" - get_resource_registry().save() - get_prediction().save() - get_measurement().save() + await get_prediction().save() + await get_measurement().save() cache_save() # keep last -def load_eos_state() -> None: +async def load_eos_state() -> None: """Load EOS state.""" cache_load() # keep first - get_measurement().load() - get_prediction().load() - get_resource_registry().load() + await get_measurement().load() + await get_prediction().load() def terminate_eos() -> None: @@ -100,18 +98,18 @@ def terminate_eos() -> None: logger.info(f"🚀 EOS terminated, PID {pid}") -def save_eos_database() -> None: +async def save_eos_database() -> None: """Save EOS database.""" - get_prediction().save() - get_measurement().save() + await get_prediction().save() + await get_measurement().save() -def compact_eos_database() -> None: +async def compact_eos_database() -> None: """Compact EOS database.""" - get_prediction().db_compact() - get_measurement().db_compact() - get_prediction().db_vacuum() - get_measurement().db_vacuum() + await get_prediction().db_compact() + await get_measurement().db_compact() + await get_prediction().db_vacuum() + await get_measurement().db_vacuum() def autosave_config() -> None: @@ -134,7 +132,7 @@ async def server_shutdown_task() -> None: Finally, logs a message indicating that the EOS server has been terminated. """ - save_eos_state() + await save_eos_state() # Give EOS time to finish some work await asyncio.sleep(5) @@ -162,7 +160,7 @@ def config_eos_ready() -> bool: async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: """Lifespan manager for the app.""" # On startup - load_eos_state() + await load_eos_state() # Prepare the Manager and all task that are handled by the manager manager = RetentionManager( @@ -200,7 +198,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: await asyncio.gather(retention_manager_task, return_exceptions=True) # On shutdown - save_eos_state() + await save_eos_state() app = FastAPI( @@ -297,7 +295,7 @@ def fastapi_admin_cache_get() -> dict: @app.get("/v1/admin/database/stats", tags=["admin"]) -def fastapi_admin_database_stats_get() -> dict: +async def fastapi_admin_database_stats_get() -> dict: """Get statistics from database. Returns: @@ -306,8 +304,8 @@ def fastapi_admin_database_stats_get() -> dict: data = {} try: # Get the stats - data[get_measurement().db_namespace()] = get_measurement().db_get_stats() - data[get_prediction().__class__.__name__] = get_prediction().db_get_stats() + data[get_measurement().db_namespace()] = await get_measurement().db_get_stats() + data[get_prediction().__class__.__name__] = await get_prediction().db_get_stats() except Exception as e: trace = "".join(traceback.TracebackException.from_exception(e).format()) raise HTTPException( @@ -316,8 +314,28 @@ def fastapi_admin_database_stats_get() -> dict: return data +@app.post("/v1/admin/database/save", tags=["admin"]) +async def fastapi_admin_database_save_post() -> dict: + """Save in memory data to database. + + Returns: + data (dict): The database stats after saving the records. + """ + data = {} + try: + await get_measurement().save() + await get_prediction().save() + # Get the stats + data[get_measurement().db_namespace()] = await get_measurement().db_get_stats() + data[get_prediction().__class__.__name__] = await get_prediction().db_get_stats() + except Exception as e: + trace = "".join(traceback.TracebackException.from_exception(e).format()) + raise HTTPException(status_code=400, detail=f"Error on database save: {e}\n{trace}") + return data + + @app.post("/v1/admin/database/vacuum", tags=["admin"]) -def fastapi_admin_database_vacuum_post() -> dict: +async def fastapi_admin_database_vacuum_post() -> dict: """Remove old records from database. Returns: @@ -325,11 +343,13 @@ def fastapi_admin_database_vacuum_post() -> dict: """ data = {} try: - get_measurement().db_vacuum() - get_prediction().db_vacuum() + await get_measurement().db_vacuum() + await get_prediction().db_vacuum() # Get the stats - data[get_measurement().db_namespace()] = get_measurement().db_get_stats() - data[get_prediction().__class__.__name__] = get_prediction().db_get_stats() + measuremet_stats = await get_measurement().db_get_stats() + data[get_measurement().db_namespace()] = measuremet_stats + prediction_stats = await get_prediction().db_get_stats() + data[get_prediction().__class__.__name__] = prediction_stats except Exception as e: trace = "".join(traceback.TracebackException.from_exception(e).format()) raise HTTPException(status_code=400, detail=f"Error on database vacuum: {e}\n{trace}") @@ -342,7 +362,7 @@ async def fastapi_admin_server_restart_post() -> dict: Restart EOS properly by starting a new instance before exiting the old one. """ - save_eos_state() + await save_eos_state() # Start a new EOS (Uvicorn) process logger.info("🔄 Restarting EOS...") @@ -525,7 +545,11 @@ def fastapi_config_put(settings: SettingsEOS) -> ConfigEOS: get_config().merge_settings(settings) return get_config() except Exception as e: - raise HTTPException(status_code=400, detail=f"Error on update of configuration: {e}") + trace = "".join(traceback.TracebackException.from_exception(e).format()) + raise HTTPException( + status_code=400, + detail=f"Error on update of configuration '{settings}':\n{e}\n{trace}", + ) @app.put("/v1/config/{path:path}", tags=["config"]) @@ -687,14 +711,14 @@ def fastapi_measurement_keys_get() -> list[str]: @app.get("/v1/measurement/series", tags=["measurement"]) -def fastapi_measurement_series_get( +async def fastapi_measurement_series_get( key: Annotated[str, Query(description="Measurement key.")], ) -> PydanticDateTimeSeries: """Get the measurements of given key as series.""" try: if key not in get_measurement().record_keys: raise HTTPException(status_code=404, detail=f"Key '{key}' is not available.") - pdseries = get_measurement().key_to_series(key=key) + pdseries = await get_measurement().key_to_series(key=key) return PydanticDateTimeSeries.from_series(pdseries) except HTTPException: # Re-raise HTTP exceptions @@ -710,7 +734,7 @@ def fastapi_measurement_series_get( @app.put("/v1/measurement/value", tags=["measurement"]) -def fastapi_measurement_value_put( +async def fastapi_measurement_value_put( datetime: Annotated[str, Query(description="Datetime.")], key: Annotated[str, Query(description="Measurement key.")], value: Union[float | str], @@ -740,8 +764,8 @@ def fastapi_measurement_value_put( detail=f"Invalid datetime '{datetime}': {e}", ) - get_measurement().update_value(dt, key, value) - pdseries = get_measurement().key_to_series(key=key) + await get_measurement().update_value(dt, key, value) + pdseries = await get_measurement().key_to_series(key=key) return PydanticDateTimeSeries.from_series(pdseries) except HTTPException: raise @@ -755,7 +779,7 @@ def fastapi_measurement_value_put( @app.put("/v1/measurement/series", tags=["measurement"]) -def fastapi_measurement_series_put( +async def fastapi_measurement_series_put( key: Annotated[str, Query(description="Measurement key.")], series: PydanticDateTimeSeries ) -> PydanticDateTimeSeries: """Merge measurement given as series into given key.""" @@ -763,8 +787,8 @@ def fastapi_measurement_series_put( if key not in get_measurement().record_keys: raise HTTPException(status_code=404, detail=f"Key '{key}' is not available.") pdseries = series.to_series() # make pandas series from PydanticDateTimeSeries - get_measurement().key_from_series(key=key, series=pdseries) - pdseries = get_measurement().key_to_series(key=key) + await get_measurement().key_from_series(key=key, series=pdseries) + pdseries = await get_measurement().key_to_series(key=key) return PydanticDateTimeSeries.from_series(pdseries) except HTTPException: # Re-raise HTTP exceptions @@ -780,11 +804,11 @@ def fastapi_measurement_series_put( @app.put("/v1/measurement/dataframe", tags=["measurement"]) -def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None: +async def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None: """Merge the measurement data given as dataframe into EOS measurements.""" try: dataframe = data.to_dataframe() - get_measurement().import_from_dataframe(dataframe) + await get_measurement().import_from_dataframe(dataframe) except Exception as e: # Log unexpected errors trace = "".join(traceback.TracebackException.from_exception(e).format()) @@ -796,11 +820,11 @@ def fastapi_measurement_dataframe_put(data: PydanticDateTimeDataFrame) -> None: @app.put("/v1/measurement/data", tags=["measurement"]) -def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None: +async def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None: """Merge the measurement data given as datetime data into EOS measurements.""" try: datetimedata = data.to_dict() - get_measurement().import_from_dict(datetimedata) + await get_measurement().import_from_dict(datetimedata) except Exception as e: # Log unexpected errors trace = "".join(traceback.TracebackException.from_exception(e).format()) @@ -811,6 +835,49 @@ def fastapi_measurement_data_put(data: PydanticDateTimeData) -> None: ) +@app.delete("/v1/measurement/range", tags=["measurement"]) +async def fastapi_measurement_range_delete( + key: Annotated[str, Query(description="Measurement key.")], + start_datetime: Annotated[Optional[str], Query(description="Start datetime.")] = None, + end_datetime: Annotated[Optional[str], Query(description="End datetime.")] = None, +) -> PydanticDateTimeSeries: + """Delete measurement values for a key within a datetime range.""" + try: + if key not in get_measurement().record_keys: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Key '{key}' not found in measurements", + ) + + try: + start_dt = to_datetime(start_datetime) if start_datetime else None + end_dt = to_datetime(end_datetime) if end_datetime else None + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid datetime: {e}", + ) + + await get_measurement().key_delete_by_datetime( + key=key, + start_datetime=start_dt, + end_datetime=end_dt, + ) + + pdseries = await get_measurement().key_to_series(key=key) + return PydanticDateTimeSeries.from_series(pdseries) + + except HTTPException: + raise + except Exception as e: + trace = "".join(traceback.TracebackException.from_exception(e).format()) + logger.exception(f"Unexpected error deleting measurement range: {key}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Internal server error:\n{e}\n{trace}", + ) + + @app.get("/v1/prediction/providers", tags=["prediction"]) def fastapi_prediction_providers_get(enabled: Optional[bool] = None) -> list[str]: """Get a list of available prediction providers. @@ -838,7 +905,7 @@ def fastapi_prediction_keys_get() -> list[str]: @app.get("/v1/prediction/series", tags=["prediction"]) -def fastapi_prediction_series_get( +async def fastapi_prediction_series_get( key: Annotated[str, Query(description="Prediction key.")], start_datetime: Annotated[ Optional[str], @@ -868,14 +935,14 @@ def fastapi_prediction_series_get( end_datetime = get_prediction().end_datetime else: end_datetime = to_datetime(end_datetime) - pdseries = get_prediction().key_to_series( + pdseries = await get_prediction().key_to_series( key=key, start_datetime=start_datetime, end_datetime=end_datetime ) return PydanticDateTimeSeries.from_series(pdseries) @app.get("/v1/prediction/dataframe", tags=["prediction"]) -def fastapi_prediction_dataframe_get( +async def fastapi_prediction_dataframe_get( keys: Annotated[list[str], Query(description="Prediction keys.")], start_datetime: Annotated[ Optional[str], @@ -911,14 +978,14 @@ def fastapi_prediction_dataframe_get( end_datetime = get_prediction().end_datetime else: end_datetime = to_datetime(end_datetime) - df = get_prediction().keys_to_dataframe( + df = await get_prediction().keys_to_dataframe( keys=keys, start_datetime=start_datetime, end_datetime=end_datetime, interval=interval ) return PydanticDateTimeDataFrame.from_dataframe(df, tz=get_config().general.timezone) @app.get("/v1/prediction/list", tags=["prediction"]) -def fastapi_prediction_list_get( +async def fastapi_prediction_list_get( key: Annotated[str, Query(description="Prediction key.")], start_datetime: Annotated[ Optional[str], @@ -958,16 +1025,13 @@ def fastapi_prediction_list_get( interval = to_duration("1 hour") else: interval = to_duration(interval) - prediction_list = ( - get_prediction() - .key_to_array( - key=key, - start_datetime=start_datetime, - end_datetime=end_datetime, - interval=interval, - ) - .tolist() + prediction_array = await get_prediction().key_to_array( + key=key, + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=interval, ) + prediction_list = prediction_array.tolist() return prediction_list @@ -1130,22 +1194,19 @@ async def fastapi_strompreis() -> list[float]: start_datetime = to_datetime().start_of("day") end_datetime = start_datetime.add(days=2) try: - elecprice = ( - get_prediction() - .key_to_array( - key="elecprice_marketprice_wh", - start_datetime=start_datetime, - end_datetime=end_datetime, - ) - .tolist() + elecprice_array = await get_prediction().key_to_array( + key="elecprice_marketprice_wh", + start_datetime=start_datetime, + end_datetime=end_datetime, ) + elecprice_list = elecprice_array.tolist() except Exception as e: raise HTTPException( status_code=404, detail=f"Can not get the electricity price forecast: {e}.\nDid you configure the electricity price forecast provider?", ) - return elecprice + return elecprice_list class GesamtlastRequest(PydanticBaseModel): @@ -1191,7 +1252,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: # Insert measured data into EOS measurement # Convert from energy per interval to dummy energy meter readings measurement_key = "gesamtlast_emr" - get_measurement().key_delete_by_datetime( + await get_measurement().key_delete_by_datetime( key=measurement_key ) # delete all gesamtlast_emr measurements energy = {} @@ -1218,7 +1279,7 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: energy_mr_values.append(0.0) energy_mr_dates.append(dt) energy_mr_values.append(energy_mr) - get_measurement().key_from_lists(measurement_key, energy_mr_dates, energy_mr_values) + await get_measurement().key_from_lists(measurement_key, energy_mr_dates, energy_mr_values) # Ensure there is only one optimization/ energy management run at a time try: @@ -1236,15 +1297,12 @@ async def fastapi_gesamtlast(request: GesamtlastRequest) -> list[float]: start_datetime = to_datetime().start_of("day") end_datetime = start_datetime.add(days=2) try: - prediction_list = ( - get_prediction() - .key_to_array( - key="loadforecast_power_w", - start_datetime=start_datetime, - end_datetime=end_datetime, - ) - .tolist() + prediction_array = await get_prediction().key_to_array( + key="loadforecast_power_w", + start_datetime=start_datetime, + end_datetime=end_datetime, ) + prediction_list = prediction_array.tolist() except Exception as e: raise HTTPException( status_code=404, @@ -1299,15 +1357,12 @@ async def fastapi_gesamtlast_simple(year_energy: float) -> list[float]: start_datetime = to_datetime().start_of("day") end_datetime = start_datetime.add(days=2) try: - prediction_list = ( - get_prediction() - .key_to_array( - key="loadforecast_power_w", - start_datetime=start_datetime, - end_datetime=end_datetime, - ) - .tolist() + prediction_array = await get_prediction().key_to_array( + key="loadforecast_power_w", + start_datetime=start_datetime, + end_datetime=end_datetime, ) + prediction_list = prediction_array.tolist() except Exception as e: raise HTTPException( status_code=404, @@ -1358,24 +1413,18 @@ async def fastapi_pvforecast() -> ForecastResponse: start_datetime = to_datetime().start_of("day") end_datetime = start_datetime.add(days=2) try: - ac_power = ( - get_prediction() - .key_to_array( - key="pvforecast_ac_power", - start_datetime=start_datetime, - end_datetime=end_datetime, - ) - .tolist() + ac_power_array = await get_prediction().key_to_array( + key="pvforecast_ac_power", + start_datetime=start_datetime, + end_datetime=end_datetime, ) - temp_air = ( - get_prediction() - .key_to_array( - key="pvforecastakkudoktor_temp_air", - start_datetime=start_datetime, - end_datetime=end_datetime, - ) - .tolist() + ac_power_list = ac_power_array.tolist() + temp_air_array = await get_prediction().key_to_array( + key="pvforecastakkudoktor_temp_air", + start_datetime=start_datetime, + end_datetime=end_datetime, ) + temp_air_list = temp_air_array.tolist() except Exception as e: raise HTTPException( status_code=404, @@ -1383,7 +1432,7 @@ async def fastapi_pvforecast() -> ForecastResponse: ) # Return both forecasts as a JSON response - return ForecastResponse(temperature=temp_air, pvpower=ac_power) + return ForecastResponse(temperature=temp_air_list, pvpower=ac_power_list) @app.post("/optimize", tags=["optimize"]) diff --git a/tests/single_test_optimization.py b/tests/single_test_optimization.py index f8856dc0..772ba09d 100755 --- a/tests/single_test_optimization.py +++ b/tests/single_test_optimization.py @@ -24,7 +24,7 @@ prediction_eos = get_prediction() ems_eos = get_ems() -def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: +async def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: """Prepare and return optimization parameters with real world data. Returns: @@ -43,6 +43,7 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: "optimization": { "horizon_hours": 24, "interval": 3600, + "algorithm": "GENETIC", "genetic": { "individuals": 300, "generations": 400, @@ -129,10 +130,10 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: print( f"Real data prediction from {prediction_eos.ems_start_datetime} to {prediction_eos.end_datetime}" ) - prediction_eos.update_data() + await prediction_eos.update_data() # PV Forecast (in W) - pv_forecast = prediction_eos.key_to_array( + pv_forecast = await prediction_eos.key_to_array( key="pvforecast_ac_power", start_datetime=prediction_eos.ems_start_datetime, end_datetime=prediction_eos.end_datetime, @@ -140,7 +141,7 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: print(f"pv_forecast: {pv_forecast}") # Temperature Forecast (in degree C) - temperature_forecast = prediction_eos.key_to_array( + temperature_forecast = await prediction_eos.key_to_array( key="weather_temp_air", start_datetime=prediction_eos.ems_start_datetime, end_datetime=prediction_eos.end_datetime, @@ -148,7 +149,7 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: print(f"temperature_forecast: {temperature_forecast}") # Electricity Price (in Euro per Wh) - strompreis_euro_pro_wh = prediction_eos.key_to_array( + strompreis_euro_pro_wh = await prediction_eos.key_to_array( key="elecprice_marketprice_wh", start_datetime=prediction_eos.ems_start_datetime, end_datetime=prediction_eos.end_datetime, @@ -156,7 +157,7 @@ def prepare_optimization_real_parameters() -> GeneticOptimizationParameters: print(f"strompreis_euro_pro_wh: {strompreis_euro_pro_wh}") # Overall System Load (in W) - gesamtlast = prediction_eos.key_to_array( + gesamtlast = await prediction_eos.key_to_array( key="load_mean", start_datetime=prediction_eos.ems_start_datetime, end_datetime=prediction_eos.end_datetime, @@ -215,6 +216,7 @@ def prepare_optimization_parameters() -> GeneticOptimizationParameters: "optimization": { "horizon_hours": 48, "interval": 3600, + "algorithm": "GENETIC", "genetic": { "individuals": 300, "generations": 400, @@ -414,7 +416,7 @@ def run_optimization( with open(parameters_file, "r") as f: parameters = GeneticOptimizationParameters(**json.load(f)) elif real_world: - parameters = prepare_optimization_real_parameters() + parameters = asyncio.run(prepare_optimization_real_parameters()) else: parameters = prepare_optimization_parameters() logger.info("Optimization Parameters:") diff --git a/tests/single_test_prediction.py b/tests/single_test_prediction.py index c08fae46..1f677520 100644 --- a/tests/single_test_prediction.py +++ b/tests/single_test_prediction.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import asyncio import cProfile import pstats import sys @@ -159,7 +160,7 @@ def run_prediction(provider_id: str, verbose: bool = False) -> str: provider = prediction_eos.provider_by_id(provider_id) - prediction_eos.update_data() + asyncio.run(prediction_eos.update_data()) # Return result of prediction if verbose: @@ -176,7 +177,7 @@ def run_prediction(provider_id: str, verbose: bool = False) -> str: print(f"enabled: {provider.enabled()}") for key in provider.record_keys: print(f"\n{key}\n----------") - print(f"Array: {provider.key_to_array(key)}") + print(f"Array: {asyncio.run(provider.key_to_array(key))}") return provider.model_dump_json(indent=4) @@ -225,4 +226,4 @@ def main(): if __name__ == "__main__": - main() + asyncio.run(main()) diff --git a/tests/test_adapternodered.py b/tests/test_adapternodered.py index 115d70a3..c85b8cdc 100644 --- a/tests/test_adapternodered.py +++ b/tests/test_adapternodered.py @@ -38,6 +38,7 @@ def adapter(config_eos, mock_ems: MagicMock) -> NodeREDAdapter: return ad +@pytest.mark.asyncio class TestNodeREDAdapter: def test_provider_id(self, adapter: NodeREDAdapter): @@ -52,37 +53,37 @@ class TestNodeREDAdapter: assert adapter.enabled() is True @patch("requests.get") - def test_update_datetime(self, mock_get, adapter: NodeREDAdapter): + async def test_update_datetime(self, mock_get, adapter: NodeREDAdapter): adapter.ems.stage.return_value = EnergyManagementStage.DATA_ACQUISITION mock_get.return_value.status_code = 200 mock_get.return_value.json.return_value = {"foo": "bar"} now = to_datetime() - adapter.update_data(force_enable=True) + await adapter.update_data(force_enable=True) mock_get.assert_called_once() assert compare_datetimes(adapter.update_datetime, now).approximately_equal @patch("requests.get") - def test_update_data_data_acquisition_success(self, mock_get , adapter: NodeREDAdapter): + async def test_update_data_data_acquisition_success(self, mock_get , adapter: NodeREDAdapter): adapter.ems.stage.return_value = EnergyManagementStage.DATA_ACQUISITION mock_get.return_value.status_code = 200 mock_get.return_value.json.return_value = {"foo": "bar"} - adapter.update_data(force_enable=True) + await adapter.update_data(force_enable=True) mock_get.assert_called_once() url, = mock_get.call_args[0] assert "/eos/data_aquisition" in url @patch("requests.get", side_effect=Exception("boom")) - def test_update_data_data_acquisition_failure(self, mock_get, adapter: NodeREDAdapter): + async def test_update_data_data_acquisition_failure(self, mock_get, adapter: NodeREDAdapter): adapter.ems.stage.return_value = EnergyManagementStage.DATA_ACQUISITION with pytest.raises(RuntimeError): - adapter.update_data(force_enable=True) + await adapter.update_data(force_enable=True) @patch("requests.post") - def test_update_data_control_dispatch_instructions(self, mock_post, adapter: NodeREDAdapter): + async def test_update_data_control_dispatch_instructions(self, mock_post, adapter: NodeREDAdapter): adapter.ems.stage.return_value = EnergyManagementStage.CONTROL_DISPATCH instr1 = DDBCInstruction( @@ -98,7 +99,7 @@ class TestNodeREDAdapter: mock_post.return_value.status_code = 200 mock_post.return_value.json.return_value = {} - adapter.update_data(force_enable=True) + await adapter.update_data(force_enable=True) _, kwargs = mock_post.call_args payload = kwargs["json"] @@ -110,18 +111,18 @@ class TestNodeREDAdapter: assert "/eos/control_dispatch" in url @patch("requests.post") - def test_update_data_disabled_provider(self, mock_post, adapter: NodeREDAdapter): + async def test_update_data_disabled_provider(self, mock_post, adapter: NodeREDAdapter): adapter.config.adapter.provider = ["HomeAssistant"] # NodeRED disabled - adapter.update_data(force_enable=False) + await adapter.update_data(force_enable=False) mock_post.assert_not_called() @patch("requests.post") - def test_update_data_force_enable_overrides_disabled(self, mock_post, adapter: NodeREDAdapter): + async def test_update_data_force_enable_overrides_disabled(self, mock_post, adapter: NodeREDAdapter): adapter.config.adapter.provider = ["HomeAssistant"] adapter.ems.stage.return_value = EnergyManagementStage.CONTROL_DISPATCH mock_post.return_value.status_code = 200 mock_post.return_value.json.return_value = {} - adapter.update_data(force_enable=True) + await adapter.update_data(force_enable=True) mock_post.assert_called_once() diff --git a/tests/test_dataabc.py b/tests/test_dataabc.py deleted file mode 100644 index 0f31dc18..00000000 --- a/tests/test_dataabc.py +++ /dev/null @@ -1,1627 +0,0 @@ -import json -from datetime import datetime, timezone -from typing import Any, ClassVar, List, Optional, Union - -import numpy as np -import pandas as pd -import pendulum -import pytest -from pydantic import Field, ValidationError - -from akkudoktoreos.config.configabc import SettingsBaseModel -from akkudoktoreos.core.coreabc import get_ems -from akkudoktoreos.core.dataabc import ( - DataABC, - DataContainer, - DataImportProvider, - DataProvider, - DataRecord, - DataSequence, -) -from akkudoktoreos.core.databaseabc import DatabaseTimestamp -from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration - -# Derived classes for testing -# --------------------------- - -class DerivedConfig(SettingsBaseModel): - env_var: Optional[int] = Field(default=None, description="Test config by environment var") - instance_field: Optional[str] = Field(default=None, description="Test config by instance field") - class_constant: Optional[int] = Field(default=None, description="Test config by class constant") - - -class DerivedBase(DataABC): - instance_field: Optional[str] = Field(default=None, description="Field Value") - class_constant: ClassVar[int] = 30 - - -class DerivedRecord(DataRecord): - """Date Record derived from base class DataRecord. - - The derived data record got the - - `data_value` field and the - - `dish_washer_emr`, `solar_power`, `temp` configurable field like data. - """ - - data_value: Optional[float] = Field(default=None, description="Data Value") - - @classmethod - def configured_data_keys(cls) -> Optional[list[str]]: - return ["dish_washer_emr", "solar_power", "temp"] - - -class DerivedSequence(DataSequence): - # overload - records: List[DerivedRecord] = Field( - default_factory=list, description="List of DerivedRecord records" - ) - - @classmethod - def record_class(cls) -> Any: - return DerivedRecord - -class DerivedSequence2(DataSequence): - # overload - records: List[DerivedRecord] = Field( - default_factory=list, description="List of DerivedRecord records" - ) - - @classmethod - def record_class(cls) -> Any: - return DerivedRecord - -class DerivedDataProvider(DataProvider): - """A concrete subclass of DataProvider for testing purposes.""" - - # overload - records: List[DerivedRecord] = Field( - default_factory=list, description="List of DerivedRecord records" - ) - provider_enabled: ClassVar[bool] = False - provider_updated: ClassVar[bool] = False - - @classmethod - def record_class(cls) -> Any: - return DerivedRecord - - # Implement abstract methods for test purposes - def provider_id(self) -> str: - return "DerivedDataProvider" - - def enabled(self) -> bool: - return self.provider_enabled - - def _update_data(self, force_update: Optional[bool] = False) -> None: - # Simulate update logic - DerivedDataProvider.provider_updated = True - - -class DerivedDataImportProvider(DataImportProvider): - """A concrete subclass of DataImportProvider for testing purposes.""" - - # overload - records: List[DerivedRecord] = Field( - default_factory=list, description="List of DerivedRecord records" - ) - provider_enabled: ClassVar[bool] = False - provider_updated: ClassVar[bool] = False - - @classmethod - def record_class(cls) -> Any: - return DerivedRecord - - # Implement abstract methods for test purposes - def provider_id(self) -> str: - return "DerivedDataImportProvider" - - def enabled(self) -> bool: - return self.provider_enabled - - def _update_data(self, force_update: Optional[bool] = False) -> None: - # Simulate update logic - DerivedDataImportProvider.provider_updated = True - - -class DerivedDataContainer(DataContainer): - providers: List[Union[DerivedDataProvider, DataProvider]] = Field( - default_factory=list, description="List of data providers" - ) - - -# Tests -# ---------- - - -class TestDataABC: - @pytest.fixture - def base(self): - # Provide default values for configuration - derived = DerivedBase() - return derived - - def test_get_config_value_key_error(self, base): - with pytest.raises(AttributeError): - base.config.non_existent_key - - -class TestDataRecord: - def create_test_record(self, date, value): - """Helper function to create a test DataRecord.""" - return DerivedRecord(date_time=date, data_value=value) - - @pytest.fixture - def record(self): - """Fixture to create a sample DerivedDataRecord with some data set.""" - rec = DerivedRecord(date_time=to_datetime("1967-01-11"), data_value=10.0) - rec.configured_data = {"dish_washer_emr": 123.0, "solar_power": 456.0} - return rec - - def test_getitem(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - assert record["data_value"] == 10.0 - - def test_setitem(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - record["data_value"] = 20.0 - assert record.data_value == 20.0 - - def test_delitem(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - record.data_value = 20.0 - del record["data_value"] - assert record.data_value is None - - def test_len(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - record.date_time = None - record.data_value = 20.0 - assert len(record) == 5 # 2 regular fields + 3 configured data "fields" - - def test_to_dict(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - record.data_value = 20.0 - record_dict = record.to_dict() - assert "data_value" in record_dict - assert record_dict["data_value"] == 20.0 - record2 = DerivedRecord.from_dict(record_dict) - assert record2.model_dump() == record.model_dump() - - def test_to_json(self): - record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) - record.data_value = 20.0 - json_str = record.to_json() - assert "data_value" in json_str - assert "20.0" in json_str - record2 = DerivedRecord.from_json(json_str) - assert record2.model_dump() == record.model_dump() - - def test_record_keys_includes_configured_data_keys(self, record): - """Ensure record_keys includes all configured configured data keys.""" - assert set(record.record_keys()) >= set(record.configured_data_keys()) - - def test_record_keys_writable_includes_configured_data_keys(self, record): - """Ensure record_keys_writable includes all configured configured data keys.""" - assert set(record.record_keys_writable()) >= set(record.configured_data_keys()) - - def test_getitem_existing_field(self, record): - """Test that __getitem__ returns correct value for existing native field.""" - record.date_time = "2024-01-01T00:00:00+00:00" - assert record["date_time"] is not None - - def test_getitem_existing_configured_data(self, record): - """Test that __getitem__ retrieves existing configured data values.""" - assert record["dish_washer_emr"] == 123.0 - assert record["solar_power"] == 456.0 - - def test_getitem_missing_configured_data_returns_none(self, record): - """Test that __getitem__ returns None for missing but known configured data keys.""" - assert record["temp"] is None - - def test_getitem_raises_keyerror(self, record): - """Test that __getitem__ raises KeyError for completely unknown keys.""" - with pytest.raises(KeyError): - _ = record["nonexistent"] - - def test_setitem_field(self, record): - """Test setting a native field using __setitem__.""" - record["date_time"] = "2025-01-01T12:00:00+00:00" - assert str(record.date_time).startswith("2025-01-01") - - def test_setitem_configured_data(self, record): - """Test setting a known configured data key using __setitem__.""" - record["temp"] = 25.5 - assert record.configured_data["temp"] == 25.5 - - def test_setitem_invalid_key_raises(self, record): - """Test that __setitem__ raises KeyError for unknown keys.""" - with pytest.raises(KeyError): - record["unknown_key"] = 123 - - def test_delitem_field(self, record): - """Test deleting a native field using __delitem__.""" - record["date_time"] = "2025-01-01T12:00:00+00:00" - del record["date_time"] - assert record.date_time is None - - def test_delitem_configured_data(self, record): - """Test deleting a known configured data key using __delitem__.""" - del record["solar_power"] - assert "solar_power" not in record.configured_data - - def test_delitem_unknown_raises(self, record): - """Test that __delitem__ raises KeyError for unknown keys.""" - with pytest.raises(KeyError): - del record["nonexistent"] - - def test_attribute_get_existing_field(self, record): - """Test accessing a native field via attribute.""" - record.date_time = "2025-01-01T12:00:00+00:00" - assert record.date_time is not None - - def test_attribute_get_existing_configured_data(self, record): - """Test accessing an existing configured data via attribute.""" - assert record.dish_washer_emr == 123.0 - - def test_attribute_get_missing_configured_data(self, record): - """Test accessing a missing but known configured data returns None.""" - assert record.temp is None - - def test_attribute_get_invalid_raises(self, record): - """Test accessing an unknown attribute raises AttributeError.""" - with pytest.raises(AttributeError): - _ = record.nonexistent - - def test_attribute_set_existing_field(self, record): - """Test setting a native field via attribute.""" - record.date_time = "2025-06-25T12:00:00+00:00" - assert record.date_time is not None - - def test_attribute_set_existing_configured_data(self, record): - """Test setting a known configured data key via attribute.""" - record.temp = 99.9 - assert record.configured_data["temp"] == 99.9 - - def test_attribute_set_invalid_raises(self, record): - """Test setting an unknown attribute raises AttributeError.""" - with pytest.raises(AttributeError): - record.invalid = 123 - - def test_delattr_field(self, record): - """Test deleting a native field via attribute.""" - record.date_time = "2025-06-25T12:00:00+00:00" - del record.date_time - assert record.date_time is None - - def test_delattr_configured_data(self, record): - """Test deleting a known configured data key via attribute.""" - record.temp = 88.0 - del record.temp - assert "temp" not in record.configured_data - - def test_delattr_ignored_missing_configured_data_key(self, record): - """Test deleting a known configured data key that was never set is a no-op.""" - del record.temp - assert "temp" not in record.configured_data - - def test_len_and_iter(self, record): - """Test that __len__ and __iter__ behave as expected.""" - keys = list(iter(record)) - assert set(record.record_keys_writable()) == set(keys) - assert len(record) == len(keys) - - def test_in_operator_includes_configured_data(self, record): - """Test that 'in' operator includes configured data keys.""" - assert "dish_washer_emr" in record - assert "temp" in record # known key, even if not yet set - assert "nonexistent" not in record - - def test_hasattr_behavior(self, record): - """Test that hasattr returns True for fields and known configured dataWs.""" - assert hasattr(record, "date_time") - assert hasattr(record, "dish_washer_emr") - assert hasattr(record, "temp") # allowed, even if not yet set - assert not hasattr(record, "nonexistent") - - def test_model_validate_roundtrip(self, record): - """Test that MeasurementDataRecord can be serialized and revalidated.""" - dumped = record.model_dump() - restored = DerivedRecord.model_validate(dumped) - assert restored.dish_washer_emr == 123.0 - assert restored.solar_power == 456.0 - assert restored.temp is None # not set - - def test_copy_preserves_configured_data(self, record): - """Test that copying preserves configured data values.""" - record.temp = 22.2 - copied = record.model_copy() - assert copied.dish_washer_emr == 123.0 - assert copied.temp == 22.2 - assert copied is not record - - def test_equality_includes_configured_data(self, record): - """Test that equality includes the `configured data` content.""" - other = record.model_copy() - assert record == other - - def test_inequality_differs_with_configured_data(self, record): - """Test that records with different configured datas are not equal.""" - other = record.model_copy(deep=True) - # Modify one configured data value in the copy - other.configured_data["dish_washer_emr"] = 999.9 - assert record != other - - def test_in_operator_for_configured_data_and_fields(self, record): - """Ensure 'in' works for both fields and configured configured data keys.""" - assert "dish_washer_emr" in record - assert "solar_power" in record - assert "date_time" in record # standard field - assert "temp" in record # allowed but not yet set - assert "unknown" not in record - - def test_hasattr_equivalence_to_getattr(self, record): - """hasattr should return True for all valid keys/configured datas.""" - assert hasattr(record, "dish_washer_emr") - assert hasattr(record, "temp") - assert hasattr(record, "date_time") - assert not hasattr(record, "nonexistent") - - def test_dir_includes_configured_data_keys(self, record): - """`dir(record)` should include configured data keys for introspection. - It shall not include the internal 'configured datas' attribute. - """ - keys = dir(record) - assert "configured datas" not in keys - for key in record.configured_data_keys(): - assert key in keys - - def test_init_configured_field_like_data_applies_before_model_init(self): - """Test that keys listed in `_configured_data_keys` are moved to `configured_data` at init time.""" - record = DerivedRecord( - date_time="2024-01-03T00:00:00+00:00", - data_value=42.0, - dish_washer_emr=111.1, - solar_power=222.2, - temp=333.3 # assume `temp` is also a valid configured key - ) - - assert record.data_value == 42.0 - assert record.configured_data == { - "dish_washer_emr": 111.1, - "solar_power": 222.2, - "temp": 333.3, - } - - -class TestDataSequence: - @pytest.fixture - def sequence(self): - sequence0 = DerivedSequence() - assert len(sequence0) == 0 - return sequence0 - - @pytest.fixture - def sequence2(self): - sequence = DerivedSequence() - record1 = self.create_test_record(datetime(1970, 1, 1), 1970) - record2 = self.create_test_record(datetime(1971, 1, 1), 1971) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - assert len(sequence) == 2 - return sequence - - def create_test_record(self, date, value): - """Helper function to create a test DataRecord.""" - return DerivedRecord(date_time=date, data_value=value) - - # Test cases - @pytest.mark.parametrize("tz_name", ["UTC", "Europe/Berlin", "Atlantic/Canary"]) - def test_min_max_datetime_timezone_and_order(self, sequence, tz_name, monkeypatch, config_eos): - # Monkeypatch the read-only timezone property - monkeypatch.setattr(config_eos.general.__class__, "timezone", property(lambda self: tz_name)) - - # Create timezone-aware datetimes using the patched config - dt_early = to_datetime("2024-01-01T00:00:00", in_timezone=config_eos.general.timezone) - dt_late = to_datetime("2024-01-02T00:00:00", in_timezone=config_eos.general.timezone) - - # Insert in reverse order to verify sorting - record1 = self.create_test_record(dt_late, 1) - record2 = self.create_test_record(dt_early, 2) - - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - min_dt = sequence.min_datetime - max_dt = sequence.max_datetime - - # --- Basic correctness --- - assert min_dt == dt_early - assert max_dt == dt_late - - # --- Must be timezone aware --- - assert min_dt.tzinfo is not None - assert max_dt.tzinfo is not None - - # --- Must preserve timezone --- - assert min_dt.tzinfo.name == tz_name - assert max_dt.tzinfo.name == tz_name - - - def test_getitem(self, sequence): - assert len(sequence) == 0 - dt = to_datetime("2024-01-01 00:00:00") - record = self.create_test_record(dt, 0) - sequence.insert_by_datetime(record) - assert isinstance(sequence.get_by_datetime(dt), DerivedRecord) - - def test_setitem(self, sequence2): - dt = to_datetime("2024-01-03", in_timezone="UTC") - record = self.create_test_record(dt, 1) - sequence2.insert_by_datetime(record) - assert sequence2.records[2].date_time == dt - - def test_insert_reversed_date_record(self, sequence2): - dt1 = to_datetime("2023-11-05", in_timezone="UTC") - dt2 = to_datetime("2024-01-03", in_timezone="UTC") - record1 = self.create_test_record(dt2, 0.8) - record2 = self.create_test_record(dt1, 0.9) # reversed date - sequence2.insert_by_datetime(record1) - assert sequence2.records[2].date_time == dt2 - sequence2.insert_by_datetime(record2) - assert len(sequence2) == 4 - assert sequence2.records[2] == record2 - - def test_insert_duplicate_date_record(self, sequence): - dt1 = to_datetime("2023-11-05") - record1 = self.create_test_record(dt1, 0.8) - record2 = self.create_test_record(dt1, 0.9) # Duplicate date - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - assert len(sequence) == 1 - assert sequence.get_by_datetime(dt1).data_value == 0.9 # Record should have merged with new value - - def test_key_to_series(self, sequence): - dt = to_datetime(datetime(2023, 11, 6)) - record = self.create_test_record(dt, 0.8) - sequence.insert_by_datetime(record) - series = sequence.key_to_series("data_value") - assert isinstance(series, pd.Series) - - retrieved_record = sequence.get_by_datetime(dt) - assert retrieved_record is not None - assert retrieved_record.data_value == 0.8 - - def test_key_from_series(self, sequence): - dt1 = to_datetime(datetime(2023, 11, 5)) - dt2 = to_datetime(datetime(2023, 11, 6)) - - series = pd.Series( - data=[0.8, 0.9], index=pd.to_datetime([dt1, dt2]) - ) - sequence.key_from_series("data_value", series) - assert len(sequence) == 2 - - record1 = sequence.get_by_datetime(dt1) - assert record1 is not None - assert record1.data_value == 0.8 - - record2 = sequence.get_by_datetime(dt2) - assert record2 is not None - assert record2.data_value == 0.9 - - def test_key_to_array(self, sequence): - interval = to_duration("1 day") - start_datetime = to_datetime("2023-11-6") - last_datetime = to_datetime("2023-11-8") - end_datetime = to_datetime("2023-11-9") - - record1 = self.create_test_record(start_datetime, float(start_datetime.day)) - sequence.insert_by_datetime(record1) - record2 = self.create_test_record(last_datetime, float(last_datetime.day)) - sequence.insert_by_datetime(record2) - - retrieved_record1 = sequence.get_by_datetime(start_datetime) - assert retrieved_record1 is not None - assert retrieved_record1.data_value == 6.0 - - retrieved_record2 = sequence.get_by_datetime(last_datetime) - assert retrieved_record2 is not None - assert retrieved_record2.data_value == 8.0 - - series = sequence.key_to_series( - key="data_value", start_datetime=start_datetime, end_datetime=end_datetime - ) - assert len(series) == 2 - assert series[to_datetime("2023-11-6")] == 6 - assert series[to_datetime("2023-11-8")] == 8 - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_datetime, - end_datetime=end_datetime, - interval=interval, - ) - assert isinstance(array, np.ndarray) - np.testing.assert_equal(array, [6.0, 7.0, 8.0]) - - def test_key_to_array_linear_interpolation(self, sequence): - """Test key_to_array with linear interpolation for numeric data.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) - record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) # Gap of 2 hours - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 3), - interval=interval, - fill_method="linear", - ) - assert len(array) == 3 - assert array[0] == 0.8 - assert array[1] == 0.9 # Interpolated value - assert array[2] == 1.0 - - - def test_key_to_array_linear_interpolation_out_of_grid(self, sequence): - """Test key_to_array with linear interpolation out of grid.""" - interval = to_duration("1 hour") - start_datetime= to_datetime("2023-11-06T00:30:00") # out of grid - end_datetime=to_datetime("2023-11-06T01:30:00") # out of grid - - record1_datetime = to_datetime("2023-11-06T00:00:00") - record1 = self.create_test_record(record1_datetime, 1.0) - - record2_datetime = to_datetime("2023-11-06T02:00:00") - record2 = self.create_test_record(record2_datetime, 2.0) # Gap of 2 hours - - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - # Check test setup - record1_timestamp = DatabaseTimestamp.from_datetime(record1_datetime) - record2_timestamp = DatabaseTimestamp.from_datetime(record2_datetime) - start_timestamp = DatabaseTimestamp.from_datetime(start_datetime) - end_timestamp = DatabaseTimestamp.from_datetime(end_datetime) - - start_previous_timestamp = sequence.db_previous_timestamp(start_timestamp) - assert start_previous_timestamp == record1_timestamp - end_next_timestamp = sequence.db_next_timestamp(end_timestamp) - assert end_next_timestamp == record2_timestamp - - # Test - array = sequence.key_to_array( - key="data_value", - start_datetime=start_datetime, - end_datetime=end_datetime, - interval=interval, - fill_method="linear", - boundary="context", - ) - np.testing.assert_equal(array, [1.5]) - - def test_key_to_array_ffill(self, sequence): - """Test key_to_array with forward filling for missing values.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) - record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 3), - interval=interval, - fill_method="ffill", - ) - assert len(array) == 3 - assert array[0] == 0.8 - assert array[1] == 0.8 # Forward-filled value - assert array[2] == 1.0 - - def test_key_to_array_ffill_one_value(self, sequence): - """Test key_to_array with forward filling for missing values and only one value at end available.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) - sequence.insert_by_datetime(record1) - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 4), - interval=interval, - fill_method="ffill", - ) - assert len(array) == 4 - assert array[0] == 1.0 # Backward-filled value - assert array[1] == 1.0 # Backward-filled value - assert array[2] == 1.0 - assert array[2] == 1.0 # Forward-filled value - - def test_key_to_array_bfill(self, sequence): - """Test key_to_array with backward filling for missing values.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) - record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 3), - interval=interval, - fill_method="bfill", - ) - assert len(array) == 3 - assert array[0] == 0.8 - assert array[1] == 1.0 # Backward-filled value - assert array[2] == 1.0 - - def test_key_to_array_with_truncation(self, sequence): - """Test truncation behavior in key_to_array.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 5, 23), 0.8) - record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 1), 1.0) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - #assert sequence is None - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 5, 23), - end_datetime=pendulum.datetime(2023, 11, 6, 2), - interval=interval, - ) - - assert len(array) == 3 - assert array[0] == 0.8 - assert array[1] == 0.9 # Interpolated from previous day - assert array[2] == 1.0 - - def test_key_to_array_with_none(self, sequence): - """Test handling of empty series in key_to_array.""" - interval = to_duration("1 hour") - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 3), - interval=interval, - ) - assert isinstance(array, np.ndarray) - assert np.all(array == None) - - def test_key_to_array_with_one(self, sequence): - """Test handling of one element series in key_to_array.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 5, 23), 0.8) - sequence.insert_by_datetime(record1) - - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 5, 23), - end_datetime=pendulum.datetime(2023, 11, 6, 2), - interval=interval, - ) - assert len(array) == 3 - assert array[0] == 0.8 - assert array[1] == 0.8 # Interpolated from previous day - assert array[2] == 0.8 # Interpolated from previous day - - def test_key_to_array_invalid_fill_method(self, sequence): - """Test invalid fill_method raises an error.""" - interval = to_duration("1 hour") - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) - sequence.insert_by_datetime(record1) - - with pytest.raises(ValueError, match="Unsupported fill method: invalid"): - sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6), - end_datetime=pendulum.datetime(2023, 11, 6, 1), - interval=interval, - fill_method="invalid", - ) - - def test_key_to_array_resample_mean(self, sequence): - """Test that numeric resampling uses mean when multiple values fall into one interval.""" - interval = to_duration("1 hour") - # Insert values every 15 minutes within the same hour - record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 0), 1.0) - record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 15), 2.0) - record3 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 30), 3.0) - record4 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 45), 4.0) - - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - sequence.insert_by_datetime(record4) - - # Resample to hourly interval, expecting the mean of the 4 values - array = sequence.key_to_array( - key="data_value", - start_datetime=pendulum.datetime(2023, 11, 6, 0), - end_datetime=pendulum.datetime(2023, 11, 6, 1), - interval=interval, - ) - - assert isinstance(array, np.ndarray) - assert len(array) == 1 # one interval: 0:00-1:00 - # The first interval mean = (1+2+3+4)/4 = 2.5 - assert array[0] == pytest.approx(2.5) - - # ------------------------------------------------------------------ - # key_to_array — align_to_interval parameter - # ------------------------------------------------------------------ - # - # The existing tests above use start_datetime values that already sit on - # clean hour/day boundaries, so the default alignment (origin=query_start) - # and clock alignment (origin=epoch-floor) produce identical results. - # The tests below specifically use off-boundary start times to expose - # the difference and verify the new parameter. - - def test_key_to_array_align_false_origin_is_query_start(self, sequence): - """Without align_to_interval the first bucket sits at query_start, not a clock boundary. - - With start_datetime at 10:07:00 and 15-min interval the first resampled - bucket must be at 10:07:00 (origin = query_start), NOT at 10:00:00 or 10:15:00. - """ - # Off-boundary start: 10:07 - start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") - - # Records every 15 min so the resampled mean equals the input values - for m in range(0, 120, 15): - dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=False, - ) - - assert len(array) > 0 - # Reconstruct the pandas index that key_to_array used: origin=start_dt - idx = pd.date_range(start=start_dt, periods=len(array), freq="900s") - # First bucket must be exactly at start_dt (10:07) - assert idx[0].minute == 7 - assert idx[0].second == 0 - - def test_key_to_array_align_true_15min_buckets_on_quarter_hours(self, sequence): - """align_to_interval=True produces timestamps on :00/:15/:30/:45 boundaries.""" - # Off-boundary start: 10:07 - start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") - - # 1-min records across the window so resampling has data to work with - for m in range(0, 121): - dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert len(array) > 0 - # Reconstruct the epoch-aligned index that key_to_array must have used - import math - epoch = int(start_dt.timestamp()) - floored_epoch = (epoch // 900) * 900 # floor to nearest 15-min boundary - idx = pd.date_range( - start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), - periods=len(array), - freq="900s", - ) - # Every bucket must land on a :00/:15/:30/:45 minute mark with zero seconds - for ts in idx: - assert ts.minute % 15 == 0, ( - f"Bucket at {ts} is not on a 15-min boundary (minute={ts.minute})" - ) - assert ts.second == 0, ( - f"Bucket at {ts} has non-zero seconds ({ts.second})" - ) - - def test_key_to_array_align_true_1hour_buckets_on_the_hour(self, sequence): - """align_to_interval=True with 1-hour interval produces on-the-hour timestamps.""" - # Off-boundary start: 10:23 - start_dt = pendulum.datetime(2024, 6, 1, 10, 23, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 15, 23, tz="UTC") - - for m in range(0, 301, 15): - dt = pendulum.datetime(2024, 6, 1, 10, 23, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("1 hour"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert len(array) > 0 - epoch = int(start_dt.timestamp()) - floored_epoch = (epoch // 3600) * 3600 # floor to nearest hour - idx = pd.date_range( - start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), - periods=len(array), - freq="1h", - ) - for ts in idx: - assert ts.minute == 0, ( - f"Bucket at {ts} should be on the hour (minute={ts.minute})" - ) - assert ts.second == 0, ( - f"Bucket at {ts} has non-zero seconds ({ts.second})" - ) - - def test_key_to_array_align_true_when_start_already_on_boundary(self, sequence): - """align_to_interval=True is a no-op when start_datetime is exactly on a boundary. - - With start at a clean 15-min mark both modes must produce identical arrays. - """ - # Exactly on boundary: 10:00:00 - start_dt = pendulum.datetime(2024, 6, 1, 10, 0, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 12, 0, tz="UTC") - - for m in range(0, 121, 15): - dt = pendulum.datetime(2024, 6, 1, 10, 0, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - arr_aligned = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - arr_default = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=False, - ) - - assert len(arr_aligned) == len(arr_default) - np.testing.assert_array_almost_equal(arr_aligned, arr_default, decimal=6) - - def test_key_to_array_align_true_without_start_datetime(self, sequence): - """align_to_interval=True with no start_datetime must not raise. - - Without a query_start there is no origin to snap; behaviour falls back - to 'start_day' (same as default). No exception is expected. - """ - for m in range(0, 121, 15): - dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=None, - end_datetime=pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC"), - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert isinstance(array, np.ndarray) - assert len(array) > 0 - - def test_key_to_array_align_true_output_within_requested_window(self, sequence): - """align_to_interval=True truncates output to [start_datetime, end_datetime). - - The epoch-floor origin may generate a bucket before start_datetime (e.g. 10:00 - when start is 10:07), but key_to_array must truncate it away. The surviving - buckets are verified directly by reconstructing the index from the first - surviving timestamp (the first epoch-aligned bucket >= start_datetime). - - Also checks that all surviving buckets are on 15-min clock boundaries. - """ - start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 13, 7, tz="UTC") - - for m in range(0, 181): - dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert len(array) > 0 - - # The first surviving bucket is the first epoch-aligned timestamp >= start_dt. - # Compute it the same way key_to_array does: floor then step forward if needed. - epoch = int(start_dt.timestamp()) - floored_epoch = (epoch // 900) * 900 - first_bucket = pd.Timestamp(floored_epoch, unit="s", tz="UTC") - if first_bucket < pd.Timestamp(start_dt): - first_bucket += pd.Timedelta(seconds=900) - - idx = pd.date_range(start=first_bucket, periods=len(array), freq="900s") - - start_pd = pd.Timestamp(start_dt) - end_pd = pd.Timestamp(end_dt) - for ts in idx: - assert ts >= start_pd, f"Bucket {ts} is before start_datetime {start_pd}" - assert ts < end_pd, f"Bucket {ts} is at or after end_datetime {end_pd}" - assert ts.minute % 15 == 0, f"Bucket {ts} is not on a 15-min boundary" - assert ts.second == 0, f"Bucket {ts} has non-zero seconds" - - def test_key_to_array_align_true_preserves_mean_values(self, sequence): - """align_to_interval=True does not corrupt resampled values. - - A constant-valued series must resample to the same constant regardless - of bucket alignment. - """ - # 1-min records with constant value 42.0, starting off-boundary - start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") - end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") - - for m in range(0, 121): - dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, 42.0)) - - array = sequence.key_to_array( - key="data_value", - start_datetime=start_dt, - end_datetime=end_dt, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert len(array) > 0 - for v in array: - if v is not None: - assert abs(v - 42.0) < 1e-6, f"Expected 42.0, got {v}" - - def test_key_to_array_align_true_compaction_call_pattern(self, sequence): - """Verify the call pattern used by _db_compact_tier produces clock-aligned timestamps. - - _db_compact_tier calls key_to_array with boundary='strict', fill_method='time', - align_to_interval=True on a window whose start has arbitrary sub-second precision. - All output buckets must land on 15-min boundaries so that compacted records are - stored at predictable, human-readable timestamps. - """ - # Non-round base time: 08:43 — chosen to expose any origin-alignment bug - base_dt = pendulum.datetime(2024, 6, 1, 8, 43, tz="UTC") - window_end = pendulum.datetime(2024, 6, 1, 11, 43, tz="UTC") - - for m in range(0, 181): - dt = base_dt.add(minutes=m) - sequence.insert_by_datetime(self.create_test_record(dt, float(m))) - - array = sequence.key_to_array( - key="data_value", - start_datetime=base_dt, - end_datetime=window_end, - interval=to_duration("15 minutes"), - fill_method="time", - boundary="strict", - align_to_interval=True, - ) - - assert len(array) > 0 - epoch = int(base_dt.timestamp()) - floored_epoch = (epoch // 900) * 900 - idx = pd.date_range( - start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), - periods=len(array), - freq="900s", - ) - for ts in idx: - assert ts.minute % 15 == 0, ( - f"Compacted record at {ts} is not on a 15-min boundary (minute={ts.minute})" - ) - assert ts.second == 0, ( - f"Compacted record at {ts} has non-zero seconds ({ts.second})" - ) - - def test_delete_by_datetime_range(self, sequence): - dt1 = to_datetime("2023-11-05") - dt2 = to_datetime("2023-11-06") - dt3 = to_datetime("2023-11-07") - record1 = self.create_test_record(dt1, 0.8) - record2 = self.create_test_record(dt2, 0.9) - record3 = self.create_test_record(dt3, 1.0) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - assert len(sequence) == 3 - sequence.delete_by_datetime(start_datetime=dt2, end_datetime=dt3) - assert len(sequence) == 2 - assert sequence.records[0].date_time == dt1 - assert sequence.records[1].date_time == dt3 - - def test_delete_by_datetime_start(self, sequence): - dt1 = to_datetime("2023-11-05") - dt2 = to_datetime("2023-11-06") - record1 = self.create_test_record(dt1, 0.8) - record2 = self.create_test_record(dt2, 0.9) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - assert len(sequence) == 2 - sequence.delete_by_datetime(start_datetime=dt2) - assert len(sequence) == 1 - assert sequence.records[0].date_time == dt1 - - def test_delete_by_datetime_end(self, sequence): - dt1 = to_datetime("2023-11-05") - dt2 = to_datetime("2023-11-06") - record1 = self.create_test_record(dt1, 0.8) - record2 = self.create_test_record(dt2, 0.9) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - assert len(sequence) == 2 - sequence.delete_by_datetime(end_datetime=dt2) - assert len(sequence) == 1 - assert sequence.records[0].date_time == dt2 - - def test_to_dict(self, sequence): - dt = to_datetime("2023-11-06") - record = self.create_test_record(dt, 0.8) - sequence.insert_by_datetime(record) - data_dict = sequence.to_dict() - assert isinstance(data_dict, dict) - # We need a new class - Sequences are singletons - sequence2 = DerivedSequence2.from_dict(data_dict) - assert sequence2.model_dump() == sequence.model_dump() - - def test_to_json(self, sequence): - dt = to_datetime("2023-11-06") - record = self.create_test_record(dt, 0.8) - sequence.insert_by_datetime(record) - json_str = sequence.to_json() - assert isinstance(json_str, str) - assert "2023-11-06" in json_str - assert ": 0.8" in json_str - - def test_from_json(self, sequence, sequence2): - json_str = sequence2.to_json() - sequence = sequence.from_json(json_str) - assert len(sequence) == len(sequence2) - assert sequence.records[0].date_time == sequence2.records[0].date_time - assert sequence.records[0].data_value == sequence2.records[0].data_value - - def test_key_to_value_exact_match(self, sequence): - """Test key_to_value returns exact match when datetime matches a record.""" - dt = to_datetime("2023-11-05") - record = self.create_test_record(dt, 0.75) - sequence.insert_by_datetime(record) - result = sequence.key_to_value("data_value", dt) - assert result == 0.75 - - def test_key_to_value_nearest(self, sequence): - """Test key_to_value returns value closest in time to the given datetime.""" - record1 = self.create_test_record(datetime(2023, 11, 5, 12), 0.6) - record2 = self.create_test_record(datetime(2023, 11, 6, 12), 0.9) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - dt = datetime(2023, 11, 6, 10) # closer to record2 - result = sequence.key_to_value("data_value", dt, time_window=to_duration("48 hours")) - assert result == 0.9 - - def test_key_to_value_nearest_after(self, sequence): - """Test key_to_value returns value nearest after the given datetime.""" - record1 = self.create_test_record(datetime(2023, 11, 5, 10), 0.7) - record2 = self.create_test_record(datetime(2023, 11, 5, 15), 0.8) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - dt = datetime(2023, 11, 5, 14) # closer to record2 - result = sequence.key_to_value("data_value", dt, time_window=to_duration("48 hours")) - assert result == 0.8 - - def test_key_to_value_empty_sequence(self, sequence): - """Test key_to_value returns None when sequence is empty.""" - result = sequence.key_to_value("data_value", datetime(2023, 11, 5)) - assert result is None - - def test_key_to_value_missing_key(self, sequence): - """Test key_to_value returns None when key is missing in records.""" - record = self.create_test_record(datetime(2023, 11, 5), None) - sequence.insert_by_datetime(record) - result = sequence.key_to_value("data_value", datetime(2023, 11, 5)) - assert result is None - - def test_key_to_value_multiple_records_with_none(self, sequence): - """Test key_to_value skips records with None values.""" - r1 = self.create_test_record(datetime(2023, 11, 5), None) - r2 = self.create_test_record(datetime(2023, 11, 6), 1.0) - sequence.insert_by_datetime(r1) - sequence.insert_by_datetime(r2) - result = sequence.key_to_value("data_value", datetime(2023, 11, 5, 12), time_window=to_duration("48 hours")) - assert result == 1.0 - - def test_key_to_dict(self, sequence): - record1 = self.create_test_record(datetime(2023, 11, 5), 0.8) - record2 = self.create_test_record(datetime(2023, 11, 6), 0.9) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - data_dict = sequence.key_to_dict("data_value") - assert isinstance(data_dict, dict) - assert data_dict[to_datetime(datetime(2023, 11, 5), as_string=True)] == 0.8 - assert data_dict[to_datetime(datetime(2023, 11, 6), as_string=True)] == 0.9 - - def test_key_to_lists(self, sequence): - record1 = self.create_test_record(datetime(2023, 11, 5), 0.8) - record2 = self.create_test_record(datetime(2023, 11, 6), 0.9) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - dates, values = sequence.key_to_lists("data_value") - assert dates == [to_datetime(datetime(2023, 11, 5)), to_datetime(datetime(2023, 11, 6))] - assert values == [0.8, 0.9] - - def test_to_dataframe_full_data(self, sequence): - """Test conversion of all records to a DataFrame without filtering.""" - record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) - record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) - record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - - df = sequence.to_dataframe() - - # Validate DataFrame structure - assert isinstance(df, pd.DataFrame) - assert not df.empty - assert len(df) == 3 # All records should be included - assert "data_value" in df.columns - - def test_to_dataframe_with_filter(self, sequence): - """Test filtering records by datetime range.""" - record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) - record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) - record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - - start = to_datetime("2024-01-01T12:30:00Z") - end = to_datetime("2024-01-01T14:00:00Z") - - df = sequence.to_dataframe(start_datetime=start, end_datetime=end) - - assert isinstance(df, pd.DataFrame) - assert not df.empty - assert len(df) == 1 # Only one record should match the range - assert df.index[0] == pd.Timestamp("2024-01-01T13:00:00Z") - - def test_to_dataframe_no_matching_records(self, sequence): - """Test when no records match the given datetime filter.""" - record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) - record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - - start = to_datetime("2024-01-01T14:00:00Z") # Start time after all records - end = to_datetime("2024-01-01T15:00:00Z") - - df = sequence.to_dataframe(start_datetime=start, end_datetime=end) - - assert isinstance(df, pd.DataFrame) - assert df.empty # No records should match - - def test_to_dataframe_empty_sequence(self, sequence): - """Test when DataSequence has no records.""" - sequence = DataSequence(records=[]) - - df = sequence.to_dataframe() - - assert isinstance(df, pd.DataFrame) - assert df.empty # Should return an empty DataFrame - - def test_to_dataframe_no_start_datetime(self, sequence): - """Test when only end_datetime is given (all past records should be included).""" - record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) - record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) - record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - - end = to_datetime("2024-01-01T13:00:00Z") # Include only first record - - df = sequence.to_dataframe(end_datetime=end) - - assert isinstance(df, pd.DataFrame) - assert not df.empty - assert len(df) == 1 - assert df.index[0] == pd.Timestamp("2024-01-01T12:00:00Z") - - def test_to_dataframe_no_end_datetime(self, sequence): - """Test when only start_datetime is given (all future records should be included).""" - record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) - record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) - record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) - sequence.insert_by_datetime(record1) - sequence.insert_by_datetime(record2) - sequence.insert_by_datetime(record3) - - start = to_datetime("2024-01-01T13:00:00Z") # Include last two records - - df = sequence.to_dataframe(start_datetime=start) - - assert isinstance(df, pd.DataFrame) - assert not df.empty - assert len(df) == 2 - assert df.index[0] == pd.Timestamp("2024-01-01T13:00:00Z") - - -class TestDataProvider: - # Fixtures and helper functions - @pytest.fixture - def provider(self): - """Fixture to provide an instance of TestDataProvider for testing.""" - DerivedDataProvider.provider_enabled = True - DerivedDataProvider.provider_updated = False - return DerivedDataProvider() - - @pytest.fixture - def sample_start_datetime(self): - """Fixture for a sample start datetime.""" - return to_datetime(datetime(2024, 11, 1, 12, 0)) - - def create_test_record(self, date, value): - """Helper function to create a test DataRecord.""" - return DerivedRecord(date_time=date, data_value=value) - - # Tests - - def test_singleton_behavior(self, provider): - """Test that DataProvider enforces singleton behavior.""" - instance1 = provider - instance2 = DerivedDataProvider() - assert instance1 is instance2, ( - "Singleton pattern is not enforced; instances are not the same." - ) - - def test_update_method_with_defaults(self, provider, sample_start_datetime, monkeypatch): - """Test the `update` method with default parameters.""" - ems_eos = get_ems() - - ems_eos.set_start_datetime(sample_start_datetime) - provider.update_data() - - assert provider.ems_start_datetime == sample_start_datetime - - def test_update_method_force_enable(self, provider, monkeypatch): - """Test that `update` executes when `force_enable` is True, even if `enabled` is False.""" - # Override enabled to return False for this test - DerivedDataProvider.provider_enabled = False - DerivedDataProvider.provider_updated = False - provider.update_data(force_enable=True) - assert provider.enabled() is False, "Provider should be disabled, but enabled() is True." - assert DerivedDataProvider.provider_updated is True, ( - "Provider should have been executed, but was not." - ) - - def test_delete_by_datetime(self, provider, sample_start_datetime): - """Test `delete_by_datetime` method for removing records by datetime range.""" - # Add records to the provider for deletion testing - records = [ - self.create_test_record(sample_start_datetime - to_duration("3 hours"), 1), - self.create_test_record(sample_start_datetime - to_duration("1 hour"), 2), - self.create_test_record(sample_start_datetime + to_duration("1 hour"), 3), - ] - for record in records: - provider.insert_by_datetime(record) - - provider.delete_by_datetime( - start_datetime=sample_start_datetime - to_duration("2 hours"), - end_datetime=sample_start_datetime + to_duration("2 hours"), - ) - assert len(provider.records) == 1, ( - "Only one record should remain after deletion by datetime." - ) - assert provider.records[0].date_time == sample_start_datetime - to_duration("3 hours"), ( - "Unexpected record remains." - ) - - -class NewTestDataImportProvider: - - # Fixtures and helper functions - @pytest.fixture - def provider(self): - """Fixture to provide an instance of DerivedDataImportProvider for testing.""" - DerivedDataImportProvider.provider_enabled = True - DerivedDataImportProvider.provider_updated = True - return DerivedDataImportProvider() - -# --------------------------------------------------------------------------- -# import_from_dict -# --------------------------------------------------------------------------- - - def test_import_from_dict_basic(self, provider): - data = { - "start_datetime": "2024-01-01 00:00:00", - "interval": "1 hour", - "power": [1, 2, 3], - } - - provider.import_from_dict(data) - - assert provider.records is not None - assert provider.records[0]["power"] == 1 - assert provider.records[1]["power"] == 2 - - - def test_import_from_dict_default_start_and_interval(self, provider): - data = { - "power": [10, 20], - } - - provider.import_from_dict(data) - - assert len(provider._updates) == 2 - - - def test_import_from_dict_with_prefix(self, provider): - data = { - "load_power": [1, 2], - "other": [5, 6], - } - - provider.import_from_dict(data, key_prefix="load") - - assert len(provider._updates) == 2 - assert all(update[1] == "load_power" for update in provider._updates) - - - def test_import_from_dict_mismatching_lengths(self, provider): - data = { - "power": [1, 2], - "voltage": [1], - } - - with pytest.raises(ValueError): - provider.import_from_dict(data) - - - def test_import_from_dict_invalid_interval(self, provider): - data = { - "interval": "17 minutes", # does not divide hour - "power": [1, 2, 3], - } - - with pytest.raises(NotImplementedError): - provider.import_from_dict(data) - - - def test_import_from_dict_skips_none_and_nan(self, provider): - data = { - "power": [1, None, np.nan, 4], - } - - provider.import_from_dict(data) - - # only 1 and 4 should be written - assert len(provider._updates) == 2 - assert provider._updates[0][2] == 1 - assert provider._updates[1][2] == 4 - - - def test_import_from_dict_invalid_value_type(self, provider): - data = { - "power": "not a list" - } - - with pytest.raises(ValueError): - provider.import_from_dict(data) - - -# --------------------------------------------------------------------------- -# import_from_dataframe -# --------------------------------------------------------------------------- - - def test_import_from_dataframe_with_datetime_index(self, provider): - index = pd.date_range("2024-01-01", periods=3, freq="H") - df = pd.DataFrame({"power": [1, 2, 3]}, index=index) - - provider.import_from_dataframe(df) - - assert len(provider._updates) == 3 - assert provider._updates[0][2] == 1 - - - def test_import_from_dataframe_without_datetime_index(self, provider): - df = pd.DataFrame({"power": [5, 6, 7]}) - - provider.import_from_dataframe( - df, - start_datetime=datetime(2024, 1, 1), - interval=to_duration("1 hour"), - ) - - assert len(provider._updates) == 3 - - - def test_import_from_dataframe_prefix_filter(self, provider): - df = pd.DataFrame({ - "load_power": [1, 2], - "other": [3, 4], - }) - - provider.import_from_dataframe(df, key_prefix="load") - - assert len(provider._updates) == 2 - assert all(update[1] == "load_power" for update in provider._updates) - - - def test_import_from_dataframe_invalid_input(self, provider): - with pytest.raises(ValueError): - provider.import_from_dataframe("not a dataframe") - - -# --------------------------------------------------------------------------- -# import_from_json -# --------------------------------------------------------------------------- - - def test_import_from_json_simple_dict(self, provider): - json_str = json.dumps({ - "power": [1, 2, 3] - }) - - provider.import_from_json(json_str) - - assert len(provider._updates) == 3 - - - def test_import_from_json_invalid(self, provider): - with pytest.raises(ValueError): - provider.import_from_json("this is not json") - - -# --------------------------------------------------------------------------- -# import_from_file -# --------------------------------------------------------------------------- - - def test_import_from_file(self, provider, tmp_path): - file_path = tmp_path / "data.json" - - file_path.write_text(json.dumps({ - "power": [1, 2] - })) - - provider.import_from_file(file_path) - - assert len(provider._updates) == 2 - - - -class TestDataContainer: - # Fixture and helpers - @pytest.fixture - def container(self): - container = DerivedDataContainer() - return container - - @pytest.fixture - def container_with_providers(self): - record1 = self.create_test_record(datetime(2023, 11, 5), 1) - record2 = self.create_test_record(datetime(2023, 11, 6), 2) - record3 = self.create_test_record(datetime(2023, 11, 7), 3) - provider = DerivedDataProvider() - provider.delete_by_datetime(start_datetime=None, end_datetime=None) - assert len(provider) == 0 - provider.insert_by_datetime(record1) - provider.insert_by_datetime(record2) - provider.insert_by_datetime(record3) - assert len(provider) == 3 - container = DerivedDataContainer() - container.providers.clear() - assert len(container.providers) == 0 - container.providers.append(provider) - assert len(container.providers) == 1 - return container - - def create_test_record(self, date, value): - """Helper function to create a test DataRecord.""" - return DerivedRecord(date_time=date, data_value=value) - - def test_append_provider(self, container): - assert len(container.providers) == 0 - container.providers.append(DerivedDataProvider()) - assert len(container.providers) == 1 - assert isinstance(container.providers[0], DerivedDataProvider) - - @pytest.mark.skip(reason="type check not implemented") - def test_append_provider_invalid_type(self, container): - with pytest.raises(ValueError, match="must be an instance of DataProvider"): - container.providers.append("not_a_provider") - - def test_getitem_existing_key(self, container_with_providers): - assert len(container_with_providers.providers) == 1 - # check all keys are available (don't care for position) - for key in ["data_value", "date_time"]: - assert key in list(container_with_providers.keys()) - series = container_with_providers["data_value"] - assert isinstance(series, pd.Series) - assert series.name == "data_value" - assert series.tolist() == [1.0, 2.0, 3.0] - - def test_getitem_non_existing_key(self, container_with_providers): - with pytest.raises(KeyError, match="No data found for key 'non_existent_key'"): - container_with_providers["non_existent_key"] - - def test_setitem_existing_key(self, container_with_providers): - new_series = container_with_providers["data_value"] - new_series[:] = [4, 5, 6] - container_with_providers["data_value"] = new_series - series = container_with_providers["data_value"] - assert series.name == "data_value" - assert series.tolist() == [4, 5, 6] - - def test_setitem_invalid_value(self, container_with_providers): - with pytest.raises(ValueError, match="Value must be an instance of pd.Series"): - container_with_providers["test_key"] = "not_a_series" - - def test_setitem_non_existing_key(self, container_with_providers): - new_series = pd.Series([4, 5, 6], name="non_existent_key") - with pytest.raises(KeyError, match="Key 'non_existent_key' not found"): - container_with_providers["non_existent_key"] = new_series - - def test_delitem_existing_key(self, container_with_providers): - del container_with_providers["data_value"] - series = container_with_providers["data_value"] - assert series.name == "data_value" - assert series.tolist() == [] - - def test_delitem_non_existing_key(self, container_with_providers): - with pytest.raises(KeyError, match="Key 'non_existent_key' not found"): - del container_with_providers["non_existent_key"] - - def test_len(self, container_with_providers): - assert len(container_with_providers) == 5 - - def test_repr(self, container_with_providers): - representation = repr(container_with_providers) - assert representation.startswith("DerivedDataContainer(") - assert "DerivedDataProvider" in representation - - def test_to_json(self, container_with_providers): - json_str = container_with_providers.to_json() - container_other = DerivedDataContainer.from_json(json_str) - assert container_other == container_with_providers - - def test_from_json(self, container_with_providers): - json_str = container_with_providers.to_json() - container = DerivedDataContainer.from_json(json_str) - assert isinstance(container, DerivedDataContainer) - assert len(container.providers) == 1 - assert container.providers[0] == container_with_providers.providers[0] - - def test_provider_by_id(self, container_with_providers): - provider = container_with_providers.provider_by_id("DerivedDataProvider") - assert isinstance(provider, DerivedDataProvider) diff --git a/tests/test_dataabccompact.py b/tests/test_dataabccompact.py index 08662180..40d8dd74 100644 --- a/tests/test_dataabccompact.py +++ b/tests/test_dataabccompact.py @@ -9,12 +9,15 @@ A temporary SQLite database is configured for the entire test session via the use the real Database singleton via DatabaseMixin — have a working backend. """ +import asyncio from typing import List, Optional, Type import numpy as np import pytest +import pytest_asyncio from pydantic import Field +from akkudoktoreos.core.coreabc import get_database from akkudoktoreos.core.dataabc import ( DataContainer, DataProvider, @@ -91,7 +94,7 @@ class EnergyProvider(DataProvider): def enabled(self) -> bool: return True - def _update_data(self, force_update=False) -> None: + async def _update_data(self, force_update=False) -> None: pass def db_namespace(self) -> str: @@ -114,7 +117,7 @@ class PriceProvider(DataProvider): def enabled(self) -> bool: return True - def _update_data(self, force_update=False) -> None: + async def _update_data(self, force_update=False) -> None: pass def db_namespace(self) -> str: @@ -144,7 +147,7 @@ def _aligned_base(now: DateTime, interval_minutes: int = 15) -> DateTime: return now.subtract(seconds=epoch % interval_sec).set(microsecond=0) -def _fill_sequence( +async def _fill_sequence( seq: DataSequence, base: DateTime, count: int, @@ -160,8 +163,13 @@ def _fill_sequence( for i in range(count): dt = base.add(minutes=i * interval_minutes) rec = EnergyRecord(date_time=dt, power_w=power_w + i, price_eur=price_eur) - seq.db_insert_record(rec) - seq.db_save_records() + await seq.db_insert_record(rec) + await seq.db_save_records() + + records_count = await seq.db_count_records() + assert records_count == count + + assert len(seq.records) == count def _reset_singletons() -> None: @@ -178,8 +186,8 @@ def _reset_singletons() -> None: pass -@pytest.fixture(autouse=True) -def configure_database(tmp_path): +@pytest_asyncio.fixture(autouse=True) +async def configure_database(tmp_path, config_eos): """Configure a fresh temporary SQLite database for every test. DataSequence uses the real Database singleton via DatabaseMixin. @@ -198,23 +206,29 @@ def configure_database(tmp_path): # Reset the Database singleton itself Database.reset_instance() - # Patch config to use SQLite in tmp_path - db = Database() - db.config.database.provider = "SQLite" - db.config.general.data_folder_path = tmp_path - db.open() + # Config to use SQLite in tmp_path + config_eos.database.provider = "SQLite" + config_eos.general.data_folder_path = tmp_path + + db = get_database() + + await db.open() + + assert db.provider_id() == "SQLite" + assert db.is_open is True yield # Teardown try: - db.close() + await db.close() finally: _reset_singletons() try: Database.reset_instance() except Exception: pass + config_eos.database.provider = None # --------------------------------------------------------------------------- @@ -222,45 +236,133 @@ def configure_database(tmp_path): # --------------------------------------------------------------------------- -@pytest.fixture -def energy_seq(): +@pytest_asyncio.fixture +async def energy_seq(): """Fresh EnergySequence with no data.""" - return EnergySequence() + seq = EnergySequence() + + # wipe all records + await seq.delete_by_datetime() + count = await seq.db_count_records() + assert count == 0 + + # Wipe DB metadata + await seq._db_init_metadata() + + # Ensure compaction metadata is not left over from prior singleton + assert seq._db_metadata is not None + assert seq._db_metadata.get('last_compact_cutoff_900') == None + assert seq._db_metadata.get('last_compact_cutoff_3600') == None + + return seq -@pytest.fixture -def dense_energy_seq(): +@pytest_asyncio.fixture +async def dense_energy_seq(): """EnergySequence with 4 weeks of 15-min records (~2688 records). The base timestamp is floored to a 15-min boundary so compacted bucket timestamps are deterministic and on clock-round marks. """ + records_count = 4 * 7 * 24 * 4 seq = EnergySequence() + + # wipe all records + await seq.delete_by_datetime() + count = await seq.db_count_records() + assert count == 0 + + # Wipe DB metadata + await seq._db_init_metadata() + + # Ensure compaction metadata is not left over from prior singleton + assert seq._db_metadata is not None + assert seq._db_metadata.get('last_compact_cutoff_900') == None + assert seq._db_metadata.get('last_compact_cutoff_3600') == None + now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(seq, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(seq, base, count=records_count, interval_minutes=15) + + count = await seq.db_count_records() + assert count == records_count + return seq, now -@pytest.fixture -def dense_price_seq(): +@pytest_asyncio.fixture +async def price_seq(): + """Fresh PriceSequence with no data.""" + seq = PriceSequence() + + # wipe all records + await seq.delete_by_datetime() + count = await seq.db_count_records() + assert count == 0 + + # Wipe DB metadata + await seq._db_init_metadata() + + # Ensure compaction metadata is not left over from prior singleton + assert seq._db_metadata is not None + assert seq._db_metadata.get('last_compact_cutoff_900') == None + assert seq._db_metadata.get('last_compact_cutoff_3600') == None + + return seq + + +@pytest_asyncio.fixture +async def dense_price_seq(): """PriceSequence with 4 weeks of 15-min records. The base timestamp is floored to a 15-min boundary so compacted bucket timestamps are deterministic and on clock-round marks. """ + records_count = 4 * 7 * 24 * 4 seq = PriceSequence() + + # wipe all records + await seq.delete_by_datetime() + count = await seq.db_count_records() + assert count == 0 + + # Wipe DB metadata + await seq._db_init_metadata() + + # Ensure compaction metadata is not left over from prior singleton + assert seq._db_metadata is not None + assert seq._db_metadata.get('last_compact_cutoff_900') == None + assert seq._db_metadata.get('last_compact_cutoff_3600') == None + now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(seq, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(seq, base, count=records_count, interval_minutes=15) + + count = await seq.db_count_records() + assert count == records_count + return seq, now -@pytest.fixture -def energy_container(energy_seq): +@pytest_asyncio.fixture +async def energy_and_price_container(): """DataContainer with one EnergyProvider and one PriceProvider.""" ep = EnergyProvider() + # wipe all records + await ep.delete_by_datetime() + count = await ep.db_count_records() + assert count == 0 + # Wipe DB metadata + await ep._db_init_metadata() + pp = PriceProvider() + # wipe all records + await pp.delete_by_datetime() + count = await pp.db_count_records() + assert count == 0 + # Wipe DB metadata + await pp._db_init_metadata() + container = EnergyContainer(providers=[ep, pp]) return container, ep, pp @@ -270,34 +372,43 @@ def energy_container(energy_seq): # --------------------------------------------------------------------------- +@pytest.mark.asyncio class TestDataSequenceCompactTiers: - def test_default_tiers_two_entries(self, energy_seq): + async def test_default_tiers_two_entries(self, energy_seq): tiers = energy_seq.db_compact_tiers() assert len(tiers) == 2 - def test_default_first_tier_2h_15min(self, energy_seq): + async def test_default_first_tier_2h_15min(self, energy_seq): tiers = energy_seq.db_compact_tiers() age_sec = tiers[0][0].total_seconds() interval_sec = tiers[0][1].total_seconds() assert age_sec == 2 * 3600 assert interval_sec == 15 * 60 - def test_default_second_tier_2weeks_1h(self, energy_seq): + async def test_default_second_tier_2weeks_1h(self, energy_seq): tiers = energy_seq.db_compact_tiers() age_sec = tiers[1][0].total_seconds() interval_sec = tiers[1][1].total_seconds() assert age_sec == 14 * 24 * 3600 assert interval_sec == 3600 - def test_price_sequence_overrides_to_single_tier(self): - seq = PriceSequence() + async def test_price_sequence_overrides_to_single_tier(self, price_seq): + seq = price_seq tiers = seq.db_compact_tiers() assert len(tiers) == 1 assert tiers[0][0].total_seconds() == 14 * 24 * 3600 assert tiers[0][1].total_seconds() == 3600 - def test_empty_tiers_disables_compaction(self): + +# --------------------------------------------------------------------------- +# DataSequence — compaction behaviour +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +class TestDataSequenceCompact: + + async def test_empty_tiers_disables_compaction(self): class NoCompact(EnergySequence): def db_compact_tiers(self): return [] @@ -305,31 +416,35 @@ class TestDataSequenceCompactTiers: seq = NoCompact() now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(seq, base, count=500, interval_minutes=15) - assert seq.db_compact() == 0 + await _fill_sequence(seq, base, count=500, interval_minutes=15) + compacted = await seq.db_compact() + assert compacted == 0 + async def test_empty_sequence_returns_zero(self, energy_seq): + db_enabled = energy_seq.db_enabled + assert db_enabled == True, "database must be enabled" -# --------------------------------------------------------------------------- -# DataSequence — compaction behaviour -# --------------------------------------------------------------------------- + compacted = await energy_seq.db_compact() + assert compacted == 0 - -class TestDataSequenceCompact: - - def test_empty_sequence_returns_zero(self, energy_seq): - assert energy_seq.db_compact() == 0 - - def test_dense_data_reduces_count(self, dense_energy_seq): + async def test_dense_data_reduces_count(self, dense_energy_seq): seq, _ = dense_energy_seq - before = seq.db_count_records() - deleted = seq.db_compact() - assert deleted > 0 - assert seq.db_count_records() < before + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" - def test_all_fields_compacted(self, dense_energy_seq): + before = await seq.db_count_records() + deleted = await seq.db_compact() + assert deleted > 0 + after = await seq.db_count_records() + assert after < before + + async def test_all_fields_compacted(self, dense_energy_seq): """Both power_w and price_eur should be present on compacted records.""" seq, now = dense_energy_seq - seq.db_compact() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" + + await seq.db_compact() cutoff = now.subtract(weeks=2) old_records = [r for r in seq.records if r.date_time and r.date_time < cutoff] @@ -339,9 +454,12 @@ class TestDataSequenceCompact: assert rec.power_w is not None, "power_w must survive compaction" assert rec.price_eur is not None, "price_eur must survive compaction" - def test_recent_records_untouched(self, dense_energy_seq): + async def test_recent_records_untouched(self, dense_energy_seq): """Records within 2 hours of now must not be compacted.""" seq, now = dense_energy_seq + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" + cutoff = now.subtract(hours=2) # Snapshot recent values @@ -351,7 +469,7 @@ class TestDataSequenceCompact: if r.date_time and r.date_time >= cutoff } - seq.db_compact() + await seq.db_compact() recent_after = { DatabaseTimestamp.from_datetime(r.date_time): r.power_w @@ -361,20 +479,26 @@ class TestDataSequenceCompact: assert recent_before == recent_after - def test_idempotent(self, dense_energy_seq): + async def test_idempotent(self, dense_energy_seq): seq, _ = dense_energy_seq - seq.db_compact() - after_first = seq.db_count_records() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" - seq.db_compact() - after_second = seq.db_count_records() + await seq.db_compact() + after_first = await seq.db_count_records() + + await seq.db_compact() + after_second = await seq.db_count_records() assert after_first == after_second - def test_price_sequence_preserves_15min_in_recent_2weeks(self, dense_price_seq): + async def test_price_sequence_preserves_15min_in_recent_2weeks(self, dense_price_seq): """PriceSequence keeps 15-min resolution for data younger than 2 weeks.""" seq, now = dense_price_seq - seq.db_compact() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" + + await seq.db_compact() two_weeks_ago = now.subtract(weeks=2) recent_records = [ @@ -394,10 +518,13 @@ class TestDataSequenceCompact: f"Expected ~15min spacing in recent 2 weeks, got {avg_spacing/60:.1f} min" ) - def test_price_sequence_compacts_older_than_2weeks_to_1h(self, dense_price_seq): + async def test_price_sequence_compacts_older_than_2weeks_to_1h(self, dense_price_seq): """PriceSequence compacts data older than 2 weeks to 1-hour resolution.""" seq, now = dense_price_seq - seq.db_compact() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" + + await seq.db_compact() two_weeks_ago = now.subtract(weeks=2) old_records = sorted( @@ -415,19 +542,23 @@ class TestDataSequenceCompact: f"Expected ~1h spacing for old price data, got {avg_spacing/60:.1f} min" ) - def test_compact_with_custom_tiers_argument(self, dense_energy_seq): + async def test_compact_with_custom_tiers_argument(self, dense_energy_seq): """db_compact(compact_tiers=...) overrides the instance's tiers.""" seq, _ = dense_energy_seq - before = seq.db_count_records() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" - deleted = seq.db_compact( + before = await seq.db_count_records() + + deleted = await seq.db_compact( compact_tiers=[(to_duration("1 day"), to_duration("1 hour"))] ) - assert deleted > 0 - assert seq.db_count_records() < before - def test_compacted_timestamps_are_clock_aligned(self, dense_energy_seq): + after = await seq.db_count_records() + assert after < before + + async def test_compacted_timestamps_are_clock_aligned(self, dense_energy_seq): """All timestamps produced by compaction must sit on UTC clock boundaries. _db_compact_tier floors its cutoff timestamps to interval boundaries, so @@ -439,11 +570,14 @@ class TestDataSequenceCompact: - Records younger than floored 2h cutoff → unchanged """ seq, now = dense_energy_seq - seq.db_compact() + db_enabled = seq.db_enabled + assert db_enabled == True, "database must be enabled" + + await seq.db_compact() # _db_compact_tier floors new_cutoff from db_max, not from wall-clock now. # Compute the same floored cutoffs that the implementation used. - _, db_max_ts = seq.db_timestamp_range() + _, db_max_ts = await seq.db_timestamp_range() # DatabaseTimestamp already imported at top of file db_max_epoch = int(DatabaseTimestamp.to_datetime(db_max_ts).timestamp()) two_weeks_cutoff_epoch = ((db_max_epoch - 14*24*3600) // 3600) * 3600 @@ -467,7 +601,7 @@ class TestDataSequenceCompact: # DataSequence — data integrity after compaction # --------------------------------------------------------------------------- - +@pytest.mark.asyncio class TestDataSequenceCompactIntegrity: @staticmethod @@ -483,19 +617,19 @@ class TestDataSequenceCompactIntegrity: floored_epoch = (raw_epoch // interval_seconds) * interval_seconds return now.__class__.fromtimestamp(floored_epoch, tz=now.tzinfo) - def test_constant_power_preserved(self): + async def test_constant_power_preserved(self, energy_seq): """Mean resampling of a constant must equal the constant.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") # Use aligned base so bucket boundaries are deterministic base = _aligned_base(now.subtract(hours=6), interval_minutes=15) for i in range(6 * 60): # 1-min records for 6 hours dt = base.add(minutes=i) - seq.db_insert_record(EnergyRecord(date_time=dt, power_w=500.0, price_eur=0.30)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=dt, power_w=500.0, price_eur=0.30)) + await seq.db_save_records() - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) cutoff = now.subtract(hours=2) for rec in seq.records: @@ -503,27 +637,27 @@ class TestDataSequenceCompactIntegrity: assert rec.power_w == pytest.approx(500.0, abs=1e-3) assert rec.price_eur == pytest.approx(0.30, abs=1e-6) - def test_record_count_monotonically_decreases(self): + async def test_record_count_monotonically_decreases(self, energy_seq): """Each successive tier run should never increase record count.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(seq, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(seq, base, count=4 * 7 * 24 * 4, interval_minutes=15) - counts = [seq.db_count_records()] + counts = [await seq.db_count_records()] for age, interval in reversed(seq.db_compact_tiers()): - seq._db_compact_tier(age, interval) - counts.append(seq.db_count_records()) + await seq._db_compact_tier(age, interval) + counts.append(await seq.db_count_records()) for i in range(1, len(counts)): assert counts[i] <= counts[i - 1], ( f"Record count increased from {counts[i-1]} to {counts[i]} at tier {i}" ) - def test_no_duplicate_timestamps_after_compaction(self, dense_energy_seq): + async def test_no_duplicate_timestamps_after_compaction(self, dense_energy_seq): """Compaction must not create duplicate timestamps.""" seq, _ = dense_energy_seq - seq.db_compact() + await seq.db_compact() timestamps = [ DatabaseTimestamp.from_datetime(r.date_time) @@ -532,15 +666,15 @@ class TestDataSequenceCompactIntegrity: ] assert len(timestamps) == len(set(timestamps)), "Duplicate timestamps after compaction" - def test_timestamps_remain_sorted(self, dense_energy_seq): + async def test_timestamps_remain_sorted(self, dense_energy_seq): """Records must remain in ascending order after compaction.""" seq, _ = dense_energy_seq - seq.db_compact() + await seq.db_compact() dts = [r.date_time for r in seq.records if r.date_time is not None] assert dts == sorted(dts) - def test_compacted_old_timestamps_on_1h_boundaries(self, dense_energy_seq): + async def test_compacted_old_timestamps_on_1h_boundaries(self, dense_energy_seq): """Records older than the floored 2-week cutoff must be on whole-hour UTC boundaries. _db_compact_tier floors new_cutoff to the interval boundary, so we must @@ -549,11 +683,11 @@ class TestDataSequenceCompactIntegrity: 15-min resolution from the previous tier. """ seq, now = dense_energy_seq - seq.db_compact() + await seq.db_compact() # _db_compact_tier floors new_cutoff from db_max (the newest record), # not from wall-clock now. Derive the same floored cutoff here. - _, db_max_ts = seq.db_timestamp_range() + _, db_max_ts = await seq.db_timestamp_range() # DatabaseTimestamp already imported at top of file db_max_epoch = int(DatabaseTimestamp.to_datetime(db_max_ts).timestamp()) two_weeks_cutoff_epoch = ((db_max_epoch - 14*24*3600) // 3600) * 3600 @@ -568,7 +702,7 @@ class TestDataSequenceCompactIntegrity: f"Old record at {rec.date_time} is not on an hour boundary" ) - def test_compacted_mid_timestamps_on_15min_boundaries(self): + async def test_compacted_mid_timestamps_on_15min_boundaries(self, energy_seq): """Records compacted by the 15-min tier must land on 15-min UTC boundaries. We run _db_compact_tier directly with the 2h/15min tier on a sequence @@ -579,17 +713,17 @@ class TestDataSequenceCompactIntegrity: We replicate that exact calculation to identify which records were in the compaction window. """ - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(hours=6), interval_minutes=15) # 1-min records for 6 hours; newest record is at base + 359 min for i in range(6 * 60): dt = base.add(minutes=i) - seq.db_insert_record(EnergyRecord(date_time=dt, power_w=500.0, price_eur=0.30)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=dt, power_w=500.0, price_eur=0.30)) + await seq.db_save_records() - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) # Replicate the implementation's floored cutoff exactly: # newest_dt = last inserted record = base + 359min @@ -617,17 +751,17 @@ class TestDataSequenceCompactIntegrity: f"is not on a 15-min boundary (epoch % 900 = {epoch % 900})" ) - def test_no_compacted_timestamps_between_boundaries(self, dense_energy_seq): + async def test_no_compacted_timestamps_between_boundaries(self, dense_energy_seq): """After compaction no record timestamp must fall between expected bucket boundaries. Records older than the floored 2-week cutoff (processed by the 1h tier) must be on hour marks. Records in the 15-min band must be on 15-min marks. """ seq, now = dense_energy_seq - seq.db_compact() + await seq.db_compact() # Derive floored cutoffs from db_max — same reference as the implementation. - _, db_max_ts = seq.db_timestamp_range() + _, db_max_ts = await seq.db_timestamp_range() # DatabaseTimestamp already imported at top of file db_max_epoch = int(DatabaseTimestamp.to_datetime(db_max_ts).timestamp()) two_weeks_cutoff_epoch = ((db_max_epoch - 14*24*3600) // 3600) * 3600 @@ -651,39 +785,41 @@ class TestDataSequenceCompactIntegrity: # DataContainer — delegation # --------------------------------------------------------------------------- - +@pytest.mark.asyncio class TestDataContainerCompact: - def test_compact_delegates_to_all_providers(self, energy_container): - container, ep, pp = energy_container + async def test_compact_delegates_to_all_providers(self, energy_and_price_container): + container, ep, pp = energy_and_price_container now = to_datetime().in_timezone("UTC") # Fill both providers with 4 weeks of 15-min data base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(ep, base, count=4 * 7 * 24 * 4, interval_minutes=15) - _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(ep, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) - ep_before = ep.db_count_records() - pp_before = pp.db_count_records() + ep_before = await ep.db_count_records() + pp_before = await pp.db_count_records() - container.db_compact() + await container.db_compact() - assert ep.db_count_records() < ep_before, "EnergyProvider records should be compacted" - assert pp.db_count_records() < pp_before, "PriceProvider records should be compacted" + ep_after = await ep.db_count_records() + assert ep_after < ep_before, "EnergyProvider records should be compacted" + pp_after = await pp.db_count_records() + assert pp_after < pp_before, "PriceProvider records should be compacted" - def test_compact_empty_container_no_error(self): + async def test_compact_empty_container_no_error(self): container = EnergyContainer(providers=[]) - container.db_compact() # must not raise + await container.db_compact() # must not raise - def test_compact_provider_tiers_respected(self, energy_container): + async def test_compact_provider_tiers_respected(self, energy_and_price_container): """PriceProvider with single 2-week tier must not compact recent 15-min data.""" - container, ep, pp = energy_container + container, ep, pp = energy_and_price_container now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) - container.db_compact() + await container.db_compact() # Price data in last 2 weeks should still be at 15-min resolution two_weeks_ago = now.subtract(weeks=2) @@ -697,14 +833,14 @@ class TestDataContainerCompact: f"PriceProvider recent data should be ~15min, got {diff/60:.1f} min" ) - def test_compact_raises_on_provider_failure(self): + async def test_compact_raises_on_provider_failure(self): """A provider that raises during compaction must bubble up as RuntimeError. Monkey-patching is blocked by Pydantic v2's __setattr__ validation, so we use a subclass that overrides db_compact instead. """ class BrokenProvider(EnergyProvider): - def db_compact(self, *args, **kwargs): + async def db_compact(self, *args, **kwargs): raise ValueError("simulated failure") def provider_id(self) -> str: @@ -718,22 +854,24 @@ class TestDataContainerCompact: container = EnergyContainer(providers=[bp]) with pytest.raises(RuntimeError, match="fails on db_compact"): - container.db_compact() + await container.db_compact() - def test_compact_idempotent_on_container(self, energy_container): - container, ep, pp = energy_container + async def test_compact_idempotent_on_container(self, energy_and_price_container): + container, ep, pp = energy_and_price_container now = to_datetime().in_timezone("UTC") base = _aligned_base(now.subtract(weeks=4), interval_minutes=15) - _fill_sequence(ep, base, count=4 * 7 * 24 * 4, interval_minutes=15) - _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(ep, base, count=4 * 7 * 24 * 4, interval_minutes=15) + await _fill_sequence(pp, base, count=4 * 7 * 24 * 4, interval_minutes=15) - container.db_compact() - ep_after_first = ep.db_count_records() - pp_after_first = pp.db_count_records() + await container.db_compact() + ep_after_first = await ep.db_count_records() + pp_after_first = await pp.db_count_records() - container.db_compact() - assert ep.db_count_records() == ep_after_first - assert pp.db_count_records() == pp_after_first + await container.db_compact() + ep_after_second = await ep.db_count_records() + assert ep_after_second == ep_after_first + pp_after_second = await pp.db_count_records() + assert pp_after_second == pp_after_first # --------------------------------------------------------------------------- @@ -750,19 +888,19 @@ class TestDataContainerCompact: # are merged key-by-key; count decreases by 1 # --------------------------------------------------------------------------- - +@pytest.mark.asyncio class TestDataSequenceSparseGuard: # ------------------------------------------------------------------ # Case 1: sparse + already aligned → pure skip # ------------------------------------------------------------------ - def test_sparse_aligned_data_not_modified(self): + async def test_sparse_aligned_data_not_modified(self, energy_seq): """Sparse records that already sit on interval boundaries must not be touched. deleted must be 0 and record count must be unchanged. """ - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=4) @@ -771,25 +909,26 @@ class TestDataSequenceSparseGuard: raw = base.add(days=offset_days) # Floor to nearest hour boundary so timestamp is already aligned aligned = raw.set(minute=0, second=0, microsecond=0) - seq.db_insert_record(EnergyRecord(date_time=aligned, power_w=100.0)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=aligned, power_w=100.0)) + await seq.db_save_records() - before = seq.db_count_records() - deleted = seq.db_compact() + before = await seq.db_count_records() + deleted = await seq.db_compact() assert deleted == 0, "Aligned sparse records must not be deleted" - assert seq.db_count_records() == before, "Record count must not change" + after = await seq.db_count_records() + assert after == before, "Record count must not change" - def test_sparse_aligned_data_values_untouched(self): + async def test_sparse_aligned_data_values_untouched(self, energy_seq): """Values of aligned sparse records must be preserved exactly.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=4).set(minute=0, second=0, microsecond=0) - seq.db_insert_record(EnergyRecord(date_time=base, power_w=42.0, price_eur=0.99)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=base, power_w=42.0, price_eur=0.99)) + await seq.db_save_records() - seq.db_compact() + await seq.db_compact() remaining = [r for r in seq.records if r.date_time == base] assert len(remaining) == 1 @@ -801,7 +940,7 @@ class TestDataSequenceSparseGuard: # ------------------------------------------------------------------ @staticmethod - def _make_snapping_seq(now, offsets_minutes, interval_minutes=10, age_minutes=30): + async def _make_snapping_seq(seq, now, offsets_minutes, interval_minutes=10, age_minutes=30): """Build a sequence guaranteed to enter the sparse-snapping path. Key insight: _db_compact_tier measures age_threshold from db_max (the @@ -849,34 +988,34 @@ class TestDataSequenceSparseGuard: base_epoch = (raw_base_epoch // interval_sec) * interval_sec base = DateTime.fromtimestamp(base_epoch, tz="UTC") - seq = EnergySequence() dts = [] for off in offsets_minutes: dt = base.add(minutes=off) - seq.db_insert_record(EnergyRecord(date_time=dt, power_w=float(off * 10))) + await seq.db_insert_record(EnergyRecord(date_time=dt, power_w=float(off * 10))) dts.append(dt) # Newest anchor: makes db_max ≈ now so cutoff = now - age_threshold anchor = now.subtract(seconds=1) - seq.db_insert_record(EnergyRecord(date_time=anchor, power_w=0.0)) - seq.db_save_records() - return seq, age_td, interval_td, dts + await seq.db_insert_record(EnergyRecord(date_time=anchor, power_w=0.0)) + await seq.db_save_records() + return age_td, interval_td, dts - def test_sparse_misaligned_records_are_snapped(self): + async def test_sparse_misaligned_records_are_snapped(self, energy_seq): """Sparse misaligned records must be moved to the nearest boundary. Uses a tight window (30 min age, 10 min interval → 3 resampled buckets) with 4 misaligned records so existing_count(4) > resampled_count(3) and the snapping path is entered deterministically. """ + seq = energy_seq now = to_datetime().in_timezone("UTC") # 4 records at :03, :08, :13, :18 — all misaligned for a 10-min interval - seq, age_td, interval_td, dts = self._make_snapping_seq( - now, offsets_minutes=[3, 8, 13, 18] + age_td, interval_td, dts = await self._make_snapping_seq( + seq, now, offsets_minutes=[3, 8, 13, 18] ) n_test_records = len([3, 8, 13, 18]) - deleted = seq._db_compact_tier(age_td, interval_td) - after = seq.db_count_records() + deleted = await seq._db_compact_tier(age_td, interval_td) + after = await seq.db_count_records() assert deleted == n_test_records, ( f"All {n_test_records} in-window records must be deleted (whole-window delete); " @@ -898,20 +1037,21 @@ class TestDataSequenceSparseGuard: f"got {after}" ) - def test_sparse_misaligned_timestamps_become_aligned(self): + async def test_sparse_misaligned_timestamps_become_aligned(self, energy_seq): """After snapping, in-window timestamps must be on the target interval boundary. The anchor record lives outside the compaction window (it is younger than age_threshold) and is intentionally misaligned — it must NOT be checked. """ + seq = energy_seq now = to_datetime().in_timezone("UTC") interval_minutes = 10 age_minutes = 30 - seq, age_td, interval_td, dts = self._make_snapping_seq( - now, offsets_minutes=[3, 8, 13, 18], interval_minutes=interval_minutes, + age_td, interval_td, dts = await self._make_snapping_seq( + seq, now, offsets_minutes=[3, 8, 13, 18], interval_minutes=interval_minutes, age_minutes=age_minutes, ) - seq._db_compact_tier(age_td, interval_td) + await seq._db_compact_tier(age_td, interval_td) # Compute window_end the same way _db_compact_tier does # (anchor is db_max; raw_cutoff = anchor - age_threshold ≈ now - 30min) @@ -931,16 +1071,16 @@ class TestDataSequenceSparseGuard: f"{interval_minutes}-min boundary (epoch % {interval_sec} = {epoch % interval_sec})" ) - def test_sparse_misaligned_values_preserved_after_snap(self): + async def test_sparse_misaligned_values_preserved_after_snap(self, energy_seq): """Snapping must not alter the field values of sparse records.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") # Single misaligned record, old enough for both tiers dt = now.subtract(weeks=4).set(minute=7, second=0, microsecond=0) - seq.db_insert_record(EnergyRecord(date_time=dt, power_w=777.0, price_eur=0.55)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=dt, power_w=777.0, price_eur=0.55)) + await seq.db_save_records() - seq.db_compact() + await seq.db_compact() # Exactly one record must remain and its values must be unchanged assert len(seq.records) == 1 @@ -951,7 +1091,7 @@ class TestDataSequenceSparseGuard: # Case 3: two sparse records collide on the same snapped bucket # ------------------------------------------------------------------ - def test_sparse_collision_merges_records(self): + async def test_sparse_collision_merges_records(self, energy_seq): """Two sparse records that snap to the same bucket must be merged. Records at :03 and :04 both round to :00 with a 10-min interval. @@ -959,6 +1099,7 @@ class TestDataSequenceSparseGuard: A newest-anchor record at now-1s pushes db_max ≈ now so the compaction cutoff lands at now-30min, which is after all test records. """ + seq = energy_seq now = to_datetime().in_timezone("UTC") age_td = to_duration("30 minutes") interval_td = to_duration("10 minutes") @@ -969,19 +1110,18 @@ class TestDataSequenceSparseGuard: raw_base = now.subtract(minutes=52).set(second=0, microsecond=0) base = raw_base.subtract(seconds=int(raw_base.timestamp()) % interval_sec) - seq = EnergySequence() - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), power_w=100.0, price_eur=None)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=4), + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=4), power_w=None, price_eur=0.25)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=17), power_w=20.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=17), power_w=20.0)) # Anchor: makes db_max ≈ now → cutoff = now - 30min (after all test records) - seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) + await seq.db_save_records() # existing_count in window = 4, resampled_count = 3 → snapping path - seq._db_compact_tier(age_td, interval_td) + await seq._db_compact_tier(age_td, interval_td) snapped_epoch = int(base.timestamp()) snapped = [ @@ -992,7 +1132,7 @@ class TestDataSequenceSparseGuard: assert snapped[0].power_w == pytest.approx(100.0), "power_w from :03 must survive" assert snapped[0].price_eur == pytest.approx(0.25), "price_eur from :04 must survive" - def test_sparse_collision_keeps_first_value_for_shared_key(self): + async def test_sparse_collision_keeps_first_value_for_shared_key(self, energy_seq): """When two sparse records floor to the same bucket, the earlier value wins. Two records at :03 (power_w=111) and :04 (power_w=222) both floor to :00 @@ -1001,19 +1141,19 @@ class TestDataSequenceSparseGuard: snapping path is taken rather than full resampling. The merged record at :00 must carry power_w=111 because the chronologically earlier record wins. """ + seq = energy_seq now = to_datetime().in_timezone("UTC") interval_sec = 600 # Place both records 52 min ago so they are before window_end ≈ now - 30min. # Only 2 test records → existing_count(2) <= resampled_count → sparse path. raw_base = now.subtract(minutes=52).set(second=0, microsecond=0) base = raw_base.subtract(seconds=int(raw_base.timestamp()) % interval_sec) - seq = EnergySequence() - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), power_w=111.0)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=4), power_w=222.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), power_w=111.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=4), power_w=222.0)) # Anchor at now-1s: makes db_max ≈ now so cutoff = now - 30min - seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) - seq.db_save_records() - seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) + await seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) + await seq.db_save_records() + await seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) snapped_epoch = int(base.timestamp()) snapped = [ r for r in seq.records @@ -1022,7 +1162,7 @@ class TestDataSequenceSparseGuard: assert len(snapped) == 1, ":03 and :04 must floor-snap into one :00 record" assert snapped[0].power_w == pytest.approx(111.0), "Earlier record's value must win" - def test_sparse_collision_with_existing_aligned_record(self): + async def test_sparse_collision_with_existing_aligned_record(self, energy_seq): """A misaligned record that snaps onto an already-aligned record must merge into it without raising ValueError. The aligned record's existing values win. @@ -1030,6 +1170,7 @@ class TestDataSequenceSparseGuard: power_w=None, price_eur=0.30) both map to :00. Result: power_w=50 (aligned wins) and price_eur=0.30 (filled from :03). """ + seq = energy_seq now = to_datetime().in_timezone("UTC") interval_sec = 600 # base must be far enough back that all records (+17min max) land before @@ -1037,19 +1178,18 @@ class TestDataSequenceSparseGuard: raw_base = now.subtract(minutes=52).set(second=0, microsecond=0) base = raw_base.subtract(seconds=int(raw_base.timestamp()) % interval_sec) - seq = EnergySequence() - seq.db_insert_record(EnergyRecord(date_time=base, + await seq.db_insert_record(EnergyRecord(date_time=base, power_w=50.0, price_eur=None)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=3), power_w=None, price_eur=0.30)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=17), power_w=20.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=17), power_w=20.0)) # Anchor: db_max ≈ now → cutoff = now - 30min, after all test records - seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) + await seq.db_save_records() # Must not raise ValueError - seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) + await seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) snapped_epoch = int(base.timestamp()) snapped = [ @@ -1063,13 +1203,14 @@ class TestDataSequenceSparseGuard: assert rec.date_time is not None assert int(rec.date_time.timestamp()) % interval_sec == 0 - def test_sparse_no_duplicate_timestamps_after_collision(self): + async def test_sparse_no_duplicate_timestamps_after_collision(self, energy_seq): """After collision merging, no duplicate timestamps must remain. Three records at :02, :03, :04 all round to :00 with a 10-min interval. Together with a record at :13 this gives existing_count(4) > resampled_count(3) so the snapping path is entered. """ + seq = energy_seq now = to_datetime().in_timezone("UTC") interval_sec = 600 # base must be far enough back that all records (+17min max) land before @@ -1077,17 +1218,16 @@ class TestDataSequenceSparseGuard: raw_base = now.subtract(minutes=52).set(second=0, microsecond=0) base = raw_base.subtract(seconds=int(raw_base.timestamp()) % interval_sec) - seq = EnergySequence() for offset_min in [2, 3, 4]: # all snap to :00 - seq.db_insert_record(EnergyRecord( + await seq.db_insert_record(EnergyRecord( date_time=base.add(minutes=offset_min), power_w=float(offset_min) )) - seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) + await seq.db_insert_record(EnergyRecord(date_time=base.add(minutes=13), power_w=10.0)) # Anchor: db_max ≈ now → cutoff = now - 30min, after all test records - seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) - seq.db_save_records() + await seq.db_insert_record(EnergyRecord(date_time=now.subtract(seconds=1), power_w=0.0)) + await seq.db_save_records() - seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) + await seq._db_compact_tier(to_duration("30 minutes"), to_duration("10 minutes")) timestamps = [ int(r.date_time.timestamp()) @@ -1100,30 +1240,34 @@ class TestDataSequenceSparseGuard: # Existing tier-skip tests (unchanged semantics) # ------------------------------------------------------------------ - def test_hourly_data_skips_1h_tier(self): + async def test_hourly_data_skips_1h_tier(self, energy_seq): """Data already at 1-hour resolution and aligned must not be re-compacted.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") # Use an hour-aligned base so records are on clean boundaries base = now.subtract(weeks=3).set(minute=0, second=0, microsecond=0) - _fill_sequence(seq, base, count=3 * 7 * 24, interval_minutes=60) + await _fill_sequence(seq, base, count=3 * 7 * 24, interval_minutes=60) - before = seq.db_count_records() - deleted = seq._db_compact_tier(to_duration("14 days"), to_duration("1 hour")) + before = await seq.db_count_records() + deleted = await seq._db_compact_tier(to_duration("14 days"), to_duration("1 hour")) assert deleted == 0 - assert seq.db_count_records() == before - def test_15min_data_younger_than_2weeks_skips_1h_tier(self): + after = await seq.db_count_records() + assert after == before + + async def test_15min_data_younger_than_2weeks_skips_1h_tier(self, energy_seq): """15-min data between 2h and 2weeks old must NOT be compacted by the 1h tier.""" - seq = EnergySequence() + seq = energy_seq now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=1).set(minute=0, second=0, microsecond=0) - _fill_sequence(seq, base, count=7 * 24 * 4, interval_minutes=15) + await _fill_sequence(seq, base, count=7 * 24 * 4, interval_minutes=15) - before = seq.db_count_records() - deleted = seq._db_compact_tier(to_duration("14 days"), to_duration("1 hour")) + before = await seq.db_count_records() + deleted = await seq._db_compact_tier(to_duration("14 days"), to_duration("1 hour")) assert deleted == 0 - assert seq.db_count_records() == before + + after = await seq.db_count_records() + assert after == before diff --git a/tests/test_dataabccontainer.py b/tests/test_dataabccontainer.py new file mode 100644 index 00000000..b878bd7a --- /dev/null +++ b/tests/test_dataabccontainer.py @@ -0,0 +1,308 @@ +import asyncio +import json +from datetime import datetime, timezone +from typing import Any, ClassVar, List, Optional, Union + +import numpy as np +import pandas as pd +import pendulum +import pytest +import pytest_asyncio +from pydantic import Field, PrivateAttr + +from akkudoktoreos.config.configabc import SettingsBaseModel +from akkudoktoreos.core.coreabc import get_ems +from akkudoktoreos.core.dataabc import ( + DataABC, + DataContainer, + DataImportProvider, + DataProvider, + DataRecord, + DataSequence, +) +from akkudoktoreos.core.databaseabc import DatabaseTimestamp +from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration + +# --------------------------------------------------------------------------- +# Derived classes for testing +# --------------------------------------------------------------------------- + +class DerivedRecord(DataRecord): + """DataRecord with a numeric field and configured field-like data.""" + + data_value: Optional[float] = Field(default=None, description="Data Value") + + @classmethod + def configured_data_keys(cls) -> Optional[list[str]]: + return ["dish_washer_emr", "solar_power", "temp"] + + +class DerivedDataProvider(DataProvider): + """Concrete DataProvider for testing.""" + + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + provider_enabled: ClassVar[bool] = True + provider_updated: ClassVar[bool] = False + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedDataProvider" + + def provider_id(self) -> str: + return "DerivedDataProvider" + + def enabled(self) -> bool: + return self.provider_enabled + + async def _update_data(self, force_update: Optional[bool] = False) -> None: + DerivedDataProvider.provider_updated = True + + +class DerivedDataContainer(DataContainer): + providers: List[Union[DerivedDataProvider, DataProvider]] = Field( + default_factory=list, description="List of data providers" + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def make_record(date, value: float) -> DerivedRecord: + return DerivedRecord(date_time=to_datetime(date), data_value=value) + + +async def make_provider_with_records() -> DerivedDataProvider: + """Return a fresh provider with three hourly records.""" + provider = DerivedDataProvider() + await provider.delete_by_datetime() # wipe singleton state + await provider.insert_by_datetime(make_record(datetime(2024, 1, 1, 0), 1.0)) + await provider.insert_by_datetime(make_record(datetime(2024, 1, 1, 1), 2.0)) + await provider.insert_by_datetime(make_record(datetime(2024, 1, 1, 2), 3.0)) + return provider + + +async def make_container() -> DerivedDataContainer: + """Return a container with one populated provider.""" + provider = await make_provider_with_records() + container = DerivedDataContainer() + container.providers.clear() + container.providers.append(provider) + return container + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +class TestDataContainer: + + # ----------------------------------------------------------------------- + # Fixtures + # ----------------------------------------------------------------------- + + @pytest_asyncio.fixture + async def container(self): + """Empty container (no providers).""" + c = DerivedDataContainer() + c.providers.clear() + return c + + @pytest_asyncio.fixture + async def populated(self): + """Container with one provider holding three records.""" + return await make_container() + + # ----------------------------------------------------------------------- + # Provider management + # ----------------------------------------------------------------------- + + async def test_append_provider(self, container): + assert len(container.providers) == 0 + provider = DerivedDataProvider() + container.providers.append(provider) + assert len(container.providers) == 1 + assert isinstance(container.providers[0], DerivedDataProvider) + + async def test_enabled_providers_reflects_enabled_flag(self, populated): + DerivedDataProvider.provider_enabled = True + assert len(populated.enabled_providers) == 1 + + DerivedDataProvider.provider_enabled = False + assert len(populated.enabled_providers) == 0 + + DerivedDataProvider.provider_enabled = True # restore + + async def test_provider_by_id_found(self, populated): + provider = populated.provider_by_id("DerivedDataProvider") + assert isinstance(provider, DerivedDataProvider) + + async def test_provider_by_id_unknown_raises(self, populated): + with pytest.raises(ValueError, match="Unknown provider id"): + populated.provider_by_id("NonExistentProvider") + + # ----------------------------------------------------------------------- + # record_keys / record_keys_writable + # ----------------------------------------------------------------------- + + async def test_record_keys_contains_expected_fields(self, populated): + keys = populated.record_keys + assert "data_value" in keys + assert "date_time" in keys + # configured keys + for k in ("dish_washer_emr", "solar_power", "temp"): + assert k in keys + + async def test_record_keys_writable_contains_expected_fields(self, populated): + keys = populated.record_keys_writable + assert "data_value" in keys + for k in ("dish_washer_emr", "solar_power", "temp"): + assert k in keys + + async def test_record_keys_empty_when_no_providers(self, container): + assert container.record_keys == [] + assert container.record_keys_writable == [] + + # ----------------------------------------------------------------------- + # iter / len / repr / keys() + # ----------------------------------------------------------------------- + + async def test_iter_yields_record_keys(self, populated): + keys = list(populated) + assert "data_value" in keys + + async def test_len_equals_number_of_record_keys(self, populated): + assert len(populated) == len(populated.record_keys) + + async def test_repr_contains_class_and_provider(self, populated): + r = repr(populated) + assert r.startswith("DerivedDataContainer(") + assert "DerivedDataProvider" in r + + async def test_keys_view(self, populated): + kv = populated.keys() + assert "data_value" in kv + + # ----------------------------------------------------------------------- + # key_to_series + # ----------------------------------------------------------------------- + + async def test_key_to_series_returns_series(self, populated): + series = await populated.key_to_series("data_value") + assert isinstance(series, pd.Series) + assert series.name == "data_value" + + async def test_key_to_series_values(self, populated): + series = await populated.key_to_series("data_value") + assert sorted(series.tolist()) == [1.0, 2.0, 3.0] + + async def test_key_to_series_with_datetime_range(self, populated): + start = to_datetime(datetime(2024, 1, 1, 1)) + end = to_datetime(datetime(2024, 1, 1, 3)) + series = await populated.key_to_series("data_value", start_datetime=start, end_datetime=end) + assert len(series) == 2 + assert sorted(series.tolist()) == [2.0, 3.0] + + async def test_key_to_series_unknown_key_raises(self, populated): + with pytest.raises(KeyError, match="No data found for key"): + await populated.key_to_series("non_existent_key") + + async def test_key_to_series_no_enabled_providers_raises(self, populated): + DerivedDataProvider.provider_enabled = False + try: + with pytest.raises(KeyError, match="No data found for key"): + await populated.key_to_series("data_value") + finally: + DerivedDataProvider.provider_enabled = True + + # ----------------------------------------------------------------------- + # key_to_array + # ----------------------------------------------------------------------- + + async def test_key_to_array_returns_ndarray(self, populated): + start = to_datetime(datetime(2024, 1, 1, 0)) + end = to_datetime(datetime(2024, 1, 1, 3)) + array = await populated.key_to_array("data_value", start_datetime=start, end_datetime=end) + assert isinstance(array, np.ndarray) + assert len(array) == 3 + + async def test_key_to_array_unknown_key_raises(self, populated): + with pytest.raises(KeyError, match="No data found for key"): + await populated.key_to_array("non_existent_key") + + # ----------------------------------------------------------------------- + # update_data + # ----------------------------------------------------------------------- + + async def test_update_data_calls_provider(self, populated): + DerivedDataProvider.provider_updated = False + DerivedDataProvider.provider_enabled = True + await populated.update_data(force_enable=True) + assert DerivedDataProvider.provider_updated is True + + async def test_update_data_skips_disabled_provider(self, populated): + DerivedDataProvider.provider_enabled = False + DerivedDataProvider.provider_updated = False + await populated.update_data() + assert DerivedDataProvider.provider_updated is False + DerivedDataProvider.provider_enabled = True # restore + + async def test_update_data_force_enable_runs_disabled_provider(self, populated): + DerivedDataProvider.provider_enabled = False + DerivedDataProvider.provider_updated = False + await populated.update_data(force_enable=True) + assert DerivedDataProvider.provider_updated is True + DerivedDataProvider.provider_enabled = True # restore + + # ----------------------------------------------------------------------- + # save / load + # ----------------------------------------------------------------------- + + async def test_save_and_load_roundtrip(self, populated): + """Save then wipe in-memory records and verify load restores data if db is available.""" + start = to_datetime(datetime(2024, 1, 1, 0)) + end = to_datetime(datetime(2024, 1, 1, 3)) + + # Confirm data is present before save + series_before = await populated.key_to_series( + "data_value", start_datetime=start, end_datetime=end + ) + assert sorted(series_before.tolist()) == [1.0, 2.0, 3.0] + + for provider in populated.providers: + assert provider.db_enabled == False + + saved = await populated.save() + + if not saved: + # No database configured — verify save correctly reported nothing was persisted + pytest.skip("No database configured, skipping roundtrip persistence check") + + # Wipe in-memory state only (not the database) + for provider in populated.providers: + provider.records.clear() + assert all(len(p.records) == 0 for p in populated.providers) + + loaded = await populated.load() + assert loaded is True + + # Verify data is restored via the public async API + series_after = await populated.key_to_series( + "data_value", start_datetime=start, end_datetime=end + ) + assert sorted(series_after.tolist()) == [1.0, 2.0, 3.0] + + # ----------------------------------------------------------------------- + # db_get_stats + # ----------------------------------------------------------------------- + + async def test_db_get_stats_returns_dict(self, populated): + stats = await populated.db_get_stats() + assert isinstance(stats, dict) + assert "DerivedDataProvider" in stats diff --git a/tests/test_dataabcprovider.py b/tests/test_dataabcprovider.py new file mode 100644 index 00000000..ad614d13 --- /dev/null +++ b/tests/test_dataabcprovider.py @@ -0,0 +1,332 @@ +import asyncio +import json +from datetime import datetime, timezone +from typing import Any, ClassVar, List, Optional, Union + +import numpy as np +import pandas as pd +import pendulum +import pytest +from pydantic import Field, PrivateAttr, ValidationError + +from akkudoktoreos.config.configabc import SettingsBaseModel +from akkudoktoreos.core.coreabc import get_ems +from akkudoktoreos.core.dataabc import ( + DataABC, + DataContainer, + DataImportProvider, + DataProvider, + DataRecord, + DataSequence, +) +from akkudoktoreos.core.databaseabc import DatabaseTimestamp +from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration + +# Derived classes for testing +# --------------------------- + +class DerivedConfig(SettingsBaseModel): + env_var: Optional[int] = Field(default=None, description="Test config by environment var") + instance_field: Optional[str] = Field(default=None, description="Test config by instance field") + class_constant: Optional[int] = Field(default=None, description="Test config by class constant") + + +class DerivedBase(DataABC): + instance_field: Optional[str] = Field(default=None, description="Field Value") + class_constant: ClassVar[int] = 30 + + +class DerivedRecord(DataRecord): + """Date Record derived from base class DataRecord. + + The derived data record got the + - `data_value` field and the + - `dish_washer_emr`, `solar_power`, `temp` configurable field like data. + """ + + data_value: Optional[float] = Field(default=None, description="Data Value") + + @classmethod + def configured_data_keys(cls) -> Optional[list[str]]: + return ["dish_washer_emr", "solar_power", "temp"] + + +class DerivedSequence(DataSequence): + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedSequence" + + +class DerivedSequence2(DataSequence): + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedSequence2" + + +class DerivedDataProvider(DataProvider): + """A concrete subclass of DataProvider for testing purposes.""" + + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + provider_enabled: ClassVar[bool] = False + provider_updated: ClassVar[bool] = False + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedDataProvider" + + # Implement abstract methods for test purposes + def provider_id(self) -> str: + return "DerivedDataProvider" + + def enabled(self) -> bool: + return self.provider_enabled + + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Simulate update logic + DerivedDataProvider.provider_updated = True + + +class DerivedDataImportProvider(DataImportProvider): + """A concrete subclass of DataImportProvider for testing purposes.""" + + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + provider_enabled: ClassVar[bool] = False + provider_updated: ClassVar[bool] = False + _updates: list = PrivateAttr(default_factory=list) + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + # Implement abstract methods for test purposes + def provider_id(self) -> str: + return "DerivedDataImportProvider" + + def enabled(self) -> bool: + return self.provider_enabled + + async def _update_data(self, force_update: Optional[bool] = False) -> None: + # Simulate update logic + DerivedDataProvider.provider_updated = True + + async def _update_value(self, date, *args, **kwargs) -> None: + # Simulate update logic + self._updates.append((date, args, kwargs)) + await super()._update_value(date, *args, **kwargs) + + +class DerivedDataContainer(DataContainer): + providers: List[Union[DerivedDataProvider, DataProvider]] = Field( + default_factory=list, description="List of data providers" + ) + + +# Tests +# ---------- + + +@pytest.mark.asyncio +class TestDataProvider: + # Fixtures and helper functions + @pytest.fixture + def provider(self): + """Fixture to provide an instance of TestDataProvider for testing.""" + DerivedDataProvider.provider_enabled = True + DerivedDataProvider.provider_updated = False + return DerivedDataProvider() + + @pytest.fixture + def sample_start_datetime(self): + """Fixture for a sample start datetime.""" + return to_datetime(datetime(2024, 11, 1, 12, 0)) + + def create_test_record(self, date, value): + """Helper function to create a test DataRecord.""" + return DerivedRecord(date_time=date, data_value=value) + + # Tests + + async def test_singleton_behavior(self, provider): + """Test that DataProvider enforces singleton behavior.""" + instance1 = provider + instance2 = DerivedDataProvider() + assert instance1 is instance2, ( + "Singleton pattern is not enforced; instances are not the same." + ) + + async def test_update_method_with_defaults(self, provider, sample_start_datetime, monkeypatch): + """Test the `update` method with default parameters.""" + ems_eos = get_ems() + + ems_eos.set_start_datetime(sample_start_datetime) + await provider.update_data() + + assert provider.ems_start_datetime == sample_start_datetime + + async def test_update_method_force_enable(self, provider, monkeypatch): + """Test that `update` executes when `force_enable` is True, even if `enabled` is False.""" + # Override enabled to return False for this test + DerivedDataProvider.provider_enabled = False + DerivedDataProvider.provider_updated = False + await provider.update_data(force_enable=True) + assert provider.enabled() is False, "Provider should be disabled, but enabled() is True." + assert DerivedDataProvider.provider_updated is True, ( + "Provider should have been executed, but was not." + ) + + async def test_delete_by_datetime(self, provider, sample_start_datetime): + """Test `delete_by_datetime` method for removing records by datetime range.""" + # Add records to the provider for deletion testing + records = [ + self.create_test_record(sample_start_datetime - to_duration("3 hours"), 1), + self.create_test_record(sample_start_datetime - to_duration("1 hour"), 2), + self.create_test_record(sample_start_datetime + to_duration("1 hour"), 3), + ] + for record in records: + await provider.insert_by_datetime(record) + + await provider.delete_by_datetime( + start_datetime=sample_start_datetime - to_duration("2 hours"), + end_datetime=sample_start_datetime + to_duration("2 hours"), + ) + assert len(provider.records) == 1, ( + "Only one record should remain after deletion by datetime." + ) + assert provider.records[0].date_time == sample_start_datetime - to_duration("3 hours"), ( + "Unexpected record remains." + ) + + +@pytest.mark.asyncio +class TestDataImportProvider: + + @pytest.fixture + def provider(self): + DerivedDataImportProvider.provider_enabled = True + DerivedDataImportProvider.provider_updated = True + p = DerivedDataImportProvider() + p._updates.clear() + p.records.clear() + return p + + async def test_import_from_dict_basic(self, provider): + data = { + "start_datetime": "2024-01-01 00:00:00", + "interval": "1 hour", + "solar_power": [1, 2, 3], + } + await provider.import_from_dict(data) + assert provider.records is not None + assert provider.records[0]["solar_power"] == 1 + assert provider.records[1]["solar_power"] == 2 + + async def test_import_from_dict_default_start_and_interval(self, provider): + data = {"solar_power": [10, 20]} + await provider.import_from_dict(data) + assert len(provider._updates) == 2 + + async def test_import_from_dict_with_prefix(self, provider): + data = { + "dish_washer_emr": [1, 2], + "data_value": [5, 6], + } + await provider.import_from_dict(data, key_prefix="dish") + assert len(provider._updates) == 2 + assert all(update[1][0] == "dish_washer_emr" for update in provider._updates) + + async def test_import_from_dict_mismatching_lengths(self, provider): + data = { + "solar_power": [1, 2], + "temp": [1], + } + with pytest.raises(ValueError): + await provider.import_from_dict(data) + + async def test_import_from_dict_invalid_interval(self, provider): + data = { + "interval": "17 minutes", + "solar_power": [1, 2, 3], + } + with pytest.raises(NotImplementedError): + await provider.import_from_dict(data) + + async def test_import_from_dict_skips_none_and_nan(self, provider): + data = {"solar_power": [1, None, np.nan, 4]} + await provider.import_from_dict(data) + assert len(provider._updates) == 2 + assert provider._updates[0][1][1] == 1 + assert provider._updates[1][1][1] == 4 + + async def test_import_from_dict_invalid_value_type(self, provider): + data = {"solar_power": "not a list"} + with pytest.raises(ValueError): + await provider.import_from_dict(data) + + async def test_import_from_dataframe_with_datetime_index(self, provider): + index = pd.date_range("2024-01-01", periods=3, freq="h") + df = pd.DataFrame({"solar_power": [1, 2, 3]}, index=index) + await provider.import_from_dataframe(df) + assert len(provider._updates) == 3 + assert provider._updates[0][1][1] == 1 + + async def test_import_from_dataframe_without_datetime_index(self, provider): + df = pd.DataFrame({"solar_power": [5, 6, 7]}) + await provider.import_from_dataframe( + df, + start_datetime=to_datetime(datetime(2024, 1, 1)), + interval=to_duration("1 hour"), + ) + assert len(provider._updates) == 3 + + async def test_import_from_dataframe_prefix_filter(self, provider): + df = pd.DataFrame({ + "dish_washer_emr": [1, 2], + "data_value": [3, 4], + }) + await provider.import_from_dataframe(df, key_prefix="dish") + assert len(provider._updates) == 2 + assert all(update[1][0] == "dish_washer_emr" for update in provider._updates) + + async def test_import_from_dataframe_invalid_input(self, provider): + with pytest.raises(ValueError): + await provider.import_from_dataframe("not a dataframe") + + async def test_import_from_json_simple_dict(self, provider): + json_str = json.dumps({"solar_power": [1, 2, 3]}) + await provider.import_from_json(json_str) + assert len(provider._updates) == 3 + + async def test_import_from_json_invalid(self, provider): + with pytest.raises(ValueError): + await provider.import_from_json("this is not json") + + async def test_import_from_file(self, provider, tmp_path): + file_path = tmp_path / "data.json" + file_path.write_text(json.dumps({"solar_power": [1, 2]})) + await provider.import_from_file(file_path) + assert len(provider._updates) == 2 diff --git a/tests/test_dataabcrecord.py b/tests/test_dataabcrecord.py new file mode 100644 index 00000000..541ee3d6 --- /dev/null +++ b/tests/test_dataabcrecord.py @@ -0,0 +1,312 @@ +"""Pytest test for data records fro dataabc module.""" + +import asyncio +import json +from datetime import datetime, timezone +from typing import Any, ClassVar, List, Optional, Union + +import numpy as np +import pandas as pd +import pendulum +import pytest +from pydantic import Field, ValidationError + +from akkudoktoreos.config.configabc import SettingsBaseModel +from akkudoktoreos.core.dataabc import ( + DataABC, + DataRecord, +) +from akkudoktoreos.core.databaseabc import DatabaseTimestamp +from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration + +# Derived classes for testing +# --------------------------- + +class DerivedConfig(SettingsBaseModel): + env_var: Optional[int] = Field(default=None, description="Test config by environment var") + instance_field: Optional[str] = Field(default=None, description="Test config by instance field") + class_constant: Optional[int] = Field(default=None, description="Test config by class constant") + + +class DerivedBase(DataABC): + instance_field: Optional[str] = Field(default=None, description="Field Value") + class_constant: ClassVar[int] = 30 + + +class DerivedRecord(DataRecord): + """Date Record derived from base class DataRecord. + + The derived data record got the + - `data_value` field and the + - `dish_washer_emr`, `solar_power`, `temp` configurable field like data. + """ + + data_value: Optional[float] = Field(default=None, description="Data Value") + + @classmethod + def configured_data_keys(cls) -> Optional[list[str]]: + return ["dish_washer_emr", "solar_power", "temp"] + + +# Tests +# ---------- + + +class TestDataABC: + @pytest.fixture + def base(self): + # Provide default values for configuration + derived = DerivedBase() + return derived + + def test_get_config_value_key_error(self, base): + with pytest.raises(AttributeError): + base.config.non_existent_key + + +class TestDataRecord: + def create_test_record(self, date, value): + """Helper function to create a test DataRecord.""" + return DerivedRecord(date_time=date, data_value=value) + + @pytest.fixture + def record(self): + """Fixture to create a sample DerivedDataRecord with some data set.""" + rec = DerivedRecord(date_time=to_datetime("1967-01-11"), data_value=10.0) + rec.configured_data = {"dish_washer_emr": 123.0, "solar_power": 456.0} + return rec + + def test_getitem(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + assert record["data_value"] == 10.0 + + def test_setitem(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + record["data_value"] = 20.0 + assert record.data_value == 20.0 + + def test_delitem(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + record.data_value = 20.0 + del record["data_value"] + assert record.data_value is None + + def test_len(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + record.date_time = None + record.data_value = 20.0 + assert len(record) == 5 # 2 regular fields + 3 configured data "fields" + + def test_to_dict(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + record.data_value = 20.0 + record_dict = record.to_dict() + assert "data_value" in record_dict + assert record_dict["data_value"] == 20.0 + record2 = DerivedRecord.from_dict(record_dict) + assert record2.model_dump() == record.model_dump() + + def test_to_json(self): + record = self.create_test_record(datetime(2024, 1, 3, tzinfo=timezone.utc), 10.0) + record.data_value = 20.0 + json_str = record.to_json() + assert "data_value" in json_str + assert "20.0" in json_str + record2 = DerivedRecord.from_json(json_str) + assert record2.model_dump() == record.model_dump() + + def test_record_keys_includes_configured_data_keys(self, record): + """Ensure record_keys includes all configured configured data keys.""" + assert set(record.record_keys()) >= set(record.configured_data_keys()) + + def test_record_keys_writable_includes_configured_data_keys(self, record): + """Ensure record_keys_writable includes all configured configured data keys.""" + assert set(record.record_keys_writable()) >= set(record.configured_data_keys()) + + def test_getitem_existing_field(self, record): + """Test that __getitem__ returns correct value for existing native field.""" + record.date_time = "2024-01-01T00:00:00+00:00" + assert record["date_time"] is not None + + def test_getitem_existing_configured_data(self, record): + """Test that __getitem__ retrieves existing configured data values.""" + assert record["dish_washer_emr"] == 123.0 + assert record["solar_power"] == 456.0 + + def test_getitem_missing_configured_data_returns_none(self, record): + """Test that __getitem__ returns None for missing but known configured data keys.""" + assert record["temp"] is None + + def test_getitem_raises_keyerror(self, record): + """Test that __getitem__ raises KeyError for completely unknown keys.""" + with pytest.raises(KeyError): + _ = record["nonexistent"] + + def test_setitem_field(self, record): + """Test setting a native field using __setitem__.""" + record["date_time"] = "2025-01-01T12:00:00+00:00" + assert str(record.date_time).startswith("2025-01-01") + + def test_setitem_configured_data(self, record): + """Test setting a known configured data key using __setitem__.""" + record["temp"] = 25.5 + assert record.configured_data["temp"] == 25.5 + + def test_setitem_invalid_key_raises(self, record): + """Test that __setitem__ raises KeyError for unknown keys.""" + with pytest.raises(KeyError): + record["unknown_key"] = 123 + + def test_delitem_field(self, record): + """Test deleting a native field using __delitem__.""" + record["date_time"] = "2025-01-01T12:00:00+00:00" + del record["date_time"] + assert record.date_time is None + + def test_delitem_configured_data(self, record): + """Test deleting a known configured data key using __delitem__.""" + del record["solar_power"] + assert "solar_power" not in record.configured_data + + def test_delitem_unknown_raises(self, record): + """Test that __delitem__ raises KeyError for unknown keys.""" + with pytest.raises(KeyError): + del record["nonexistent"] + + def test_attribute_get_existing_field(self, record): + """Test accessing a native field via attribute.""" + record.date_time = "2025-01-01T12:00:00+00:00" + assert record.date_time is not None + + def test_attribute_get_existing_configured_data(self, record): + """Test accessing an existing configured data via attribute.""" + assert record.dish_washer_emr == 123.0 + + def test_attribute_get_missing_configured_data(self, record): + """Test accessing a missing but known configured data returns None.""" + assert record.temp is None + + def test_attribute_get_invalid_raises(self, record): + """Test accessing an unknown attribute raises AttributeError.""" + with pytest.raises(AttributeError): + _ = record.nonexistent + + def test_attribute_set_existing_field(self, record): + """Test setting a native field via attribute.""" + record.date_time = "2025-06-25T12:00:00+00:00" + assert record.date_time is not None + + def test_attribute_set_existing_configured_data(self, record): + """Test setting a known configured data key via attribute.""" + record.temp = 99.9 + assert record.configured_data["temp"] == 99.9 + + def test_attribute_set_invalid_raises(self, record): + """Test setting an unknown attribute raises AttributeError.""" + with pytest.raises(AttributeError): + record.invalid = 123 + + def test_delattr_field(self, record): + """Test deleting a native field via attribute.""" + record.date_time = "2025-06-25T12:00:00+00:00" + del record.date_time + assert record.date_time is None + + def test_delattr_configured_data(self, record): + """Test deleting a known configured data key via attribute.""" + record.temp = 88.0 + del record.temp + assert "temp" not in record.configured_data + + def test_delattr_ignored_missing_configured_data_key(self, record): + """Test deleting a known configured data key that was never set is a no-op.""" + del record.temp + assert "temp" not in record.configured_data + + def test_len_and_iter(self, record): + """Test that __len__ and __iter__ behave as expected.""" + keys = list(iter(record)) + assert set(record.record_keys_writable()) == set(keys) + assert len(record) == len(keys) + + def test_in_operator_includes_configured_data(self, record): + """Test that 'in' operator includes configured data keys.""" + assert "dish_washer_emr" in record + assert "temp" in record # known key, even if not yet set + assert "nonexistent" not in record + + def test_hasattr_behavior(self, record): + """Test that hasattr returns True for fields and known configured dataWs.""" + assert hasattr(record, "date_time") + assert hasattr(record, "dish_washer_emr") + assert hasattr(record, "temp") # allowed, even if not yet set + assert not hasattr(record, "nonexistent") + + def test_model_validate_roundtrip(self, record): + """Test that MeasurementDataRecord can be serialized and revalidated.""" + dumped = record.model_dump() + restored = DerivedRecord.model_validate(dumped) + assert restored.dish_washer_emr == 123.0 + assert restored.solar_power == 456.0 + assert restored.temp is None # not set + + def test_copy_preserves_configured_data(self, record): + """Test that copying preserves configured data values.""" + record.temp = 22.2 + copied = record.model_copy() + assert copied.dish_washer_emr == 123.0 + assert copied.temp == 22.2 + assert copied is not record + + def test_equality_includes_configured_data(self, record): + """Test that equality includes the `configured data` content.""" + other = record.model_copy() + assert record == other + + def test_inequality_differs_with_configured_data(self, record): + """Test that records with different configured datas are not equal.""" + other = record.model_copy(deep=True) + # Modify one configured data value in the copy + other.configured_data["dish_washer_emr"] = 999.9 + assert record != other + + def test_in_operator_for_configured_data_and_fields(self, record): + """Ensure 'in' works for both fields and configured configured data keys.""" + assert "dish_washer_emr" in record + assert "solar_power" in record + assert "date_time" in record # standard field + assert "temp" in record # allowed but not yet set + assert "unknown" not in record + + def test_hasattr_equivalence_to_getattr(self, record): + """hasattr should return True for all valid keys/configured datas.""" + assert hasattr(record, "dish_washer_emr") + assert hasattr(record, "temp") + assert hasattr(record, "date_time") + assert not hasattr(record, "nonexistent") + + def test_dir_includes_configured_data_keys(self, record): + """`dir(record)` should include configured data keys for introspection. + It shall not include the internal 'configured datas' attribute. + """ + keys = dir(record) + assert "configured datas" not in keys + for key in record.configured_data_keys(): + assert key in keys + + def test_init_configured_field_like_data_applies_before_model_init(self): + """Test that keys listed in `_configured_data_keys` are moved to `configured_data` at init time.""" + record = DerivedRecord( + date_time="2024-01-03T00:00:00+00:00", + data_value=42.0, + dish_washer_emr=111.1, + solar_power=222.2, + temp=333.3 # assume `temp` is also a valid configured key + ) + + assert record.data_value == 42.0 + assert record.configured_data == { + "dish_washer_emr": 111.1, + "solar_power": 222.2, + "temp": 333.3, + } diff --git a/tests/test_dataabcsequence.py b/tests/test_dataabcsequence.py new file mode 100644 index 00000000..d24e4a43 --- /dev/null +++ b/tests/test_dataabcsequence.py @@ -0,0 +1,979 @@ +import asyncio +import json +from datetime import datetime, timezone +from typing import Any, ClassVar, List, Optional, Union + +import numpy as np +import pandas as pd +import pendulum +import pytest +import pytest_asyncio +from pydantic import Field, ValidationError + +from akkudoktoreos.config.configabc import SettingsBaseModel +from akkudoktoreos.core.coreabc import get_ems +from akkudoktoreos.core.dataabc import ( + DataABC, + DataContainer, + DataImportProvider, + DataProvider, + DataRecord, + DataSequence, +) +from akkudoktoreos.core.databaseabc import DatabaseTimestamp +from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime, to_duration + +# Derived classes for testing +# --------------------------- + +class DerivedConfig(SettingsBaseModel): + env_var: Optional[int] = Field(default=None, description="Test config by environment var") + instance_field: Optional[str] = Field(default=None, description="Test config by instance field") + class_constant: Optional[int] = Field(default=None, description="Test config by class constant") + + +class DerivedBase(DataABC): + instance_field: Optional[str] = Field(default=None, description="Field Value") + class_constant: ClassVar[int] = 30 + + +class DerivedRecord(DataRecord): + """Date Record derived from base class DataRecord. + + The derived data record got the + - `data_value` field and the + - `dish_washer_emr`, `solar_power`, `temp` configurable field like data. + """ + + data_value: Optional[float] = Field(default=None, description="Data Value") + + @classmethod + def configured_data_keys(cls) -> Optional[list[str]]: + return ["dish_washer_emr", "solar_power", "temp"] + + +class DerivedSequence(DataSequence): + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedSequence" + + +class DerivedSequence2(DataSequence): + # overload + records: List[DerivedRecord] = Field( + default_factory=list, description="List of DerivedRecord records" + ) + + @classmethod + def record_class(cls) -> Any: + return DerivedRecord + + def db_namespace(self) -> str: + return "DerivedSequence2" + + +# Tests +# ---------- + + +@pytest.mark.asyncio +class TestDataSequence: + + @pytest_asyncio.fixture + async def sequence(self): + sequence0 = DerivedSequence() + mem_len = len(sequence0) + db_len = await sequence0.db_count_records() + assert mem_len == 0 + assert db_len == 0 + return sequence0 + + @pytest_asyncio.fixture + async def sequence2(self): + sequence = DerivedSequence() + record1 = self.create_test_record(datetime(1970, 1, 1), 1970) + record2 = self.create_test_record(datetime(1971, 1, 1), 1971) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + mem_len = len(sequence) + db_len = await sequence.db_count_records() + assert mem_len == 2 + assert db_len == 2 + return sequence + + def create_test_record(self, date, value): + """Helper function to create a test DataRecord.""" + return DerivedRecord(date_time=date, data_value=value) + + # Test cases + @pytest.mark.parametrize("tz_name", ["UTC", "Europe/Berlin", "Atlantic/Canary"]) + async def test_min_max_datetime_timezone_and_order(self, sequence, tz_name, monkeypatch, config_eos): + # Monkeypatch the read-only timezone property + monkeypatch.setattr(config_eos.general.__class__, "timezone", property(lambda self: tz_name)) + + # Create timezone-aware datetimes using the patched config + dt_early = to_datetime("2024-01-01T00:00:00", in_timezone=config_eos.general.timezone) + dt_late = to_datetime("2024-01-02T00:00:00", in_timezone=config_eos.general.timezone) + + # Insert in reverse order to verify sorting + record1 = self.create_test_record(dt_late, 1) + record2 = self.create_test_record(dt_early, 2) + + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + min_dt = await sequence.min_datetime() + max_dt = await sequence.max_datetime() + + # --- Basic correctness --- + assert min_dt == dt_early + assert max_dt == dt_late + + # --- Must be timezone aware --- + assert min_dt.tzinfo is not None + assert max_dt.tzinfo is not None + + # --- Must preserve timezone --- + assert min_dt.tzinfo.name == tz_name + assert max_dt.tzinfo.name == tz_name + + async def test_get_by_datetime(self, sequence): + assert len(sequence) == 0 + dt = to_datetime("2024-01-01 00:00:00") + record = self.create_test_record(dt, 0) + await sequence.insert_by_datetime(record) + item = await sequence.get_by_datetime(dt) + assert isinstance(item, DerivedRecord) + + async def test_insert_by_datetime(self, sequence2): + dt = to_datetime("2024-01-03", in_timezone="UTC") + record = self.create_test_record(dt, 1) + await sequence2.insert_by_datetime(record) + assert sequence2.records[2].date_time == dt + + async def test_insert_reversed_date_record(self, sequence2): + dt1 = to_datetime("2023-11-05", in_timezone="UTC") + dt2 = to_datetime("2024-01-03", in_timezone="UTC") + record1 = self.create_test_record(dt2, 0.8) + record2 = self.create_test_record(dt1, 0.9) # reversed date + await sequence2.insert_by_datetime(record1) + assert sequence2.records[2].date_time == dt2 + await sequence2.insert_by_datetime(record2) + assert len(sequence2) == 4 + assert sequence2.records[2] == record2 + + async def test_insert_duplicate_date_record(self, sequence): + dt1 = to_datetime("2023-11-05") + record1 = self.create_test_record(dt1, 0.8) + record2 = self.create_test_record(dt1, 0.9) # Duplicate date + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + assert len(sequence) == 1 + retrieved_record = await sequence.get_by_datetime(dt1) + assert retrieved_record.data_value == 0.9 # Record should have merged with new value + + async def test_key_to_series(self, sequence): + dt = to_datetime(datetime(2023, 11, 6)) + record = self.create_test_record(dt, 0.8) + await sequence.insert_by_datetime(record) + series = await sequence.key_to_series("data_value") + assert isinstance(series, pd.Series) + + retrieved_record = await sequence.get_by_datetime(dt) + assert retrieved_record is not None + assert retrieved_record.data_value == 0.8 + + async def test_key_from_series(self, sequence): + dt1 = to_datetime(datetime(2023, 11, 5)) + dt2 = to_datetime(datetime(2023, 11, 6)) + + series = pd.Series( + data=[0.8, 0.9], index=pd.to_datetime([dt1, dt2]) + ) + await sequence.key_from_series("data_value", series) + assert len(sequence) == 2 + + record1 = await sequence.get_by_datetime(dt1) + assert record1 is not None + assert record1.data_value == 0.8 + + record2 = await sequence.get_by_datetime(dt2) + assert record2 is not None + assert record2.data_value == 0.9 + + async def test_key_to_array(self, sequence): + interval = to_duration("1 day") + start_datetime = to_datetime("2023-11-6") + last_datetime = to_datetime("2023-11-8") + end_datetime = to_datetime("2023-11-9") + + record1 = self.create_test_record(start_datetime, float(start_datetime.day)) + await sequence.insert_by_datetime(record1) + record2 = self.create_test_record(last_datetime, float(last_datetime.day)) + await sequence.insert_by_datetime(record2) + + retrieved_record1 = await sequence.get_by_datetime(start_datetime) + assert retrieved_record1 is not None + assert retrieved_record1.data_value == 6.0 + + retrieved_record2 = await sequence.get_by_datetime(last_datetime) + assert retrieved_record2 is not None + assert retrieved_record2.data_value == 8.0 + + series = await sequence.key_to_series( + key="data_value", start_datetime=start_datetime, end_datetime=end_datetime + ) + assert len(series) == 2 + assert series[to_datetime("2023-11-6")] == 6 + assert series[to_datetime("2023-11-8")] == 8 + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=interval, + ) + assert isinstance(array, np.ndarray) + np.testing.assert_equal(array, [6.0, 7.0, 8.0]) + + async def test_key_to_array_linear_interpolation(self, sequence): + """Test key_to_array with linear interpolation for numeric data.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) + record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) # Gap of 2 hours + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 3), + interval=interval, + fill_method="linear", + ) + assert len(array) == 3 + assert array[0] == 0.8 + assert array[1] == 0.9 # Interpolated value + assert array[2] == 1.0 + + + async def test_key_to_array_linear_interpolation_out_of_grid(self, sequence): + """Test key_to_array with linear interpolation out of grid.""" + interval = to_duration("1 hour") + start_datetime= to_datetime("2023-11-06T00:30:00") # out of grid + end_datetime=to_datetime("2023-11-06T01:30:00") # out of grid + + record1_datetime = to_datetime("2023-11-06T00:00:00") + record1 = self.create_test_record(record1_datetime, 1.0) + + record2_datetime = to_datetime("2023-11-06T02:00:00") + record2 = self.create_test_record(record2_datetime, 2.0) # Gap of 2 hours + + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + # Check test setup + record1_timestamp = DatabaseTimestamp.from_datetime(record1_datetime) + record2_timestamp = DatabaseTimestamp.from_datetime(record2_datetime) + start_timestamp = DatabaseTimestamp.from_datetime(start_datetime) + end_timestamp = DatabaseTimestamp.from_datetime(end_datetime) + + start_previous_timestamp = await sequence.db_previous_timestamp(start_timestamp) + assert start_previous_timestamp == record1_timestamp + end_next_timestamp = await sequence.db_next_timestamp(end_timestamp) + assert end_next_timestamp == record2_timestamp + + # Test + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_datetime, + end_datetime=end_datetime, + interval=interval, + fill_method="linear", + boundary="context", + ) + np.testing.assert_equal(array, [1.5]) + + async def test_key_to_array_ffill(self, sequence): + """Test key_to_array with forward filling for missing values.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) + record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 3), + interval=interval, + fill_method="ffill", + ) + assert len(array) == 3 + assert array[0] == 0.8 + assert array[1] == 0.8 # Forward-filled value + assert array[2] == 1.0 + + async def test_key_to_array_ffill_one_value(self, sequence): + """Test key_to_array with forward filling for missing values and only one value at end available.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) + await sequence.insert_by_datetime(record1) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 4), + interval=interval, + fill_method="ffill", + ) + assert len(array) == 4 + assert array[0] == 1.0 # Backward-filled value + assert array[1] == 1.0 # Backward-filled value + assert array[2] == 1.0 + assert array[2] == 1.0 # Forward-filled value + + async def test_key_to_array_bfill(self, sequence): + """Test key_to_array with backward filling for missing values.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) + record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 2), 1.0) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 3), + interval=interval, + fill_method="bfill", + ) + assert len(array) == 3 + assert array[0] == 0.8 + assert array[1] == 1.0 # Backward-filled value + assert array[2] == 1.0 + + async def test_key_to_array_with_truncation(self, sequence): + """Test truncation behavior in key_to_array.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 5, 23), 0.8) + record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 1), 1.0) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + #assert sequence is None + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 5, 23), + end_datetime=pendulum.datetime(2023, 11, 6, 2), + interval=interval, + ) + + assert len(array) == 3 + assert array[0] == 0.8 + assert array[1] == 0.9 # Interpolated from previous day + assert array[2] == 1.0 + + async def test_key_to_array_with_none(self, sequence): + """Test handling of empty series in key_to_array.""" + interval = to_duration("1 hour") + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 3), + interval=interval, + ) + assert isinstance(array, np.ndarray) + assert np.all(array == None) + + async def test_key_to_array_with_one(self, sequence): + """Test handling of one element series in key_to_array.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 5, 23), 0.8) + await sequence.insert_by_datetime(record1) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 5, 23), + end_datetime=pendulum.datetime(2023, 11, 6, 2), + interval=interval, + ) + assert len(array) == 3 + assert array[0] == 0.8 + assert array[1] == 0.8 # Interpolated from previous day + assert array[2] == 0.8 # Interpolated from previous day + + async def test_key_to_array_invalid_fill_method(self, sequence): + """Test invalid fill_method raises an error.""" + interval = to_duration("1 hour") + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0), 0.8) + await sequence.insert_by_datetime(record1) + + with pytest.raises(ValueError, match="Unsupported fill method: invalid"): + await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6), + end_datetime=pendulum.datetime(2023, 11, 6, 1), + interval=interval, + fill_method="invalid", + ) + + async def test_key_to_array_resample_mean(self, sequence): + """Test that numeric resampling uses mean when multiple values fall into one interval.""" + interval = to_duration("1 hour") + # Insert values every 15 minutes within the same hour + record1 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 0), 1.0) + record2 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 15), 2.0) + record3 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 30), 3.0) + record4 = self.create_test_record(pendulum.datetime(2023, 11, 6, 0, 45), 4.0) + + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + await sequence.insert_by_datetime(record4) + + # Resample to hourly interval, expecting the mean of the 4 values + array = await sequence.key_to_array( + key="data_value", + start_datetime=pendulum.datetime(2023, 11, 6, 0), + end_datetime=pendulum.datetime(2023, 11, 6, 1), + interval=interval, + ) + + assert isinstance(array, np.ndarray) + assert len(array) == 1 # one interval: 0:00-1:00 + # The first interval mean = (1+2+3+4)/4 = 2.5 + assert array[0] == pytest.approx(2.5) + + # ------------------------------------------------------------------ + # key_to_array — align_to_interval parameter + # ------------------------------------------------------------------ + # + # The existing tests above use start_datetime values that already sit on + # clean hour/day boundaries, so the default alignment (origin=query_start) + # and clock alignment (origin=epoch-floor) produce identical results. + # The tests below specifically use off-boundary start times to expose + # the difference and verify the new parameter. + + async def test_key_to_array_align_false_origin_is_query_start(self, sequence): + """Without align_to_interval the first bucket sits at query_start, not a clock boundary. + + With start_datetime at 10:07:00 and 15-min interval the first resampled + bucket must be at 10:07:00 (origin = query_start), NOT at 10:00:00 or 10:15:00. + """ + # Off-boundary start: 10:07 + start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") + + # Records every 15 min so the resampled mean equals the input values + for m in range(0, 120, 15): + dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=False, + ) + + assert len(array) > 0 + # Reconstruct the pandas index that key_to_array used: origin=start_dt + idx = pd.date_range(start=start_dt, periods=len(array), freq="900s") + # First bucket must be exactly at start_dt (10:07) + assert idx[0].minute == 7 + assert idx[0].second == 0 + + async def test_key_to_array_align_true_15min_buckets_on_quarter_hours(self, sequence): + """align_to_interval=True produces timestamps on :00/:15/:30/:45 boundaries.""" + # Off-boundary start: 10:07 + start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") + + # 1-min records across the window so resampling has data to work with + for m in range(0, 121): + dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert len(array) > 0 + # Reconstruct the epoch-aligned index that key_to_array must have used + import math + epoch = int(start_dt.timestamp()) + floored_epoch = (epoch // 900) * 900 # floor to nearest 15-min boundary + idx = pd.date_range( + start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), + periods=len(array), + freq="900s", + ) + # Every bucket must land on a :00/:15/:30/:45 minute mark with zero seconds + for ts in idx: + assert ts.minute % 15 == 0, ( + f"Bucket at {ts} is not on a 15-min boundary (minute={ts.minute})" + ) + assert ts.second == 0, ( + f"Bucket at {ts} has non-zero seconds ({ts.second})" + ) + + async def test_key_to_array_align_true_1hour_buckets_on_the_hour(self, sequence): + """align_to_interval=True with 1-hour interval produces on-the-hour timestamps.""" + # Off-boundary start: 10:23 + start_dt = pendulum.datetime(2024, 6, 1, 10, 23, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 15, 23, tz="UTC") + + for m in range(0, 301, 15): + dt = pendulum.datetime(2024, 6, 1, 10, 23, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("1 hour"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert len(array) > 0 + epoch = int(start_dt.timestamp()) + floored_epoch = (epoch // 3600) * 3600 # floor to nearest hour + idx = pd.date_range( + start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), + periods=len(array), + freq="1h", + ) + for ts in idx: + assert ts.minute == 0, ( + f"Bucket at {ts} should be on the hour (minute={ts.minute})" + ) + assert ts.second == 0, ( + f"Bucket at {ts} has non-zero seconds ({ts.second})" + ) + + async def test_key_to_array_align_true_when_start_already_on_boundary(self, sequence): + """align_to_interval=True is a no-op when start_datetime is exactly on a boundary. + + With start at a clean 15-min mark both modes must produce identical arrays. + """ + # Exactly on boundary: 10:00:00 + start_dt = pendulum.datetime(2024, 6, 1, 10, 0, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 12, 0, tz="UTC") + + for m in range(0, 121, 15): + dt = pendulum.datetime(2024, 6, 1, 10, 0, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + arr_aligned = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + arr_default = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=False, + ) + + assert len(arr_aligned) == len(arr_default) + np.testing.assert_array_almost_equal(arr_aligned, arr_default, decimal=6) + + async def test_key_to_array_align_true_without_start_datetime(self, sequence): + """align_to_interval=True with no start_datetime must not raise. + + Without a query_start there is no origin to snap; behaviour falls back + to 'start_day' (same as default). No exception is expected. + """ + for m in range(0, 121, 15): + dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=None, + end_datetime=pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC"), + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert isinstance(array, np.ndarray) + assert len(array) > 0 + + async def test_key_to_array_align_true_output_within_requested_window(self, sequence): + """align_to_interval=True truncates output to [start_datetime, end_datetime). + + The epoch-floor origin may generate a bucket before start_datetime (e.g. 10:00 + when start is 10:07), but key_to_array must truncate it away. The surviving + buckets are verified directly by reconstructing the index from the first + surviving timestamp (the first epoch-aligned bucket >= start_datetime). + + Also checks that all surviving buckets are on 15-min clock boundaries. + """ + start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 13, 7, tz="UTC") + + for m in range(0, 181): + dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert len(array) > 0 + + # The first surviving bucket is the first epoch-aligned timestamp >= start_dt. + # Compute it the same way key_to_array does: floor then step forward if needed. + epoch = int(start_dt.timestamp()) + floored_epoch = (epoch // 900) * 900 + first_bucket = pd.Timestamp(floored_epoch, unit="s", tz="UTC") + if first_bucket < pd.Timestamp(start_dt): + first_bucket += pd.Timedelta(seconds=900) + + idx = pd.date_range(start=first_bucket, periods=len(array), freq="900s") + + start_pd = pd.Timestamp(start_dt) + end_pd = pd.Timestamp(end_dt) + for ts in idx: + assert ts >= start_pd, f"Bucket {ts} is before start_datetime {start_pd}" + assert ts < end_pd, f"Bucket {ts} is at or after end_datetime {end_pd}" + assert ts.minute % 15 == 0, f"Bucket {ts} is not on a 15-min boundary" + assert ts.second == 0, f"Bucket {ts} has non-zero seconds" + + async def test_key_to_array_align_true_preserves_mean_values(self, sequence): + """align_to_interval=True does not corrupt resampled values. + + A constant-valued series must resample to the same constant regardless + of bucket alignment. + """ + # 1-min records with constant value 42.0, starting off-boundary + start_dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC") + end_dt = pendulum.datetime(2024, 6, 1, 12, 7, tz="UTC") + + for m in range(0, 121): + dt = pendulum.datetime(2024, 6, 1, 10, 7, tz="UTC").add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, 42.0)) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=start_dt, + end_datetime=end_dt, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert len(array) > 0 + for v in array: + if v is not None: + assert abs(v - 42.0) < 1e-6, f"Expected 42.0, got {v}" + + async def test_key_to_array_align_true_compaction_call_pattern(self, sequence): + """Verify the call pattern used by _db_compact_tier produces clock-aligned timestamps. + + _db_compact_tier calls key_to_array with boundary='strict', fill_method='time', + align_to_interval=True on a window whose start has arbitrary sub-second precision. + All output buckets must land on 15-min boundaries so that compacted records are + stored at predictable, human-readable timestamps. + """ + # Non-round base time: 08:43 — chosen to expose any origin-alignment bug + base_dt = pendulum.datetime(2024, 6, 1, 8, 43, tz="UTC") + window_end = pendulum.datetime(2024, 6, 1, 11, 43, tz="UTC") + + for m in range(0, 181): + dt = base_dt.add(minutes=m) + await sequence.insert_by_datetime(self.create_test_record(dt, float(m))) + + array = await sequence.key_to_array( + key="data_value", + start_datetime=base_dt, + end_datetime=window_end, + interval=to_duration("15 minutes"), + fill_method="time", + boundary="strict", + align_to_interval=True, + ) + + assert len(array) > 0 + epoch = int(base_dt.timestamp()) + floored_epoch = (epoch // 900) * 900 + idx = pd.date_range( + start=pd.Timestamp(floored_epoch, unit="s", tz="UTC"), + periods=len(array), + freq="900s", + ) + for ts in idx: + assert ts.minute % 15 == 0, ( + f"Compacted record at {ts} is not on a 15-min boundary (minute={ts.minute})" + ) + assert ts.second == 0, ( + f"Compacted record at {ts} has non-zero seconds ({ts.second})" + ) + + async def test_delete_by_datetime_range(self, sequence): + dt1 = to_datetime("2023-11-05") + dt2 = to_datetime("2023-11-06") + dt3 = to_datetime("2023-11-07") + record1 = self.create_test_record(dt1, 0.8) + record2 = self.create_test_record(dt2, 0.9) + record3 = self.create_test_record(dt3, 1.0) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + assert len(sequence) == 3 + await sequence.delete_by_datetime(start_datetime=dt2, end_datetime=dt3) + assert len(sequence) == 2 + assert sequence.records[0].date_time == dt1 + assert sequence.records[1].date_time == dt3 + + async def test_delete_by_datetime_start(self, sequence): + dt1 = to_datetime("2023-11-05") + dt2 = to_datetime("2023-11-06") + record1 = self.create_test_record(dt1, 0.8) + record2 = self.create_test_record(dt2, 0.9) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + assert len(sequence) == 2 + await sequence.delete_by_datetime(start_datetime=dt2) + assert len(sequence) == 1 + assert sequence.records[0].date_time == dt1 + + async def test_delete_by_datetime_end(self, sequence): + dt1 = to_datetime("2023-11-05") + dt2 = to_datetime("2023-11-06") + record1 = self.create_test_record(dt1, 0.8) + record2 = self.create_test_record(dt2, 0.9) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + assert len(sequence) == 2 + await sequence.delete_by_datetime(end_datetime=dt2) + assert len(sequence) == 1 + assert sequence.records[0].date_time == dt2 + + async def test_to_dict_async(self, sequence): + dt = to_datetime("2023-11-06") + record = self.create_test_record(dt, 0.8) + await sequence.insert_by_datetime(record) + data_dict = await sequence.to_dict_async() + assert isinstance(data_dict, dict) + # We need a new class - Sequences are singletons + sequence2 = await DerivedSequence2.from_dict_async(data_dict) + assert sequence2.model_dump() == sequence.model_dump() + + async def test_to_json_async(self, sequence): + dt = to_datetime("2023-11-06") + record = self.create_test_record(dt, 0.8) + await sequence.insert_by_datetime(record) + json_str = await sequence.to_json_async() + assert isinstance(json_str, str) + assert "2023-11-06" in json_str + assert ": 0.8" in json_str + + async def test_from_json_async(self, sequence, sequence2): + json_str = sequence2.to_json() + sequence = await sequence.from_json_async(json_str) + assert len(sequence) == len(sequence2) + assert sequence.records[0].date_time == sequence2.records[0].date_time + assert sequence.records[0].data_value == sequence2.records[0].data_value + + async def test_key_to_value_exact_match(self, sequence): + """Test key_to_value returns exact match when datetime matches a record.""" + dt = to_datetime("2023-11-05") + record = self.create_test_record(dt, 0.75) + await sequence.insert_by_datetime(record) + result = await sequence.key_to_value("data_value", dt) + assert result == 0.75 + + async def test_key_to_value_nearest(self, sequence): + """Test key_to_value returns value closest in time to the given datetime.""" + record1 = self.create_test_record(datetime(2023, 11, 5, 12), 0.6) + record2 = self.create_test_record(datetime(2023, 11, 6, 12), 0.9) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + dt = datetime(2023, 11, 6, 10) # closer to record2 + result = await sequence.key_to_value("data_value", dt, time_window=to_duration("48 hours")) + assert result == 0.9 + + async def test_key_to_value_nearest_after(self, sequence): + """Test key_to_value returns value nearest after the given datetime.""" + record1 = self.create_test_record(datetime(2023, 11, 5, 10), 0.7) + record2 = self.create_test_record(datetime(2023, 11, 5, 15), 0.8) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + dt = datetime(2023, 11, 5, 14) # closer to record2 + result = await sequence.key_to_value("data_value", dt, time_window=to_duration("48 hours")) + assert result == 0.8 + + async def test_key_to_value_empty_sequence(self, sequence): + """Test key_to_value returns None when sequence is empty.""" + result = await sequence.key_to_value("data_value", datetime(2023, 11, 5)) + assert result is None + + async def test_key_to_value_missing_key(self, sequence): + """Test key_to_value returns None when key is missing in records.""" + record = self.create_test_record(datetime(2023, 11, 5), None) + await sequence.insert_by_datetime(record) + result = await sequence.key_to_value("data_value", datetime(2023, 11, 5)) + assert result is None + + async def test_key_to_value_multiple_records_with_none(self, sequence): + """Test key_to_value skips records with None values.""" + r1 = self.create_test_record(datetime(2023, 11, 5), None) + r2 = self.create_test_record(datetime(2023, 11, 6), 1.0) + await sequence.insert_by_datetime(r1) + await sequence.insert_by_datetime(r2) + result = await sequence.key_to_value("data_value", datetime(2023, 11, 5, 12), time_window=to_duration("48 hours")) + assert result == 1.0 + + async def test_key_to_dict(self, sequence): + record1 = self.create_test_record(datetime(2023, 11, 5), 0.8) + record2 = self.create_test_record(datetime(2023, 11, 6), 0.9) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + data_dict = await sequence.key_to_dict("data_value") + assert isinstance(data_dict, dict) + assert data_dict[to_datetime(datetime(2023, 11, 5), as_string=True)] == 0.8 + assert data_dict[to_datetime(datetime(2023, 11, 6), as_string=True)] == 0.9 + + async def test_key_to_lists(self, sequence): + record1 = self.create_test_record(datetime(2023, 11, 5), 0.8) + record2 = self.create_test_record(datetime(2023, 11, 6), 0.9) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + dates, values = await sequence.key_to_lists("data_value") + assert dates == [to_datetime(datetime(2023, 11, 5)), to_datetime(datetime(2023, 11, 6))] + assert values == [0.8, 0.9] + + async def test_to_dataframe_full_data(self, sequence): + """Test conversion of all records to a DataFrame without filtering.""" + record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) + record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) + record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + + df = await sequence.to_dataframe() + + # Validate DataFrame structure + assert isinstance(df, pd.DataFrame) + assert not df.empty + assert len(df) == 3 # All records should be included + assert "data_value" in df.columns + + async def test_to_dataframe_with_filter(self, sequence): + """Test filtering records by datetime range.""" + record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) + record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) + record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + + start = to_datetime("2024-01-01T12:30:00Z") + end = to_datetime("2024-01-01T14:00:00Z") + + df = await sequence.to_dataframe(start_datetime=start, end_datetime=end) + + assert isinstance(df, pd.DataFrame) + assert not df.empty + assert len(df) == 1 # Only one record should match the range + assert df.index[0] == pd.Timestamp("2024-01-01T13:00:00Z") + + async def test_to_dataframe_no_matching_records(self, sequence): + """Test when no records match the given datetime filter.""" + record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) + record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + + start = to_datetime("2024-01-01T14:00:00Z") # Start time after all records + end = to_datetime("2024-01-01T15:00:00Z") + + df = await sequence.to_dataframe(start_datetime=start, end_datetime=end) + + assert isinstance(df, pd.DataFrame) + assert df.empty # No records should match + + async def test_to_dataframe_empty_sequence(self, sequence): + """Test when DataSequence has no records.""" + sequence = DataSequence(records=[]) + + df = await sequence.to_dataframe() + + assert isinstance(df, pd.DataFrame) + assert df.empty # Should return an empty DataFrame + + async def test_to_dataframe_no_start_datetime(self, sequence): + """Test when only end_datetime is given (all past records should be included).""" + record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) + record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) + record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + + end = to_datetime("2024-01-01T13:00:00Z") # Include only first record + + df = await sequence.to_dataframe(end_datetime=end) + + assert isinstance(df, pd.DataFrame) + assert not df.empty + assert len(df) == 1 + assert df.index[0] == pd.Timestamp("2024-01-01T12:00:00Z") + + async def test_to_dataframe_no_end_datetime(self, sequence): + """Test when only start_datetime is given (all future records should be included).""" + record1 = self.create_test_record("2024-01-01T12:00:00Z", 10) + record2 = self.create_test_record("2024-01-01T13:00:00Z", 20) + record3 = self.create_test_record("2024-01-01T14:00:00Z", 30) + await sequence.insert_by_datetime(record1) + await sequence.insert_by_datetime(record2) + await sequence.insert_by_datetime(record3) + + start = to_datetime("2024-01-01T13:00:00Z") # Include last two records + + df = await sequence.to_dataframe(start_datetime=start) + + assert isinstance(df, pd.DataFrame) + assert not df.empty + assert len(df) == 2 + assert df.index[0] == pd.Timestamp("2024-01-01T13:00:00Z") diff --git a/tests/test_dataabcsequencedb.py b/tests/test_dataabcsequencedb.py new file mode 100644 index 00000000..29250548 --- /dev/null +++ b/tests/test_dataabcsequencedb.py @@ -0,0 +1,701 @@ +"""Pytest tests for async DataSequence with persistence. + +Tests the async DataSequence with database persistence. +""" + +from __future__ import annotations + +import asyncio +import shutil +import tempfile +import time +from pathlib import Path +from typing import AsyncIterator, Optional, Type + +import pytest +import pytest_asyncio +from pydantic import Field + +from akkudoktoreos.core.coreabc import get_database +from akkudoktoreos.core.dataabc import DataProvider, DataRecord, DataSequence +from akkudoktoreos.core.database import Database, LMDBDatabase, SQLiteDatabase +from akkudoktoreos.core.databaseabc import ( + DatabaseRecordProtocolLoadPhase, + DatabaseTimestamp, +) +from akkudoktoreos.utils.datetimeutil import ( + DateTime, + Duration, + to_datetime, + to_duration, +) + +# ==================== Test Fixtures ==================== + +@pytest.fixture +def temp_dir(): + """Create a temporary directory for test databases.""" + temp_path = Path(tempfile.mkdtemp()) + yield temp_path + shutil.rmtree(temp_path, ignore_errors=True) + + +@pytest.fixture(params=["LMDB", "SQLite"]) +def database_provider(request) -> str: + """Parametrize all database backend tests.""" + return request.param + + +@pytest_asyncio.fixture +async def async_database_instance( + config_eos, + database_provider: str, +) -> AsyncIterator[Database]: + """Open a database instance for testing and close it afterwards.""" + config_eos.database.compression_level = 6 + config_eos.database.provider = database_provider + + db = get_database() + + await db.open() + + assert db.provider_id() == database_provider + assert db.is_open is True + + yield db + + await db.close() + + config_eos.database.provider = None + + +# ==================== Helpers ==================== + +async def _clear_sequence_state(sequence) -> None: + """Clear runtime DB state without re-instantiating the singleton. + + Does _NOT_ initialize the DB state. + """ + await sequence.db_delete_records() + try: + sequence._db_metadata = None + await sequence.database().set_metadata(None, namespace=sequence.db_namespace()) + except Exception: + # Database may not be available, just skip + pass + try: + del sequence._db_initialized + except Exception: + # May not be set + pass + + +async def _reset_sequence_state(sequence) -> None: + """Reset runtime DB state without re-instantiating the singleton.""" + try: + sequence.records = [] + del sequence._db_initialized + except Exception: + # May not be set + pass + await sequence._db_ensure_initialized() + + +# Sample Data + +class SampleDataRecord(DataRecord): + """Minimal DataRecord for testing.""" + temperature: float = Field(default=0.0) + humidity: float = Field(default=0.0) + pressure: float = Field(default=0.0) + + +class SampleDataSequence(DataSequence): + """DataSequence subclass with database support.""" + records: list[SampleDataRecord] = Field(default_factory=list) + + @classmethod + def record_class(cls) -> Type[SampleDataRecord]: + return SampleDataRecord + + def db_namespace(self) -> str: + return "SampleDataSequence" + + +class SampleDataProvider(DataProvider): + """DataProvider subclass with database support.""" + records: list[SampleDataRecord] = Field(default_factory=list) + + @classmethod + def record_class(cls) -> Type[SampleDataRecord]: + return SampleDataRecord + + def provider_id(self) -> str: + return "SampleDataProvider" + + def enabled(self) -> bool: + return True + + async def _update_data(self, force_update: Optional[bool] = False) -> None: + pass + + def db_namespace(self) -> str: + return "SampleDataProvider" + + +# ==================== DatabaseRecordProtocolMixin Tests ==================== + +@pytest.mark.asyncio +class TestDataSequenceDatabaseProtocol: + """Tests for DatabaseRecordProtocolMixin via SampleDataSequence.""" + + async def test_db_enabled_when_db_open(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + assert sequence.db_enabled is True + + async def test_db_disabled_when_db_closed(self, config_eos): + config_eos.database.provider = None + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + assert sequence.db_enabled is False + + async def test_insert_and_save_records(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) + ) + + # All 10 are dirty/new, none persisted yet + assert len(sequence.records) == 10 + assert len(sequence._db_new_timestamps) == 10 + + saved = await sequence.db_save_records() + assert saved == 10 # 10 inserts + 0 deletes + assert len(sequence._db_dirty_timestamps) == 0 + assert len(sequence._db_new_timestamps) == 0 + + async def test_save_returns_insert_plus_delete_count(self, async_database_instance): + """db_save_records() return value = saved_inserts + deleted_count.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(5): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + # Persist the 5 records + await sequence.db_save_records() + + # Delete 2 of them + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=4)) + deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + # Insert 3 new ones + for i in range(10, 13): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + + result = await sequence.db_save_records() + # 3 inserts + 2 deletes = 5 + assert result == 5 + + async def test_load_records_from_db(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) + ) + await sequence.db_save_records() + + # Clear memory, then reload from DB + await _reset_sequence_state(sequence) + loaded = await sequence.db_load_records() + + assert loaded == 10 + assert len(sequence.records) == 10 + for i, record in enumerate(sequence.records): + assert record.temperature == 20.0 + i + + async def test_load_records_with_range(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # Load [hours=3, hours=7) → 4 records (3, 4, 5, 6) + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) + loaded = await sequence.db_load_records(start_timestamp=db_start, end_timestamp=db_end) + assert loaded == 4 + assert sequence.records[0].temperature == 23.0 + assert sequence.records[-1].temperature == 26.0 + + async def test_iterate_records_triggers_lazy_load(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # db_iterate_records calls _db_ensure_loaded internally + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + assert len(records) == 3 + assert all(base_time.add(hours=2) <= r.date_time < base_time.add(hours=5) for r in records) + + async def test_delete_records(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(6): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0) + ) + await sequence.db_save_records() + + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5)) + deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + assert deleted == 3 + + # Persist the deletions + await sequence.db_save_records() + + await _reset_sequence_state(sequence) + await sequence.db_load_records() + assert len(sequence.records) == 3 + + async def test_delete_tombstone_prevents_resurrection(self, async_database_instance): + """Deleted records must not re-appear when db_load_records is called.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(3): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + + # Delete middle record + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=1)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + assert deleted == 1 + + # Do NOT persist yet — tombstone lives only in memory + # Loading should not resurrect the tombstoned record + loaded = await sequence.db_load_records() + assert all(r.date_time != base_time.add(hours=1) for r in sequence.records) + + async def test_insert_after_delete_clears_tombstone(self, async_database_instance): + """Re-inserting a deleted datetime must clear its tombstone.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + dt = base_time.add(hours=5) + + await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=10.0)) + await sequence.db_save_records() + + db_start = DatabaseTimestamp.from_datetime(dt) + db_end = sequence._db_timestamp_after(db_start) + deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + assert deleted == 1 + + await sequence.db_save_records() + + # Re-insert the same datetime + await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=99.0)) + assert dt not in sequence._db_deleted_timestamps + await sequence.db_save_records() + + await _reset_sequence_state(sequence) + await sequence.db_load_records() + assert any(r.date_time == dt and r.temperature == 99.0 for r in sequence.records) + + async def test_db_count_records_memory_only(self): + """When db is disabled, count reflects memory only.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + + # Without a live DB, db_enabled is False + if sequence.db_enabled: + pytest.skip("DB is open; this test requires it to be closed") + + base_time = to_datetime("2024-01-01T00:00:00Z") + for i in range(5): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)), + mark_dirty=False, + ) + count = await sequence.db_count_records() + assert count == 5 + + async def test_db_count_records_combined(self, async_database_instance): + """db_count_records = storage + new_unpersisted - pending_deletes.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + # Persist 10 records + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + + # Add 3 new unpersisted records + for i in range(10, 13): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + + # Delete 2 persisted records (not yet saved) + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + assert deleted == 2 + + # storage=10, new=3, pending_deletes=2 → expected=11 + count = await sequence.db_count_records() + assert count == 11 + + async def test_db_timestamp_range_empty(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + min_dt, max_dt = await sequence.db_timestamp_range() + assert min_dt is None + assert max_dt is None + + async def test_db_timestamp_range_with_records(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for hours in [0, 5, 10]: + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=hours), temperature=20.0) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + min_dt, max_dt = await sequence.db_timestamp_range() + assert min_dt == DatabaseTimestamp.from_datetime(base_time) + assert max_dt == DatabaseTimestamp.from_datetime(base_time.add(hours=10)) + + async def test_db_mark_dirty_triggers_save(self, async_database_instance): + """Marking a record dirty causes it to be re-saved.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + record = SampleDataRecord(date_time=base_time, temperature=20.0) + await sequence.db_insert_record(record) + await sequence.db_save_records() + + # Mutate and mark dirty + record.temperature = 99.0 + await sequence.db_mark_dirty_record(record) + await sequence.db_save_records() + + # Reload and verify update was persisted + await _reset_sequence_state(sequence) + await sequence.db_load_records() + assert sequence.records[0].temperature == 99.0 + + async def test_db_vacuum_keep_hours(self, async_database_instance): + """db_vacuum(keep_hours=N) retains only the last N hours of records.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + # 240 hourly records = 10 days + for i in range(240): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + keep_hours = 5 * 24 # keep last 5 days + deleted = await sequence.db_vacuum(keep_hours=keep_hours) + + assert deleted == 240 - keep_hours + count = await sequence.db_count_records() + assert count == keep_hours + + async def test_db_vacuum_keep_timestamp(self, async_database_instance): + """db_vacuum(keep_timestamp=T) deletes everything before T (exclusive).""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # Keep from hours=5 onward — delete [0, 5), i.e. 5 records + cutoff = base_time.add(hours=5) + db_cutoff = DatabaseTimestamp.from_datetime(cutoff) + deleted = await sequence.db_vacuum(keep_timestamp=db_cutoff) + + assert deleted == 5 + count = await sequence.db_count_records() + assert count == 5 + + # Verify the boundary record (hours=5) was NOT deleted + await _reset_sequence_state(sequence) + await sequence.db_load_records() + assert any(r.date_time == cutoff for r in sequence.records) + + async def test_db_vacuum_no_argument(self, async_database_instance, config_eos): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + record = SampleDataRecord(date_time=base_time, temperature=20.0) + await sequence.db_insert_record(record) + await sequence.db_save_records() + + config_eos.database.keep_duration_h = None + deleted = await sequence.db_vacuum() + assert deleted == 0 + + config_eos.database.keep_duration_h = 0 + deleted = await sequence.db_vacuum() + assert deleted == 1 + + async def test_db_vacuum_keep_hours_zero_deletes_all(self, async_database_instance): + """keep_hours=0 should delete all records.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(5): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + deleted = await sequence.db_vacuum(keep_hours=0) + assert deleted == 5 + count = await sequence.db_count_records() + assert count == 0 + + async def test_db_get_stats(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + stats = await sequence.db_get_stats() + + assert stats["enabled"] is True + assert "backend" in stats + assert "path" in stats + assert "memory_records" in stats + assert "total_records" in stats + assert "compression_enabled" in stats + assert "timestamp_range" in stats + assert stats["timestamp_range"]["min"] == "None" + assert stats["timestamp_range"]["max"] == "None" + + async def test_db_get_stats_disabled(self, config_eos): + config_eos.database.provider = None + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + stats = await sequence.db_get_stats() + assert stats == {"enabled": False} + + async def test_lazy_load_phase_none_to_initial(self, async_database_instance): + """Phase transitions from NONE to INITIAL when a range is loaded via ensure_loaded.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE + + base_time = to_datetime("2024-01-01T00:00:00Z") + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + + assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL + + async def test_lazy_load_phase_initial_to_full(self, async_database_instance): + """Phase transitions from INITIAL to FULL when iterate is called without range.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # Load partial range → INITIAL + # Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL + + # Iterate without range → escalates to FULL + records = [record async for record in sequence.db_iterate_records()] + assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL + + async def test_range_covered_skips_redundant_load(self, async_database_instance): + """_db_range_covered prevents a second DB query for the same range.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(10): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=8)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + + # Loaded range is now set + assert sequence._db_loaded_range is not None + assert sequence._db_range_covered(db_start, db_end) is True + + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=20)) + assert sequence._db_range_covered(db_start, db_end) is False + + async def test_loaded_range_not_clobbered_by_expansion(self, async_database_instance): + """Expanding left or right must not narrow the tracked loaded range.""" + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T00:00:00Z") + + for i in range(24): + await sequence.db_insert_record( + SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) + ) + await sequence.db_save_records() + await _reset_sequence_state(sequence) + + # Initial window: hours 8–16 + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=8)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + + assert sequence._db_loaded_range is not None + initial_start, initial_end = sequence._db_loaded_range + assert initial_start is not None + assert initial_end is not None + + # Expand left: load hours 4–8 + db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=4)) + db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16)) + records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)] + + assert sequence._db_loaded_range is not None + expanded_start, expanded_end = sequence._db_loaded_range + assert expanded_start is not None + assert expanded_end is not None + + # Left boundary must have moved left; right must not have shrunk + assert expanded_start <= initial_start + assert expanded_end >= initial_end + + async def test_duplicate_insert_raises(self, async_database_instance): + sequence = SampleDataSequence() + await _reset_sequence_state(sequence) + dt = to_datetime("2024-01-01T00:00:00Z") + + await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=1.0)) + with pytest.raises(ValueError, match="Duplicate timestamp"): + await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=2.0)) + + async def test_metadata_round_trip(self, async_database_instance): + """Metadata can be saved and loaded back correctly.""" + sequence = SampleDataSequence() + + await _clear_sequence_state(sequence) + assert sequence._db_metadata is None + + await _reset_sequence_state(sequence) + assert sequence._db_metadata is not None + created = sequence._db_metadata["created"] + assert sequence._db_metadata["version"] == 1 + + await _reset_sequence_state(sequence) + assert sequence._db_metadata is not None + assert sequence._db_metadata["created"] == created + assert sequence._db_metadata["version"] == 1 + + async def test_initial_load_window_respected(self, async_database_instance): + """db_initial_time_window limits the initial load from DB.""" + + class WindowedSequence(SampleDataSequence): + def db_namespace(self) -> str: + return "WindowedSequence" + + def db_initial_time_window(self) -> Optional[Duration]: + return to_duration("2 hours") + + sequence = WindowedSequence() + await _reset_sequence_state(sequence) + base_time = to_datetime("2024-01-01T12:00:00Z") + + # Store 24 hourly records centred on base_time + for i in range(24): + await sequence.db_insert_record( + SampleDataRecord( + date_time=base_time.subtract(hours=12).add(hours=i), + temperature=float(i), + ) + ) + await sequence.db_save_records() + + await _reset_sequence_state(sequence) + + # Trigger initial window load centred on base_time + sequence.config.database.initial_load_window_h = 2 + db_center = DatabaseTimestamp.from_datetime(base_time) + await sequence._db_load_initial_window(center_timestamp=db_center) + + # Only records within ±2h of base_time should be in memory + assert len(sequence.records) <= 5 # at most 4h window = 4–5 records + assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL diff --git a/tests/test_database.py b/tests/test_database.py index 1e555cf0..a6b88919 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,43 +1,24 @@ -"""Pytest tests for database persistence module. +"""Pytest tests for async database persistence module. -Tests the abstract Database interface and concrete implementations (LMDB, SQLite). -Also tests the database integration with DataSequence/DataProvider classes via -DatabaseRecordProtocolMixin. - -Design constraints honoured by these tests: -- DatabaseRecordProtocolMixin subclasses are singletons; tests reset state via - _db_reset_state() helpers rather than re-instantiating. -- db_save_records() has no clear_memory or start/end parameters; memory management - is separate from persistence. -- db_delete_records() has no clear_memory parameter. -- _db_ensure_loaded() is private; public callers use db_iterate_records() or - db_load_records() which trigger loading internally. -- db_count_records() correctly combines storage_count + new_count - pending_deletes. -- db_vacuum() end_timestamp is already exclusive; no +1ms offset applied. -- db_save_records() returns saved_count + deleted_count. +Tests the async Database interface and concrete implementations (LMDB, SQLite). """ -import pickle +from __future__ import annotations + +import asyncio import shutil import tempfile import time from pathlib import Path -from typing import Iterator, List, Optional, Type +from typing import AsyncIterator, Optional, Type import pytest +import pytest_asyncio from pydantic import Field from akkudoktoreos.core.coreabc import get_database -from akkudoktoreos.core.dataabc import ( - DataProvider, - DataRecord, - DataSequence, -) -from akkudoktoreos.core.database import ( - Database, - LMDBDatabase, - SQLiteDatabase, -) +from akkudoktoreos.core.dataabc import DataProvider, DataRecord, DataSequence +from akkudoktoreos.core.database import Database, LMDBDatabase, SQLiteDatabase from akkudoktoreos.core.databaseabc import ( DatabaseRecordProtocolLoadPhase, DatabaseTimestamp, @@ -49,39 +30,9 @@ from akkudoktoreos.utils.datetimeutil import ( to_duration, ) -# ==================== Helpers ==================== - -def _clear_sequence_state(sequence) -> None: - """Clear runtime DB state without re-instantiating the singleton. - - Does _NOT_ initialize the DB state. - """ - sequence.db_delete_records() - try: - sequence._db_metadata = None - sequence.database().set_metadata(None, namespace=sequence.db_namespace()) - except Exception: - # Database may not be available, just skip - pass - try: - del sequence._db_initialized - except Exception: - # May not be set - pass - -def _reset_sequence_state(sequence) -> None: - """Reset runtime DB state without re-instantiating the singleton.""" - try: - sequence.records = [] - del sequence._db_initialized - except Exception: - # May not be set - pass - sequence._db_ensure_initialized() - - # ==================== Test Fixtures ==================== + @pytest.fixture def temp_dir(): """Create a temporary directory for test databases.""" @@ -96,28 +47,631 @@ def database_provider(request) -> str: return request.param -@pytest.fixture -def database_instance(config_eos, database_provider: str) -> Iterator[Database]: - """Open a database instance for testing and close it afterwards. - - Note: Database is a singleton — we configure and use it, then restore - the provider to None so subsequent tests start clean. - """ +@pytest_asyncio.fixture +async def async_database_instance( + config_eos, + database_provider: str, +) -> AsyncIterator[Database]: + """Open a database instance for testing and close it afterwards.""" config_eos.database.compression_level = 6 config_eos.database.provider = database_provider + db = get_database() - assert db.is_open is True + await db.open() + assert db.provider_id() == database_provider + assert db.is_open is True yield db - # Teardown: close and reset provider so next fixture gets a fresh state - db.close() + await db.close() + config_eos.database.provider = None -# ==================== Test Data Models ==================== +# ==================== Database Backend Tests ==================== + + +@pytest.mark.asyncio +class TestDatabase: + """Tests for the async Database interface.""" + + async def test_database_creation(self, config_eos, database_provider): + config_eos.database.compression_level = 6 + config_eos.database.provider = database_provider + + db = get_database() + + await db.open() + + assert db.is_open is True + assert db.compression is True + assert db.compression_level == 6 + + assert db.storage_path == ( + config_eos.general.data_folder_path + / "db" + / db._db.__class__.__name__.lower() + ) + + await db.close() + + async def test_database_open_close(self, async_database_instance): + assert async_database_instance.is_open is True + assert async_database_instance._db.connection is not None + + await async_database_instance.close() + + assert async_database_instance._db.is_open is False + + async def test_save_and_load_single_record(self, async_database_instance): + key = b"2024-01-01T00:00:00+00:00" + value = b"test_data_12345" + + saved = await async_database_instance.save_records([(key, value)]) + + assert saved == 1 + + records = [ + record + async for record in async_database_instance.iterate_records( + key, + key + b"\xff", + ) + ] + + assert len(records) == 1 + assert records[0] == (key, value) + + async def test_save_multiple_records(self, async_database_instance): + records = [ + (b"2024-01-01T00:00:00+00:00", b"data1"), + (b"2024-01-02T00:00:00+00:00", b"data2"), + (b"2024-01-03T00:00:00+00:00", b"data3"), + ] + + saved = await async_database_instance.save_records(records) + + assert saved == len(records) + + loaded = [ + record + async for record in async_database_instance.iterate_records() + ] + + assert len(loaded) == len(records) + + for expected, actual in zip(records, loaded): + assert expected == actual + + async def test_load_records_with_range(self, async_database_instance): + records = [ + (b"2024-01-01T00:00:00+00:00", b"data1"), + (b"2024-01-02T00:00:00+00:00", b"data2"), + (b"2024-01-03T00:00:00+00:00", b"data3"), + (b"2024-01-04T00:00:00+00:00", b"data4"), + (b"2024-01-05T00:00:00+00:00", b"data5"), + ] + + await async_database_instance.save_records(records) + + start_key = b"2024-01-02T00:00:00+00:00" + end_key = b"2024-01-04T00:00:00+00:00" + + loaded = [ + record + async for record in async_database_instance.iterate_records( + start_key, + end_key, + ) + ] + + assert len(loaded) == 2 + assert loaded[0][0] == b"2024-01-02T00:00:00+00:00" + assert loaded[1][0] == b"2024-01-03T00:00:00+00:00" + + async def test_delete_record(self, async_database_instance): + key = b"2024-01-01T00:00:00+00:00" + + await async_database_instance.save_records([(key, b"test_data")]) + + assert await async_database_instance.count_records() == 1 + + deleted = await async_database_instance.delete_records([key]) + + assert deleted == 1 + assert await async_database_instance.count_records() == 0 + + deleted = await async_database_instance.delete_records([key]) + + assert deleted == 0 + + async def test_count_records(self, async_database_instance): + assert await async_database_instance.count_records() == 0 + + for i in range(10): + key = f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() + + await async_database_instance.save_records([(key, b"data")]) + + assert await async_database_instance.count_records() == 10 + + async def test_get_key_range_empty(self, async_database_instance): + min_key, max_key = await async_database_instance.get_key_range() + + assert min_key is None + assert max_key is None + + async def test_get_key_range_with_records(self, async_database_instance): + keys = [ + b"2024-01-01T00:00:00+00:00", + b"2024-01-05T00:00:00+00:00", + b"2024-01-03T00:00:00+00:00", + ] + + for key in keys: + await async_database_instance.save_records([(key, b"data")]) + + min_key, max_key = await async_database_instance.get_key_range() + + assert min_key == b"2024-01-01T00:00:00+00:00" + assert max_key == b"2024-01-05T00:00:00+00:00" + + async def test_iterate_records_forward(self, async_database_instance): + keys = [ + b"2024-01-01T00:00:00+00:00", + b"2024-01-02T00:00:00+00:00", + b"2024-01-03T00:00:00+00:00", + ] + + for key in keys: + await async_database_instance.save_records([(key, b"data")]) + + result_keys = [ + k + async for k, _ in async_database_instance.iterate_records() + ] + + assert result_keys == keys + + async def test_iterate_records_reverse(self, async_database_instance): + keys = [ + b"2024-01-01T00:00:00+00:00", + b"2024-01-02T00:00:00+00:00", + b"2024-01-03T00:00:00+00:00", + ] + + for key in keys: + await async_database_instance.save_records([(key, b"data")]) + + result_keys = [ + k + async for k, _ in async_database_instance.iterate_records(reverse=True) + ] + + assert result_keys == list(reversed(keys)) + + async def test_reverse_iteration_with_bounds(self, async_database_instance): + keys = [f"{i:03d}".encode() for i in range(10)] + + await async_database_instance.save_records( + [(k, b"v") for k in keys] + ) + + result = [ + k + async for k, _ in async_database_instance.iterate_records( + start_key=b"003", + end_key=b"007", + reverse=True, + ) + ] + + assert result == [ + b"006", + b"005", + b"004", + b"003", + ] + + async def test_empty_iteration(self, async_database_instance): + result = [ + record + async for record in async_database_instance.iterate_records() + ] + + assert result == [] + + async def test_compression_reduces_size(self, config_eos, async_database_instance): + large_data = b"A" * 10_000 + + config_eos.database.compression_level = 9 + compressed = async_database_instance.serialize_data(large_data) + assert async_database_instance.deserialize_data(compressed) == large_data + + config_eos.database.compression_level = 0 + uncompressed = async_database_instance.serialize_data(large_data) + assert async_database_instance.deserialize_data(uncompressed) == large_data + + assert len(compressed) < len(uncompressed) + + async def test_flush(self, async_database_instance): + key = b"2024-01-01T00:00:00+00:00" + + await async_database_instance.save_records([(key, b"test_data")]) + + await async_database_instance.flush() + + loaded = [ + record + async for record in async_database_instance.iterate_records() + ] + + assert len(loaded) == 1 + assert loaded[0] == (key, b"test_data") + + async def test_backend_stats(self, async_database_instance): + stats = await async_database_instance.get_backend_stats() + + assert isinstance(stats, dict) + assert "backend" in stats + + for i in range(10): + key = f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() + + await async_database_instance.save_records( + [(key, b"data" * 100)] + ) + + stats = await async_database_instance.get_backend_stats() + + assert stats is not None + + async def test_metadata_roundtrip(self, async_database_instance): + await async_database_instance.set_metadata( + b"metadata", + namespace="test", + ) + + result = await async_database_instance.get_metadata( + namespace="test", + ) + + assert result == b"metadata" + + async def test_metadata_excluded_from_count(self, async_database_instance): + await async_database_instance.save_records( + [(b"2024-01-01T00:00:00+00:00", b"data")] + ) + + count = await async_database_instance.count_records() + + assert count == 1 + + async def test_namespace_isolation(self, async_database_instance): + await async_database_instance.save_records( + [(b"k1", b"ns1")], + namespace="a", + ) + + await async_database_instance.save_records( + [(b"k1", b"ns2")], + namespace="b", + ) + + records_a = [ + r + async for r in async_database_instance.iterate_records( + namespace="a" + ) + ] + + records_b = [ + r + async for r in async_database_instance.iterate_records( + namespace="b" + ) + ] + + assert records_a == [(b"k1", b"ns1")] + assert records_b == [(b"k1", b"ns2")] + + async def test_concurrent_writes(self, async_database_instance): + async def writer(start: int): + records = [ + (f"{i:08d}".encode(), b"data") + for i in range(start, start + 100) + ] + + await async_database_instance.save_records(records) + + await asyncio.gather( + writer(0), + writer(1000), + writer(2000), + ) + + count = await async_database_instance.count_records() + + assert count == 300 + + async def test_concurrent_reads_and_writes(self, async_database_instance): + async def writer(): + for i in range(100): + await async_database_instance.save_records( + [(f"{i:08d}".encode(), b"data")] + ) + + async def reader(): + total = 0 + + for _ in range(20): + records = [ + r + async for r in async_database_instance.iterate_records() + ] + + total += len(records) + + return total + + results = await asyncio.gather( + writer(), + reader(), + reader(), + ) + + assert results[1] >= 0 + assert results[2] >= 0 + + final_count = await async_database_instance.count_records() + + assert final_count == 100 + + async def test_delete_multiple_records(self, async_database_instance): + records = [ + (b"k1", b"v1"), + (b"k2", b"v2"), + (b"k3", b"v3"), + ] + + await async_database_instance.save_records(records) + + deleted = await async_database_instance.delete_records( + [b"k1", b"k3"] + ) + + assert deleted == 2 + + remaining = [ + r + async for r in async_database_instance.iterate_records() + ] + + assert remaining == [(b"k2", b"v2")] + + async def test_delete_empty_input(self, async_database_instance): + deleted = await async_database_instance.delete_records([]) + + assert deleted == 0 + + async def test_save_empty_input(self, async_database_instance): + saved = await async_database_instance.save_records([]) + + assert saved == 0 + + async def test_count_with_bounds(self, async_database_instance): + keys = [f"{i:03d}".encode() for i in range(10)] + + await async_database_instance.save_records( + [(k, b"v") for k in keys] + ) + + count = await async_database_instance.count_records( + start_key=b"003", + end_key=b"007", + ) + + assert count == 4 + + +# ==================== Backend-Specific Tests ==================== + + +class TestLMDBDatabase: + """LMDB-specific tests.""" + + @pytest.mark.asyncio + async def test_lmdb_compact(self, config_eos): + config_eos.database.compression_level = 0 + config_eos.database.provider = "LMDB" + + db = get_database() + + await db.open() + + assert db.is_open + + for i in range(1000): + key = f"2024-01-01T{i:06d}+00:00".encode() + + await db.save_records([(key, b"X" * 1000)]) + + for i in range(500): + key = f"2024-01-01T{i:06d}+00:00".encode() + + await db.delete_records([key]) + + assert await db.count_records() == 500 + + lmdb = db._db + + assert isinstance(lmdb, LMDBDatabase) + + # compact() itself is synchronous + await asyncio.to_thread(lmdb.compact) + + assert await db.count_records() == 500 + + await db.close() + + @pytest.mark.asyncio + async def test_lmdb_namespace_isolation(self, config_eos): + """Records in different namespaces must not interfere.""" + + config_eos.database.provider = "LMDB" + + db = get_database() + + await db.open() + + assert db.is_open + + key = b"2024-01-01T00:00:00+00:00" + + await db.save_records( + [(key, b"ns_a_data")], + namespace="ns_a", + ) + + await db.save_records( + [(key, b"ns_b_data")], + namespace="ns_b", + ) + + ns_a = [ + record + async for record in db.iterate_records(namespace="ns_a") + ] + + ns_b = [ + record + async for record in db.iterate_records(namespace="ns_b") + ] + + assert ns_a[0][1] == b"ns_a_data" + assert ns_b[0][1] == b"ns_b_data" + + await db.close() + + +class TestSQLiteDatabase: + """SQLite-specific tests.""" + + @pytest.mark.asyncio + async def test_sqlite_vacuum(self, config_eos): + config_eos.database.compression_level = 0 + config_eos.database.provider = "SQLite" + + db = get_database() + + await db.open() + + assert db.is_open + + records = [ + ( + f"2024-01-{i + 1:02d}T00:00:00+00:00".encode(), + b"data" * 100, + ) + for i in range(100) + ] + + await db.save_records(records) + + keys_to_delete = [ + f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() + for i in range(50) + ] + + await db.delete_records(keys_to_delete) + + assert await db.count_records() == 50 + + sqlitedb = db._db + + assert isinstance(sqlitedb, SQLiteDatabase) + + # vacuum() itself is synchronous + await asyncio.to_thread(sqlitedb.vacuum) + + assert await db.count_records() == 50 + + await db.close() + + @pytest.mark.asyncio + async def test_sqlite_namespace_isolation(self, config_eos): + """Records in different namespaces must not interfere.""" + + config_eos.database.provider = "SQLite" + + db = get_database() + + await db.open() + + assert db.is_open + + key = b"2024-01-01T00:00:00+00:00" + + await db.save_records( + [(key, b"ns_a_data")], + namespace="ns_a", + ) + + await db.save_records( + [(key, b"ns_b_data")], + namespace="ns_b", + ) + + ns_a = [ + record + async for record in db.iterate_records(namespace="ns_a") + ] + + ns_b = [ + record + async for record in db.iterate_records(namespace="ns_b") + ] + + assert ns_a[0][1] == b"ns_a_data" + assert ns_b[0][1] == b"ns_b_data" + + await db.close() + +# Helpers + +async def _clear_sequence_state(sequence) -> None: + """Clear runtime DB state without re-instantiating the singleton. + + Does _NOT_ initialize the DB state. + """ + await sequence.db_delete_records() + try: + sequence._db_metadata = None + await sequence.database().set_metadata(None, namespace=sequence.db_namespace()) + except Exception: + # Database may not be available, just skip + pass + try: + del sequence._db_initialized + except Exception: + # May not be set + pass + + +async def _reset_sequence_state(sequence) -> None: + """Reset runtime DB state without re-instantiating the singleton.""" + try: + sequence.records = [] + del sequence._db_initialized + except Exception: + # May not be set + pass + await sequence._db_ensure_initialized() + + +# Sample Data class SampleDataRecord(DataRecord): """Minimal DataRecord for testing.""" @@ -128,7 +682,7 @@ class SampleDataRecord(DataRecord): class SampleDataSequence(DataSequence): """DataSequence subclass with database support.""" - records: List[SampleDataRecord] = Field(default_factory=list) + records: list[SampleDataRecord] = Field(default_factory=list) @classmethod def record_class(cls) -> Type[SampleDataRecord]: @@ -140,7 +694,7 @@ class SampleDataSequence(DataSequence): class SampleDataProvider(DataProvider): """DataProvider subclass with database support.""" - records: List[SampleDataRecord] = Field(default_factory=list) + records: list[SampleDataRecord] = Field(default_factory=list) @classmethod def record_class(cls) -> Type[SampleDataRecord]: @@ -152,925 +706,100 @@ class SampleDataProvider(DataProvider): def enabled(self) -> bool: return True - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: pass def db_namespace(self) -> str: return "SampleDataProvider" -# ==================== Database Backend Tests ==================== - -class TestDatabase: - """Tests for the raw Database interface (both backends).""" - - def test_database_creation(self, config_eos, database_provider): - config_eos.database.compression_level = 6 - config_eos.database.provider = database_provider - db = get_database() - - assert db.is_open is True - assert db.compression is True - assert db.compression_level == 6 - # storage_path uses the concrete backend class name - assert db.storage_path == ( - config_eos.general.data_folder_path / "db" / db._db.__class__.__name__.lower() - ) - - def test_database_open_close(self, database_instance): - assert database_instance.is_open is True - assert database_instance._db.connection is not None - - database_instance.close() - assert database_instance._db.is_open is False - - def test_save_and_load_single_record(self, database_instance): - key = b"2024-01-01T00:00:00+00:00" - value = b"test_data_12345" - - database_instance.save_records([(key, value)]) - records = list(database_instance.iterate_records(key, key + b"\xff")) - - assert len(records) == 1 - assert records[0] == (key, value) - - def test_save_multiple_records(self, database_instance): - records = [ - (b"2024-01-01T00:00:00+00:00", b"data1"), - (b"2024-01-02T00:00:00+00:00", b"data2"), - (b"2024-01-03T00:00:00+00:00", b"data3"), - ] - saved = database_instance.save_records(records) - assert saved == len(records) - - loaded = list(database_instance.iterate_records()) - assert len(loaded) == len(records) - for expected, actual in zip(records, loaded): - assert expected == actual - - def test_load_records_with_range(self, database_instance): - records = [ - (b"2024-01-01T00:00:00+00:00", b"data1"), - (b"2024-01-02T00:00:00+00:00", b"data2"), - (b"2024-01-03T00:00:00+00:00", b"data3"), - (b"2024-01-04T00:00:00+00:00", b"data4"), - (b"2024-01-05T00:00:00+00:00", b"data5"), - ] - database_instance.save_records(records) - - # Range is half-open: [2024-01-02, 2024-01-04) - start_key = b"2024-01-02T00:00:00+00:00" - end_key = b"2024-01-04T00:00:00+00:00" - loaded = list(database_instance.iterate_records(start_key, end_key)) - - assert len(loaded) == 2 - assert loaded[0][0] == b"2024-01-02T00:00:00+00:00" - assert loaded[1][0] == b"2024-01-03T00:00:00+00:00" - - def test_delete_record(self, database_instance): - key = b"2024-01-01T00:00:00+00:00" - database_instance.save_records([(key, b"test_data")]) - assert database_instance.count_records() == 1 - - deleted = database_instance.delete_records([key]) - assert deleted == 1 - assert database_instance.count_records() == 0 - - # Deleting a non-existent key returns 0 - deleted = database_instance.delete_records([key]) - assert deleted == 0 - - def test_count_records(self, database_instance): - assert database_instance.count_records() == 0 - - for i in range(10): - key = f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() - database_instance.save_records([(key, b"data")]) - - assert database_instance.count_records() == 10 - - def test_get_key_range_empty(self, database_instance): - min_key, max_key = database_instance.get_key_range() - assert min_key is None - assert max_key is None - - def test_get_key_range_with_records(self, database_instance): - keys = [ - b"2024-01-01T00:00:00+00:00", - b"2024-01-05T00:00:00+00:00", - b"2024-01-03T00:00:00+00:00", - ] - for key in keys: - database_instance.save_records([(key, b"data")]) - - min_key, max_key = database_instance.get_key_range() - assert min_key == b"2024-01-01T00:00:00+00:00" - assert max_key == b"2024-01-05T00:00:00+00:00" - - def test_iterate_records_forward(self, database_instance): - keys = [ - b"2024-01-01T00:00:00+00:00", - b"2024-01-02T00:00:00+00:00", - b"2024-01-03T00:00:00+00:00", - ] - for key in keys: - database_instance.save_records([(key, b"data")]) - - result_keys = [k for k, _ in database_instance.iterate_records()] - assert result_keys == keys - - def test_iterate_records_reverse(self, database_instance): - keys = [ - b"2024-01-01T00:00:00+00:00", - b"2024-01-02T00:00:00+00:00", - b"2024-01-03T00:00:00+00:00", - ] - for key in keys: - database_instance.save_records([(key, b"data")]) - - result_keys = [k for k, _ in database_instance.iterate_records(reverse=True)] - assert result_keys == list(reversed(keys)) - - def test_compression_reduces_size(self, config_eos, database_provider): - large_data = b"A" * 10_000 - - config_eos.database.provider = database_provider - config_eos.database.compression_level = 9 - compressed = get_database().serialize_data(large_data) - assert get_database().deserialize_data(compressed) == large_data - - config_eos.database.compression_level = 0 - uncompressed = get_database().serialize_data(large_data) - assert get_database().deserialize_data(uncompressed) == large_data - - assert len(compressed) < len(uncompressed) - - def test_flush(self, database_instance): - key = b"2024-01-01T00:00:00+00:00" - database_instance.save_records([(key, b"test_data")]) - database_instance.flush() - - loaded = list(database_instance.iterate_records()) - assert len(loaded) == 1 - assert loaded[0] == (key, b"test_data") - - def test_backend_stats(self, database_instance): - stats = database_instance.get_backend_stats() - assert isinstance(stats, dict) - assert "backend" in stats - - for i in range(10): - key = f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() - database_instance.save_records([(key, b"data" * 100)]) - - stats = database_instance.get_backend_stats() - assert stats is not None - - def test_metadata_excluded_from_count(self, database_instance): - """Metadata record stored under DATABASE_METADATA_KEY must not appear in count.""" - # Save a normal record - database_instance.save_records([(b"2024-01-01T00:00:00+00:00", b"data")]) - count = database_instance.count_records() - assert count == 1 # metadata excluded by backend implementation - - -# ==================== DatabaseRecordProtocolMixin Tests ==================== - -class TestDataSequenceDatabaseProtocol: - """Tests for DatabaseRecordProtocolMixin via SampleDataSequence.""" - - def test_db_enabled_when_db_open(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - assert sequence.db_enabled is True - - def test_db_disabled_when_db_closed(self, config_eos): - config_eos.database.provider = None - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - assert sequence.db_enabled is False - - def test_insert_and_save_records(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) - ) - - # All 10 are dirty/new, none persisted yet - assert len(sequence.records) == 10 - assert len(sequence._db_new_timestamps) == 10 - - saved = sequence.db_save_records() - assert saved == 10 # 10 inserts + 0 deletes - assert len(sequence._db_dirty_timestamps) == 0 - assert len(sequence._db_new_timestamps) == 0 - - def test_save_returns_insert_plus_delete_count(self, database_instance): - """db_save_records() return value = saved_inserts + deleted_count.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(5): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - # Persist the 5 records - sequence.db_save_records() - - # Delete 2 of them - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=4)) - deleted = sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) - # Insert 3 new ones - for i in range(10, 13): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - - result = sequence.db_save_records() - # 3 inserts + 2 deletes = 5 - assert result == 5 - - def test_load_records_from_db(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) - ) - sequence.db_save_records() - - # Clear memory, then reload from DB - _reset_sequence_state(sequence) - loaded = sequence.db_load_records() - - assert loaded == 10 - assert len(sequence.records) == 10 - for i, record in enumerate(sequence.records): - assert record.temperature == 20.0 + i - - def test_load_records_with_range(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # Load [hours=3, hours=7) → 4 records (3, 4, 5, 6) - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) - loaded = sequence.db_load_records(start_timestamp=db_start, end_timestamp=db_end) - assert loaded == 4 - assert sequence.records[0].temperature == 23.0 - assert sequence.records[-1].temperature == 26.0 - - def test_iterate_records_triggers_lazy_load(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # db_iterate_records calls _db_ensure_loaded internally - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5)) - records = list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - assert len(records) == 3 - assert all(base_time.add(hours=2) <= r.date_time < base_time.add(hours=5) for r in records) - - def test_delete_records(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(6): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0) - ) - sequence.db_save_records() - - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5)) - deleted = sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) - assert deleted == 3 - - # Persist the deletions - sequence.db_save_records() - - _reset_sequence_state(sequence) - sequence.db_load_records() - assert len(sequence.records) == 3 - - def test_delete_tombstone_prevents_resurrection(self, database_instance): - """Deleted records must not re-appear when db_load_records is called.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(3): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - - # Delete middle record - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=1)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - deleted = sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) - assert deleted == 1 - - # Do NOT persist yet — tombstone lives only in memory - # Loading should not resurrect the tombstoned record - loaded = sequence.db_load_records() - assert all(r.date_time != base_time.add(hours=1) for r in sequence.records) - - def test_insert_after_delete_clears_tombstone(self, database_instance): - """Re-inserting a deleted datetime must clear its tombstone.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - dt = base_time.add(hours=5) - - sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=10.0)) - sequence.db_save_records() - - db_start = DatabaseTimestamp.from_datetime(dt) - db_end = sequence._db_timestamp_after(db_start) - deleted = sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) - assert deleted == 1 - - sequence.db_save_records() - - # Re-insert the same datetime - sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=99.0)) - assert dt not in sequence._db_deleted_timestamps - sequence.db_save_records() - - _reset_sequence_state(sequence) - sequence.db_load_records() - assert any(r.date_time == dt and r.temperature == 99.0 for r in sequence.records) - - def test_db_count_records_memory_only(self): - """When db is disabled, count reflects memory only.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - - # Without a live DB, db_enabled is False - if sequence.db_enabled: - pytest.skip("DB is open; this test requires it to be closed") - - base_time = to_datetime("2024-01-01T00:00:00Z") - for i in range(5): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)), - mark_dirty=False, - ) - assert sequence.db_count_records() == 5 - - def test_db_count_records_combined(self, database_instance): - """db_count_records = storage + new_unpersisted - pending_deletes.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - # Persist 10 records - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - - # Add 3 new unpersisted records - for i in range(10, 13): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - - # Delete 2 persisted records (not yet saved) - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - deleted = sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) - assert deleted == 2 - - # storage=10, new=3, pending_deletes=2 → expected=11 - assert sequence.db_count_records() == 11 - - def test_db_timestamp_range_empty(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - min_dt, max_dt = sequence.db_timestamp_range() - assert min_dt is None - assert max_dt is None - - def test_db_timestamp_range_with_records(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for hours in [0, 5, 10]: - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=hours), temperature=20.0) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - min_dt, max_dt = sequence.db_timestamp_range() - assert min_dt == DatabaseTimestamp.from_datetime(base_time) - assert max_dt == DatabaseTimestamp.from_datetime(base_time.add(hours=10)) - - def test_db_mark_dirty_triggers_save(self, database_instance): - """Marking a record dirty causes it to be re-saved.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - record = SampleDataRecord(date_time=base_time, temperature=20.0) - sequence.db_insert_record(record) - sequence.db_save_records() - - # Mutate and mark dirty - record.temperature = 99.0 - sequence.db_mark_dirty_record(record) - sequence.db_save_records() - - # Reload and verify update was persisted - _reset_sequence_state(sequence) - sequence.db_load_records() - assert sequence.records[0].temperature == 99.0 - - def test_db_vacuum_keep_hours(self, database_instance): - """db_vacuum(keep_hours=N) retains only the last N hours of records.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - # 240 hourly records = 10 days - for i in range(240): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - keep_hours = 5 * 24 # keep last 5 days - deleted = sequence.db_vacuum(keep_hours=keep_hours) - - assert deleted == 240 - keep_hours - assert sequence.db_count_records() == keep_hours - - def test_db_vacuum_keep_timestamp(self, database_instance): - """db_vacuum(keep_timestamp=T) deletes everything before T (exclusive).""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # Keep from hours=5 onward — delete [0, 5), i.e. 5 records - cutoff = base_time.add(hours=5) - db_cutoff = DatabaseTimestamp.from_datetime(cutoff) - deleted = sequence.db_vacuum(keep_timestamp=db_cutoff) - - assert deleted == 5 - assert sequence.db_count_records() == 5 - - # Verify the boundary record (hours=5) was NOT deleted - _reset_sequence_state(sequence) - sequence.db_load_records() - assert any(r.date_time == cutoff for r in sequence.records) - - def test_db_vacuum_no_argument(self, database_instance, config_eos): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - record = SampleDataRecord(date_time=base_time, temperature=20.0) - sequence.db_insert_record(record) - sequence.db_save_records() - - config_eos.database.keep_duration_h = None - assert sequence.db_vacuum() == 0 - - config_eos.database.keep_duration_h = 0 - assert sequence.db_vacuum() == 1 - - def test_db_vacuum_keep_hours_zero_deletes_all(self, database_instance): - """keep_hours=0 should delete all records.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(5): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - deleted = sequence.db_vacuum(keep_hours=0) - assert deleted == 5 - assert sequence.db_count_records() == 0 - - def test_db_get_stats(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - stats = sequence.db_get_stats() - - assert stats["enabled"] is True - assert "backend" in stats - assert "path" in stats - assert "memory_records" in stats - assert "total_records" in stats - assert "compression_enabled" in stats - assert "timestamp_range" in stats - assert stats["timestamp_range"]["min"] == "None" - assert stats["timestamp_range"]["max"] == "None" - - def test_db_get_stats_disabled(self, config_eos): - config_eos.database.provider = None - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - stats = sequence.db_get_stats() - assert stats == {"enabled": False} - - def test_lazy_load_phase_none_to_initial(self, database_instance): - """Phase transitions from NONE to INITIAL when a range is loaded via ensure_loaded.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE - - base_time = to_datetime("2024-01-01T00:00:00Z") - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) - list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - - assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL - - def test_lazy_load_phase_initial_to_full(self, database_instance): - """Phase transitions from INITIAL to FULL when iterate is called without range.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # Load partial range → INITIAL - # Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7)) - list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL - - # Iterate without range → escalates to FULL - list(sequence.db_iterate_records()) - assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL - - def test_range_covered_skips_redundant_load(self, database_instance): - """_db_range_covered prevents a second DB query for the same range.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(10): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=8)) - list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - - # Loaded range is now set - assert sequence._db_loaded_range is not None - assert sequence._db_range_covered(db_start, db_end) is True - - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=20)) - assert sequence._db_range_covered(db_start, db_end) is False - - def test_loaded_range_not_clobbered_by_expansion(self, database_instance): - """Expanding left or right must not narrow the tracked loaded range.""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(24): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - sequence.db_save_records() - _reset_sequence_state(sequence) - - # Initial window: hours 8–16 - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=8)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16)) - list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - - assert sequence._db_loaded_range is not None - initial_start, initial_end = sequence._db_loaded_range - assert initial_start is not None - assert initial_end is not None - - # Expand left: load hours 4–8 - db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=4)) - db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16)) - list(sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)) - - assert sequence._db_loaded_range is not None - expanded_start, expanded_end = sequence._db_loaded_range - assert expanded_start is not None - assert expanded_end is not None - - # Left boundary must have moved left; right must not have shrunk - assert expanded_start <= initial_start - assert expanded_end >= initial_end - - def test_duplicate_insert_raises(self, database_instance): - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - dt = to_datetime("2024-01-01T00:00:00Z") - - sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=1.0)) - with pytest.raises(ValueError, match="Duplicate timestamp"): - sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=2.0)) - - def test_autosave_delegates_to_save_records(self, database_instance): - """db_autosave() is equivalent to db_save_records().""" - sequence = SampleDataSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T00:00:00Z") - - for i in range(3): - sequence.db_insert_record( - SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) - ) - - saved = sequence.db_autosave() - assert saved == 3 - assert len(sequence._db_dirty_timestamps) == 0 - - def test_metadata_round_trip(self, database_instance): - """Metadata can be saved and loaded back correctly.""" - sequence = SampleDataSequence() - - _clear_sequence_state(sequence) - assert sequence._db_metadata is None - - _reset_sequence_state(sequence) - assert sequence._db_metadata is not None - created = sequence._db_metadata["created"] - assert sequence._db_metadata["version"] == 1 - - _reset_sequence_state(sequence) - assert sequence._db_metadata is not None - assert sequence._db_metadata["created"] == created - assert sequence._db_metadata["version"] == 1 - - def test_initial_load_window_respected(self, database_instance): - """db_initial_time_window limits the initial load from DB.""" - - class WindowedSequence(SampleDataSequence): - def db_namespace(self) -> str: - return "WindowedSequence" - - def db_initial_time_window(self) -> Optional[Duration]: - return to_duration("2 hours") - - sequence = WindowedSequence() - _reset_sequence_state(sequence) - base_time = to_datetime("2024-01-01T12:00:00Z") - - # Store 24 hourly records centred on base_time - for i in range(24): - sequence.db_insert_record( - SampleDataRecord( - date_time=base_time.subtract(hours=12).add(hours=i), - temperature=float(i), - ) - ) - sequence.db_save_records() - - _reset_sequence_state(sequence) - - # Trigger initial window load centred on base_time - sequence.config.database.initial_load_window_h = 2 - db_center = DatabaseTimestamp.from_datetime(base_time) - sequence._db_load_initial_window(center_timestamp=db_center) - - # Only records within ±2h of base_time should be in memory - assert len(sequence.records) <= 5 # at most 4h window = 4–5 records - assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL - - -# ==================== Backend-Specific Tests ==================== - -class TestLMDBDatabase: - """LMDB-specific tests.""" - - def test_lmdb_compact(self, config_eos): - config_eos.database.compression_level = 0 - config_eos.database.provider = "LMDB" - db = get_database() - assert db.is_open - - for i in range(1000): - key = f"2024-01-01T{i:06d}+00:00".encode() - db.save_records([(key, b"X" * 1000)]) - - for i in range(500): - key = f"2024-01-01T{i:06d}+00:00".encode() - db.delete_records([key]) - - lmdb = db._database() - assert isinstance(lmdb, LMDBDatabase) - lmdb.compact() - assert db.count_records() == 500 - db.close() - - def test_lmdb_namespace_isolation(self, config_eos): - """Records in different namespaces must not interfere.""" - config_eos.database.provider = "LMDB" - db = get_database() - assert db.is_open - - key = b"2024-01-01T00:00:00+00:00" - db.save_records([(key, b"ns_a_data")], namespace="ns_a") - db.save_records([(key, b"ns_b_data")], namespace="ns_b") - - ns_a = list(db.iterate_records(namespace="ns_a")) - ns_b = list(db.iterate_records(namespace="ns_b")) - - assert ns_a[0][1] == b"ns_a_data" - assert ns_b[0][1] == b"ns_b_data" - db.close() - - -class TestSQLiteDatabase: - """SQLite-specific tests.""" - - def test_sqlite_vacuum(self, config_eos): - config_eos.database.compression_level = 0 - config_eos.database.provider = "SQLite" - db = get_database() - assert db.is_open - - records = [ - (f"2024-01-{i + 1:02d}T00:00:00+00:00".encode(), b"data" * 100) - for i in range(100) - ] - db.save_records(records) - - keys_to_delete = [f"2024-01-{i + 1:02d}T00:00:00+00:00".encode() for i in range(50)] - db.delete_records(keys_to_delete) - - sqlitedb = db._database() - assert isinstance(sqlitedb, SQLiteDatabase) - sqlitedb.vacuum() - - assert db.count_records() == 50 - db.close() - - def test_sqlite_namespace_isolation(self, config_eos): - """Records in different namespaces must not interfere.""" - config_eos.database.provider = "SQLite" - db = get_database() - assert db.is_open - - key = b"2024-01-01T00:00:00+00:00" - db.save_records([(key, b"ns_a_data")], namespace="ns_a") - db.save_records([(key, b"ns_b_data")], namespace="ns_b") - - ns_a = list(db.iterate_records(namespace="ns_a")) - ns_b = list(db.iterate_records(namespace="ns_b")) - - assert ns_a[0][1] == b"ns_a_data" - assert ns_b[0][1] == b"ns_b_data" - db.close() - - # ==================== Integration Tests ==================== +@pytest.mark.asyncio class TestIntegration: """Full end-to-end workflow tests.""" - def test_full_workflow(self, config_eos, database_instance): + async def test_full_workflow(self, config_eos, async_database_instance): """Save → partial load → update → vacuum → verify.""" sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) base_time = to_datetime("2024-01-01T00:00:00Z") # Step 1: Insert 100 records and persist for i in range(100): - sequence.db_insert_record( + await sequence.db_insert_record( SampleDataRecord( date_time=base_time.add(hours=i), temperature=20.0 + i * 0.1, humidity=60.0, ) ) - sequence.db_save_records() + await sequence.db_save_records() - storage_count = sequence.database.count_records(namespace="SampleDataSequence") + storage_count = await sequence.database.count_records(namespace="SampleDataSequence") assert storage_count == 100 - assert sequence.db_count_records() == 100 + assert await sequence.db_count_records() == 100 # Step 2: Clear memory and load a specific range - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=20)) db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=40)) - loaded = sequence.db_load_records(db_start, db_end) + loaded = await sequence.db_load_records(db_start, db_end) assert loaded == 20 assert len(sequence.records) == 20 # Step 3: Update records in memory and persist for record in sequence.records: record.humidity = 75.0 - sequence.db_mark_dirty_record(record) - sequence.db_save_records() + await sequence.db_mark_dirty_record(record) + await sequence.db_save_records() # Step 4: Reload the range and verify updates - _reset_sequence_state(sequence) - sequence.db_load_records(db_start, db_end) + await _reset_sequence_state(sequence) + await sequence.db_load_records(db_start, db_end) assert all(r.humidity == 75.0 for r in sequence.records) # Step 5: Vacuum — keep from hours=75 onward (delete first 75) db_cutoff = DatabaseTimestamp.from_datetime(base_time.add(hours=75)) - deleted = sequence.db_vacuum(keep_timestamp=db_cutoff) + deleted = await sequence.db_vacuum(keep_timestamp=db_cutoff) assert deleted == 75 - assert sequence.db_count_records() == 25 + assert await sequence.db_count_records() == 25 # Step 6: Stats reflect vacuum result - _reset_sequence_state(sequence) - stats = sequence.db_get_stats() + await _reset_sequence_state(sequence) + stats = await sequence.db_get_stats() assert stats["total_records"] == 25 - def test_error_handling_db_disabled(self, config_eos): + async def test_error_handling_db_disabled(self, config_eos): """Operations on a disabled DB raise clearly.""" config_eos.database.provider = None sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) assert sequence.db_enabled is False # Save is a no-op and returns 0 when disabled — no RuntimeError # (mixin returns 0 early when not enabled) - result = sequence.db_save_records() + result = await sequence.db_save_records() assert result == 0 - def test_persistence_across_resets(self, database_instance): + async def test_persistence_across_resets(self, async_database_instance): """Data written in one memory session is available after reset.""" sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) base_time = to_datetime("2024-06-01T00:00:00Z") for i in range(20): - sequence.db_insert_record( + await sequence.db_insert_record( SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)) ) - sequence.db_save_records() + await sequence.db_save_records() # Simulate a restart: reset memory state - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) assert len(sequence.records) == 0 - loaded = sequence.db_load_records() + loaded = await sequence.db_load_records() assert loaded == 20 assert sequence.records[0].temperature == 0.0 assert sequence.records[-1].temperature == 19.0 @@ -1078,18 +807,19 @@ class TestIntegration: # ==================== Performance Tests ==================== +@pytest.mark.asyncio class TestPerformance: """Throughput benchmarks — not correctness tests.""" - def test_insert_throughput(self, config_eos, database_instance): + async def test_insert_throughput(self, config_eos, async_database_instance): sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) base_time = to_datetime("2024-01-01T00:00:00Z") n = 10_000 start = time.perf_counter() for i in range(n): - sequence.db_insert_record( + await sequence.db_insert_record( SampleDataRecord( date_time=base_time.add(minutes=i), temperature=20.0 + (i % 100) * 0.1, @@ -1101,14 +831,14 @@ class TestPerformance: assert len(sequence.records) == n - def test_save_throughput(self, config_eos, database_instance): + async def test_save_throughput(self, config_eos, async_database_instance): sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) base_time = to_datetime("2024-01-01T00:00:00Z") n = 10_000 for i in range(n): - sequence.db_insert_record( + await sequence.db_insert_record( SampleDataRecord( date_time=base_time.add(minutes=i), temperature=20.0 + (i % 100) * 0.1, @@ -1116,31 +846,31 @@ class TestPerformance: ) start = time.perf_counter() - saved = sequence.db_save_records() + saved = await sequence.db_save_records() save_duration = time.perf_counter() - start assert saved == n print(f"\nSaved {n} records in {save_duration:.2f}s " f"({n / save_duration:.0f} rec/s)") - def test_load_throughput(self, config_eos, database_instance): + async def test_load_throughput(self, config_eos, async_database_instance): sequence = SampleDataSequence() - _reset_sequence_state(sequence) + await _reset_sequence_state(sequence) base_time = to_datetime("2024-01-01T00:00:00Z") n = 10_000 for i in range(n): - sequence.db_insert_record( + await sequence.db_insert_record( SampleDataRecord( date_time=base_time.add(minutes=i), temperature=20.0 + (i % 100) * 0.1, ) ) - sequence.db_save_records() - _reset_sequence_state(sequence) + await sequence.db_save_records() + await _reset_sequence_state(sequence) start = time.perf_counter() - loaded = sequence.db_load_records() + loaded = await sequence.db_load_records() load_duration = time.perf_counter() - start assert loaded == n diff --git a/tests/test_databaseabc.py b/tests/test_databaseabc.py index e15fc5d2..4a92a452 100644 --- a/tests/test_databaseabc.py +++ b/tests/test_databaseabc.py @@ -1,11 +1,28 @@ -from typing import Any, Iterator, Literal, Optional, Type, cast +"""Tests for DatabaseRecordProtocolMixin. + +All public and internal mixin methods that are ``async def`` are exercised +with ``async def`` test functions marked with ``@pytest.mark.asyncio``. +Synchronous helpers (_db_get_compact_state, db_compact_tiers) are tested +without the decorator. + +The fake ``SampleDatabase`` exposes the same interface as the real +``Database`` wrapper: every method called via ``await self.database.*`` +in the mixin is defined as ``async def`` here. +""" + +from __future__ import annotations + +import pickle +from typing import Any, AsyncIterator, Iterator, Literal, Optional, Type, cast import pytest +import pytest_asyncio from numpydantic import NDArray, Shape from pydantic import BaseModel, Field from akkudoktoreos.core.databaseabc import ( DATABASE_METADATA_KEY, + DatabaseRecordProtocolLoadPhase, DatabaseRecordProtocolMixin, DatabaseTimestamp, _DatabaseTimestampUnbound, @@ -33,17 +50,27 @@ class SampleRecord(BaseModel): return self.date_time if key == "value": return self.value - assert key is None - return None + raise KeyError(key) + + def model_dump(self) -> dict: + return {"date_time": self.date_time, "value": self.value} # --------------------------------------------------------------------------- -# Fake database backend +# Fake async database backend +# +# Every method that the mixin calls via ``await self.database.*`` must be +# ``async def``. The real ``Database`` wrapper (database.py) is fully async; +# the plain synchronous ``DatabaseBackendABC`` subclasses are never accessed +# directly from the mixin. # --------------------------------------------------------------------------- class SampleDatabase: - def __init__(self): + """Minimal async-compatible in-memory database for unit testing.""" + + def __init__(self) -> None: + # namespace -> {key: value} self._data: dict[Optional[str], dict[bytes, bytes]] = {} self._metadata: Optional[bytes] = None self.is_open = True @@ -51,7 +78,9 @@ class SampleDatabase: self.compression_level = 0 self.storage_path = "/fake" - # serialization (pass-through) + # ------------------------------------------------------------------ + # Serialisation helpers (pass-through; no compression in tests) + # ------------------------------------------------------------------ def serialize_data(self, data: bytes) -> bytes: return data @@ -59,18 +88,27 @@ class SampleDatabase: def deserialize_data(self, data: bytes) -> bytes: return data - # metadata + # ------------------------------------------------------------------ + # Metadata — async + # ------------------------------------------------------------------ - def set_metadata(self, metadata: Optional[bytes], *, namespace: Optional[str] = None) -> None: + async def set_metadata( + self, metadata: Optional[bytes], *, namespace: Optional[str] = None + ) -> None: self._metadata = metadata - def get_metadata(self, namespace: Optional[str] = None) -> Optional[bytes]: + async def get_metadata(self, *, namespace: Optional[str] = None) -> Optional[bytes]: return self._metadata - # write + # ------------------------------------------------------------------ + # Write operations — async + # ------------------------------------------------------------------ - def save_records( - self, records: list[tuple[bytes, bytes]], namespace: Optional[str] = None + async def save_records( + self, + records: list[tuple[bytes, bytes]], + *, + namespace: Optional[str] = None, ) -> int: ns = self._data.setdefault(namespace, {}) saved = 0 @@ -79,8 +117,11 @@ class SampleDatabase: saved += 1 return saved - def delete_records( - self, keys: Iterator[bytes], namespace: Optional[str] = None + async def delete_records( + self, + keys: Iterator[bytes], + *, + namespace: Optional[str] = None, ) -> int: ns_data = self._data.get(namespace, {}) deleted = 0 @@ -90,29 +131,39 @@ class SampleDatabase: deleted += 1 return deleted - # read + # ------------------------------------------------------------------ + # Read operations — async + # ------------------------------------------------------------------ - def iterate_records( + async def iterate_records( self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, + *, namespace: Optional[str] = None, reverse: bool = False, - ) -> Iterator[tuple[bytes, bytes]]: + ) -> AsyncIterator[tuple[bytes, bytes]]: + """Return a snapshot list so callers can iterate without holding a cursor.""" items = self._data.get(namespace, {}) keys = sorted(items, reverse=reverse) + result = [] for k in keys: if k == DATABASE_METADATA_KEY: continue - if start_key and k < start_key: + if start_key is not None and k < start_key: continue - if end_key and k >= end_key: + if end_key is not None and k >= end_key: continue - yield k, items[k] + result.append((k, items[k])) - # stats + for item in result: + yield item - def count_records( + # ------------------------------------------------------------------ + # Stats — async + # ------------------------------------------------------------------ + + async def count_records( self, start_key: Optional[bytes] = None, end_key: Optional[bytes] = None, @@ -124,26 +175,27 @@ class SampleDatabase: for k in items: if k == DATABASE_METADATA_KEY: continue - if start_key and k < start_key: + if start_key is not None and k < start_key: continue - if end_key and k >= end_key: + if end_key is not None and k >= end_key: continue count += 1 return count - def get_key_range( + async def get_key_range( self, namespace: Optional[str] = None ) -> tuple[Optional[bytes], Optional[bytes]]: + """Called as ``await self.database.get_key_range(self.db_namespace())`` (positional).""" items = self._data.get(namespace, {}) keys = sorted(k for k in items if k != DATABASE_METADATA_KEY) if not keys: return None, None return keys[0], keys[-1] - def get_backend_stats(self, namespace: Optional[str] = None) -> dict: + async def get_backend_stats(self, *, namespace: Optional[str] = None) -> dict: return {} - def flush(self, namespace: Optional[str] = None) -> None: + async def flush(self, *, namespace: Optional[str] = None) -> None: pass @@ -155,7 +207,7 @@ class SampleDatabase: class SampleSequence(DatabaseRecordProtocolMixin[SampleRecord]): """Minimal concrete implementation for unit-testing the mixin.""" - def __init__(self): + def __init__(self) -> None: self.records: list[SampleRecord] = [] self._db_record_index: dict[DatabaseTimestamp, SampleRecord] = {} self._db_sorted_timestamps: list[DatabaseTimestamp] = [] @@ -166,7 +218,6 @@ class SampleSequence(DatabaseRecordProtocolMixin[SampleRecord]): self._db_storage_initialized: bool = False self._db_metadata: Optional[dict] = None self._db_loaded_range = None - from akkudoktoreos.core.databaseabc import DatabaseRecordProtocolLoadPhase self._db_load_phase = DatabaseRecordProtocolLoadPhase.NONE self._db_version: int = 1 @@ -198,18 +249,10 @@ class SampleSequence(DatabaseRecordProtocolMixin[SampleRecord]): @property def record_keys_writable(self) -> list[str]: - """Return writable field names of SampleRecord. - - Required by _db_compact_tier which iterates record_keys_writable - to decide which fields to resample. Must match exactly what - key_to_array accepts — only 'value' here, not 'date_time'. - """ + """Writable field names — ``date_time`` excluded so key_to_array only sees ``value``.""" return ["value"] - # Override key_to_array for the mixin tests — the full DataSequence - # implementation lives in dataabc.py; here we provide a minimal version - # that resamples the single `value` field to demonstrate compaction. - def key_to_array( + async def key_to_array( self, key: str, start_datetime: Optional[DateTime] = None, @@ -220,6 +263,7 @@ class SampleSequence(DatabaseRecordProtocolMixin[SampleRecord]): boundary: Literal["strict", "context"] = "context", align_to_interval: bool = False, ) -> NDArray[Shape["*"], Any]: + """Minimal resampling stub sufficient for compaction tests.""" import numpy as np import pandas as pd @@ -257,23 +301,23 @@ class SampleSequence(DatabaseRecordProtocolMixin[SampleRecord]): # --------------------------------------------------------------------------- -# Helpers +# Async helpers # --------------------------------------------------------------------------- -def _insert_records_every_n_minutes( +async def _insert_records_every_n_minutes( seq: SampleSequence, base: DateTime, count: int, interval_minutes: int, value_fn=None, ) -> None: - """Insert `count` records spaced `interval_minutes` apart starting at `base`.""" + """Insert ``count`` records spaced ``interval_minutes`` apart starting at ``base``.""" for i in range(count): dt = base.add(minutes=i * interval_minutes) value = value_fn(i) if value_fn else float(i) - seq.db_insert_record(SampleRecord(date_time=dt, value=value)) - seq.db_save_records() + await seq.db_insert_record(SampleRecord(date_time=dt, value=value)) + await seq.db_save_records() # --------------------------------------------------------------------------- @@ -282,41 +326,42 @@ def _insert_records_every_n_minutes( @pytest.fixture -def seq(): +def seq() -> SampleSequence: return SampleSequence() -@pytest.fixture -def seq_with_15min_data(): +@pytest_asyncio.fixture +async def seq_with_15min_data(): """Sequence with 15-min records spanning 4 weeks, so both tiers have data.""" s = SampleSequence() now = to_datetime().in_timezone("UTC") - # 4 weeks × 7 days × 24 h × 4 records/h = 2688 records + # 4 weeks × 7 days × 24 h × 4 records/h = 2 688 records base = now.subtract(weeks=4) - _insert_records_every_n_minutes(s, base, count=2688, interval_minutes=15) + await _insert_records_every_n_minutes(s, base, count=2688, interval_minutes=15) return s, now -@pytest.fixture -def seq_sparse(): - """Sequence with only 3 records spread over 4 weeks — sparse, no compaction benefit.""" +@pytest_asyncio.fixture +async def seq_sparse(): + """Sequence with only 3 records spread over 4 weeks — sparse, below the compaction guard.""" s = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=4) for offset_days in [0, 14, 27]: dt = base.add(days=offset_days) - s.db_insert_record(SampleRecord(date_time=dt, value=float(offset_days))) - s.db_save_records() + await s.db_insert_record(SampleRecord(date_time=dt, value=float(offset_days))) + await s.db_save_records() return s, now # --------------------------------------------------------------------------- -# Existing tests (unchanged) +# Core mixin tests # --------------------------------------------------------------------------- class TestDatabaseRecordProtocolMixin: + @pytest.mark.asyncio @pytest.mark.parametrize( "start_str, value_count, interval_seconds", [ @@ -326,7 +371,7 @@ class TestDatabaseRecordProtocolMixin: ("2024-10-27 00:00:00", 24, 3600), ], ) - def test_db_generate_timestamps_utc_spacing( + async def test_db_generate_timestamps_utc_spacing( self, seq, start_str, value_count, interval_seconds ): start_dt = to_datetime(start_str, in_timezone="Europe/Berlin") @@ -349,93 +394,188 @@ class TestDatabaseRecordProtocolMixin: delta = (current_dt - last_dt).total_seconds() assert delta == interval_seconds, f"Spacing mismatch at index {i}: {delta}s" - def test_insert_and_memory_range(self, seq): + @pytest.mark.asyncio + async def test_insert_and_memory_range(self, seq): t0 = to_datetime() t1 = t0.add(hours=1) - seq.db_insert_record(SampleRecord(date_time=t0, value=1)) - seq.db_insert_record(SampleRecord(date_time=t1, value=2)) + await seq.db_insert_record(SampleRecord(date_time=t0, value=1)) + await seq.db_insert_record(SampleRecord(date_time=t1, value=2)) assert seq.records[0].date_time == t0 assert seq.records[-1].date_time == t1 assert len(seq.records) == 2 - def test_roundtrip_reload(self): + @pytest.mark.asyncio + async def test_roundtrip_reload(self): seq = SampleSequence() t0 = to_datetime() t1 = t0.add(hours=1) - seq.db_insert_record(SampleRecord(date_time=t0, value=1)) - seq.db_insert_record(SampleRecord(date_time=t1, value=2)) - assert seq.db_save_records() == 2 + await seq.db_insert_record(SampleRecord(date_time=t0, value=1)) + await seq.db_insert_record(SampleRecord(date_time=t1, value=2)) + assert await seq.db_save_records() == 2 db = seq.database seq2 = SampleSequence() seq2.database = db - loaded = seq2.db_load_records() + loaded = await seq2.db_load_records() assert loaded == 2 assert len(seq2.records) == 2 - def test_db_count_records(self, seq): + @pytest.mark.asyncio + async def test_db_count_records(self, seq): t0 = to_datetime() - seq.db_insert_record(SampleRecord(date_time=t0, value=1)) - assert seq.db_count_records() == 1 - seq.db_save_records() - assert seq.db_count_records() == 1 + await seq.db_insert_record(SampleRecord(date_time=t0, value=1)) + assert await seq.db_count_records() == 1 + await seq.db_save_records() + assert await seq.db_count_records() == 1 - def test_delete_range(self, seq): + @pytest.mark.asyncio + async def test_delete_range(self, seq): base = to_datetime() for i in range(5): - seq.db_insert_record(SampleRecord(date_time=base.add(minutes=i), value=i)) + await seq.db_insert_record(SampleRecord(date_time=base.add(minutes=i), value=i)) db_start = DatabaseTimestamp.from_datetime(base.add(minutes=1)) db_end = DatabaseTimestamp.from_datetime(base.add(minutes=4)) - deleted = seq.db_delete_records(start_timestamp=db_start, end_timestamp=db_end) + deleted = await seq.db_delete_records( + start_timestamp=db_start, end_timestamp=db_end + ) assert deleted == 3 assert [r.value for r in seq.records] == [0, 4] - def test_db_count_records_memory_only_multiple(self): + @pytest.mark.asyncio + async def test_db_count_records_memory_only_multiple(self): seq = SampleSequence() base = to_datetime() for i in range(3): - seq.db_insert_record(SampleRecord(date_time=base.add(minutes=i), value=i)) - assert seq.db_count_records() == 3 + await seq.db_insert_record(SampleRecord(date_time=base.add(minutes=i), value=i)) + assert await seq.db_count_records() == 3 - def test_db_count_records_memory_newer_than_db(self): + @pytest.mark.asyncio + async def test_db_count_records_memory_newer_than_db(self): seq = SampleSequence() base = to_datetime() - seq.db_insert_record(SampleRecord(date_time=base, value=1)) - seq.db_save_records() - seq.db_insert_record(SampleRecord(date_time=base.add(hours=1), value=2)) - seq.db_insert_record(SampleRecord(date_time=base.add(hours=2), value=3)) - assert seq.db_count_records() == 3 + await seq.db_insert_record(SampleRecord(date_time=base, value=1)) + await seq.db_save_records() + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=1), value=2)) + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=2), value=3)) + assert await seq.db_count_records() == 3 - def test_db_count_records_memory_older_than_db(self): + @pytest.mark.asyncio + async def test_db_count_records_memory_older_than_db(self): seq = SampleSequence() base = to_datetime() - seq.db_insert_record(SampleRecord(date_time=base.add(hours=1), value=2)) - seq.db_save_records() - seq.db_insert_record(SampleRecord(date_time=base, value=1)) - assert seq.db_count_records() == 2 + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=1), value=2)) + await seq.db_save_records() + await seq.db_insert_record(SampleRecord(date_time=base, value=1)) + assert await seq.db_count_records() == 2 - def test_db_count_records_empty_everywhere(self): + @pytest.mark.asyncio + async def test_db_count_records_empty_everywhere(self): seq = SampleSequence() - assert seq.db_count_records() == 0 + assert await seq.db_count_records() == 0 - def test_metadata_not_counted(self, seq): + @pytest.mark.asyncio + async def test_metadata_not_counted(self, seq): + """Metadata key must not be counted as a data record.""" seq.database._data.setdefault("test", {})[DATABASE_METADATA_KEY] = b"meta" - assert seq.db_count_records() == 0 + assert await seq.db_count_records() == 0 - def test_key_range_excludes_metadata(self, seq): + @pytest.mark.asyncio + async def test_key_range_excludes_metadata(self, seq): ns = seq.db_namespace() seq.database._data.setdefault(ns, {})[DATABASE_METADATA_KEY] = b"meta" - assert seq.database.get_key_range(ns) == (None, None) + assert await seq.database.get_key_range(ns) == (None, None) + + @pytest.mark.asyncio + async def test_duplicate_timestamp_raises(self, seq): + t0 = to_datetime() + await seq.db_insert_record(SampleRecord(date_time=t0, value=1)) + with pytest.raises(ValueError, match="Duplicate"): + await seq.db_insert_record(SampleRecord(date_time=t0, value=2)) + + @pytest.mark.asyncio + async def test_timestamp_range_empty(self, seq): + min_ts, max_ts = await seq.db_timestamp_range() + assert min_ts is None + assert max_ts is None + + @pytest.mark.asyncio + async def test_timestamp_range_reflects_inserts(self, seq): + base = to_datetime() + await seq.db_insert_record(SampleRecord(date_time=base, value=0)) + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=2), value=1)) + min_ts, max_ts = await seq.db_timestamp_range() + assert min_ts is not None + assert max_ts is not None + assert min_ts < max_ts + + @pytest.mark.asyncio + async def test_get_record_exact_match(self, seq): + t0 = to_datetime() + await seq.db_insert_record(SampleRecord(date_time=t0, value=42.0)) + db_ts = DatabaseTimestamp.from_datetime(t0) + record = await seq.db_get_record(db_ts) + assert record is not None + assert record.value == pytest.approx(42.0) + + @pytest.mark.asyncio + async def test_get_record_no_match_returns_none(self, seq): + t0 = to_datetime() + db_ts = DatabaseTimestamp.from_datetime(t0) + record = await seq.db_get_record(db_ts) + assert record is None + + @pytest.mark.asyncio + async def test_iterate_records_full_range(self, seq): + base = to_datetime() + expected_values = [float(i) for i in range(4)] + for i, v in enumerate(expected_values): + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=i), value=v)) + + collected = [] + async for record in seq.db_iterate_records(): + collected.append(record.value) + + assert collected == expected_values + + @pytest.mark.asyncio + async def test_iterate_records_bounded_range(self, seq): + base = to_datetime() + for i in range(6): + await seq.db_insert_record(SampleRecord(date_time=base.add(hours=i), value=float(i))) + + start_ts = DatabaseTimestamp.from_datetime(base.add(hours=2)) + end_ts = DatabaseTimestamp.from_datetime(base.add(hours=5)) + + collected = [] + async for record in seq.db_iterate_records( + start_timestamp=start_ts, end_timestamp=end_ts + ): + collected.append(record.value) + + assert collected == [2.0, 3.0, 4.0] + + @pytest.mark.asyncio + async def test_mark_dirty_record(self, seq): + t0 = to_datetime() + await seq.db_insert_record(SampleRecord(date_time=t0, value=1.0)) + await seq.db_save_records() # clears dirty set + + record = seq.records[0] + record.value = 99.0 + await seq.db_mark_dirty_record(record) + + saved = await seq.db_save_records() + assert saved >= 1 # --------------------------------------------------------------------------- -# Compaction tests +# Compact tier configuration tests (sync — db_compact_tiers is not async) # --------------------------------------------------------------------------- @@ -453,19 +593,15 @@ class TestCompactTiers: def test_default_tiers_first_is_2h_to_15min(self, seq): tiers = seq.db_compact_tiers() - age_sec, interval_sec = ( - tiers[0][0].total_seconds(), - tiers[0][1].total_seconds(), - ) + age_sec = tiers[0][0].total_seconds() + interval_sec = tiers[0][1].total_seconds() assert age_sec == 2 * 3600 assert interval_sec == 15 * 60 def test_default_tiers_second_is_2weeks_to_1h(self, seq): tiers = seq.db_compact_tiers() - age_sec, interval_sec = ( - tiers[1][0].total_seconds(), - tiers[1][1].total_seconds(), - ) + age_sec = tiers[1][0].total_seconds() + interval_sec = tiers[1][1].total_seconds() assert age_sec == 14 * 24 * 3600 assert interval_sec == 3600 @@ -479,7 +615,8 @@ class TestCompactTiers: assert len(tiers) == 1 assert tiers[0][1].total_seconds() == 3600 - def test_empty_tiers_disables_compaction(self): + @pytest.mark.asyncio + async def test_empty_tiers_disables_compaction(self): class NoCompactSeq(SampleSequence): def db_compact_tiers(self): return [] @@ -487,12 +624,17 @@ class TestCompactTiers: s = NoCompactSeq() now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=4) - _insert_records_every_n_minutes(s, base, count=100, interval_minutes=15) + await _insert_records_every_n_minutes(s, base, count=100, interval_minutes=15) - deleted = s.db_compact() + deleted = await s.db_compact() assert deleted == 0 +# --------------------------------------------------------------------------- +# Compact state persistence tests +# --------------------------------------------------------------------------- + + class TestCompactState: """Tests for _db_get_compact_state / _db_set_compact_state.""" @@ -500,17 +642,19 @@ class TestCompactState: interval = to_duration("1 hour") assert seq._db_get_compact_state(interval) is None - def test_set_and_get_state_roundtrip(self, seq): + @pytest.mark.asyncio + async def test_set_and_get_state_roundtrip(self, seq): interval = to_duration("1 hour") now = to_datetime().in_timezone("UTC") ts = DatabaseTimestamp.from_datetime(now) - seq._db_set_compact_state(interval, ts) + await seq._db_set_compact_state(interval, ts) retrieved = seq._db_get_compact_state(interval) assert retrieved == ts - def test_state_is_per_tier(self, seq): + @pytest.mark.asyncio + async def test_state_is_per_tier(self, seq): """Different tier intervals must not overwrite each other.""" interval_15min = to_duration("15 minutes") interval_1h = to_duration("1 hour") @@ -519,76 +663,73 @@ class TestCompactState: ts_15 = DatabaseTimestamp.from_datetime(now) ts_1h = DatabaseTimestamp.from_datetime(now.subtract(days=1)) - seq._db_set_compact_state(interval_15min, ts_15) - seq._db_set_compact_state(interval_1h, ts_1h) + await seq._db_set_compact_state(interval_15min, ts_15) + await seq._db_set_compact_state(interval_1h, ts_1h) assert seq._db_get_compact_state(interval_15min) == ts_15 assert seq._db_get_compact_state(interval_1h) == ts_1h - def test_state_persists_in_metadata(self, seq): - """State must survive a metadata reload.""" + @pytest.mark.asyncio + async def test_state_persists_in_metadata(self, seq): + """State must survive a metadata reload into a fresh sequence instance.""" interval = to_duration("1 hour") now = to_datetime().in_timezone("UTC") ts = DatabaseTimestamp.from_datetime(now) - seq._db_set_compact_state(interval, ts) + await seq._db_set_compact_state(interval, ts) # Reload metadata from fake DB seq2 = SampleSequence() seq2.database = seq.database - seq2._db_metadata = seq2._db_load_metadata() + seq2._db_metadata = await seq2._db_load_metadata() assert seq2._db_get_compact_state(interval) == ts -class TestCompactSparseGuard: - """The inflation guard must skip compaction when records are already sparse.""" +# --------------------------------------------------------------------------- +# Sparse data guard tests +# --------------------------------------------------------------------------- - def test_sparse_data_aligns_but_does_not_reduce_cardinality(self, seq_sparse): + +class TestCompactSparseGuard: + + @pytest.mark.asyncio + async def test_sparse_data_aligns_but_does_not_reduce_cardinality(self, seq_sparse): """Sparse data must be aligned to the target interval for all records that were modified.""" seq, _ = seq_sparse - + seq, _ = seq_sparse interval = to_duration("15 minutes") interval_sec = int(interval.total_seconds()) - # Snapshot original timestamps - before_epochs = { - int(r.date_time.timestamp()) - for r in seq.records - } + before_epochs = {int(r.date_time.timestamp()) for r in seq.records} - seq._db_compact_tier( - to_duration("30 minutes"), - interval, - ) + await seq._db_compact_tier(to_duration("30 minutes"), interval) - after_epochs = { - int(r.date_time.timestamp()) - for r in seq.records - } + after_epochs = {int(r.date_time.timestamp()) for r in seq.records} # Cardinality must not increase assert len(after_epochs) <= len(before_epochs) # Any timestamp that changed must now be aligned changed_epochs = after_epochs - before_epochs - for epoch in changed_epochs: assert epoch % interval_sec == 0 - def test_sparse_guard_advances_cutoff(self, seq_sparse): - """Even when skipped, the cutoff should be stored so next run skips the same window.""" + @pytest.mark.asyncio + async def test_sparse_guard_advances_cutoff(self, seq_sparse): + """Even when skipped, the cutoff should be stored so the next run skips the same window.""" seq, _ = seq_sparse interval_1h = to_duration("1 hour") interval_15min = to_duration("15 minutes") - seq.db_compact() + await seq.db_compact() # Both tiers should have stored a cutoff even though nothing was deleted assert seq._db_get_compact_state(interval_1h) is not None assert seq._db_get_compact_state(interval_15min) is not None - def test_exactly_at_boundary_remains_stable(self, seq): + @pytest.mark.asyncio + async def test_exactly_at_boundary_remains_stable(self, seq): now = to_datetime().in_timezone("UTC") interval = to_duration("1 hour") @@ -596,147 +737,148 @@ class TestCompactSparseGuard: base = raw_base.subtract(seconds=int(raw_base.timestamp()) % 3600) for i in range(4): - seq.db_insert_record( - SampleRecord( - date_time=base.add(hours=i), - value=float(i), - ) + await seq.db_insert_record( + SampleRecord(date_time=base.add(hours=i), value=float(i)) ) - seq.db_insert_record( + await seq.db_insert_record( SampleRecord(date_time=now.subtract(seconds=1), value=0.0) ) - seq.db_save_records() + await seq.db_save_records() - before = [ - (int(r.date_time.timestamp()), r.value) - for r in seq.records - ] + before = [(int(r.date_time.timestamp()), r.value) for r in seq.records] - seq._db_compact_tier( - to_duration("30 minutes"), - interval, - ) + await seq._db_compact_tier(to_duration("30 minutes"), interval) - after = [ - (int(r.date_time.timestamp()), r.value) - for r in seq.records - ] + after = [(int(r.date_time.timestamp()), r.value) for r in seq.records] assert before == after +# --------------------------------------------------------------------------- +# Single-tier worker tests +# --------------------------------------------------------------------------- + + class TestCompactTierWorker: """Unit tests for _db_compact_tier directly.""" - def test_empty_sequence_returns_zero(self, seq): + @pytest.mark.asyncio + async def test_empty_sequence_returns_zero(self, seq): age = to_duration("2 hours") interval = to_duration("15 minutes") - assert seq._db_compact_tier(age, interval) == 0 + assert await seq._db_compact_tier(age, interval) == 0 - def test_all_records_too_recent_skipped(self): + @pytest.mark.asyncio + async def test_all_records_too_recent_skipped(self): """Records within the age threshold must not be touched.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") # Insert 10 records from 30 minutes ago — all within 2h threshold base = now.subtract(minutes=30) - _insert_records_every_n_minutes(seq, base, count=10, interval_minutes=1) + await _insert_records_every_n_minutes(seq, base, count=10, interval_minutes=1) - before = seq.db_count_records() - deleted = seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + before = await seq.db_count_records() + deleted = await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) assert deleted == 0 - assert seq.db_count_records() == before + assert await seq.db_count_records() == before - def test_compaction_reduces_record_count(self): - """Dense 1-min records older than 2h should be downsampled to 15-min.""" + @pytest.mark.asyncio + async def test_compaction_reduces_record_count(self): + """Dense 1-min records older than 2 h should be downsampled to 15-min.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") # Insert 1-min records for 6 hours ending 3 hours ago base = now.subtract(hours=9) - _insert_records_every_n_minutes(seq, base, count=6 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, base, count=6 * 60, interval_minutes=1) - before = seq.db_count_records() - deleted = seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + before = await seq.db_count_records() + deleted = await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) - after = seq.db_count_records() + after = await seq.db_count_records() assert deleted > 0 assert after < before - def test_records_within_threshold_preserved(self): + @pytest.mark.asyncio + async def test_records_within_threshold_preserved(self): """Records newer than age_threshold must remain untouched after compaction.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") # Old dense records (will be compacted) old_base = now.subtract(hours=6) - _insert_records_every_n_minutes(seq, old_base, count=4 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, old_base, count=4 * 60, interval_minutes=1) # Recent records (must not be touched) — insert 5 records in the last hour recent_base = now.subtract(minutes=50) - _insert_records_every_n_minutes(seq, recent_base, count=5, interval_minutes=10) + await _insert_records_every_n_minutes(seq, recent_base, count=5, interval_minutes=10) - recent_before = [ - r for r in seq.records - if r.date_time and r.date_time >= recent_base - ] + recent_before = [r for r in seq.records if r.date_time and r.date_time >= recent_base] - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) - recent_after = [ - r for r in seq.records - if r.date_time and r.date_time >= recent_base - ] + recent_after = [r for r in seq.records if r.date_time and r.date_time >= recent_base] assert len(recent_after) == len(recent_before) - def test_incremental_cutoff_prevents_recompaction(self): + @pytest.mark.asyncio + async def test_incremental_cutoff_prevents_recompaction(self): """Running compaction twice must not re-compact already-compacted data.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(hours=8) - _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) age = to_duration("2 hours") interval = to_duration("15 minutes") - deleted_first = seq._db_compact_tier(age, interval) - count_after_first = seq.db_count_records() + deleted_first = await seq._db_compact_tier(age, interval) + count_after_first = await seq.db_count_records() - deleted_second = seq._db_compact_tier(age, interval) - count_after_second = seq.db_count_records() + deleted_second = await seq._db_compact_tier(age, interval) + count_after_second = await seq.db_count_records() assert deleted_first > 0 assert deleted_second == 0, "Second run must be a no-op" assert count_after_first == count_after_second - def test_cutoff_stored_after_compaction(self): + @pytest.mark.asyncio + async def test_cutoff_stored_after_compaction(self): """Cutoff timestamp must be persisted after a successful compaction run.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(hours=8) - _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) + + await _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) interval = to_duration("15 minutes") - seq._db_compact_tier(to_duration("2 hours"), interval) + await seq._db_compact_tier(to_duration("2 hours"), interval) assert seq._db_get_compact_state(interval) is not None +# --------------------------------------------------------------------------- +# Public db_compact() integration tests +# --------------------------------------------------------------------------- + + class TestDbCompact: """Integration tests for the public db_compact() entry point.""" - def test_compact_dense_data_both_tiers(self, seq_with_15min_data): + @pytest.mark.asyncio + async def test_compact_dense_data_both_tiers(self, seq_with_15min_data): """4 weeks of 15-min data should be reduced by both tiers.""" seq, _ = seq_with_15min_data - before = seq.db_count_records() + before = await seq.db_count_records() - total_deleted = seq.db_compact() + total_deleted = await seq.db_compact() - after = seq.db_count_records() + after = await seq.db_count_records() assert total_deleted > 0 assert after < before - def test_compact_coarsest_tier_runs_first(self, seq_with_15min_data): + @pytest.mark.asyncio + async def test_compact_coarsest_tier_runs_first(self, seq_with_15min_data): """The 1-hour tier (coarsest) must run before the 15-min tier. If coarsest ran last it would re-compact records the 15-min tier @@ -744,7 +886,7 @@ class TestDbCompact: cutoff is not later than the 15-min cutoff. """ seq, _ = seq_with_15min_data - seq.db_compact() + await seq.db_compact() cutoff_1h = seq._db_get_compact_state(to_duration("1 hour")) cutoff_15min = seq._db_get_compact_state(to_duration("15 minutes")) @@ -754,36 +896,42 @@ class TestDbCompact: # The 1h tier covers older data → its cutoff must be earlier than 15min tier assert cutoff_1h <= cutoff_15min - def test_compact_idempotent(self, seq_with_15min_data): + @pytest.mark.asyncio + async def test_compact_idempotent(self, seq_with_15min_data): """Running db_compact twice must not change record count.""" seq, _ = seq_with_15min_data - seq.db_compact() - after_first = seq.db_count_records() + await seq.db_compact() + after_first = await seq.db_count_records() - seq.db_compact() - after_second = seq.db_count_records() + await seq.db_compact() + after_second = await seq.db_count_records() assert after_first == after_second - def test_compact_empty_sequence_returns_zero(self, seq): - assert seq.db_compact() == 0 + @pytest.mark.asyncio + async def test_compact_empty_sequence_returns_zero(self, seq): + assert await seq.db_compact() == 0 - def test_compact_with_override_tiers(self): + @pytest.mark.asyncio + async def test_compact_with_override_tiers(self): """Passing compact_tiers directly must override db_compact_tiers().""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(weeks=3) - _insert_records_every_n_minutes(seq, base, count=3 * 7 * 24 * 4, interval_minutes=15) + await _insert_records_every_n_minutes( + seq, base, count=3 * 7 * 24 * 4, interval_minutes=15 + ) - before = seq.db_count_records() - deleted = seq.db_compact( + before = await seq.db_count_records() + deleted = await seq.db_compact( compact_tiers=[(to_duration("1 day"), to_duration("1 hour"))] ) assert deleted > 0 - assert seq.db_count_records() < before + assert await seq.db_count_records() < before - def test_compact_only_processes_new_window_on_second_call(self): + @pytest.mark.asyncio + async def test_compact_only_processes_new_window_on_second_call(self): """Second call processes only the new window, not the full history.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") @@ -792,63 +940,69 @@ class TestDbCompact: now_floored = now.set(second=0, microsecond=0) base = now_floored.subtract(weeks=3) # Dense 1-min data for 3 weeks - _insert_records_every_n_minutes(seq, base, count=3 * 7 * 24 * 60, interval_minutes=1) + await _insert_records_every_n_minutes( + seq, base, count=3 * 7 * 24 * 60, interval_minutes=1 + ) - seq.db_compact() - count_after_first = seq.db_count_records() + await seq.db_compact() + count_after_first = await seq.db_count_records() # Start 2 days before `base` and insert only 1 day worth of records, # so the window [extra_base, extra_base + 1439min] stays entirely # before `base - 1day` and never collides with compacted timestamps # that were snapped to clean hour/15-min boundaries inside the original range. extra_base = now_floored.subtract(weeks=3).subtract(days=2) - _insert_records_every_n_minutes(seq, extra_base, count=24 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, extra_base, count=24 * 60, interval_minutes=1) - seq.db_compact() - count_after_second = seq.db_count_records() + await seq.db_compact() + count_after_second = await seq.db_count_records() # Second compact should have processed the newly added old data # Record count may change but should not exceed first compacted count by much assert count_after_second >= 0 # basic sanity -class TestCompactDataIntegrity: - """Verify value integrity is preserved after compaction.""" +# --------------------------------------------------------------------------- +# Data integrity tests +# --------------------------------------------------------------------------- - def test_constant_value_preserved(self): + +class TestCompactDataIntegrity: + + @pytest.mark.asyncio + async def test_constant_value_preserved(self): """Constant value field must survive mean-resampling unchanged.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(hours=6) - # All values = 42.0 - _insert_records_every_n_minutes( + await _insert_records_every_n_minutes( seq, base, count=6 * 60, interval_minutes=1, value_fn=lambda _: 42.0 ) - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) for record in seq.records: if record.date_time and record.date_time < now.subtract(hours=2): assert record.value == pytest.approx(42.0, abs=1e-6) - def test_recent_records_not_modified(self): + @pytest.mark.asyncio + async def test_recent_records_not_modified(self): """Records newer than the age threshold must have unchanged values.""" seq = SampleSequence() now = to_datetime().in_timezone("UTC") old_base = now.subtract(hours=6) - _insert_records_every_n_minutes(seq, old_base, count=3 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, old_base, count=3 * 60, interval_minutes=1) - # Known recent values recent_base = now.subtract(minutes=30) expected = {i * 10: float(100 + i) for i in range(3)} for offset, val in expected.items(): dt = recent_base.add(minutes=offset) - seq.db_insert_record(SampleRecord(date_time=dt, value=val)) - seq.db_save_records() + await seq.db_insert_record(SampleRecord(date_time=dt, value=val)) + await seq.db_save_records() - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) for record in seq.records: if record.date_time and record.date_time >= recent_base: @@ -856,7 +1010,8 @@ class TestCompactDataIntegrity: if offset in expected: assert record.value == pytest.approx(expected[offset], abs=1e-6) - def test_compacted_timestamps_spacing(self): + @pytest.mark.asyncio + async def test_compacted_timestamps_spacing(self): """Resampled records must be fewer than original and span the compaction window. Exact per-bucket spacing depends on the full DataSequence.key_to_array @@ -867,10 +1022,10 @@ class TestCompactDataIntegrity: seq = SampleSequence() now = to_datetime().in_timezone("UTC") base = now.subtract(hours=6) - _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) + await _insert_records_every_n_minutes(seq, base, count=5 * 60, interval_minutes=1) - before = seq.db_count_records() - seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) + before = await seq.db_count_records() + await seq._db_compact_tier(to_duration("2 hours"), to_duration("15 minutes")) cutoff = now.subtract(hours=2) compacted = sorted( @@ -889,6 +1044,4 @@ class TestCompactDataIntegrity: tz="UTC", ) assert compacted[0].date_time >= expected_window_start - - # Last compacted record must be before the cutoff assert compacted[-1].date_time < cutoff diff --git a/tests/test_docsphinx.py b/tests/test_docsphinx.py index 0fe67352..01509ab6 100644 --- a/tests/test_docsphinx.py +++ b/tests/test_docsphinx.py @@ -20,16 +20,52 @@ DIR_SRC = DIR_PROJECT_ROOT / "src" HASH_FILE = DIR_BUILD / ".sphinx_hash.json" -def find_sphinx_build() -> str: - venv = os.getenv("VIRTUAL_ENV") - paths = [Path(venv)] if venv else [] - paths.append(DIR_PROJECT_ROOT / ".venv") +import os +import subprocess +from pathlib import Path - for base in paths: - cmd = base / ("Scripts" if os.name == "nt" else "bin") / ("sphinx-build.exe" if os.name == "nt" else "sphinx-build") - if cmd.exists(): - return str(cmd) - return "sphinx-build" + +def find_sphinx_build() -> list[str]: + """Return command to invoke sphinx-build via virtualenv, uv, or globally.""" + candidates = [] + + # 1️⃣ Currently active virtualenv + venv = os.getenv("VIRTUAL_ENV") + if venv: + candidates.append(Path(venv)) + + # 2️⃣ uv‑managed virtualenv + uv_venv = Path(".uv") / "venv" + if uv_venv.exists(): + candidates.append(uv_venv) + + # 3️⃣ traditional .venv + dot_venv = Path(".venv") + if dot_venv.exists(): + candidates.append(dot_venv) + + # Check each candidate for the sphinx‑build binary + for base in candidates: + sphinx_build_path = base / ("Scripts" if os.name == "nt" else "bin") / ( + "sphinx-build.exe" if os.name == "nt" else "sphinx-build" + ) + if sphinx_build_path.exists(): + return [str(sphinx_build_path)] + + # 4️⃣ fallback to uv run sphinx‑build + try: + subprocess.run( + ["uv", "run", "sphinx-build", "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return ["uv", "run", "sphinx-build"] + except (subprocess.CalledProcessError, FileNotFoundError): + pass + + # 5️⃣ final fallback to system sphinx‑build + return ["sphinx-build"] @pytest.fixture(scope="session") @@ -61,8 +97,7 @@ class TestSphinxDocumentation: Ensures no major warnings are emitted. """ - SPHINX_CMD = [ - find_sphinx_build(), + SPHINX_CMD = find_sphinx_build() + [ "-M", "html", str(DIR_DOCS), @@ -105,8 +140,6 @@ class TestSphinxDocumentation: env["EOS_CONFIG_DIR"] = eos_dir try: - # Run sphinx-build - project_dir = Path(__file__).parent.parent process = subprocess.run( self.SPHINX_CMD, check=True, @@ -114,13 +147,15 @@ class TestSphinxDocumentation: stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - cwd=project_dir, + cwd=DIR_PROJECT_ROOT, # use the existing constant ) - # Combine output output = process.stdout + "\n" + process.stderr returncode = process.returncode - except: - output = f"ERROR: Could not start sphinx-build - {self.SPHINX_CMD}" + except subprocess.CalledProcessError as e: + output = e.stdout + "\n" + e.stderr if e.stdout else "" + returncode = e.returncode + except Exception as e: + output = f"Failed to execute command: {e}" returncode = -1 # Remove temporary EOS_DIR diff --git a/tests/test_elecpriceakkudoktor.py b/tests/test_elecpriceakkudoktor.py index 732e48b5..fd4dd90e 100644 --- a/tests/test_elecpriceakkudoktor.py +++ b/tests/test_elecpriceakkudoktor.py @@ -1,3 +1,4 @@ +import asyncio import json from pathlib import Path from unittest.mock import Mock, patch @@ -47,177 +48,181 @@ def cache_store(): return CacheFileStore() -# ------------------------------------------------ -# General forecast -# ------------------------------------------------ +class TestElecPriceAkkudokor: + # ------------------------------------------------ + # General forecast + # ------------------------------------------------ + + def test_singleton_instance(self, provider): + """Test that ElecPriceForecast behaves as a singleton.""" + another_instance = ElecPriceAkkudoktor() + assert provider is another_instance -def test_singleton_instance(provider): - """Test that ElecPriceForecast behaves as a singleton.""" - another_instance = ElecPriceAkkudoktor() - assert provider is another_instance + def test_invalid_provider(self, provider, monkeypatch): + """Test requesting an unsupported provider.""" + monkeypatch.setenv("EOS_ELECPRICE__ELECPRICE_PROVIDER", "") + provider.config.reset_settings() + assert not provider.enabled() -def test_invalid_provider(provider, monkeypatch): - """Test requesting an unsupported provider.""" - monkeypatch.setenv("EOS_ELECPRICE__ELECPRICE_PROVIDER", "") - provider.config.reset_settings() - assert not provider.enabled() + # ------------------------------------------------ + # Akkudoktor + # ------------------------------------------------ -# ------------------------------------------------ -# Akkudoktor -# ------------------------------------------------ + @patch("akkudoktoreos.prediction.elecpriceakkudoktor.logger.error") + def test_validate_data_invalid_format(self, mock_logger, provider): + """Test validation for invalid Akkudoktor data.""" + invalid_data = '{"invalid": "data"}' + with pytest.raises(ValueError): + provider._validate_data(invalid_data) + mock_logger.assert_called_once_with(mock_logger.call_args[0][0]) -@patch("akkudoktoreos.prediction.elecpriceakkudoktor.logger.error") -def test_validate_data_invalid_format(mock_logger, provider): - """Test validation for invalid Akkudoktor data.""" - invalid_data = '{"invalid": "data"}' - with pytest.raises(ValueError): - provider._validate_data(invalid_data) - mock_logger.assert_called_once_with(mock_logger.call_args[0][0]) + @patch("requests.get") + def test_request_forecast(self, mock_get, provider, sample_akkudoktor_1_json): + """Test requesting forecast from Akkudoktor.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_akkudoktor_1_json) + mock_get.return_value = mock_response + + # Test function + akkudoktor_data = provider._request_forecast() + + assert isinstance(akkudoktor_data, AkkudoktorElecPrice) + assert akkudoktor_data.values[0].model_dump() == AkkudoktorElecPriceValue( + start_timestamp=1733785200000, + end_timestamp=1733788800000, + start="2024-12-09T23:00:00.000Z", + end="2024-12-10T00:00:00.000Z", + marketprice=92.85, + unit="Eur/MWh", + marketpriceEurocentPerKWh=9.29, + ).model_dump() -@patch("requests.get") -def test_request_forecast(mock_get, provider, sample_akkudoktor_1_json): - """Test requesting forecast from Akkudoktor.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_akkudoktor_1_json) - mock_get.return_value = mock_response + @pytest.mark.asyncio + @patch("requests.get") + async def test_update_data(self, mock_get, provider, sample_akkudoktor_1_json, cache_store): + """Test fetching forecast from Akkudoktor.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_akkudoktor_1_json) + mock_get.return_value = mock_response - # Test function - akkudoktor_data = provider._request_forecast() + cache_store.clear(clear_all=True) - assert isinstance(akkudoktor_data, AkkudoktorElecPrice) - assert akkudoktor_data.values[0].model_dump() == AkkudoktorElecPriceValue( - start_timestamp=1733785200000, - end_timestamp=1733788800000, - start="2024-12-09T23:00:00.000Z", - end="2024-12-10T00:00:00.000Z", - marketprice=92.85, - unit="Eur/MWh", - marketpriceEurocentPerKWh=9.29, - ).model_dump() + # Call the method + ems_eos = get_ems() + ems_eos.set_start_datetime(to_datetime("2024-12-11 00:00:00", in_timezone="Europe/Berlin")) + await provider.update_data(force_enable=True, force_update=True) + + # Assert: Verify the result is as expected + mock_get.assert_called_once() + assert ( + len(provider) == 73 + ) # we have 48 datasets in the api response, we want to know 48h into the future. The data we get has already 23h into the future so we need only 25h more. 48+25=73 + + # Assert we get hours prioce values by resampling + np_price_array = await provider.key_to_array( + key="elecprice_marketprice_wh", + start_datetime=provider.ems_start_datetime, + end_datetime=provider.end_datetime, + ) + assert len(np_price_array) == provider.total_hours + + # with open(FILE_TESTDATA_ELECPRICEAKKUDOKTOR_2_JSON, "w") as f_out: + # f_out.write(provider.to_json()) -@patch("requests.get") -def test_update_data(mock_get, provider, sample_akkudoktor_1_json, cache_store): - """Test fetching forecast from Akkudoktor.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_akkudoktor_1_json) - mock_get.return_value = mock_response + @pytest.mark.asyncio + @patch("requests.get") + async def test_update_data_with_incomplete_forecast(self, mock_get, provider): + """Test `_update_data` with incomplete or missing forecast data.""" + incomplete_data: dict = {"meta": {}, "values": []} + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(incomplete_data) + mock_get.return_value = mock_response + logger.info("The following errors are intentional and part of the test.") + with pytest.raises(ValueError): + await provider._update_data(force_update=True) - cache_store.clear(clear_all=True) - # Call the method - ems_eos = get_ems() - ems_eos.set_start_datetime(to_datetime("2024-12-11 00:00:00", in_timezone="Europe/Berlin")) - provider.update_data(force_enable=True, force_update=True) - - # Assert: Verify the result is as expected - mock_get.assert_called_once() - assert ( - len(provider) == 73 - ) # we have 48 datasets in the api response, we want to know 48h into the future. The data we get has already 23h into the future so we need only 25h more. 48+25=73 - - # Assert we get hours prioce values by resampling - np_price_array = provider.key_to_array( - key="elecprice_marketprice_wh", - start_datetime=provider.ems_start_datetime, - end_datetime=provider.end_datetime, + @pytest.mark.parametrize( + "status_code, exception", + [(400, requests.exceptions.HTTPError), (500, requests.exceptions.HTTPError), (200, None)], ) - assert len(np_price_array) == provider.total_hours - - # with open(FILE_TESTDATA_ELECPRICEAKKUDOKTOR_2_JSON, "w") as f_out: - # f_out.write(provider.to_json()) - - -@patch("requests.get") -def test_update_data_with_incomplete_forecast(mock_get, provider): - """Test `_update_data` with incomplete or missing forecast data.""" - incomplete_data: dict = {"meta": {}, "values": []} - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(incomplete_data) - mock_get.return_value = mock_response - logger.info("The following errors are intentional and part of the test.") - with pytest.raises(ValueError): - provider._update_data(force_update=True) - - -@pytest.mark.parametrize( - "status_code, exception", - [(400, requests.exceptions.HTTPError), (500, requests.exceptions.HTTPError), (200, None)], -) -@patch("requests.get") -def test_request_forecast_status_codes( - mock_get, provider, sample_akkudoktor_1_json, status_code, exception -): - """Test handling of various API status codes.""" - mock_response = Mock() - mock_response.status_code = status_code - mock_response.content = json.dumps(sample_akkudoktor_1_json) - mock_response.raise_for_status.side_effect = ( - requests.exceptions.HTTPError if exception else None - ) - mock_get.return_value = mock_response - if exception: - with pytest.raises(exception): + @patch("requests.get") + def test_request_forecast_status_codes( + self, mock_get, provider, sample_akkudoktor_1_json, status_code, exception + ): + """Test handling of various API status codes.""" + mock_response = Mock() + mock_response.status_code = status_code + mock_response.content = json.dumps(sample_akkudoktor_1_json) + mock_response.raise_for_status.side_effect = ( + requests.exceptions.HTTPError if exception else None + ) + mock_get.return_value = mock_response + if exception: + with pytest.raises(exception): + provider._request_forecast() + else: provider._request_forecast() - else: - provider._request_forecast() -@patch("requests.get") -@patch("akkudoktoreos.core.cache.CacheFileStore") -def test_cache_integration(mock_cache, mock_get, provider, sample_akkudoktor_1_json): - """Test caching of 8-day electricity price data.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_akkudoktor_1_json) - mock_get.return_value = mock_response + @pytest.mark.asyncio + @patch("requests.get") + @patch("akkudoktoreos.core.cache.CacheFileStore") + async def test_cache_integration(self, mock_cache, mock_get, provider, sample_akkudoktor_1_json): + """Test caching of 8-day electricity price data.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_akkudoktor_1_json) + mock_get.return_value = mock_response - # Mock cache object - mock_cache_instance = mock_cache.return_value - mock_cache_instance.get.return_value = None # Simulate no cache + # Mock cache object + mock_cache_instance = mock_cache.return_value + mock_cache_instance.get.return_value = None # Simulate no cache - provider._update_data(force_update=True) - mock_cache_instance.create.assert_called_once() - mock_cache_instance.get.assert_called_once() + await provider._update_data(force_update=True) + mock_cache_instance.create.assert_called_once() + mock_cache_instance.get.assert_called_once() -def test_key_to_array_resampling(provider): - """Test resampling of forecast data to NumPy array.""" - provider.update_data(force_update=True) - array = provider.key_to_array( - key="elecprice_marketprice_wh", - start_datetime=provider.ems_start_datetime, - end_datetime=provider.end_datetime, - ) - assert isinstance(array, np.ndarray) - assert len(array) == provider.total_hours + @pytest.mark.asyncio + async def test_key_to_array_resampling(self, provider): + """Test resampling of forecast data to NumPy array.""" + await provider.update_data(force_update=True) + array = await provider.key_to_array( + key="elecprice_marketprice_wh", + start_datetime=provider.ems_start_datetime, + end_datetime=provider.end_datetime, + ) + assert isinstance(array, np.ndarray) + assert len(array) == provider.total_hours -# ------------------------------------------------ -# Development Akkudoktor -# ------------------------------------------------ + # ------------------------------------------------ + # Development Akkudoktor + # ------------------------------------------------ -@pytest.mark.skip(reason="For development only") -def test_akkudoktor_development_forecast_data(provider): - """Fetch data from real Akkudoktor server.""" - # Preset, as this is usually done by update_data() - provider.ems_start_datetime = to_datetime("2024-10-26 00:00:00") + @pytest.mark.skip(reason="For development only") + def test_akkudoktor_development_forecast_data(self, provider): + """Fetch data from real Akkudoktor server.""" + # Preset, as this is usually done by update_data() + provider.ems_start_datetime = to_datetime("2024-10-26 00:00:00") - akkudoktor_data = provider._request_forecast() + akkudoktor_data = provider._request_forecast() - with FILE_TESTDATA_ELECPRICEAKKUDOKTOR_1_JSON.open( - "w", encoding="utf-8", newline="\n" - ) as f_out: - json.dump(akkudoktor_data, f_out, indent=4) + with FILE_TESTDATA_ELECPRICEAKKUDOKTOR_1_JSON.open( + "w", encoding="utf-8", newline="\n" + ) as f_out: + json.dump(akkudoktor_data, f_out, indent=4) diff --git a/tests/test_elecpriceenergycharts.py b/tests/test_elecpriceenergycharts.py index fdfbcd32..71b8d922 100644 --- a/tests/test_elecpriceenergycharts.py +++ b/tests/test_elecpriceenergycharts.py @@ -1,3 +1,4 @@ +import asyncio import json from pathlib import Path from unittest.mock import Mock, patch @@ -51,204 +52,205 @@ def cache_store(): return CacheFileStore() -# ------------------------------------------------ -# General forecast -# ------------------------------------------------ +@pytest.mark.asyncio +class TestElecPriceEnergyCharts: + # ------------------------------------------------ + # General forecast + # ------------------------------------------------ + + def test_singleton_instance(self, provider): + """Test that ElecPriceForecast behaves as a singleton.""" + another_instance = ElecPriceEnergyCharts() + assert provider is another_instance -def test_singleton_instance(provider): - """Test that ElecPriceForecast behaves as a singleton.""" - another_instance = ElecPriceEnergyCharts() - assert provider is another_instance + def test_invalid_provider(self, provider, monkeypatch): + """Test requesting an unsupported provider.""" + monkeypatch.setenv("EOS_ELECPRICE__ELECPRICE_PROVIDER", "") + provider.config.reset_settings() + assert not provider.enabled() -def test_invalid_provider(provider, monkeypatch): - """Test requesting an unsupported provider.""" - monkeypatch.setenv("EOS_ELECPRICE__ELECPRICE_PROVIDER", "") - provider.config.reset_settings() - assert not provider.enabled() + # ------------------------------------------------ + # Akkudoktor + # ------------------------------------------------ -# ------------------------------------------------ -# Akkudoktor -# ------------------------------------------------ + @patch("akkudoktoreos.prediction.elecpriceenergycharts.logger.error") + def test_validate_data_invalid_format(self, mock_logger, provider): + """Test validation for invalid Energy-Charts data.""" + invalid_data = '{"invalid": "data"}' + with pytest.raises(ValueError): + provider._validate_data(invalid_data) + mock_logger.assert_called_once_with(mock_logger.call_args[0][0]) -@patch("akkudoktoreos.prediction.elecpriceenergycharts.logger.error") -def test_validate_data_invalid_format(mock_logger, provider): - """Test validation for invalid Energy-Charts data.""" - invalid_data = '{"invalid": "data"}' - with pytest.raises(ValueError): - provider._validate_data(invalid_data) - mock_logger.assert_called_once_with(mock_logger.call_args[0][0]) + @patch("requests.get") + def test_request_forecast(self, mock_get, provider, sample_energycharts_json): + """Test requesting forecast from Energy-Charts.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_energycharts_json) + mock_get.return_value = mock_response + + # Test function + energy_charts_data = provider._request_forecast() + + assert isinstance(energy_charts_data, EnergyChartsElecPrice) + assert energy_charts_data.unix_seconds[0] == 1733785200 + assert energy_charts_data.price[0] == 92.85 -@patch("requests.get") -def test_request_forecast(mock_get, provider, sample_energycharts_json): - """Test requesting forecast from Energy-Charts.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_energycharts_json) - mock_get.return_value = mock_response + @patch("requests.get") + async def test_update_data(self, mock_get, provider, sample_energycharts_json, cache_store): + """Test fetching forecast from Energy-Charts.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_energycharts_json) + mock_get.return_value = mock_response - # Test function - energy_charts_data = provider._request_forecast() + cache_store.clear(clear_all=True) - assert isinstance(energy_charts_data, EnergyChartsElecPrice) - assert energy_charts_data.unix_seconds[0] == 1733785200 - assert energy_charts_data.price[0] == 92.85 + # Call the method + ems_eos = get_ems() + ems_eos.set_start_datetime(to_datetime("2024-12-11 00:00:00", in_timezone="Europe/Berlin")) + await provider.update_data(force_enable=True, force_update=True) + + # Assert: Verify the result is as expected + mock_get.assert_called_once() + assert ( + len(provider) == 73 + ) # we have 48 datasets in the api response, we want to know 48h into the future. The data we get has already 23h into the future so we need only 25h more. 48+25=73 + + # Assert we get hours prioce values by resampling + np_price_array = await provider.key_to_array( + key="elecprice_marketprice_wh", + start_datetime=provider.ems_start_datetime, + end_datetime=provider.end_datetime, + ) + assert len(np_price_array) == provider.total_hours -@patch("requests.get") -def test_update_data(mock_get, provider, sample_energycharts_json, cache_store): - """Test fetching forecast from Energy-Charts.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_energycharts_json) - mock_get.return_value = mock_response + @patch("requests.get") + async def test_update_data_with_incomplete_forecast(self, mock_get, provider): + """Test `_update_data` with incomplete or missing forecast data.""" + incomplete_data: dict = {"license_info": "", "unix_seconds": [], "price": [], "unit": "", "deprecated": False} + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(incomplete_data) + mock_get.return_value = mock_response + logger.info("The following errors are intentional and part of the test.") + with pytest.raises(ValueError): + await provider._update_data(force_update=True) - cache_store.clear(clear_all=True) - # Call the method - ems_eos = get_ems() - ems_eos.set_start_datetime(to_datetime("2024-12-11 00:00:00", in_timezone="Europe/Berlin")) - provider.update_data(force_enable=True, force_update=True) - - # Assert: Verify the result is as expected - mock_get.assert_called_once() - assert ( - len(provider) == 73 - ) # we have 48 datasets in the api response, we want to know 48h into the future. The data we get has already 23h into the future so we need only 25h more. 48+25=73 - - # Assert we get hours prioce values by resampling - np_price_array = provider.key_to_array( - key="elecprice_marketprice_wh", - start_datetime=provider.ems_start_datetime, - end_datetime=provider.end_datetime, + @pytest.mark.parametrize( + "status_code, exception", + [(400, requests.exceptions.HTTPError), (500, requests.exceptions.HTTPError), (200, None)], ) - assert len(np_price_array) == provider.total_hours - - -@patch("requests.get") -def test_update_data_with_incomplete_forecast(mock_get, provider): - """Test `_update_data` with incomplete or missing forecast data.""" - incomplete_data: dict = {"license_info": "", "unix_seconds": [], "price": [], "unit": "", "deprecated": False} - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(incomplete_data) - mock_get.return_value = mock_response - logger.info("The following errors are intentional and part of the test.") - with pytest.raises(ValueError): - provider._update_data(force_update=True) - - -@pytest.mark.parametrize( - "status_code, exception", - [(400, requests.exceptions.HTTPError), (500, requests.exceptions.HTTPError), (200, None)], -) -@patch("requests.get") -def test_request_forecast_status_codes( - mock_get, provider, sample_energycharts_json, status_code, exception -): - """Test handling of various API status codes.""" - mock_response = Mock() - mock_response.status_code = status_code - mock_response.content = json.dumps(sample_energycharts_json) - mock_response.raise_for_status.side_effect = ( - requests.exceptions.HTTPError if exception else None - ) - mock_get.return_value = mock_response - if exception: - with pytest.raises(exception): + @patch("requests.get") + def test_request_forecast_status_codes( + self, mock_get, provider, sample_energycharts_json, status_code, exception + ): + """Test handling of various API status codes.""" + mock_response = Mock() + mock_response.status_code = status_code + mock_response.content = json.dumps(sample_energycharts_json) + mock_response.raise_for_status.side_effect = ( + requests.exceptions.HTTPError if exception else None + ) + mock_get.return_value = mock_response + if exception: + with pytest.raises(exception): + provider._request_forecast() + else: provider._request_forecast() - else: - provider._request_forecast() -@patch("requests.get") -@patch("akkudoktoreos.core.cache.CacheFileStore") -def test_cache_integration(mock_cache, mock_get, provider, sample_energycharts_json): - """Test caching of 8-day electricity price data.""" - # Mock response object - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_energycharts_json) - mock_get.return_value = mock_response + @patch("requests.get") + @patch("akkudoktoreos.core.cache.CacheFileStore") + async def test_cache_integration(self, mock_cache, mock_get, provider, sample_energycharts_json): + """Test caching of 8-day electricity price data.""" + # Mock response object + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_energycharts_json) + mock_get.return_value = mock_response - # Mock cache object - mock_cache_instance = mock_cache.return_value - mock_cache_instance.get.return_value = None # Simulate no cache + # Mock cache object + mock_cache_instance = mock_cache.return_value + mock_cache_instance.get.return_value = None # Simulate no cache - provider._update_data(force_update=True) - mock_cache_instance.create.assert_called_once() - mock_cache_instance.get.assert_called_once() + await provider._update_data(force_update=True) + mock_cache_instance.create.assert_called_once() + mock_cache_instance.get.assert_called_once() -def test_key_to_array_resampling(provider): - """Test resampling of forecast data to NumPy array.""" - provider.update_data(force_update=True) - array = provider.key_to_array( - key="elecprice_marketprice_wh", - start_datetime=provider.ems_start_datetime, - end_datetime=provider.end_datetime, - ) - assert isinstance(array, np.ndarray) - assert len(array) == provider.total_hours + async def test_key_to_array_resampling(self, provider): + """Test resampling of forecast data to NumPy array.""" + await provider.update_data(force_update=True) + array = await provider.key_to_array( + key="elecprice_marketprice_wh", + start_datetime=provider.ems_start_datetime, + end_datetime=provider.end_datetime, + ) + assert isinstance(array, np.ndarray) + assert len(array) == provider.total_hours -@patch("requests.get") -def test_request_forecast_url_bidding_zone_is_value(mock_get, provider, sample_energycharts_json): - """Test that the bidding zone in the API URL uses the enum *value* (e.g. 'DE-LU'), - not the enum repr (e.g. 'EnergyChartsBiddingZones.DE_LU'). + @patch("requests.get") + def test_request_forecast_url_bidding_zone_is_value(self, mock_get, provider, sample_energycharts_json): + """Test that the bidding zone in the API URL uses the enum *value* (e.g. 'DE-LU'), + not the enum repr (e.g. 'EnergyChartsBiddingZones.DE_LU'). - Regression test for: bzn=EnergyChartsBiddingZones.DE_LU appearing in the URL - instead of bzn=DE-LU, which caused a 400 Bad Request from the Energy-Charts API. - """ - mock_response = Mock() - mock_response.status_code = 200 - mock_response.content = json.dumps(sample_energycharts_json) - mock_get.return_value = mock_response + Regression test for: bzn=EnergyChartsBiddingZones.DE_LU appearing in the URL + instead of bzn=DE-LU, which caused a 400 Bad Request from the Energy-Charts API. + """ + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = json.dumps(sample_energycharts_json) + mock_get.return_value = mock_response - provider._request_forecast(force_update=True) + provider._request_forecast(force_update=True) - assert mock_get.called, "requests.get was never called" - actual_url: str = mock_get.call_args[0][0] + assert mock_get.called, "requests.get was never called" + actual_url: str = mock_get.call_args[0][0] - # Extract the bzn= query parameter value from the URL - from urllib.parse import parse_qs, urlparse - parsed = urlparse(actual_url) - query_params = parse_qs(parsed.query) + # Extract the bzn= query parameter value from the URL + from urllib.parse import parse_qs, urlparse + parsed = urlparse(actual_url) + query_params = parse_qs(parsed.query) - assert "bzn" in query_params, f"'bzn' parameter missing from URL: {actual_url}" - bzn_value = query_params["bzn"][0] + assert "bzn" in query_params, f"'bzn' parameter missing from URL: {actual_url}" + bzn_value = query_params["bzn"][0] - # Must be the raw enum value, never contain a class name or dot notation - assert "." not in bzn_value, ( - f"Bidding zone in URL looks like an enum repr: '{bzn_value}'. " - f"Use .value when building the URL, not str(enum)." - ) - assert bzn_value == provider.config.elecprice.energycharts.bidding_zone.value, ( - f"Expected bzn='{provider.config.elecprice.energycharts.bidding_zone.value}' " - f"but got bzn='{bzn_value}' in URL: {actual_url}" - ) + # Must be the raw enum value, never contain a class name or dot notation + assert "." not in bzn_value, ( + f"Bidding zone in URL looks like an enum repr: '{bzn_value}'. " + f"Use .value when building the URL, not str(enum)." + ) + assert bzn_value == provider.config.elecprice.energycharts.bidding_zone.value, ( + f"Expected bzn='{provider.config.elecprice.energycharts.bidding_zone.value}' " + f"but got bzn='{bzn_value}' in URL: {actual_url}" + ) -# ------------------------------------------------ -# Development Energy Charts -# ------------------------------------------------ + # ------------------------------------------------ + # Development Energy Charts + # ------------------------------------------------ -@pytest.mark.skip(reason="For development only") -def test_energycharts_development_forecast_data(provider): - """Fetch data from real Energy-Charts server.""" - # Preset, as this is usually done by update_data() - provider.ems_start_datetime = to_datetime("2024-10-26 00:00:00") + @pytest.mark.skip(reason="For development only") + def test_energycharts_development_forecast_data(self, provider): + """Fetch data from real Energy-Charts server.""" + # Preset, as this is usually done by update_data() + provider.ems_start_datetime = to_datetime("2024-10-26 00:00:00") - energy_charts_data = provider._request_forecast() + energy_charts_data = provider._request_forecast() - with FILE_TESTDATA_ELECPRICE_ENERGYCHARTS_JSON.open( - "w", encoding="utf-8", newline="\n" - ) as f_out: - json.dump(energy_charts_data, f_out, indent=4) + with FILE_TESTDATA_ELECPRICE_ENERGYCHARTS_JSON.open( + "w", encoding="utf-8", newline="\n" + ) as f_out: + json.dump(energy_charts_data, f_out, indent=4) diff --git a/tests/test_elecpricefixed.py b/tests/test_elecpricefixed.py index 5a70098e..86a1f8c7 100644 --- a/tests/test_elecpricefixed.py +++ b/tests/test_elecpricefixed.py @@ -1,5 +1,6 @@ """Tests for fixed electricity price prediction module.""" +import asyncio import json from pathlib import Path from unittest.mock import Mock, patch @@ -91,6 +92,7 @@ def cache_store(): return CacheFileStore() +@pytest.mark.asyncio class TestElecPriceFixed: """Tests for ElecPriceFixed provider.""" @@ -109,7 +111,7 @@ class TestElecPriceFixed: provider.config.reset_settings() assert not provider.enabled() - def test_update_data_hourly_intervals(self, provider, config_eos): + async def test_update_data_hourly_intervals(self, provider, config_eos): """Test updating data with hourly intervals (3600s).""" # Set start datetime ems_eos = get_ems() @@ -121,7 +123,7 @@ class TestElecPriceFixed: config_eos.prediction.hours = 24 # Update data - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Verify data was generated assert len(provider) == 24 # 24 hours * 1 interval per hour @@ -140,7 +142,7 @@ class TestElecPriceFixed: for i in range(8, 24): assert abs(records[i].elecprice_marketprice_wh - 0.00034) < 1e-6 - def test_update_data_15min_intervals(self, provider, config_eos): + async def test_update_data_15min_intervals(self, provider, config_eos): """Test updating data with 15-minute intervals (900s).""" ems_eos = get_ems() start_dt = to_datetime("2024-01-01 00:00:00", in_timezone="Europe/Berlin") @@ -149,7 +151,7 @@ class TestElecPriceFixed: config_eos.optimization.interval = 900 config_eos.prediction.hours = 10 # spans both windows: 00:00–10:00 = 40 intervals - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # 10 hours * 4 intervals per hour = 40 intervals assert len(provider) == 40 @@ -173,7 +175,7 @@ class TestElecPriceFixed: f"Expected day rate at interval {i}, got {records[i].elecprice_marketprice_wh}" ) - def test_update_data_30min_intervals(self, provider, config_eos): + async def test_update_data_30min_intervals(self, provider, config_eos): """Test updating data with 30-minute intervals (1800s).""" ems_eos = get_ems() start_dt = to_datetime("2024-01-01 00:00:00", in_timezone="Europe/Berlin") @@ -182,7 +184,7 @@ class TestElecPriceFixed: config_eos.optimization.interval = 1800 config_eos.prediction.hours = 10 # spans both windows: 00:00–10:00 = 20 intervals - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # 10 hours * 2 intervals per hour = 20 intervals assert len(provider) == 20 @@ -206,24 +208,24 @@ class TestElecPriceFixed: f"Expected day rate at interval {i}, got {records[i].elecprice_marketprice_wh}" ) - def test_update_data_without_config(self, provider, config_eos): + async def test_update_data_without_config(self, provider, config_eos): """Test update_data fails without configuration.""" # Remove elecpricefixed settings config_eos.elecprice.elecpricefixed = {} with pytest.raises(ValueError, match="No time windows configured"): - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) - def test_update_data_without_time_windows(self, provider, config_eos): + async def test_update_data_without_time_windows(self, provider, config_eos): """Test update_data fails without time windows.""" # Set empty time windows empty_settings = ElecPriceFixedCommonSettings(time_windows=ValueTimeWindowSequence(windows=[])) config_eos.elecprice.elecpricefixed = empty_settings with pytest.raises(ValueError, match="No time windows configured"): - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) - def test_key_to_array_resampling(self, provider, config_eos): + async def test_key_to_array_resampling(self, provider, config_eos): """Test that key_to_array can resample to different intervals.""" # Setup provider with hourly data ems_eos = get_ems() @@ -233,10 +235,10 @@ class TestElecPriceFixed: config_eos.optimization.interval = 3600 config_eos.prediction.hours = 24 - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Get data as hourly array (original) - hourly_array = provider.key_to_array( + hourly_array = await provider.key_to_array( key="elecprice_marketprice_wh", start_datetime=start_dt, end_datetime=start_dt.add(hours=24) @@ -247,7 +249,7 @@ class TestElecPriceFixed: assert abs(hourly_array[8] - 0.00034) < 1e-6 # Day rate # Resample to 15-minute intervals - quarter_hour_array = provider.key_to_array( + quarter_hour_array = await provider.key_to_array( key="elecprice_marketprice_wh", start_datetime=start_dt, end_datetime=start_dt.add(hours=24), @@ -260,7 +262,7 @@ class TestElecPriceFixed: assert abs(quarter_hour_array[i] - 0.000288) < 1e-6 # Resample to 30-minute intervals - half_hour_array = provider.key_to_array( + half_hour_array = await provider.key_to_array( key="elecprice_marketprice_wh", start_datetime=start_dt, end_datetime=start_dt.add(hours=24), @@ -277,7 +279,7 @@ class TestElecPriceFixedIntegration: """Integration tests for ElecPriceFixed.""" @pytest.mark.skip(reason="For development only") - def test_fixed_price_development(self, config_eos): + async def test_fixed_price_development(self, config_eos): """Test fixed price provider with real configuration.""" # Create provider with config provider = ElecPriceFixed() @@ -308,7 +310,7 @@ class TestElecPriceFixedIntegration: config_eos.optimization.interval = 900 # 15 minutes # Update data - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Verify data expected_intervals = 168 * 4 # 7 days * 24h * 4 intervals diff --git a/tests/test_elecpriceimport.py b/tests/test_elecpriceimport.py index 5ade60df..f91c564b 100644 --- a/tests/test_elecpriceimport.py +++ b/tests/test_elecpriceimport.py @@ -1,3 +1,4 @@ +import asyncio import json from pathlib import Path @@ -39,76 +40,78 @@ def sample_import_1_json(): return input_data -# ------------------------------------------------ -# General forecast -# ------------------------------------------------ +@pytest.mark.asyncio +class TestElecPriceImport: + # ------------------------------------------------ + # General forecast + # ------------------------------------------------ -def test_singleton_instance(provider): - """Test that ElecPriceForecast behaves as a singleton.""" - another_instance = ElecPriceImport() - assert provider is another_instance + def test_singleton_instance(self, provider): + """Test that ElecPriceForecast behaves as a singleton.""" + another_instance = ElecPriceImport() + assert provider is another_instance -def test_invalid_provider(provider, config_eos): - """Test requesting an unsupported provider.""" - settings = { - "elecprice": { - "provider": "", - "elecpriceimport": { - "import_file_path": str(FILE_TESTDATA_ELECPRICEIMPORT_1_JSON), - }, + def test_invalid_provider(self, provider, config_eos): + """Test requesting an unsupported provider.""" + settings = { + "elecprice": { + "provider": "", + "elecpriceimport": { + "import_file_path": str(FILE_TESTDATA_ELECPRICEIMPORT_1_JSON), + }, + } } - } - with pytest.raises(ValueError, match="not a valid electricity price provider"): - config_eos.merge_settings_from_dict(settings) + with pytest.raises(ValueError, match="not a valid electricity price provider"): + config_eos.merge_settings_from_dict(settings) -# ------------------------------------------------ -# Import -# ------------------------------------------------ + # ------------------------------------------------ + # Import + # ------------------------------------------------ -@pytest.mark.parametrize( - "start_datetime, from_file", - [ - ("2024-11-10 00:00:00", True), # No DST in Germany - ("2024-08-10 00:00:00", True), # DST in Germany - ("2024-03-31 00:00:00", True), # DST change in Germany (23 hours/ day) - ("2024-10-27 00:00:00", True), # DST change in Germany (25 hours/ day) - ("2024-11-10 00:00:00", False), # No DST in Germany - ("2024-08-10 00:00:00", False), # DST in Germany - ("2024-03-31 00:00:00", False), # DST change in Germany (23 hours/ day) - ("2024-10-27 00:00:00", False), # DST change in Germany (25 hours/ day) - ], -) -def test_import(provider, sample_import_1_json, start_datetime, from_file, config_eos): - """Test fetching forecast from Import.""" - key = "elecprice_marketprice_wh" - ems_eos = get_ems() - ems_eos.set_start_datetime(to_datetime(start_datetime, in_timezone="Europe/Berlin")) - if from_file: - config_eos.elecprice.elecpriceimport.import_json = None - assert config_eos.elecprice.elecpriceimport.import_json is None - else: - config_eos.elecprice.elecpriceimport.import_file_path = None - assert config_eos.elecprice.elecpriceimport.import_file_path is None - provider.delete_by_datetime(start_datetime=None, end_datetime=None) - - # Call the method - provider.update_data() - - # Assert: Verify the result is as expected - assert provider.ems_start_datetime is not None - assert provider.total_hours is not None - assert compare_datetimes(provider.ems_start_datetime, ems_eos.start_datetime).equal - - expected_values = sample_import_1_json[key] - result_values = provider.key_to_array( - key=key, - start_datetime=provider.ems_start_datetime, - end_datetime=provider.ems_start_datetime + to_duration(f"{len(expected_values)} hours"), - interval=to_duration("1 hour"), + @pytest.mark.parametrize( + "start_datetime, from_file", + [ + ("2024-11-10 00:00:00", True), # No DST in Germany + ("2024-08-10 00:00:00", True), # DST in Germany + ("2024-03-31 00:00:00", True), # DST change in Germany (23 hours/ day) + ("2024-10-27 00:00:00", True), # DST change in Germany (25 hours/ day) + ("2024-11-10 00:00:00", False), # No DST in Germany + ("2024-08-10 00:00:00", False), # DST in Germany + ("2024-03-31 00:00:00", False), # DST change in Germany (23 hours/ day) + ("2024-10-27 00:00:00", False), # DST change in Germany (25 hours/ day) + ], ) - # Allow for some difference due to value calculation on DST change - npt.assert_allclose(result_values, expected_values, rtol=0.001) + async def test_import(self, provider, sample_import_1_json, start_datetime, from_file, config_eos): + """Test fetching forecast from Import.""" + key = "elecprice_marketprice_wh" + ems_eos = get_ems() + ems_eos.set_start_datetime(to_datetime(start_datetime, in_timezone="Europe/Berlin")) + if from_file: + config_eos.elecprice.elecpriceimport.import_json = None + assert config_eos.elecprice.elecpriceimport.import_json is None + else: + config_eos.elecprice.elecpriceimport.import_file_path = None + assert config_eos.elecprice.elecpriceimport.import_file_path is None + await provider.delete_by_datetime(start_datetime=None, end_datetime=None) + + # Call the method + await provider.update_data() + + # Assert: Verify the result is as expected + assert provider.ems_start_datetime is not None + assert provider.total_hours is not None + assert compare_datetimes(provider.ems_start_datetime, ems_eos.start_datetime).equal + + expected_values = sample_import_1_json[key] + result_values = await provider.key_to_array( + key=key, + start_datetime=provider.ems_start_datetime, + end_datetime=provider.ems_start_datetime + to_duration(f"{len(expected_values)} hours"), + interval=to_duration("1 hour"), + ) + # Allow for some difference due to value calculation on DST change + npt.assert_allclose(result_values, expected_values, rtol=0.001) diff --git a/tests/test_geneticoptimize.py b/tests/test_geneticoptimize.py index 609e0ea8..8797f6ba 100644 --- a/tests/test_geneticoptimize.py +++ b/tests/test_geneticoptimize.py @@ -37,6 +37,7 @@ def compare_dict(actual: dict[str, Any], expected: dict[str, Any]): assert actual[key] == pytest.approx(value) +@pytest.mark.asyncio @pytest.mark.parametrize( "fn_in, fn_out, ngen, break_even", [ diff --git a/tests/test_inverter_efficiency.py b/tests/test_inverter_efficiency.py index a07885cd..c13eaccc 100644 --- a/tests/test_inverter_efficiency.py +++ b/tests/test_inverter_efficiency.py @@ -257,8 +257,8 @@ class TestAcChargingInSimulation: battery=akku, ) - sim = GeneticSimulation() - sim.prepare( + simulation = GeneticSimulation() + simulation.prepare( GeneticEnergyManagementParameters( pv_prognose_wh=[0.0] * prediction_hours, # No PV strompreis_euro_pro_wh=[0.0003] * prediction_hours, # ~30ct/kWh @@ -272,7 +272,7 @@ class TestAcChargingInSimulation: ev=None, home_appliance=None, ) - return sim, akku, inverter + return simulation, akku, inverter return _build diff --git a/tests/test_loadakkudoktor.py b/tests/test_loadakkudoktor.py index 3fae5eae..2bfdbe98 100644 --- a/tests/test_loadakkudoktor.py +++ b/tests/test_loadakkudoktor.py @@ -1,8 +1,10 @@ +import asyncio from unittest.mock import patch import numpy as np import pendulum import pytest +import pytest_asyncio from akkudoktoreos.core.coreabc import get_ems, get_measurement from akkudoktoreos.measurement.measurement import MeasurementDataRecord @@ -49,8 +51,8 @@ def loadakkudoktoradjusted(config_eos): assert config_eos.load.loadakkudoktor.loadakkudoktor_year_energy_kwh == 1000 return LoadAkkudoktorAdjusted() -@pytest.fixture -def measurement_eos(): +@pytest_asyncio.fixture +async def measurement_eos(): """Fixture to initialise the Measurement instance.""" # Load meter readings are in kWh measurement = get_measurement() @@ -59,7 +61,7 @@ def measurement_eos(): dt = to_datetime("2024-01-01T00:00:00") interval = to_duration("1 hour") for i in range(25): - measurement.insert_by_datetime( + await measurement.insert_by_datetime( MeasurementDataRecord( date_time=dt, load0_mr=load0_mr, @@ -70,8 +72,10 @@ def measurement_eos(): # 0.05 kWh = 50 Wh load0_mr += 0.05 load1_mr += 0.05 - assert compare_datetimes(measurement.min_datetime, to_datetime("2024-01-01T00:00:00")).equal - assert compare_datetimes(measurement.max_datetime, to_datetime("2024-01-02T00:00:00")).equal + min_dt = await measurement.min_datetime() + max_dt = await measurement.max_datetime() + assert compare_datetimes(min_dt, to_datetime("2024-01-01T00:00:00")).equal + assert compare_datetimes(max_dt, to_datetime("2024-01-02T00:00:00")).equal return measurement @@ -87,163 +91,166 @@ def mock_load_profiles_file(tmp_path): return load_profiles_path -def test_loadakkudoktor_settings_validator(): - """Test the field validator for `loadakkudoktor_year_energy_kwh`.""" - settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234) - assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) - assert settings.loadakkudoktor_year_energy_kwh == 1234.0 +@pytest.mark.asyncio +class TestLoadAkkudoktor: - settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234.56) - assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) - assert settings.loadakkudoktor_year_energy_kwh == 1234.56 + async def test_loadakkudoktor_settings_validator(self): + """Test the field validator for `loadakkudoktor_year_energy_kwh`.""" + settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234) + assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) + assert settings.loadakkudoktor_year_energy_kwh == 1234.0 + + settings = LoadAkkudoktorCommonSettings(loadakkudoktor_year_energy_kwh=1234.56) + assert isinstance(settings.loadakkudoktor_year_energy_kwh, float) + assert settings.loadakkudoktor_year_energy_kwh == 1234.56 + + async def test_loadakkudoktor_provider_id(self, loadakkudoktor): + """Test the `provider_id` class method.""" + assert loadakkudoktor.provider_id() == "LoadAkkudoktor" + + @patch("akkudoktoreos.prediction.loadakkudoktor.np.load") + async def test_load_data_from_mock(self, mock_np_load, mock_load_profiles_file, loadakkudoktor): + """Test the `load_data` method.""" + # Mock numpy load to return data similar to what would be in the file + mock_np_load.return_value = { + "yearly_profiles": np.ones((365, 24)), + "yearly_profiles_std": np.zeros((365, 24)), + } + + # Test data loading + data_year_energy = loadakkudoktor.load_data() + assert data_year_energy is not None + assert data_year_energy.shape == (365, 2, 24) + + async def test_load_data_from_file(self, loadakkudoktor): + """Test `load_data` loads data from the profiles file.""" + data_year_energy = loadakkudoktor.load_data() + assert data_year_energy is not None + + @patch("akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktor.load_data") + async def test_update_data(self, mock_load_data, loadakkudoktor): + """Test the `_update` method.""" + mock_load_data.return_value = np.random.rand(365, 2, 24) + + # Mock methods for updating values + ems_eos = get_ems() + ems_eos.set_start_datetime(pendulum.datetime(2024, 1, 1)) + + # Assure there are no prediction records + await loadakkudoktor.delete_by_datetime(start_datetime=None, end_datetime=None) + assert len(loadakkudoktor) == 0 + + # Execute the method + await loadakkudoktor._update_data() + + # Validate that update_value is called + assert len(loadakkudoktor) > 0 -def test_loadakkudoktor_provider_id(loadakkudoktor): - """Test the `provider_id` class method.""" - assert loadakkudoktor.provider_id() == "LoadAkkudoktor" +@pytest.mark.asyncio +class TestLoadAkkudoktorAdjusted: + async def test_calculate_adjustment(self, loadakkudoktoradjusted, measurement_eos): + """Test `_calculate_adjustment` for various scenarios.""" + data_year_energy = np.random.rand(365, 2, 24) -@patch("akkudoktoreos.prediction.loadakkudoktor.np.load") -def test_load_data_from_mock(mock_np_load, mock_load_profiles_file, loadakkudoktor): - """Test the `load_data` method.""" - # Mock numpy load to return data similar to what would be in the file - mock_np_load.return_value = { - "yearly_profiles": np.ones((365, 24)), - "yearly_profiles_std": np.zeros((365, 24)), - } + # Check the test setup + assert loadakkudoktoradjusted.measurement is measurement_eos + min_dt = await measurement_eos.min_datetime() + assert min_dt == to_datetime("2024-01-01T00:00:00") + max_dt = await measurement_eos.max_datetime() + assert max_dt == to_datetime("2024-01-02T00:00:00") + # Use same calculation as in _calculate_adjustment + compare_start = max_dt - to_duration("7 days") + if compare_datetimes(compare_start, min_dt).lt: + # Not enough measurements for 7 days - use what is available + compare_start = min_dt + compare_end = max_dt + compare_interval = to_duration("1 hour") + load_total_kwh_array = await measurement_eos.load_total_kwh( + start_datetime=compare_start, + end_datetime=compare_end, + interval=compare_interval, + ) + np.testing.assert_allclose(load_total_kwh_array, [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) - # Test data loading - data_year_energy = loadakkudoktor.load_data() - assert data_year_energy is not None - assert data_year_energy.shape == (365, 2, 24) + # Call the method and validate results + weekday_adjust, weekend_adjust = await loadakkudoktoradjusted._calculate_adjustment(data_year_energy) + assert weekday_adjust.shape == (24,) + assert weekend_adjust.shape == (24,) + data_year_energy = np.zeros((365, 2, 24)) + weekday_adjust, weekend_adjust = await loadakkudoktoradjusted._calculate_adjustment(data_year_energy) -def test_load_data_from_file(loadakkudoktor): - """Test `load_data` loads data from the profiles file.""" - data_year_energy = loadakkudoktor.load_data() - assert data_year_energy is not None + assert weekday_adjust.shape == (24,) + expected = np.array( + [ + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + 100.0, + ] + ) + np.testing.assert_allclose(weekday_adjust, expected) + assert weekend_adjust.shape == (24,) + expected = np.array( + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] + ) + np.testing.assert_array_equal(weekend_adjust, expected) -@patch("akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktor.load_data") -def test_update_data(mock_load_data, loadakkudoktor): - """Test the `_update` method.""" - mock_load_data.return_value = np.random.rand(365, 2, 24) + async def test_provider_adjustments_with_mock_data(self, loadakkudoktoradjusted): + """Test full integration of adjustments with mock data.""" + with patch( + "akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktorAdjusted._calculate_adjustment" + ) as mock_adjust: + mock_adjust.return_value = (np.zeros(24), np.zeros(24)) - # Mock methods for updating values - ems_eos = get_ems() - ems_eos.set_start_datetime(pendulum.datetime(2024, 1, 1)) - - # Assure there are no prediction records - loadakkudoktor.delete_by_datetime(start_datetime=None, end_datetime=None) - assert len(loadakkudoktor) == 0 - - # Execute the method - loadakkudoktor._update_data() - - # Validate that update_value is called - assert len(loadakkudoktor) > 0 - - -def test_calculate_adjustment(loadakkudoktoradjusted, measurement_eos): - """Test `_calculate_adjustment` for various scenarios.""" - data_year_energy = np.random.rand(365, 2, 24) - - # Check the test setup - assert loadakkudoktoradjusted.measurement is measurement_eos - assert measurement_eos.min_datetime == to_datetime("2024-01-01T00:00:00") - assert measurement_eos.max_datetime == to_datetime("2024-01-02T00:00:00") - # Use same calculation as in _calculate_adjustment - compare_start = measurement_eos.max_datetime - to_duration("7 days") - if compare_datetimes(compare_start, measurement_eos.min_datetime).lt: - # Not enough measurements for 7 days - use what is available - compare_start = measurement_eos.min_datetime - compare_end = measurement_eos.max_datetime - compare_interval = to_duration("1 hour") - load_total_kwh_array = measurement_eos.load_total_kwh( - start_datetime=compare_start, - end_datetime=compare_end, - interval=compare_interval, - ) - np.testing.assert_allclose(load_total_kwh_array, [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) - - # Call the method and validate results - weekday_adjust, weekend_adjust = loadakkudoktoradjusted._calculate_adjustment(data_year_energy) - assert weekday_adjust.shape == (24,) - assert weekend_adjust.shape == (24,) - - data_year_energy = np.zeros((365, 2, 24)) - weekday_adjust, weekend_adjust = loadakkudoktoradjusted._calculate_adjustment(data_year_energy) - - assert weekday_adjust.shape == (24,) - expected = np.array( - [ - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - 100.0, - ] - ) - np.testing.assert_allclose(weekday_adjust, expected) - - assert weekend_adjust.shape == (24,) - expected = np.array( - [ - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - ] - ) - np.testing.assert_array_equal(weekend_adjust, expected) - - -def test_provider_adjustments_with_mock_data(loadakkudoktoradjusted): - """Test full integration of adjustments with mock data.""" - with patch( - "akkudoktoreos.prediction.loadakkudoktor.LoadAkkudoktorAdjusted._calculate_adjustment" - ) as mock_adjust: - mock_adjust.return_value = (np.zeros(24), np.zeros(24)) - - # Test execution - loadakkudoktoradjusted._update_data() - assert mock_adjust.called + # Test execution + await loadakkudoktoradjusted._update_data() + assert mock_adjust.called diff --git a/tests/test_loadvrm.py b/tests/test_loadvrm.py index 9dae014b..062b148a 100644 --- a/tests/test_loadvrm.py +++ b/tests/test_loadvrm.py @@ -1,3 +1,4 @@ +import asyncio import json from unittest.mock import call, patch @@ -48,69 +49,67 @@ def mock_forecast_response(): totals={} ) +@pytest.mark.asyncio +class TestLoadVRM: -def test_update_data_calls_update_value(load_vrm_instance): - with patch.object(load_vrm_instance, "_request_forecast", return_value=mock_forecast_response()), \ - patch.object(LoadVrm, "update_value") as mock_update: + async def test_update_data_calls_update_value(self, load_vrm_instance): + with patch.object(load_vrm_instance, "_request_forecast", return_value=mock_forecast_response()), \ + patch.object(LoadVrm, "update_value") as mock_update: - load_vrm_instance._update_data() + await load_vrm_instance._update_data() - assert mock_update.call_count == 2 + assert mock_update.call_count == 2 - expected_calls = [ - call( - pendulum.datetime(2025, 1, 1, 0, 0, 0, tz='Europe/Berlin'), - {"loadforecast_power_w": 100.5,} - ), - call( - pendulum.datetime(2025, 1, 1, 1, 0, 0, tz='Europe/Berlin'), - {"loadforecast_power_w": 101.2,} - ), - ] + expected_calls = [ + call( + pendulum.datetime(2025, 1, 1, 0, 0, 0, tz='Europe/Berlin'), + {"loadforecast_power_w": 100.5,} + ), + call( + pendulum.datetime(2025, 1, 1, 1, 0, 0, tz='Europe/Berlin'), + {"loadforecast_power_w": 101.2,} + ), + ] - mock_update.assert_has_calls(expected_calls, any_order=False) + mock_update.assert_has_calls(expected_calls, any_order=False) + def test_validate_data_accepts_valid_json(self): + """Test that _validate_data doesn't raise with valid input.""" + response = mock_forecast_response() + json_data = response.model_dump_json() -def test_validate_data_accepts_valid_json(): - """Test that _validate_data doesn't raise with valid input.""" - response = mock_forecast_response() - json_data = response.model_dump_json() + validated = LoadVrm._validate_data(json_data) + assert validated.success + assert len(validated.records.vrm_consumption_fc) == 2 - validated = LoadVrm._validate_data(json_data) - assert validated.success - assert len(validated.records.vrm_consumption_fc) == 2 + def test_validate_data_raises_on_invalid_json(self): + """_validate_data should raise ValueError on schema mismatch.""" + invalid_json = json.dumps({"success": True}) # missing 'records' + with pytest.raises(ValueError) as exc_info: + LoadVrm._validate_data(invalid_json) -def test_validate_data_raises_on_invalid_json(): - """_validate_data should raise ValueError on schema mismatch.""" - invalid_json = json.dumps({"success": True}) # missing 'records' + assert "Field:" in str(exc_info.value) + assert "records" in str(exc_info.value) - with pytest.raises(ValueError) as exc_info: - LoadVrm._validate_data(invalid_json) + def test_request_forecast_raises_on_http_error(self, load_vrm_instance): + with patch("requests.get", side_effect=requests.Timeout("Request timed out")) as mock_get: + with pytest.raises(RuntimeError) as exc_info: + load_vrm_instance._request_forecast(0, 1) - assert "Field:" in str(exc_info.value) - assert "records" in str(exc_info.value) + assert "Failed to fetch load forecast" in str(exc_info.value) + mock_get.assert_called_once() + async def test_update_data_does_nothing_on_empty_forecast(self, load_vrm_instance): + empty_response = VrmForecastResponse( + success=True, + records=VrmForecastRecords(vrm_consumption_fc=[], solar_yield_forecast=[]), + totals={} + ) -def test_request_forecast_raises_on_http_error(load_vrm_instance): - with patch("requests.get", side_effect=requests.Timeout("Request timed out")) as mock_get: - with pytest.raises(RuntimeError) as exc_info: - load_vrm_instance._request_forecast(0, 1) + with patch.object(load_vrm_instance, "_request_forecast", return_value=empty_response), \ + patch.object(LoadVrm, "update_value") as mock_update: - assert "Failed to fetch load forecast" in str(exc_info.value) - mock_get.assert_called_once() + await load_vrm_instance._update_data() - -def test_update_data_does_nothing_on_empty_forecast(load_vrm_instance): - empty_response = VrmForecastResponse( - success=True, - records=VrmForecastRecords(vrm_consumption_fc=[], solar_yield_forecast=[]), - totals={} - ) - - with patch.object(load_vrm_instance, "_request_forecast", return_value=empty_response), \ - patch.object(LoadVrm, "update_value") as mock_update: - - load_vrm_instance._update_data() - - mock_update.assert_not_called() + mock_update.assert_not_called() diff --git a/tests/test_measurement.py b/tests/test_measurement.py index 4131f3b4..fe48b73d 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -1,5 +1,6 @@ import numpy as np import pytest +import pytest_asyncio from pendulum import datetime, duration from akkudoktoreos.config.config import SettingsEOS @@ -219,16 +220,17 @@ class TestMeasurementDataRecord: assert key in keys +@pytest.mark.asyncio class TestMeasurement: """Test suite for the Measuremen class.""" - @pytest.fixture - def measurement_eos(self, config_eos): + @pytest_asyncio.fixture + async def measurement_eos(self, config_eos): """Fixture to create a Measurement instance.""" # Load meter readings are in kWh config_eos.measurement.load_emr_keys = ["load0_mr", "load1_mr", "load2_mr", "load3_mr"] measurement = get_measurement() - measurement.delete_by_datetime(None, None) + await measurement.delete_by_datetime(None, None) record0 = MeasurementDataRecord( date_time=datetime(2023, 1, 1, hour=0), load0_mr=100, @@ -269,10 +271,10 @@ class TestMeasurement: ), ] for record in records: - measurement.insert_by_datetime(record) + await measurement.insert_by_datetime(record) return measurement - def test_interval_count(self, measurement_eos): + async def test_interval_count(self, measurement_eos): """Test interval count calculation.""" start = to_datetime("2023-01-01T00:00:00") end = to_datetime("2023-01-01T03:00:00") @@ -280,7 +282,7 @@ class TestMeasurement: assert measurement_eos._interval_count(start, end, interval) == 3 - def test_interval_count_invalid_end_before_start(self, measurement_eos): + async def test_interval_count_invalid_end_before_start(self, measurement_eos): """Test interval count raises ValueError when end_datetime is before start_datetime.""" start = to_datetime("2023-01-01T03:00:00") end = to_datetime("2023-01-01T00:00:00") @@ -289,7 +291,7 @@ class TestMeasurement: with pytest.raises(ValueError, match="end_datetime must be after start_datetime"): measurement_eos._interval_count(start, end, interval) - def test_interval_count_invalid_non_positive_interval(self, measurement_eos): + async def test_interval_count_invalid_non_positive_interval(self, measurement_eos): """Test interval count raises ValueError when interval is non-positive.""" start = to_datetime("2023-01-01T00:00:00") end = to_datetime("2023-01-01T03:00:00") @@ -297,21 +299,21 @@ class TestMeasurement: with pytest.raises(ValueError, match="interval must be positive"): measurement_eos._interval_count(start, end, duration(hours=0)) - def test_energy_from_meter_readings_valid_input(self, measurement_eos): + async def test_energy_from_meter_readings_valid_input(self, measurement_eos): """Test _energy_from_meter_readings with valid inputs and proper alignment of load data.""" key = "load0_mr" start_datetime = to_datetime("2023-01-01T00:00:00") end_datetime = to_datetime("2023-01-01T05:00:00") interval = duration(hours=1) - load_array = measurement_eos._energy_from_meter_readings( + load_array = await measurement_eos._energy_from_meter_readings( key, start_datetime, end_datetime, interval ) expected_load_array = np.array([50, 50, 50, 50, 50]) # Differences between consecutive readings np.testing.assert_array_equal(load_array, expected_load_array) - def test_energy_from_meter_readings_empty_array(self, measurement_eos): + async def test_energy_from_meter_readings_empty_array(self, measurement_eos): """Test _energy_from_meter_readings with no data (empty array).""" key = "load0_mr" start_datetime = to_datetime("2023-01-01T00:00:00") @@ -319,9 +321,9 @@ class TestMeasurement: interval = duration(hours=1) # Use empyt records array - measurement_eos.delete_by_datetime(start_datetime, end_datetime) + await measurement_eos.delete_by_datetime(start_datetime, end_datetime) - load_array = measurement_eos._energy_from_meter_readings( + load_array = await measurement_eos._energy_from_meter_readings( key, start_datetime, end_datetime, interval ) @@ -332,7 +334,7 @@ class TestMeasurement: expected_load_array = np.zeros(expected_size) np.testing.assert_array_equal(load_array, expected_load_array) - def test_energy_from_meter_readings_misaligned_array(self, measurement_eos): + async def test_energy_from_meter_readings_misaligned_array(self, measurement_eos): """Test _energy_from_meter_readings with misaligned array size.""" key = "load1_mr" interval = duration(hours=1) @@ -342,15 +344,15 @@ class TestMeasurement: # Use misaligned array, latest interval set to 2 hours (instead of 1 hour) latest_record_datetime = to_datetime("2023-01-01T05:00:00") new_record_datetime = to_datetime("2023-01-01T06:00:00") - record = measurement_eos.get_by_datetime(latest_record_datetime) + record = await measurement_eos.get_by_datetime(latest_record_datetime) assert record is not None - measurement_eos.delete_by_datetime(start_datetime = latest_record_datetime, - end_datetime = new_record_datetime) + await measurement_eos.delete_by_datetime(start_datetime = latest_record_datetime, + end_datetime = new_record_datetime) record.date_time = new_record_datetime - measurement_eos.insert_by_datetime(record) + await measurement_eos.insert_by_datetime(record) # Check test setup - dates, values = measurement_eos.key_to_lists(key, start_datetime, None) + dates, values = await measurement_eos.key_to_lists(key, start_datetime, None) assert dates == [ to_datetime("2023-01-01T00:00:00"), to_datetime("2023-01-01T01:00:00"), @@ -360,17 +362,17 @@ class TestMeasurement: to_datetime("2023-01-01T06:00:00"), ] assert values == [200, 250, 300, 350, 400, 450] - array = measurement_eos.key_to_array(key, start_datetime, end_datetime + interval, interval=interval) + array = await measurement_eos.key_to_array(key, start_datetime, end_datetime + interval, interval=interval) np.testing.assert_array_equal(array, [200, 250, 300, 350, 400, 425]) - load_array = measurement_eos._energy_from_meter_readings( + load_array = await measurement_eos._energy_from_meter_readings( key, start_datetime, end_datetime, interval ) expected_load_array = np.array([50., 50., 50., 50., 25.]) # Differences between consecutive readings np.testing.assert_array_equal(load_array, expected_load_array) - def test_energy_from_meter_readings_partial_data(self, measurement_eos, caplog): + async def test_energy_from_meter_readings_partial_data(self, measurement_eos, caplog): """Test _energy_from_meter_readings with partial data (misaligned but empty array).""" key = "load2_mr" start_datetime = to_datetime("2023-01-01T00:00:00") @@ -378,7 +380,7 @@ class TestMeasurement: interval = duration(hours=1) with caplog.at_level("DEBUG"): - load_array = measurement_eos._energy_from_meter_readings( + load_array = await measurement_eos._energy_from_meter_readings( key, start_datetime, end_datetime, interval ) @@ -388,7 +390,7 @@ class TestMeasurement: expected_load_array = np.zeros(expected_size) np.testing.assert_array_equal(load_array, expected_load_array) - def test_energy_from_meter_readings_negative_interval(self, measurement_eos): + async def test_energy_from_meter_readings_negative_interval(self, measurement_eos): """Test _energy_from_meter_readings with a negative interval.""" key = "load3_mr" start_datetime = to_datetime("2023-01-01T00:00:00") @@ -396,37 +398,37 @@ class TestMeasurement: interval = duration(hours=-1) with pytest.raises(ValueError, match="interval must be positive"): - measurement_eos._energy_from_meter_readings(key, start_datetime, end_datetime, interval) + await measurement_eos._energy_from_meter_readings(key, start_datetime, end_datetime, interval) - def test_load_total_kwh(self, measurement_eos): + async def test_load_total_kwh(self, measurement_eos): """Test total load calculation.""" start_datetime = to_datetime("2023-01-01T03:00:00") end_datetime = to_datetime("2023-01-01T05:00:00") interval = duration(hours=1) - result = measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) + result = await measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) # Expected total load per interval expected = np.array([100, 100]) # Differences between consecutive meter readings np.testing.assert_array_equal(result, expected) - def test_load_total_kwh_no_data(self, measurement_eos): + async def test_load_total_kwh_no_data(self, measurement_eos): """Test total load calculation with no data.""" measurement_eos.records = [] start_datetime = to_datetime("2023-01-01T00:00:00") end_datetime = to_datetime("2023-01-01T03:00:00") interval = duration(hours=1) - result = measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) + result = await measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) expected = np.zeros(3) # No data, so all intervals are zero np.testing.assert_array_equal(result, expected) - def test_load_total_kwh_partial_intervals(self, measurement_eos): + async def test_load_total_kwh_partial_intervals(self, measurement_eos): """Test total load calculation with partial intervals.""" start_datetime = to_datetime("2023-01-01T00:30:00") # Start in the middle of an interval end_datetime = to_datetime("2023-01-01T01:30:00") # End in the middle of another interval interval = duration(hours=1) - result = measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) + result = await measurement_eos.load_total_kwh(start_datetime=start_datetime, end_datetime=end_datetime, interval=interval) expected = np.array([100]) # Only one complete interval covered np.testing.assert_array_equal(result, expected) diff --git a/tests/test_prediction.py b/tests/test_prediction.py index b973017a..9d597a08 100644 --- a/tests/test_prediction.py +++ b/tests/test_prediction.py @@ -133,13 +133,14 @@ def test_prediction_repr(prediction): assert "WeatherImport" in result -def test_empty_providers(prediction, forecast_providers): +@pytest.mark.asyncio +async def test_empty_providers(prediction, forecast_providers): """Test behavior when Prediction does not have providers.""" # Clear all prediction providers from prediction providers_bkup = prediction.providers.copy() prediction.providers.clear() assert prediction.providers == [] - prediction.update_data() # Should not raise an error even with no providers + await prediction.update_data() # Should not raise an error even with no providers # Cleanup after Test prediction.providers = providers_bkup diff --git a/tests/test_predictionabc.py b/tests/test_predictionabc.py index 1e9954a0..ab22db11 100644 --- a/tests/test_predictionabc.py +++ b/tests/test_predictionabc.py @@ -5,6 +5,7 @@ from typing import Any, ClassVar, List, Optional, Union import pandas as pd import pendulum import pytest +import pytest_asyncio from pydantic import Field from akkudoktoreos.core.coreabc import get_ems @@ -69,7 +70,7 @@ class DerivedPredictionProvider(PredictionProvider): def enabled(self) -> bool: return self.provider_enabled - def _update_data(self, force_update: Optional[bool] = False) -> None: + async def _update_data(self, force_update: Optional[bool] = False) -> None: # Simulate update logic DerivedPredictionProvider.provider_updated = True @@ -127,6 +128,7 @@ class TestPredictionABC: # -------------------------------------------------------- +@pytest.mark.asyncio class TestPredictionProvider: # Fixtures and helper functions @pytest.fixture @@ -147,7 +149,7 @@ class TestPredictionProvider: # Tests - def test_singleton_behavior(self, provider): + async def test_singleton_behavior(self, provider): """Test that PredictionProvider enforces singleton behavior.""" instance1 = provider instance2 = DerivedPredictionProvider() @@ -155,7 +157,7 @@ class TestPredictionProvider: "Singleton pattern is not enforced; instances are not the same." ) - def test_update_computed_fields(self, provider, sample_start_datetime): + async def test_update_computed_fields(self, provider, sample_start_datetime): """Test that computed fields `end_datetime` and `keep_datetime` are correctly calculated.""" ems_eos = get_ems() ems_eos.set_start_datetime(sample_start_datetime) @@ -176,7 +178,7 @@ class TestPredictionProvider: "Keep datetime is not calculated correctly." ) - def test_update_method_with_defaults( + async def test_update_method_with_defaults( self, provider, sample_start_datetime, config_eos, monkeypatch ): """Test the `update` method with default parameters.""" @@ -188,7 +190,7 @@ class TestPredictionProvider: provider.config.reset_settings() ems_eos.set_start_datetime(sample_start_datetime) - provider.update_data() + await provider.update_data() assert provider.config.prediction.hours == config_eos.prediction.hours assert provider.config.prediction.historic_hours == 2 @@ -198,7 +200,7 @@ class TestPredictionProvider: ) assert provider.keep_datetime == sample_start_datetime - to_duration("2 hours") - def test_update_method_force_enable(self, provider, monkeypatch): + async def test_update_method_force_enable(self, provider, monkeypatch): """Test that `update` executes when `force_enable` is True, even if `enabled` is False.""" # Preset values that are needed by update monkeypatch.setenv("EOS_GENERAL__LATITUDE", "37.7749") @@ -207,13 +209,13 @@ class TestPredictionProvider: # Override enabled to return False for this test DerivedPredictionProvider.provider_enabled = False DerivedPredictionProvider.provider_updated = False - provider.update_data(force_enable=True) + await provider.update_data(force_enable=True) assert provider.enabled() is False, "Provider should be disabled, but enabled() is True." assert DerivedPredictionProvider.provider_updated is True, ( "Provider should have been executed, but was not." ) - def test_delete_by_datetime(self, provider, sample_start_datetime): + async def test_delete_by_datetime(self, provider, sample_start_datetime): """Test `delete_by_datetime` method for removing records by datetime range.""" # Add records to the provider for deletion testing records = [ @@ -222,9 +224,9 @@ class TestPredictionProvider: self.create_test_record(sample_start_datetime + to_duration("1 hour"), 3), ] for record in records: - provider.insert_by_datetime(record) + await provider.insert_by_datetime(record) - provider.delete_by_datetime( + await provider.delete_by_datetime( start_datetime=sample_start_datetime - to_duration("2 hours"), end_datetime=sample_start_datetime + to_duration("2 hours"), ) @@ -236,6 +238,7 @@ class TestPredictionProvider: ) +@pytest.mark.asyncio class TestPredictionContainer: # Fixture and helpers @pytest.fixture @@ -243,8 +246,8 @@ class TestPredictionContainer: container = DerivedPredictionContainer() return container - @pytest.fixture - def container_with_providers(self): + @pytest_asyncio.fixture + async def container_with_providers(self): records = [ # Test records - include 'prediction_value' key self.create_test_record(datetime(2023, 11, 5), 1), @@ -252,10 +255,10 @@ class TestPredictionContainer: self.create_test_record(datetime(2023, 11, 7), 3), ] provider = DerivedPredictionProvider() - provider.delete_by_datetime(start_datetime=None, end_datetime=None) + await provider.delete_by_datetime(start_datetime=None, end_datetime=None) assert len(provider) == 0 for record in records: - provider.insert_by_datetime(record) + await provider.insert_by_datetime(record) assert len(provider) == 3 container = DerivedPredictionContainer() container.providers.clear() @@ -282,7 +285,7 @@ class TestPredictionContainer: ("2024-10-27 00:00:00", 48, "2024-10-29 00:00:00"), # DST change (49 hours/ day) ], ) - def test_end_datetime(self, container, start, hours, end): + async def test_end_datetime(self, container, start, hours, end): """Test end datetime calculation from start datetime.""" ems_eos = get_ems() ems_eos.set_start_datetime(to_datetime(start, in_timezone="Europe/Berlin")) @@ -312,7 +315,7 @@ class TestPredictionContainer: ), ], ) - def test_keep_datetime(self, container, start, historic_hours, expected_keep): + async def test_keep_datetime(self, container, start, historic_hours, expected_keep): """Test the `keep_datetime` property.""" ems_eos = get_ems() ems_eos.set_start_datetime(to_datetime(start, in_timezone="Europe/Berlin")) @@ -334,7 +337,7 @@ class TestPredictionContainer: ("2024-10-27 00:00:00", 24, 25), # DST change in Germany (25 hours/ day) ], ) - def test_total_hours(self, container, start, hours, expected_hours): + async def test_total_hours(self, container, start, hours, expected_hours): """Test the `total_hours` property.""" ems_eos = get_ems() ems_eos.set_start_datetime(to_datetime(start, in_timezone="Europe/Berlin")) @@ -355,7 +358,7 @@ class TestPredictionContainer: ("2024-10-28 00:00:00", 24, 24), # DST change on 2024-10-27 in Germany (25 hours/ day) ], ) - def test_keep_hours(self, container, start, historic_hours, expected_hours): + async def test_keep_hours(self, container, start, historic_hours, expected_hours): """Test the `keep_hours` property.""" ems_eos = get_ems() ems_eos.set_start_datetime(to_datetime(start, in_timezone="Europe/Berlin")) @@ -367,80 +370,37 @@ class TestPredictionContainer: container.config.merge_settings_from_dict(settings) assert container.keep_hours == expected_hours - def test_append_provider(self, container): + async def test_append_provider(self, container): assert len(container.providers) == 0 container.providers.append(DerivedPredictionProvider()) assert len(container.providers) == 1 assert isinstance(container.providers[0], DerivedPredictionProvider) @pytest.mark.skip(reason="type check not implemented") - def test_append_provider_invalid_type(self, container): + async def test_append_provider_invalid_type(self, container): with pytest.raises(ValueError, match="must be an instance of PredictionProvider"): container.providers.append("not_a_provider") - def test_getitem_existing_key(self, container_with_providers): - assert len(container_with_providers.providers) == 1 - # check all keys are available (don't care for position) - for key in ["prediction_value", "date_time"]: - assert key in container_with_providers.record_keys - for key in ["prediction_value", "date_time"]: - assert key in container_with_providers.keys() - series = container_with_providers["prediction_value"] - assert isinstance(series, pd.Series) - assert series.name == "prediction_value" - assert series.tolist() == [1.0, 2.0, 3.0] - - def test_getitem_non_existing_key(self, container_with_providers): - with pytest.raises(KeyError, match="No data found for key 'non_existent_key'"): - container_with_providers["non_existent_key"] - - def test_setitem_existing_key(self, container_with_providers): - new_series = container_with_providers["prediction_value"] - new_series[:] = [4, 5, 6] - container_with_providers["prediction_value"] = new_series - series = container_with_providers["prediction_value"] - assert series.name == "prediction_value" - assert series.tolist() == [4, 5, 6] - - def test_setitem_invalid_value(self, container_with_providers): - with pytest.raises(ValueError, match="Value must be an instance of pd.Series"): - container_with_providers["test_key"] = "not_a_series" - - def test_setitem_non_existing_key(self, container_with_providers): - new_series = pd.Series([4, 5, 6], name="non_existent_key") - with pytest.raises(KeyError, match="Key 'non_existent_key' not found"): - container_with_providers["non_existent_key"] = new_series - - def test_delitem_existing_key(self, container_with_providers): - del container_with_providers["prediction_value"] - series = container_with_providers["prediction_value"] - assert series.name == "prediction_value" - assert series.tolist() == [] - - def test_delitem_non_existing_key(self, container_with_providers): - with pytest.raises(KeyError, match="Key 'non_existent_key' not found"): - del container_with_providers["non_existent_key"] - - def test_len(self, container_with_providers): + async def test_len(self, container_with_providers): assert len(container_with_providers) == 2 - def test_repr(self, container_with_providers): + async def test_repr(self, container_with_providers): representation = repr(container_with_providers) assert representation.startswith("DerivedPredictionContainer(") assert "DerivedPredictionProvider" in representation - def test_to_json(self, container_with_providers): + async def test_to_json(self, container_with_providers): json_str = container_with_providers.to_json() container_other = DerivedPredictionContainer.from_json(json_str) assert container_other == container_with_providers - def test_from_json(self, container_with_providers): + async def test_from_json(self, container_with_providers): json_str = container_with_providers.to_json() container = DerivedPredictionContainer.from_json(json_str) assert isinstance(container, DerivedPredictionContainer) assert len(container.providers) == 1 assert container.providers[0] == container_with_providers.providers[0] - def test_provider_by_id(self, container_with_providers): + async def test_provider_by_id(self, container_with_providers): provider = container_with_providers.provider_by_id("DerivedPredictionProvider") assert isinstance(provider, DerivedPredictionProvider) diff --git a/tests/test_pvforecastakkudoktor.py b/tests/test_pvforecastakkudoktor.py index 0960d8af..454af294 100644 --- a/tests/test_pvforecastakkudoktor.py +++ b/tests/test_pvforecastakkudoktor.py @@ -3,6 +3,7 @@ from pathlib import Path from unittest.mock import Mock, patch import pytest +import pytest_asyncio from loguru import logger from akkudoktoreos.core.coreabc import get_ems, get_prediction @@ -132,11 +133,11 @@ def provider(): return provider -@pytest.fixture -def provider_empty_instance(): +@pytest_asyncio.fixture +async def provider_empty_instance(): """Fixture that returns an empty instance of PVForecast.""" empty_instance = PVForecastAkkudoktor() - empty_instance.delete_by_datetime(start_datetime=None, end_datetime=None) + await empty_instance.delete_by_datetime(start_datetime=None, end_datetime=None) assert len(empty_instance) == 0 return empty_instance @@ -234,7 +235,8 @@ def test_pvforecast_akkudoktor_data_record(): ) # Assuming AC power measured is preferred -def test_pvforecast_akkudoktor_validate_data(provider_empty_instance, sample_forecast_data_raw): +@pytest.mark.asyncio +async def test_pvforecast_akkudoktor_validate_data(provider_empty_instance, sample_forecast_data_raw): """Test validation of PV forecast data on sample data.""" logger.info("The following errors are intentional and part of the test.") with pytest.raises( @@ -246,7 +248,8 @@ def test_pvforecast_akkudoktor_validate_data(provider_empty_instance, sample_for # everything worked -def test_pvforecast_akkudoktor_validate_data_single_plane( +@pytest.mark.asyncio +async def test_pvforecast_akkudoktor_validate_data_single_plane( provider_empty_instance, sample_forecast_data_single_plane_raw ): """Test validation of PV forecast data on sample data with a single plane.""" @@ -260,8 +263,9 @@ def test_pvforecast_akkudoktor_validate_data_single_plane( # everything worked +@pytest.mark.asyncio @patch("requests.get") -def test_pvforecast_akkudoktor_update_with_sample_forecast( +async def test_pvforecast_akkudoktor_update_with_sample_forecast( mock_get, sample_settings, sample_forecast_data_raw, sample_forecast_start, provider ): """Test data processing using sample forecast data.""" @@ -274,13 +278,14 @@ def test_pvforecast_akkudoktor_update_with_sample_forecast( # Test that update properly inserts data records ems_eos = get_ems() ems_eos.set_start_datetime(sample_forecast_start) - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) assert compare_datetimes(provider.ems_start_datetime, sample_forecast_start).equal assert compare_datetimes(provider.records[0].date_time, to_datetime(sample_forecast_start)).equal # Report Generation Test -def test_report_ac_power_and_measurement(provider, config_eos): +@pytest.mark.asyncio +async def test_report_ac_power_and_measurement(provider, config_eos): # Set the configuration config_eos.merge_settings_from_dict(sample_config_data) @@ -289,7 +294,7 @@ def test_report_ac_power_and_measurement(provider, config_eos): pvforecast_dc_power=450.0, pvforecast_ac_power=400.0, ) - provider.insert_by_datetime(record) + await provider.insert_by_datetime(record) report = provider.report_ac_power_and_measurement() assert "DC: 450.0" in report @@ -300,8 +305,9 @@ def test_report_ac_power_and_measurement(provider, config_eos): @pytest.mark.skipif( sys.platform.startswith("win"), reason="'other_timezone' fixture not supported on Windows" ) +@pytest.mark.asyncio @patch("requests.get") -def test_timezone_behaviour( +async def test_timezone_behaviour( mock_get, sample_settings, sample_forecast_data_raw, @@ -322,17 +328,17 @@ def test_timezone_behaviour( expected_datetime = to_datetime("2024-10-06T00:00:00+0200", in_timezone=other_timezone) assert compare_datetimes(other_start_datetime, expected_datetime).equal - provider.delete_by_datetime(start_datetime=None, end_datetime=None) + await provider.delete_by_datetime(start_datetime=None, end_datetime=None) assert len(provider) == 0 ems_eos = get_ems() ems_eos.set_start_datetime(other_start_datetime) - provider.update_data(force_update=True) + await provider.update_data(force_update=True) assert compare_datetimes(provider.ems_start_datetime, other_start_datetime).equal # Check wether first record starts at requested sample start time assert compare_datetimes(provider.records[0].date_time, sample_forecast_start).equal # Test updating AC power measurement for a specific date. - provider.update_value(sample_forecast_start, "pvforecastakkudoktor_ac_power_measured", 1000) + await provider.update_value(sample_forecast_start, "pvforecastakkudoktor_ac_power_measured", 1000) # Check wether first record was filled with ac power measurement assert provider.records[0].pvforecastakkudoktor_ac_power_measured == 1000 @@ -340,7 +346,7 @@ def test_timezone_behaviour( other_end_datetime = other_start_datetime + to_duration("24 hours") expected_end_datetime = to_datetime("2024-10-07T00:00:00+0200", in_timezone=other_timezone) assert compare_datetimes(other_end_datetime, expected_end_datetime).equal - forecast_temps = provider.key_to_series( + forecast_temps = await provider.key_to_series( "pvforecastakkudoktor_temp_air", other_start_datetime, other_end_datetime ) assert len(forecast_temps) == 23 # 24-1, first temperature is null @@ -349,7 +355,7 @@ def test_timezone_behaviour( # Test fetching AC power forecast other_end_datetime = other_start_datetime + to_duration("48 hours") - forecast_measured = provider.key_to_series( + forecast_measured = await provider.key_to_series( "pvforecastakkudoktor_ac_power_measured", other_start_datetime, other_end_datetime ) assert len(forecast_measured) == 1 diff --git a/tests/test_pvforecastimport.py b/tests/test_pvforecastimport.py index ba554b8e..b2ac4c20 100644 --- a/tests/test_pvforecastimport.py +++ b/tests/test_pvforecastimport.py @@ -72,7 +72,7 @@ def test_invalid_provider(provider, config_eos): # Import # ------------------------------------------------ - +@pytest.mark.asyncio @pytest.mark.parametrize( "start_datetime, from_file", [ @@ -86,7 +86,7 @@ def test_invalid_provider(provider, config_eos): ("2024-10-27 00:00:00", False), # DST change in Germany (25 hours/ day) ], ) -def test_import(provider, sample_import_1_json, start_datetime, from_file, config_eos): +async def test_import(provider, sample_import_1_json, start_datetime, from_file, config_eos): """Test fetching forecast from import.""" key = "pvforecast_ac_power" ems_eos = get_ems() @@ -97,10 +97,10 @@ def test_import(provider, sample_import_1_json, start_datetime, from_file, confi else: config_eos.pvforecast.provider_settings.PVForecastImport.import_file_path = None assert config_eos.pvforecast.provider_settings.PVForecastImport.import_file_path is None - provider.delete_by_datetime(start_datetime=None, end_datetime=None) + await provider.delete_by_datetime(start_datetime=None, end_datetime=None) # Call the method - provider.update_data() + await provider.update_data() # Assert: Verify the result is as expected assert provider.ems_start_datetime is not None @@ -108,7 +108,7 @@ def test_import(provider, sample_import_1_json, start_datetime, from_file, confi assert compare_datetimes(provider.ems_start_datetime, ems_eos.start_datetime).equal expected_values = sample_import_1_json[key] - result_values = provider.key_to_array( + result_values = await provider.key_to_array( key=key, start_datetime=provider.ems_start_datetime, end_datetime=provider.ems_start_datetime + to_duration(f"{len(expected_values)} hours"), diff --git a/tests/test_pvforecastvrm.py b/tests/test_pvforecastvrm.py index 73693731..00bb1e8a 100644 --- a/tests/test_pvforecastvrm.py +++ b/tests/test_pvforecastvrm.py @@ -51,11 +51,12 @@ def mock_forecast_response(): ) -def test_update_data_updates_dc_and_ac_power(pvforecast_instance): +@pytest.mark.asyncio +async def test_update_data_updates_dc_and_ac_power(pvforecast_instance): with patch.object(pvforecast_instance, "_request_forecast", return_value=mock_forecast_response()), \ patch.object(PVForecastVrm, "update_value") as mock_update: - pvforecast_instance._update_data() + await pvforecast_instance._update_data() # Check that update_value was called correctly assert mock_update.call_count == 2 @@ -103,7 +104,8 @@ def test_request_forecast_raises_on_http_error(pvforecast_instance): mock_get.assert_called_once() -def test_update_data_skips_on_empty_forecast(pvforecast_instance): +@pytest.mark.asyncio +async def test_update_data_skips_on_empty_forecast(pvforecast_instance): """Ensure no update_value calls are made if no forecast data is present.""" empty_response = VrmForecastResponse( success=True, @@ -114,5 +116,5 @@ def test_update_data_skips_on_empty_forecast(pvforecast_instance): with patch.object(pvforecast_instance, "_request_forecast", return_value=empty_response), \ patch.object(PVForecastVrm, "update_value") as mock_update: - pvforecast_instance._update_data() + await pvforecast_instance._update_data() mock_update.assert_not_called() diff --git a/tests/test_system.py b/tests/test_system.py index 79cb66a4..eb3659bd 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -1,5 +1,6 @@ import json import os +import random import signal import time from http import HTTPStatus @@ -8,12 +9,15 @@ from pathlib import Path import pytest import requests +from akkudoktoreos.utils.datetimeutil import to_datetime + DIR_TESTDATA = Path(__file__).absolute().parent.joinpath("testdata") FILE_TESTDATA_EOSSERVER_CONFIG_1 = DIR_TESTDATA.joinpath("eosserver_config_1.json") class TestSystem: + def test_prediction_brightsky(self, server_setup_for_class, is_system_test): """Test weather prediction by BrightSky.""" server = server_setup_for_class["server"] @@ -412,6 +416,283 @@ class TestSystem: f"Expected 400 for invalid datetime, got {result.status_code}" ) + def test_measurement_high_frequency_and_duplicates(self, server_setup_for_class): + """Simulate production-like high-frequency measurement updates.""" + + server = server_setup_for_class["server"] + + # ---------------------------------------------------------------------- + # 1. Configure measurement keys + # ---------------------------------------------------------------------- + config = { + "database": { + "provider": "LMDB", + }, + "measurement": { + "pv_production_emr_keys": [ + "pv1_emr_kwh", + "pv2_emr_kwh", + ], + "load_emr_keys": [ + "load1_emr_kwh", + "load2_emr_kwh", + ], + } + } + + result = requests.put(f"{server}/v1/config", json=config) + assert result.status_code == HTTPStatus.OK + + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "pv1_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "pv2_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "load1_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "load2_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + + # ---------------------------------------------------------------------- + # 2. Simulate high-frequency writes (1 Hz, mixed keys) + # ---------------------------------------------------------------------- + base_time = "2026-04-10T00:00:00" + + timestamps = [ + "2026-04-10T00:00:16", + "2026-04-10T00:00:46", + "2026-04-10T00:01:46", + "2026-04-10T00:02:32", + "2026-04-10T00:02:32", # duplicate + "2026-04-10T00:03:32", + "2026-04-10T00:03:32", # duplicate + ] + + test_cases = [ + # valid + ("pv1_emr_kwh", -687), + ("pv2_emr_kwh", 0.76), + ("load1_emr_kwh", 500), + ("load2_emr_kwh", 0.0), + + # invalid + ("invalid-key-1", 123), + ("invalid-key-2", 456), + ] + + results = [] + + for ts in timestamps: + for key, value in test_cases: + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": ts, # NOTE: naive datetime like production + "key": key, + "value": str(value), + }, + ) + results.append((ts, key, response.status_code)) + + result = requests.post(f"{server}/v1/admin/database/save") + assert result.status_code == HTTPStatus.OK + + # ---------------------------------------------------------------------- + # 3. Assertions: system must behave robustly + # ---------------------------------------------------------------------- + + # A. Invalid keys must consistently return 404 + for ts, key, status in results: + if key.startswith("invalid"): + assert status == HTTPStatus.NOT_FOUND + else: + assert status != HTTPStatus.NOT_FOUND + + # B. Valid keys must NEVER produce 500 + for ts, key, status in results: + if key in ("pv1_emr_kwh", "pv2_emr_kwh"): + assert status != HTTPStatus.INTERNAL_SERVER_ERROR, ( + f"500 error for key={key}, ts={ts}" + ) + + # C. Duplicates must be handled gracefully (200 or 409 or similar, but not 500) + duplicate_failures = [ + (ts, key, status) + for ts, key, status in results + if ts in ("2026-04-10T00:02:32", "2026-04-10T00:03:32") + and key == "pv1_emr_kwh" + and status == HTTPStatus.INTERNAL_SERVER_ERROR + ] + + assert not duplicate_failures, f"Duplicate timestamp caused 500: {duplicate_failures}" + + # ---------------------------------------------------------------------- + # 4. Verify data integrity (no explosion / corruption) + # ---------------------------------------------------------------------- + result = requests.get( + f"{server}/v1/measurement/series", + params={"key": "pv1_emr_kwh"}, + ) + assert result.status_code == HTTPStatus.OK + + data = result.json()["data"] + + # Should not contain excessive duplicates + assert len(data) <= len(set(timestamps)), "Duplicate timestamps not handled properly" + + @pytest.mark.parametrize("db_provider", ["LMDB", "SQLite", None]) + def test_measurement_realtime_stream(self, db_provider, server_setup_for_class): + """Simulate real production stream: 1 Hz updates with jitter, duplicates, and out-of-order timestamps.""" + + server = server_setup_for_class["server"] + + # ---------------------------------------------------------------------- + # 1. Configure measurement keys and measurement database + # ---------------------------------------------------------------------- + config = { + "database": { + "provider": db_provider, + }, + "measurement": { + "pv_production_emr_keys": ["pv1_emr_kwh"], + "load_emr_keys": ["load1_emr_kwh"], + } + } + + result = requests.put(f"{server}/v1/config", json=config) + assert result.status_code == HTTPStatus.OK, f"Failed: {result.status_code} {result}" + + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "pv1_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + result = requests.delete(f"{server}/v1/measurement/range", params={"key": "load1_emr_kwh"}) + assert result.status_code in (HTTPStatus.OK, HTTPStatus.NOT_FOUND) + + # ---------------------------------------------------------------------- + # 2. Real-time simulation + # ---------------------------------------------------------------------- + start = to_datetime().replace(microsecond=0) + + sent_timestamps = [] + errors = [] + + for i in range(20): # run ~20 seconds + now = to_datetime().replace(microsecond=0) + + # --- main timestamp --- + ts = str(now) + + # --- simulate normal write --- + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": ts, + "key": "pv1_emr_kwh", + "value": str(random.uniform(0, 1000)), + }, + ) + + if response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + errors.append(("main", ts, response.text)) + + sent_timestamps.append(ts) + + # ------------------------------------------------------------------ + # Inject real-world problems + # ------------------------------------------------------------------ + + # 0. Other key + if random.random() < 0.5: + now_same_second = now.add(microseconds=430) + ts_same_second = str(now_same_second) + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": ts_same_second, + "key": "load1_emr_kwh", + "value": str(random.uniform(0, 1000)), + }, + ) + if response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + errors.append(("other", ts, response.text)) + + + # 1. Duplicate timestamp (same second, slightly later) + if random.random() < 0.5: + time.sleep(0.05) # slight delay + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": ts, + "key": "pv1_emr_kwh", + "value": str(random.uniform(0, 1000)), + }, + ) + if response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + errors.append(("duplicate", ts, response.text)) + + # 2. Out-of-order timestamp (older data arrives late) + if len(sent_timestamps) > 2 and random.random() < 0.5: + old_ts = random.choice(sent_timestamps[:-1]) + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": old_ts, + "key": "pv1_emr_kwh", + "value": str(random.uniform(0, 1000)), + }, + ) + if response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + errors.append(("out_of_order", old_ts, response.text)) + + # 3. Same second burst (multiple writes in same second) + if random.random() < 0.5: + for _ in range(random.randint(2, 4)): + response = requests.put( + f"{server}/v1/measurement/value", + params={ + "datetime": ts, + "key": "pv1_emr_kwh", + "value": str(random.uniform(0, 1000)), + }, + ) + if response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: + errors.append(("burst", ts, response.text)) + + # 4. Assure database in memory data is saved to database + if i in (0, 4, 9, 14, 19): + response = requests.post(f"{server}/v1/admin/database/save") + assert response.status_code == HTTPStatus.OK + + # small delay to simulate real system (~1 Hz) + time.sleep(1) + + # ---------------------------------------------------------------------- + # 3. Assertions + # ---------------------------------------------------------------------- + + assert not errors, f"500 errors occurred: {errors}" + + # ---------------------------------------------------------------------- + # 4. Verify resulting data + # ---------------------------------------------------------------------- + result = requests.get( + f"{server}/v1/measurement/series", + params={"key": "pv1_emr_kwh"}, + ) + assert result.status_code == HTTPStatus.OK + + data = result.json()["data"] + + # sanity: should not explode in size + assert len(data) == len(set(sent_timestamps)) + + parsed = [to_datetime(ts) for ts in data.keys()] + assert all(parsed[i] <= parsed[i+1] for i in range(len(parsed)-1)), \ + "Timestamps are not sorted" + + unique_keys = set(data.keys()) + assert len(unique_keys) == len(data), \ + "Duplicate timestamps detected in API output" + def test_admin_cache(self, server_setup_for_class, is_system_test): """Test whether cache is reconstructed from cached files.""" server = server_setup_for_class["server"] diff --git a/tests/test_visualize.py b/tests/test_visualize.py index a0ca20f9..6272ea97 100644 --- a/tests/test_visualize.py +++ b/tests/test_visualize.py @@ -9,17 +9,13 @@ filename = "example_report.pdf" DIR_TESTDATA = Path(__file__).parent / "testdata" reference_file = DIR_TESTDATA / "test_example_report.pdf" +output_file = DIR_TESTDATA / "test_example_report_new.pdf" def test_generate_pdf_example(config_eos): """Test generation of example visualization report.""" - output_dir = config_eos.general.data_output_path - assert output_dir is not None - output_file = output_dir / filename - assert not output_file.exists() - # Generate PDF - generate_example_report() + generate_example_report(filename=str(output_file)) # Check if the file exists assert output_file.exists() diff --git a/tests/test_weatherbrightsky.py b/tests/test_weatherbrightsky.py index 7094a52d..c2dc7e6e 100644 --- a/tests/test_weatherbrightsky.py +++ b/tests/test_weatherbrightsky.py @@ -144,8 +144,9 @@ def test_request_forecast(mock_get, provider, sample_brightsky_1_json): } +@pytest.mark.asyncio @patch("requests.get") -def test_update_data(mock_get, provider, sample_brightsky_1_json, cache_store): +async def test_update_data(mock_get, provider, sample_brightsky_1_json, cache_store): """Test fetching forecast from BrightSky.""" # Mock response object mock_response = Mock() @@ -158,7 +159,7 @@ def test_update_data(mock_get, provider, sample_brightsky_1_json, cache_store): # Call the method ems_eos = get_ems() ems_eos.set_start_datetime(to_datetime("2024-10-26 00:00:00", in_timezone="Europe/Berlin")) - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Assert: Verify the result is as expected mock_get.assert_called_once() @@ -170,7 +171,8 @@ def test_update_data(mock_get, provider, sample_brightsky_1_json, cache_store): # ------------------------------------------------ -def test_brightsky_development_forecast_data(provider, config_eos, is_system_test): +@pytest.mark.asyncio +async def test_brightsky_development_forecast_data(provider, config_eos, is_system_test): """Fetch data from real BrightSky server.""" if not is_system_test: return @@ -186,7 +188,7 @@ def test_brightsky_development_forecast_data(provider, config_eos, is_system_tes with FILE_TESTDATA_WEATHERBRIGHTSKY_1_JSON.open("w", encoding="utf-8", newline="\n") as f_out: json.dump(brightsky_data, f_out, indent=4) - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) with FILE_TESTDATA_WEATHERBRIGHTSKY_2_JSON.open("w", encoding="utf-8", newline="\n") as f_out: f_out.write(provider.model_dump_json(indent=4)) diff --git a/tests/test_weatherclearoutside.py b/tests/test_weatherclearoutside.py index 8282b90b..caa09453 100644 --- a/tests/test_weatherclearoutside.py +++ b/tests/test_weatherclearoutside.py @@ -142,8 +142,9 @@ def test_request_forecast(mock_get, provider, sample_clearout_1_html, config_eos assert response.content == sample_clearout_1_html +@pytest.mark.asyncio @patch("requests.get") -def test_update_data(mock_get, provider, sample_clearout_1_html, sample_clearout_1_data): +async def test_update_data(mock_get, provider, sample_clearout_1_html, sample_clearout_1_data): # Mock response object mock_response = Mock() mock_response.status_code = 200 @@ -157,7 +158,7 @@ def test_update_data(mock_get, provider, sample_clearout_1_html, sample_clearout # Call the method ems_eos = get_ems() ems_eos.set_start_datetime(expected_start) - provider.update_data() + await provider.update_data() # Check for correct prediction time window assert provider.config.prediction.hours == 48 @@ -177,9 +178,10 @@ def test_update_data(mock_get, provider, sample_clearout_1_html, sample_clearout # # Check additional weather attributes as necessary +@pytest.mark.asyncio @pytest.mark.skip(reason="Test fixture to be improved") @patch("requests.get") -def test_cache_forecast(mock_get, provider, sample_clearout_1_html, cache_store): +async def test_cache_forecast(mock_get, provider, sample_clearout_1_html, cache_store): """Test that ClearOutside forecast data is cached with TTL. This can not be tested with mock_get. Mock objects are not pickable and therefor can not be @@ -193,11 +195,11 @@ def test_cache_forecast(mock_get, provider, sample_clearout_1_html, cache_store) cache_store.clear(clear_all=True) - provider.update_data() + await provider.update_data() mock_get.assert_called_once() forecast_data_first = provider.to_json() - provider.update_data() + await provider.update_data() forecast_data_second = provider.to_json() # Verify that cache returns the same object without calling the method again assert forecast_data_first == forecast_data_second @@ -210,9 +212,10 @@ def test_cache_forecast(mock_get, provider, sample_clearout_1_html, cache_store) # ------------------------------------------------ +@pytest.mark.asyncio @pytest.mark.skip(reason="For development only") @patch("requests.get") -def test_development_forecast_data(mock_get, provider, sample_clearout_1_html): +async def test_development_forecast_data(mock_get, provider, sample_clearout_1_html): # Mock response object mock_response = Mock() mock_response.status_code = 200 @@ -220,7 +223,7 @@ def test_development_forecast_data(mock_get, provider, sample_clearout_1_html): mock_get.return_value = mock_response # Fill the instance - provider.update_data(force_enable=True) + await provider.update_data(force_enable=True) with FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_DATA.open( "w", encoding="utf-8", newline="\n" diff --git a/tests/test_weatherimport.py b/tests/test_weatherimport.py index 19ccf2a6..8c3990d3 100644 --- a/tests/test_weatherimport.py +++ b/tests/test_weatherimport.py @@ -73,6 +73,7 @@ def test_invalid_provider(provider, config_eos, monkeypatch): # ------------------------------------------------ +@pytest.mark.asyncio @pytest.mark.parametrize( "start_datetime, from_file", [ @@ -86,7 +87,7 @@ def test_invalid_provider(provider, config_eos, monkeypatch): ("2024-10-27 00:00:00", False), # DST change in Germany (25 hours/ day) ], ) -def test_import(provider, sample_import_1_json, start_datetime, from_file, config_eos): +async def test_import(provider, sample_import_1_json, start_datetime, from_file, config_eos): """Test fetching forecast from Import.""" key = "weather_temp_air" ems_eos = get_ems() @@ -97,10 +98,10 @@ def test_import(provider, sample_import_1_json, start_datetime, from_file, confi else: config_eos.weather.provider_settings.WeatherImport.import_file_path = None assert config_eos.weather.provider_settings.WeatherImport.import_file_path is None - provider.delete_by_datetime(start_datetime=None, end_datetime=None) + await provider.delete_by_datetime(start_datetime=None, end_datetime=None) # Call the method - provider.update_data() + await provider.update_data() # Assert: Verify the result is as expected assert provider.ems_start_datetime is not None @@ -108,7 +109,7 @@ def test_import(provider, sample_import_1_json, start_datetime, from_file, confi assert compare_datetimes(provider.ems_start_datetime, ems_eos.start_datetime).equal expected_values = sample_import_1_json[key] - result_values = provider.key_to_array( + result_values = await provider.key_to_array( key=key, start_datetime=provider.ems_start_datetime, end_datetime=provider.ems_start_datetime + to_duration(f"{len(expected_values)} hours"), diff --git a/tests/test_weatheropenmeteo.py b/tests/test_weatheropenmeteo.py index 7e0136a4..708ca9a5 100644 --- a/tests/test_weatheropenmeteo.py +++ b/tests/test_weatheropenmeteo.py @@ -135,8 +135,9 @@ def test_request_forecast(mock_get, provider, sample_openmeteo_1_json): assert "diffuse_radiation" in openmeteo_data["hourly"] # DHI +@pytest.mark.asyncio @patch("requests.get") -def test_update_data(mock_get, provider, sample_openmeteo_1_json, cache_store): +async def test_update_data(mock_get, provider, sample_openmeteo_1_json, cache_store): """Test fetching and processing forecast from Open-Meteo.""" # Mock response object mock_response = Mock() @@ -151,7 +152,7 @@ def test_update_data(mock_get, provider, sample_openmeteo_1_json, cache_store): ems_eos = get_ems() start_datetime = to_datetime("2026-03-02 09:00:00+01:00", in_timezone="Europe/Berlin") ems_eos.set_start_datetime(start_datetime) - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Assert: Verify the result is as expected mock_get.assert_called_once() @@ -160,9 +161,12 @@ def test_update_data(mock_get, provider, sample_openmeteo_1_json, cache_store): # Verify that direct radiation values were properly mapped # Get the first record and check for irradiance values value_datetime = to_datetime("2026-03-04 09:00:00+01:00", in_timezone="Europe/Berlin") - assert provider.key_to_value("weather_ghi", target_datetime=start_datetime) == 21.8 - assert provider.key_to_value("weather_dni", target_datetime=start_datetime) == 1.2 - assert provider.key_to_value("weather_dhi", target_datetime=start_datetime) == 20.5 + weather_ghi = await provider.key_to_value("weather_ghi", target_datetime=start_datetime) + weather_dni = await provider.key_to_value("weather_dni", target_datetime=start_datetime) + weather_dhi = await provider.key_to_value("weather_dhi", target_datetime=start_datetime) + assert weather_ghi == 21.8 + assert weather_dni == 1.2 + assert weather_dhi == 20.5 # ------------------------------------------------ @@ -279,7 +283,8 @@ def test_openmeteo_request_mode_selection( # ------------------------------------------------ -def test_openmeteo_development_forecast_data(provider, config_eos, is_system_test): +@pytest.mark.asyncio +async def test_openmeteo_development_forecast_data(provider, config_eos, is_system_test): """Fetch data from real Open-Meteo server for development purposes.""" if not is_system_test: return @@ -304,7 +309,7 @@ def test_openmeteo_development_forecast_data(provider, config_eos, is_system_tes json.dump(openmeteo_data, f_out, indent=4) # Update and process data - provider.update_data(force_enable=True, force_update=True) + await provider.update_data(force_enable=True, force_update=True) # Save processed data with FILE_TESTDATA_WEATHEROPENMETEO_2_JSON.open("w", encoding="utf-8", newline="\n") as f_out: @@ -312,7 +317,7 @@ def test_openmeteo_development_forecast_data(provider, config_eos, is_system_tes # Verify radiation values if len(provider) > 0: - records = list(provider.data_records.values()) + records = list(provider.records) # Check fo radiation values available has_ghi = any(hasattr(r, 'ghi') and r.ghi is not None for r in records) diff --git a/tests/testdata/docs/_generated/config.md b/tests/testdata/docs/_generated/config.md new file mode 100644 index 00000000..ab268d98 --- /dev/null +++ b/tests/testdata/docs/_generated/config.md @@ -0,0 +1,25 @@ +```{toctree} +:maxdepth: 1 +:caption: Configuration Table + +../_generated/configadapter.md +../_generated/configcache.md +../_generated/configdatabase.md +../_generated/configdevices.md +../_generated/configelecprice.md +../_generated/configems.md +../_generated/configfeedintariff.md +../_generated/configgeneral.md +../_generated/configload.md +../_generated/configlogging.md +../_generated/configmeasurement.md +../_generated/configoptimization.md +../_generated/configprediction.md +../_generated/configpvforecast.md +../_generated/configserver.md +../_generated/configutils.md +../_generated/configweather.md +../_generated/configexample.md +``` + +Auto generated from source code. diff --git a/tests/testdata/docs/_generated/configadapter.md b/tests/testdata/docs/_generated/configadapter.md new file mode 100644 index 00000000..f7d307b3 --- /dev/null +++ b/tests/testdata/docs/_generated/configadapter.md @@ -0,0 +1,238 @@ +## Adapter Configuration + + +:::{table} adapter +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| homeassistant | `EOS_ADAPTER__HOMEASSISTANT` | `HomeAssistantAdapterCommonSettings` | `rw` | `required` | Home Assistant adapter settings. | +| nodered | `EOS_ADAPTER__NODERED` | `NodeREDAdapterCommonSettings` | `rw` | `required` | NodeRED adapter settings. | +| provider | `EOS_ADAPTER__PROVIDER` | `Optional[list[str]]` | `rw` | `None` | List of adapter provider id(s) of provider(s) to be used. | +| providers | | `list[str]` | `ro` | `N/A` | Available adapter provider ids. | +::: + + + +**Example Input** + + + +```json + { + "adapter": { + "provider": [ + "HomeAssistant" + ], + "homeassistant": { + "config_entity_ids": null, + "load_emr_entity_ids": null, + "grid_export_emr_entity_ids": null, + "grid_import_emr_entity_ids": null, + "pv_production_emr_entity_ids": null, + "device_measurement_entity_ids": null, + "device_instruction_entity_ids": null, + "solution_entity_ids": null + }, + "nodered": { + "host": "127.0.0.1", + "port": 1880 + } + } + } +``` + + + +**Example Output** + + + +```json + { + "adapter": { + "provider": [ + "HomeAssistant" + ], + "homeassistant": { + "config_entity_ids": null, + "load_emr_entity_ids": null, + "grid_export_emr_entity_ids": null, + "grid_import_emr_entity_ids": null, + "pv_production_emr_entity_ids": null, + "device_measurement_entity_ids": null, + "device_instruction_entity_ids": null, + "solution_entity_ids": null, + "homeassistant_entity_ids": [], + "eos_solution_entity_ids": [], + "eos_device_instruction_entity_ids": [] + }, + "nodered": { + "host": "127.0.0.1", + "port": 1880 + }, + "providers": [ + "HomeAssistant", + "NodeRED" + ] + } + } +``` + + +### Common settings for the NodeRED adapter + +The Node-RED adapter sends to HTTP IN nodes. + +This is the example flow: + +[HTTP In \\] -> [Function (parse payload)] -> [Debug] -> [HTTP Response] + +There are two URLs that are used: + +- GET /eos/data_aquisition + The GET is issued before the optimization. +- POST /eos/control_dispatch + The POST is issued after the optimization. + + +:::{table} adapter::nodered +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| host | `Optional[str]` | `rw` | `127.0.0.1` | Node-RED server IP address. Defaults to 127.0.0.1. | +| port | `Optional[int]` | `rw` | `1880` | Node-RED server IP port number. Defaults to 1880. | +::: + + + +**Example Input/Output** + + + +```json + { + "adapter": { + "nodered": { + "host": "127.0.0.1", + "port": 1880 + } + } + } +``` + + +### Common settings for the home assistant adapter + + +:::{table} adapter::homeassistant +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| config_entity_ids | `Optional[dict[str, str]]` | `rw` | `None` | Mapping of EOS config keys to Home Assistant entity IDs. +The config key has to be given by a ‘/’-separated path +e.g. devices/batteries/0/capacity_wh | +| device_instruction_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity IDs for device (resource) instructions to be updated by EOS. +The device ids (resource ids) have to be prepended by 'sensor.eos_' to build the entity_id. +E.g. The instruction for device id 'battery1' becomes the entity_id 'sensor.eos_battery1'. | +| device_measurement_entity_ids | `Optional[dict[str, str]]` | `rw` | `None` | Mapping of EOS measurement keys used by device (resource) simulations to Home Assistant entity IDs. | +| eos_device_instruction_entity_ids | `list[str]` | `ro` | `N/A` | Entity IDs for energy management instructions available at EOS. | +| eos_solution_entity_ids | `list[str]` | `ro` | `N/A` | Entity IDs for optimization solution available at EOS. | +| grid_export_emr_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity ID(s) of export to grid energy meter readings [kWh] | +| grid_import_emr_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity ID(s) of import from grid energy meter readings [kWh] | +| homeassistant_entity_ids | `list[str]` | `ro` | `N/A` | Entity IDs available at Home Assistant. | +| load_emr_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity ID(s) of load energy meter readings [kWh] | +| pv_production_emr_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity ID(s) of PV production energy meter readings [kWh] | +| solution_entity_ids | `Optional[list[str]]` | `rw` | `None` | Entity IDs for optimization solution keys to be updated by EOS. +The solution keys have to be prepended by 'sensor.eos_' to build the entity_id. +E.g. solution key 'battery1_idle_op_mode' becomes the entity_id 'sensor.eos_battery1_idle_op_mode'. | +::: + + + +**Example Input** + + + +```json + { + "adapter": { + "homeassistant": { + "config_entity_ids": { + "devices/batteries/0/capacity_wh": "sensor.battery1_capacity" + }, + "load_emr_entity_ids": [ + "sensor.load_energy_total_kwh" + ], + "grid_export_emr_entity_ids": [ + "sensor.grid_export_energy_total_kwh" + ], + "grid_import_emr_entity_ids": [ + "sensor.grid_import_energy_total_kwh" + ], + "pv_production_emr_entity_ids": [ + "sensor.pv_energy_total_kwh" + ], + "device_measurement_entity_ids": { + "ev11_soc_factor": "sensor.ev11_soc_factor", + "battery1_soc_factor": "sensor.battery1_soc_factor" + }, + "device_instruction_entity_ids": [ + "sensor.eos_battery1" + ], + "solution_entity_ids": [ + "sensor.eos_battery1_idle_mode_mode" + ] + } + } + } +``` + + + +**Example Output** + + + +```json + { + "adapter": { + "homeassistant": { + "config_entity_ids": { + "devices/batteries/0/capacity_wh": "sensor.battery1_capacity" + }, + "load_emr_entity_ids": [ + "sensor.load_energy_total_kwh" + ], + "grid_export_emr_entity_ids": [ + "sensor.grid_export_energy_total_kwh" + ], + "grid_import_emr_entity_ids": [ + "sensor.grid_import_energy_total_kwh" + ], + "pv_production_emr_entity_ids": [ + "sensor.pv_energy_total_kwh" + ], + "device_measurement_entity_ids": { + "ev11_soc_factor": "sensor.ev11_soc_factor", + "battery1_soc_factor": "sensor.battery1_soc_factor" + }, + "device_instruction_entity_ids": [ + "sensor.eos_battery1" + ], + "solution_entity_ids": [ + "sensor.eos_battery1_idle_mode_mode" + ], + "homeassistant_entity_ids": [], + "eos_solution_entity_ids": [], + "eos_device_instruction_entity_ids": [] + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configcache.md b/tests/testdata/docs/_generated/configcache.md new file mode 100644 index 00000000..89129d48 --- /dev/null +++ b/tests/testdata/docs/_generated/configcache.md @@ -0,0 +1,28 @@ +## Cache Configuration + + +:::{table} cache +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| cleanup_interval | `EOS_CACHE__CLEANUP_INTERVAL` | `float` | `rw` | `300.0` | Intervall in seconds for EOS file cache cleanup. | +| subpath | `EOS_CACHE__SUBPATH` | `Optional[pathlib.Path]` | `rw` | `cache` | Sub-path for the EOS cache data directory. | +::: + + + +**Example Input/Output** + + + +```json + { + "cache": { + "subpath": "cache", + "cleanup_interval": 300.0 + } + } +``` + diff --git a/tests/testdata/docs/_generated/configdatabase.md b/tests/testdata/docs/_generated/configdatabase.md new file mode 100644 index 00000000..92c6ec49 --- /dev/null +++ b/tests/testdata/docs/_generated/configdatabase.md @@ -0,0 +1,73 @@ +## Configuration model for database settings + +Attributes: + provider: Optional provider identifier (e.g. "LMDB"). + max_records_in_memory: Maximum records kept in memory before auto-save. + auto_save: Whether to auto-save when threshold exceeded. + batch_size: Batch size for batch operations. + + +:::{table} database +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| autosave_interval_sec | `EOS_DATABASE__AUTOSAVE_INTERVAL_SEC` | `Optional[int]` | `rw` | `10` | Automatic saving interval [seconds]. +Set to None to disable automatic saving. | +| batch_size | `EOS_DATABASE__BATCH_SIZE` | `int` | `rw` | `100` | Number of records to process in batch operations. | +| compaction_interval_sec | `EOS_DATABASE__COMPACTION_INTERVAL_SEC` | `Optional[int]` | `rw` | `604800` | Interval in between automatic tiered compaction runs [seconds]. +Compaction downsamples old records to reduce storage while retaining coverage. Set to None to disable automatic compaction. | +| compression_level | `EOS_DATABASE__COMPRESSION_LEVEL` | `int` | `rw` | `9` | Compression level for database record data. | +| initial_load_window_h | `EOS_DATABASE__INITIAL_LOAD_WINDOW_H` | `Optional[int]` | `rw` | `None` | Specifies the default duration of the initial load window when loading records from the database, in hours. If set to None, the full available range is loaded. The window is centered around the current time by default, unless a different center time is specified. Different database namespaces may define their own default windows. | +| keep_duration_h | `EOS_DATABASE__KEEP_DURATION_H` | `Optional[int]` | `rw` | `None` | Default maximum duration records shall be kept in database [hours, none]. +None indicates forever. Database namespaces may have diverging definitions. | +| provider | `EOS_DATABASE__PROVIDER` | `Optional[str]` | `rw` | `None` | Database provider id of provider to be used. | +| providers | | `List[str]` | `ro` | `N/A` | Return available database provider ids. | +::: + + + +**Example Input** + + + +```json + { + "database": { + "provider": "LMDB", + "compression_level": 0, + "initial_load_window_h": 48, + "keep_duration_h": 48, + "autosave_interval_sec": 5, + "compaction_interval_sec": 604800, + "batch_size": 100 + } + } +``` + + + +**Example Output** + + + +```json + { + "database": { + "provider": "LMDB", + "compression_level": 0, + "initial_load_window_h": 48, + "keep_duration_h": 48, + "autosave_interval_sec": 5, + "compaction_interval_sec": 604800, + "batch_size": 100, + "providers": [ + "LMDB", + "SQLite", + "NoDB" + ] + } + } +``` + diff --git a/tests/testdata/docs/_generated/configdevices.md b/tests/testdata/docs/_generated/configdevices.md new file mode 100644 index 00000000..38e3d691 --- /dev/null +++ b/tests/testdata/docs/_generated/configdevices.md @@ -0,0 +1,549 @@ +## Base configuration for devices simulation settings + + +:::{table} devices +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| batteries | `EOS_DEVICES__BATTERIES` | `Optional[list[akkudoktoreos.devices.devices.BatteriesCommonSettings]]` | `rw` | `None` | List of battery devices | +| electric_vehicles | `EOS_DEVICES__ELECTRIC_VEHICLES` | `Optional[list[akkudoktoreos.devices.devices.BatteriesCommonSettings]]` | `rw` | `None` | List of electric vehicle devices | +| home_appliances | `EOS_DEVICES__HOME_APPLIANCES` | `Optional[list[akkudoktoreos.devices.devices.HomeApplianceCommonSettings]]` | `rw` | `None` | List of home appliances | +| inverters | `EOS_DEVICES__INVERTERS` | `Optional[list[akkudoktoreos.devices.devices.InverterCommonSettings]]` | `rw` | `None` | List of inverters | +| max_batteries | `EOS_DEVICES__MAX_BATTERIES` | `Optional[int]` | `rw` | `None` | Maximum number of batteries that can be set | +| max_electric_vehicles | `EOS_DEVICES__MAX_ELECTRIC_VEHICLES` | `Optional[int]` | `rw` | `None` | Maximum number of electric vehicles that can be set | +| max_home_appliances | `EOS_DEVICES__MAX_HOME_APPLIANCES` | `Optional[int]` | `rw` | `None` | Maximum number of home_appliances that can be set | +| max_inverters | `EOS_DEVICES__MAX_INVERTERS` | `Optional[int]` | `rw` | `None` | Maximum number of inverters that can be set | +| measurement_keys | | `Optional[list[str]]` | `ro` | `N/A` | Return the measurement keys for the resource/ device stati that are measurements. | +::: + + + +**Example Input** + + + +```json + { + "devices": { + "batteries": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100 + } + ], + "max_batteries": 1, + "electric_vehicles": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100 + } + ], + "max_electric_vehicles": 1, + "inverters": [], + "max_inverters": 1, + "home_appliances": [], + "max_home_appliances": 1 + } + } +``` + + + +**Example Output** + + + +```json + { + "devices": { + "batteries": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100, + "measurement_key_soc_factor": "battery1-soc-factor", + "measurement_key_power_l1_w": "battery1-power-l1-w", + "measurement_key_power_l2_w": "battery1-power-l2-w", + "measurement_key_power_l3_w": "battery1-power-l3-w", + "measurement_key_power_3_phase_sym_w": "battery1-power-3-phase-sym-w", + "measurement_keys": [ + "battery1-soc-factor", + "battery1-power-l1-w", + "battery1-power-l2-w", + "battery1-power-l3-w", + "battery1-power-3-phase-sym-w" + ] + } + ], + "max_batteries": 1, + "electric_vehicles": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100, + "measurement_key_soc_factor": "battery1-soc-factor", + "measurement_key_power_l1_w": "battery1-power-l1-w", + "measurement_key_power_l2_w": "battery1-power-l2-w", + "measurement_key_power_l3_w": "battery1-power-l3-w", + "measurement_key_power_3_phase_sym_w": "battery1-power-3-phase-sym-w", + "measurement_keys": [ + "battery1-soc-factor", + "battery1-power-l1-w", + "battery1-power-l2-w", + "battery1-power-l3-w", + "battery1-power-3-phase-sym-w" + ] + } + ], + "max_electric_vehicles": 1, + "inverters": [], + "max_inverters": 1, + "home_appliances": [], + "max_home_appliances": 1, + "measurement_keys": [ + "battery1-soc-factor", + "battery1-power-l1-w", + "battery1-power-l2-w", + "battery1-power-l3-w", + "battery1-power-3-phase-sym-w", + "battery1-soc-factor", + "battery1-power-l1-w", + "battery1-power-l2-w", + "battery1-power-l3-w", + "battery1-power-3-phase-sym-w" + ] + } + } +``` + + +### Inverter devices base settings + + +:::{table} devices::inverters::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| ac_to_dc_efficiency | `float` | `rw` | `1.0` | Efficiency of AC to DC conversion for grid-to-battery AC charging (0-1). Set to 0 to disable AC charging. Default 1.0 (no additional inverter loss). | +| battery_id | `Optional[str]` | `rw` | `None` | ID of battery controlled by this inverter. | +| dc_to_ac_efficiency | `float` | `rw` | `1.0` | Efficiency of DC to AC conversion for battery discharging to AC load/grid (0-1). Default 1.0 (no additional inverter loss). | +| device_id | `str` | `rw` | `` | ID of device | +| max_ac_charge_power_w | `Optional[float]` | `rw` | `None` | Maximum AC charging power in watts. null means no additional limit. Set to 0 to disable AC charging. | +| max_power_w | `Optional[float]` | `rw` | `None` | Maximum power [W]. | +| measurement_keys | `Optional[list[str]]` | `ro` | `N/A` | Measurement keys for the inverter stati that are measurements. | +::: + + + +**Example Input** + + + +```json + { + "devices": { + "inverters": [ + { + "device_id": "battery1", + "max_power_w": 10000.0, + "battery_id": null, + "ac_to_dc_efficiency": 0.95, + "dc_to_ac_efficiency": 0.95, + "max_ac_charge_power_w": null + } + ] + } + } +``` + + + +**Example Output** + + + +```json + { + "devices": { + "inverters": [ + { + "device_id": "battery1", + "max_power_w": 10000.0, + "battery_id": null, + "ac_to_dc_efficiency": 0.95, + "dc_to_ac_efficiency": 0.95, + "max_ac_charge_power_w": null, + "measurement_keys": [] + } + ] + } + } +``` + + +### Model defining a daily or date time window with optional localization support + +Represents a time interval starting at `start_time` and lasting for `duration`. +Can restrict applicability to a specific day of the week or a specific calendar date. +Supports day names in multiple languages via locale-aware parsing. + +Timezone contract: + +``start_time`` is always **naive** (no ``tzinfo``). It is interpreted as a +local wall-clock time in whatever timezone the caller's ``date_time`` or +``reference_date`` carries. When those arguments are timezone-aware the +window boundaries are evaluated in that timezone; when they are naive, +arithmetic is performed as-is (no timezone conversion occurs). + +``date``, being a calendar ``Date`` object, is inherently timezone-free. + +This design avoids the ambiguity that arises when a stored ``start_time`` +carries its own timezone that differs from the caller's timezone, and keeps +the model serialisable without timezone state. + + +:::{table} devices::home_appliances::list::time_windows::windows::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| date | `Optional[pydantic_extra_types.pendulum_dt.Date]` | `rw` | `None` | Optional specific calendar date for the time window. Naive — matched against the local date of the datetime passed to contains(). Overrides `day_of_week` if set. | +| day_of_week | `Union[int, str, NoneType]` | `rw` | `None` | Optional day of the week restriction. Can be specified as integer (0=Monday to 6=Sunday) or localized weekday name. If None, applies every day unless `date` is set. | +| duration | `Duration` | `rw` | `required` | Duration of the time window starting from `start_time`. | +| locale | `Optional[str]` | `rw` | `None` | Locale used to parse weekday names in `day_of_week` when given as string. If not set, Pendulum's default locale is used. Examples: 'en', 'de', 'fr', etc. | +| start_time | `Time` | `rw` | `required` | Naive start time of the time window (time of day, no timezone). Interpreted in the timezone of the datetime passed to contains() or earliest_start_time(). | +::: + + + +**Example Input/Output** + + + +```json + { + "devices": { + "home_appliances": [ + { + "time_windows": { + "windows": [ + { + "start_time": "00:00:00.000000", + "duration": "2 hours", + "day_of_week": null, + "date": null, + "locale": null + } + ] + } + } + ] + } + } +``` + + +### Model representing a sequence of time windows with collective operations + +Manages multiple TimeWindow objects and provides methods to work with them +as a cohesive unit for scheduling and availability checking. + + +:::{table} devices::home_appliances::list::time_windows +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| windows | `list[akkudoktoreos.config.configabc.TimeWindow]` | `rw` | `required` | List of TimeWindow objects that make up this sequence. | +::: + + + +**Example Input/Output** + + + +```json + { + "devices": { + "home_appliances": [ + { + "time_windows": { + "windows": [] + } + } + ] + } + } +``` + + +### Home Appliance devices base settings + + +:::{table} devices::home_appliances::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| consumption_wh | `int` | `rw` | `required` | Energy consumption [Wh]. | +| device_id | `str` | `rw` | `` | ID of device | +| duration_h | `int` | `rw` | `required` | Usage duration in hours [0 ... 24]. | +| measurement_keys | `Optional[list[str]]` | `ro` | `N/A` | Measurement keys for the home appliance stati that are measurements. | +| time_windows | `Optional[akkudoktoreos.config.configabc.TimeWindowSequence]` | `rw` | `None` | Sequence of allowed time windows. Defaults to optimization general time window. | +::: + + + +**Example Input** + + + +```json + { + "devices": { + "home_appliances": [ + { + "device_id": "battery1", + "consumption_wh": 2000, + "duration_h": 1, + "time_windows": { + "windows": [ + { + "start_time": "10:00:00.000000", + "duration": "2 hours", + "day_of_week": null, + "date": null, + "locale": null + } + ] + } + } + ] + } + } +``` + + + +**Example Output** + + + +```json + { + "devices": { + "home_appliances": [ + { + "device_id": "battery1", + "consumption_wh": 2000, + "duration_h": 1, + "time_windows": { + "windows": [ + { + "start_time": "10:00:00.000000", + "duration": "2 hours", + "day_of_week": null, + "date": null, + "locale": null + } + ] + }, + "measurement_keys": [] + } + ] + } + } +``` + + +### Battery devices base settings + + +:::{table} devices::batteries::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| capacity_wh | `int` | `rw` | `8000` | Capacity [Wh]. | +| charge_rates | `Optional[list[float]]` | `rw` | `[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]` | Charge rates as factor of maximum charging power [0.00 ... 1.00]. None triggers fallback to default charge-rates. | +| charging_efficiency | `float` | `rw` | `0.88` | Charging efficiency [0.01 ... 1.00]. | +| device_id | `str` | `rw` | `` | ID of device | +| discharging_efficiency | `float` | `rw` | `0.88` | Discharge efficiency [0.01 ... 1.00]. | +| levelized_cost_of_storage_kwh | `float` | `rw` | `0.0` | Levelized cost of storage (LCOS), the average lifetime cost of delivering one kWh [€/kWh]. | +| max_charge_power_w | `Optional[float]` | `rw` | `5000` | Maximum charging power [W]. | +| max_soc_percentage | `int` | `rw` | `100` | Maximum state of charge (SOC) as percentage of capacity [%]. | +| measurement_key_power_3_phase_sym_w | `str` | `ro` | `N/A` | Measurement key for the symmetric 3 phase power the battery is charged or discharged with [W]. | +| measurement_key_power_l1_w | `str` | `ro` | `N/A` | Measurement key for the L1 power the battery is charged or discharged with [W]. | +| measurement_key_power_l2_w | `str` | `ro` | `N/A` | Measurement key for the L2 power the battery is charged or discharged with [W]. | +| measurement_key_power_l3_w | `str` | `ro` | `N/A` | Measurement key for the L3 power the battery is charged or discharged with [W]. | +| measurement_key_soc_factor | `str` | `ro` | `N/A` | Measurement key for the battery state of charge (SoC) as factor of total capacity [0.0 ... 1.0]. | +| measurement_keys | `Optional[list[str]]` | `ro` | `N/A` | Measurement keys for the battery stati that are measurements. | +| min_charge_power_w | `Optional[float]` | `rw` | `50` | Minimum charging power [W]. | +| min_soc_percentage | `int` | `rw` | `0` | Minimum state of charge (SOC) as percentage of capacity [%]. This is the target SoC for charging | +::: + + + +**Example Input** + + + +```json + { + "devices": { + "batteries": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.12, + "max_charge_power_w": 5000.0, + "min_charge_power_w": 50.0, + "charge_rates": [ + 0.0, + 0.25, + 0.5, + 0.75, + 1.0 + ], + "min_soc_percentage": 10, + "max_soc_percentage": 100 + } + ] + } + } +``` + + + +**Example Output** + + + +```json + { + "devices": { + "batteries": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.12, + "max_charge_power_w": 5000.0, + "min_charge_power_w": 50.0, + "charge_rates": [ + 0.0, + 0.25, + 0.5, + 0.75, + 1.0 + ], + "min_soc_percentage": 10, + "max_soc_percentage": 100, + "measurement_key_soc_factor": "battery1-soc-factor", + "measurement_key_power_l1_w": "battery1-power-l1-w", + "measurement_key_power_l2_w": "battery1-power-l2-w", + "measurement_key_power_l3_w": "battery1-power-l3-w", + "measurement_key_power_3_phase_sym_w": "battery1-power-3-phase-sym-w", + "measurement_keys": [ + "battery1-soc-factor", + "battery1-power-l1-w", + "battery1-power-l2-w", + "battery1-power-l3-w", + "battery1-power-3-phase-sym-w" + ] + } + ] + } + } +``` + diff --git a/tests/testdata/docs/_generated/configelecprice.md b/tests/testdata/docs/_generated/configelecprice.md new file mode 100644 index 00000000..4a3e0ef4 --- /dev/null +++ b/tests/testdata/docs/_generated/configelecprice.md @@ -0,0 +1,275 @@ +## Electricity Price Prediction Configuration + + +:::{table} elecprice +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| charges_kwh | `EOS_ELECPRICE__CHARGES_KWH` | `Optional[float]` | `rw` | `None` | Electricity price charges [€/kWh]. Will be added to variable market price. | +| elecpricefixed | `EOS_ELECPRICE__ELECPRICEFIXED` | `ElecPriceFixedCommonSettings` | `rw` | `required` | Fixed electricity price provider settings. | +| elecpriceimport | `EOS_ELECPRICE__ELECPRICEIMPORT` | `ElecPriceImportCommonSettings` | `rw` | `required` | Import provider settings. | +| energycharts | `EOS_ELECPRICE__ENERGYCHARTS` | `ElecPriceEnergyChartsCommonSettings` | `rw` | `required` | Energy Charts provider settings. | +| provider | `EOS_ELECPRICE__PROVIDER` | `Optional[str]` | `rw` | `None` | Electricity price provider id of provider to be used. | +| providers | | `list[str]` | `ro` | `N/A` | Available electricity price provider ids. | +| vat_rate | `EOS_ELECPRICE__VAT_RATE` | `Optional[float]` | `rw` | `1.19` | VAT rate factor applied to electricity price when charges are used. | +::: + + + +**Example Input** + + + +```json + { + "elecprice": { + "provider": "ElecPriceAkkudoktor", + "charges_kwh": 0.21, + "vat_rate": 1.19, + "elecpricefixed": { + "time_windows": { + "windows": [] + } + }, + "elecpriceimport": { + "import_file_path": null, + "import_json": null + }, + "energycharts": { + "bidding_zone": "DE-LU" + } + } + } +``` + + + +**Example Output** + + + +```json + { + "elecprice": { + "provider": "ElecPriceAkkudoktor", + "charges_kwh": 0.21, + "vat_rate": 1.19, + "elecpricefixed": { + "time_windows": { + "windows": [] + } + }, + "elecpriceimport": { + "import_file_path": null, + "import_json": null + }, + "energycharts": { + "bidding_zone": "DE-LU" + }, + "providers": [ + "ElecPriceAkkudoktor", + "ElecPriceEnergyCharts", + "ElecPriceFixed", + "ElecPriceImport" + ] + } + } +``` + + +### Common settings for Energy Charts electricity price provider + + +:::{table} elecprice::energycharts +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| bidding_zone | `` | `rw` | `DE-LU` | Bidding Zone: 'AT', 'BE', 'CH', 'CZ', 'DE-LU', 'DE-AT-LU', 'DK1', 'DK2', 'FR', 'HU', 'IT-NORTH', 'NL', 'NO2', 'PL', 'SE4' or 'SI' | +::: + + + +**Example Input/Output** + + + +```json + { + "elecprice": { + "energycharts": { + "bidding_zone": "AT" + } + } + } +``` + + +### Common settings for elecprice data import from file or JSON String + + +:::{table} elecprice::elecpriceimport +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| import_file_path | `Union[str, pathlib.Path, NoneType]` | `rw` | `None` | Path to the file to import elecprice data from. | +| import_json | `Optional[str]` | `rw` | `None` | JSON string, dictionary of electricity price forecast value lists. | +::: + + + +**Example Input/Output** + + + +```json + { + "elecprice": { + "elecpriceimport": { + "import_file_path": null, + "import_json": "{\"elecprice_marketprice_wh\": [0.0003384, 0.0003318, 0.0003284]}" + } + } + } +``` + + +### Value applicable during a specific time window + +This model extends `TimeWindow` by associating a value with the defined time interval. + + +:::{table} elecprice::elecpricefixed::time_windows::windows::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| date | `Optional[pydantic_extra_types.pendulum_dt.Date]` | `rw` | `None` | Optional specific calendar date for the time window. Naive — matched against the local date of the datetime passed to contains(). Overrides `day_of_week` if set. | +| day_of_week | `Union[int, str, NoneType]` | `rw` | `None` | Optional day of the week restriction. Can be specified as integer (0=Monday to 6=Sunday) or localized weekday name. If None, applies every day unless `date` is set. | +| duration | `Duration` | `rw` | `required` | Duration of the time window starting from `start_time`. | +| locale | `Optional[str]` | `rw` | `None` | Locale used to parse weekday names in `day_of_week` when given as string. If not set, Pendulum's default locale is used. Examples: 'en', 'de', 'fr', etc. | +| start_time | `Time` | `rw` | `required` | Naive start time of the time window (time of day, no timezone). Interpreted in the timezone of the datetime passed to contains() or earliest_start_time(). | +| value | `Optional[float]` | `rw` | `None` | Value applicable during this time window. | +::: + + + +**Example Input/Output** + + + +```json + { + "elecprice": { + "elecpricefixed": { + "time_windows": { + "windows": [ + { + "start_time": "00:00:00.000000", + "duration": "2 hours", + "day_of_week": null, + "date": null, + "locale": null, + "value": 0.288 + } + ] + } + } + } + } +``` + + +### Sequence of value time windows + +This model specializes `TimeWindowSequence` to ensure that all +contained windows are instances of `ValueTimeWindow`. +It provides the full set of sequence operations (containment checks, +availability, start time calculations) for value windows. + + +:::{table} elecprice::elecpricefixed::time_windows +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| windows | `list[akkudoktoreos.config.configabc.ValueTimeWindow]` | `rw` | `required` | Ordered list of value time windows. Each window defines a time interval and an associated value. | +::: + + + +**Example Input/Output** + + + +```json + { + "elecprice": { + "elecpricefixed": { + "time_windows": { + "windows": [] + } + } + } + } +``` + + +### Common configuration settings for fixed electricity pricing + +This model defines a fixed electricity price schedule using a sequence +of time windows. Each window specifies a time interval and the electricity +price applicable during that interval. + + +:::{table} elecprice::elecpricefixed +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| time_windows | `ValueTimeWindowSequence` | `rw` | `required` | Sequence of time windows defining the fixed price schedule. If not provided, no fixed pricing is applied. | +::: + + + +**Example Input/Output** + + + +```json + { + "elecprice": { + "elecpricefixed": { + "time_windows": { + "windows": [ + { + "start_time": "00:00:00.000000", + "duration": "8 hours", + "day_of_week": null, + "date": null, + "locale": null, + "value": 0.288 + }, + { + "start_time": "08:00:00.000000", + "duration": "16 hours", + "day_of_week": null, + "date": null, + "locale": null, + "value": 0.34 + } + ] + } + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configems.md b/tests/testdata/docs/_generated/configems.md new file mode 100644 index 00000000..5d6ab1e9 --- /dev/null +++ b/tests/testdata/docs/_generated/configems.md @@ -0,0 +1,30 @@ +## Energy Management Configuration + + +:::{table} ems +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| interval | `EOS_EMS__INTERVAL` | `float` | `rw` | `300.0` | Intervall between EOS energy management runs [seconds]. | +| mode | `EOS_EMS__MODE` | `` | `rw` | `required` | Energy management mode [DISABLED | OPTIMIZATION | PREDICTION]. | +| startup_delay | `EOS_EMS__STARTUP_DELAY` | `float` | `rw` | `5` | Startup delay in seconds for EOS energy management runs. | +::: + + + +**Example Input/Output** + + + +```json + { + "ems": { + "startup_delay": 5.0, + "interval": 300.0, + "mode": "OPTIMIZATION" + } + } +``` + diff --git a/tests/testdata/docs/_generated/configexample.md b/tests/testdata/docs/_generated/configexample.md new file mode 100644 index 00000000..dbc81870 --- /dev/null +++ b/tests/testdata/docs/_generated/configexample.md @@ -0,0 +1,260 @@ +## Full example Config + + +```json + { + "adapter": { + "provider": [ + "HomeAssistant" + ], + "homeassistant": { + "config_entity_ids": null, + "load_emr_entity_ids": null, + "grid_export_emr_entity_ids": null, + "grid_import_emr_entity_ids": null, + "pv_production_emr_entity_ids": null, + "device_measurement_entity_ids": null, + "device_instruction_entity_ids": null, + "solution_entity_ids": null + }, + "nodered": { + "host": "127.0.0.1", + "port": 1880 + } + }, + "cache": { + "subpath": "cache", + "cleanup_interval": 300.0 + }, + "database": { + "provider": "LMDB", + "compression_level": 0, + "initial_load_window_h": 48, + "keep_duration_h": 48, + "autosave_interval_sec": 5, + "compaction_interval_sec": 604800, + "batch_size": 100 + }, + "devices": { + "batteries": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100 + } + ], + "max_batteries": 1, + "electric_vehicles": [ + { + "device_id": "battery1", + "capacity_wh": 8000, + "charging_efficiency": 0.88, + "discharging_efficiency": 0.88, + "levelized_cost_of_storage_kwh": 0.0, + "max_charge_power_w": 5000, + "min_charge_power_w": 50, + "charge_rates": [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 1.0 + ], + "min_soc_percentage": 0, + "max_soc_percentage": 100 + } + ], + "max_electric_vehicles": 1, + "inverters": [], + "max_inverters": 1, + "home_appliances": [], + "max_home_appliances": 1 + }, + "elecprice": { + "provider": "ElecPriceAkkudoktor", + "charges_kwh": 0.21, + "vat_rate": 1.19, + "elecpricefixed": { + "time_windows": { + "windows": [] + } + }, + "elecpriceimport": { + "import_file_path": null, + "import_json": null + }, + "energycharts": { + "bidding_zone": "DE-LU" + } + }, + "ems": { + "startup_delay": 5.0, + "interval": 300.0, + "mode": "OPTIMIZATION" + }, + "feedintariff": { + "provider": "FeedInTariffFixed", + "provider_settings": { + "FeedInTariffFixed": null, + "FeedInTariffImport": null + } + }, + "general": { + "config_save_mode": "AUTOMATIC", + "config_save_interval_sec": 60, + "version": "0.0.0", + "data_folder_path": "/home/user/.local/share/net.akkudoktoreos.net", + "data_output_subpath": "output", + "latitude": 52.52, + "longitude": 13.405 + }, + "load": { + "provider": "LoadAkkudoktor", + "loadakkudoktor": { + "loadakkudoktor_year_energy_kwh": null + }, + "loadvrm": { + "load_vrm_token": "your-token", + "load_vrm_idsite": 12345 + }, + "loadimport": { + "import_file_path": null, + "import_json": null + } + }, + "logging": { + "console_level": "TRACE", + "file_level": "TRACE" + }, + "measurement": { + "historic_hours": 17520, + "load_emr_keys": [ + "load0_emr" + ], + "grid_export_emr_keys": [ + "grid_export_emr" + ], + "grid_import_emr_keys": [ + "grid_import_emr" + ], + "pv_production_emr_keys": [ + "pv1_emr" + ] + }, + "optimization": { + "horizon_hours": 24, + "interval": 3600, + "algorithm": "GENETIC", + "genetic": { + "individuals": 400, + "generations": 400, + "seed": null, + "penalties": { + "ev_soc_miss": 10 + } + } + }, + "prediction": { + "hours": 48, + "historic_hours": 48 + }, + "pvforecast": { + "provider": "PVForecastAkkudoktor", + "provider_settings": { + "PVForecastImport": null, + "PVForecastVrm": null + }, + "planes": [ + { + "surface_tilt": 10.0, + "surface_azimuth": 180.0, + "userhorizon": [ + 10.0, + 20.0, + 30.0 + ], + "peakpower": 5.0, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 0, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 6000, + "modules_per_string": 20, + "strings_per_inverter": 2 + }, + { + "surface_tilt": 20.0, + "surface_azimuth": 90.0, + "userhorizon": [ + 5.0, + 15.0, + 25.0 + ], + "peakpower": 3.5, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 1, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 4000, + "modules_per_string": 20, + "strings_per_inverter": 2 + } + ], + "max_planes": 1 + }, + "server": { + "host": "127.0.0.1", + "port": 8503, + "verbose": false, + "startup_eosdash": true, + "eosdash_host": "127.0.0.1", + "eosdash_port": 8504, + "eosdash_supervise_interval_sec": 10, + "run_as_user": null, + "reload": true + }, + "utils": {}, + "weather": { + "provider": "WeatherImport", + "provider_settings": { + "WeatherImport": null + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configfeedintariff.md b/tests/testdata/docs/_generated/configfeedintariff.md new file mode 100644 index 00000000..8e882a1e --- /dev/null +++ b/tests/testdata/docs/_generated/configfeedintariff.md @@ -0,0 +1,149 @@ +## Feed In Tariff Prediction Configuration + + +:::{table} feedintariff +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| provider | `EOS_FEEDINTARIFF__PROVIDER` | `Optional[str]` | `rw` | `None` | Feed in tariff provider id of provider to be used. | +| provider_settings | `EOS_FEEDINTARIFF__PROVIDER_SETTINGS` | `FeedInTariffCommonProviderSettings` | `rw` | `required` | Provider settings | +| providers | | `list[str]` | `ro` | `N/A` | Available feed in tariff provider ids. | +::: + + + +**Example Input** + + + +```json + { + "feedintariff": { + "provider": "FeedInTariffFixed", + "provider_settings": { + "FeedInTariffFixed": null, + "FeedInTariffImport": null + } + } + } +``` + + + +**Example Output** + + + +```json + { + "feedintariff": { + "provider": "FeedInTariffFixed", + "provider_settings": { + "FeedInTariffFixed": null, + "FeedInTariffImport": null + }, + "providers": [ + "FeedInTariffFixed", + "FeedInTariffImport" + ] + } + } +``` + + +### Common settings for feed in tariff data import from file or JSON string + + +:::{table} feedintariff::provider_settings::FeedInTariffImport +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| import_file_path | `Union[str, pathlib.Path, NoneType]` | `rw` | `None` | Path to the file to import feed in tariff data from. | +| import_json | `Optional[str]` | `rw` | `None` | JSON string, dictionary of feed in tariff forecast value lists. | +::: + + + +**Example Input/Output** + + + +```json + { + "feedintariff": { + "provider_settings": { + "FeedInTariffImport": { + "import_file_path": null, + "import_json": "{\"fead_in_tariff_wh\": [0.000078, 0.000078, 0.000023]}" + } + } + } + } +``` + + +### Common settings for elecprice fixed price + + +:::{table} feedintariff::provider_settings::FeedInTariffFixed +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| feed_in_tariff_kwh | `Optional[float]` | `rw` | `None` | Electricity price feed in tariff [€/kWH]. | +::: + + + +**Example Input/Output** + + + +```json + { + "feedintariff": { + "provider_settings": { + "FeedInTariffFixed": { + "feed_in_tariff_kwh": 0.078 + } + } + } + } +``` + + +### Feed In Tariff Prediction Provider Configuration + + +:::{table} feedintariff::provider_settings +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| FeedInTariffFixed | `Optional[akkudoktoreos.prediction.feedintarifffixed.FeedInTariffFixedCommonSettings]` | `rw` | `None` | FeedInTariffFixed settings | +| FeedInTariffImport | `Optional[akkudoktoreos.prediction.feedintariffimport.FeedInTariffImportCommonSettings]` | `rw` | `None` | FeedInTariffImport settings | +::: + + + +**Example Input/Output** + + + +```json + { + "feedintariff": { + "provider_settings": { + "FeedInTariffFixed": null, + "FeedInTariffImport": null + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configgeneral.md b/tests/testdata/docs/_generated/configgeneral.md new file mode 100644 index 00000000..73a55e96 --- /dev/null +++ b/tests/testdata/docs/_generated/configgeneral.md @@ -0,0 +1,67 @@ +## General settings + + +:::{table} general +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| config_file_path | | `Optional[pathlib.Path]` | `ro` | `N/A` | Path to EOS configuration file. | +| config_folder_path | | `Optional[pathlib.Path]` | `ro` | `N/A` | Path to EOS configuration directory. | +| config_save_interval_sec | `EOS_GENERAL__CONFIG_SAVE_INTERVAL_SEC` | `int` | `rw` | `60` | Automatic configuration file saving interval [seconds]. | +| config_save_mode | `EOS_GENERAL__CONFIG_SAVE_MODE` | `` | `rw` | `AUTOMATIC` | Configuration file save mode for configuration changes ['MANUAL', 'AUTOMATIC']. Defaults to 'AUTOMATIC'. | +| data_folder_path | `EOS_GENERAL__DATA_FOLDER_PATH` | `Path` | `rw` | `required` | Path to EOS data folder. | +| data_output_path | | `Optional[pathlib.Path]` | `ro` | `N/A` | Computed data_output_path based on data_folder_path. | +| data_output_subpath | `EOS_GENERAL__DATA_OUTPUT_SUBPATH` | `Optional[pathlib.Path]` | `rw` | `output` | Sub-path for the EOS output data folder. | +| home_assistant_addon | `EOS_GENERAL__HOME_ASSISTANT_ADDON` | `bool` | `rw` | `required` | EOS is running as home assistant add-on. | +| latitude | `EOS_GENERAL__LATITUDE` | `Optional[float]` | `rw` | `52.52` | Latitude in decimal degrees between -90 and 90. North is positive (ISO 19115) (°) | +| longitude | `EOS_GENERAL__LONGITUDE` | `Optional[float]` | `rw` | `13.405` | Longitude in decimal degrees within -180 to 180 (°) | +| timezone | | `Optional[str]` | `ro` | `N/A` | Computed timezone based on latitude and longitude. | +| version | `EOS_GENERAL__VERSION` | `Optional[str]` | `rw` | `None` | Configuration file version. | +::: + + + +**Example Input** + + + +```json + { + "general": { + "config_save_mode": "AUTOMATIC", + "config_save_interval_sec": 60, + "version": "0.0.0", + "data_folder_path": "/home/user/.local/share/net.akkudoktoreos.net", + "data_output_subpath": "output", + "latitude": 52.52, + "longitude": 13.405 + } + } +``` + + + +**Example Output** + + + +```json + { + "general": { + "config_save_mode": "AUTOMATIC", + "config_save_interval_sec": 60, + "version": "0.0.0", + "data_folder_path": "/home/user/.local/share/net.akkudoktoreos.net", + "data_output_subpath": "output", + "latitude": 52.52, + "longitude": 13.405, + "timezone": "Europe/Berlin", + "data_output_path": "/home/user/.local/share/net.akkudoktoreos.net/output", + "config_folder_path": "/home/user/.config/net.akkudoktoreos.net", + "config_file_path": "/home/user/.config/net.akkudoktoreos.net/EOS.config.json" + } + } +``` + diff --git a/tests/testdata/docs/_generated/configload.md b/tests/testdata/docs/_generated/configload.md new file mode 100644 index 00000000..95055dad --- /dev/null +++ b/tests/testdata/docs/_generated/configload.md @@ -0,0 +1,163 @@ +## Load Prediction Configuration + + +:::{table} load +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| loadakkudoktor | `EOS_LOAD__LOADAKKUDOKTOR` | `LoadAkkudoktorCommonSettings` | `rw` | `required` | LoadAkkudoktor provider settings. | +| loadimport | `EOS_LOAD__LOADIMPORT` | `LoadImportCommonSettings` | `rw` | `required` | LoadImport provider settings. | +| loadvrm | `EOS_LOAD__LOADVRM` | `LoadVrmCommonSettings` | `rw` | `required` | LoadVrm provider settings. | +| provider | `EOS_LOAD__PROVIDER` | `Optional[str]` | `rw` | `None` | Load provider id of provider to be used. | +| providers | | `list[str]` | `ro` | `N/A` | Available load provider ids. | +::: + + + +**Example Input** + + + +```json + { + "load": { + "provider": "LoadAkkudoktor", + "loadakkudoktor": { + "loadakkudoktor_year_energy_kwh": null + }, + "loadvrm": { + "load_vrm_token": "your-token", + "load_vrm_idsite": 12345 + }, + "loadimport": { + "import_file_path": null, + "import_json": null + } + } + } +``` + + + +**Example Output** + + + +```json + { + "load": { + "provider": "LoadAkkudoktor", + "loadakkudoktor": { + "loadakkudoktor_year_energy_kwh": null + }, + "loadvrm": { + "load_vrm_token": "your-token", + "load_vrm_idsite": 12345 + }, + "loadimport": { + "import_file_path": null, + "import_json": null + }, + "providers": [ + "LoadAkkudoktor", + "LoadAkkudoktorAdjusted", + "LoadVrm", + "LoadImport" + ] + } + } +``` + + +### Common settings for load forecast VRM API + + +:::{table} load::loadvrm +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| load_vrm_idsite | `int` | `rw` | `12345` | VRM-Installation-ID | +| load_vrm_token | `str` | `rw` | `your-token` | Token for Connecting VRM API | +::: + + + +**Example Input/Output** + + + +```json + { + "load": { + "loadvrm": { + "load_vrm_token": "your-token", + "load_vrm_idsite": 12345 + } + } + } +``` + + +### Common settings for load data import from file or JSON string + + +:::{table} load::loadimport +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| import_file_path | `Union[str, pathlib.Path, NoneType]` | `rw` | `None` | Path to the file to import load data from. | +| import_json | `Optional[str]` | `rw` | `None` | JSON string, dictionary of load forecast value lists. | +::: + + + +**Example Input/Output** + + + +```json + { + "load": { + "loadimport": { + "import_file_path": null, + "import_json": "{\"loadforecast_power_w\": [676.71, 876.19, 527.13]}" + } + } + } +``` + + +### Common settings for load data import from file + + +:::{table} load::loadakkudoktor +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| loadakkudoktor_year_energy_kwh | `Optional[float]` | `rw` | `None` | Yearly energy consumption (kWh). | +::: + + + +**Example Input/Output** + + + +```json + { + "load": { + "loadakkudoktor": { + "loadakkudoktor_year_energy_kwh": 40421.0 + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configlogging.md b/tests/testdata/docs/_generated/configlogging.md new file mode 100644 index 00000000..04ad1f97 --- /dev/null +++ b/tests/testdata/docs/_generated/configlogging.md @@ -0,0 +1,45 @@ +## Logging Configuration + + +:::{table} logging +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| console_level | `EOS_LOGGING__CONSOLE_LEVEL` | `Optional[str]` | `rw` | `None` | Logging level when logging to console. | +| file_level | `EOS_LOGGING__FILE_LEVEL` | `Optional[str]` | `rw` | `None` | Logging level when logging to file. | +| file_path | | `Optional[pathlib.Path]` | `ro` | `N/A` | Computed log file path based on data output path. | +::: + + + +**Example Input** + + + +```json + { + "logging": { + "console_level": "TRACE", + "file_level": "TRACE" + } + } +``` + + + +**Example Output** + + + +```json + { + "logging": { + "console_level": "TRACE", + "file_level": "TRACE", + "file_path": "/home/user/.local/share/net.akkudoktor.eos/output/eos.log" + } + } +``` + diff --git a/tests/testdata/docs/_generated/configmeasurement.md b/tests/testdata/docs/_generated/configmeasurement.md new file mode 100644 index 00000000..303e6793 --- /dev/null +++ b/tests/testdata/docs/_generated/configmeasurement.md @@ -0,0 +1,75 @@ +## Measurement Configuration + + +:::{table} measurement +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| grid_export_emr_keys | `EOS_MEASUREMENT__GRID_EXPORT_EMR_KEYS` | `Optional[list[str]]` | `rw` | `None` | The keys of the measurements that are energy meter readings of energy export to grid [kWh]. | +| grid_import_emr_keys | `EOS_MEASUREMENT__GRID_IMPORT_EMR_KEYS` | `Optional[list[str]]` | `rw` | `None` | The keys of the measurements that are energy meter readings of energy import from grid [kWh]. | +| historic_hours | `EOS_MEASUREMENT__HISTORIC_HOURS` | `Optional[int]` | `rw` | `17520` | Number of hours into the past for measurement data | +| keys | | `list[str]` | `ro` | `N/A` | The keys of the measurements that can be stored. | +| load_emr_keys | `EOS_MEASUREMENT__LOAD_EMR_KEYS` | `Optional[list[str]]` | `rw` | `None` | The keys of the measurements that are energy meter readings of a load [kWh]. | +| pv_production_emr_keys | `EOS_MEASUREMENT__PV_PRODUCTION_EMR_KEYS` | `Optional[list[str]]` | `rw` | `None` | The keys of the measurements that are PV production energy meter readings [kWh]. | +::: + + + +**Example Input** + + + +```json + { + "measurement": { + "historic_hours": 17520, + "load_emr_keys": [ + "load0_emr" + ], + "grid_export_emr_keys": [ + "grid_export_emr" + ], + "grid_import_emr_keys": [ + "grid_import_emr" + ], + "pv_production_emr_keys": [ + "pv1_emr" + ] + } + } +``` + + + +**Example Output** + + + +```json + { + "measurement": { + "historic_hours": 17520, + "load_emr_keys": [ + "load0_emr" + ], + "grid_export_emr_keys": [ + "grid_export_emr" + ], + "grid_import_emr_keys": [ + "grid_import_emr" + ], + "pv_production_emr_keys": [ + "pv1_emr" + ], + "keys": [ + "grid_export_emr", + "grid_import_emr", + "load0_emr", + "pv1_emr" + ] + } + } +``` + diff --git a/tests/testdata/docs/_generated/configoptimization.md b/tests/testdata/docs/_generated/configoptimization.md new file mode 100644 index 00000000..74651c8a --- /dev/null +++ b/tests/testdata/docs/_generated/configoptimization.md @@ -0,0 +1,104 @@ +## General Optimization Configuration + + +:::{table} optimization +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| algorithm | `EOS_OPTIMIZATION__ALGORITHM` | `str` | `rw` | `GENETIC` | The optimization algorithm. Defaults to GENETIC | +| genetic | `EOS_OPTIMIZATION__GENETIC` | `GeneticCommonSettings` | `rw` | `required` | Genetic optimization algorithm configuration. | +| horizon | | `int` | `ro` | `N/A` | Number of optimization steps. | +| horizon_hours | `EOS_OPTIMIZATION__HORIZON_HOURS` | `int` | `rw` | `24` | The general time window within which the energy optimization goal shall be achieved [h]. Defaults to 24 hours. | +| interval | `EOS_OPTIMIZATION__INTERVAL` | `int` | `rw` | `3600` | The optimization interval [sec]. Defaults to 3600 seconds (1 hour) | +| keys | | `list[str]` | `ro` | `N/A` | The keys of the solution. | +::: + + + +**Example Input** + + + +```json + { + "optimization": { + "horizon_hours": 24, + "interval": 3600, + "algorithm": "GENETIC", + "genetic": { + "individuals": 400, + "generations": 400, + "seed": null, + "penalties": { + "ev_soc_miss": 10 + } + } + } + } +``` + + + +**Example Output** + + + +```json + { + "optimization": { + "horizon_hours": 24, + "interval": 3600, + "algorithm": "GENETIC", + "genetic": { + "individuals": 400, + "generations": 400, + "seed": null, + "penalties": { + "ev_soc_miss": 10 + } + }, + "keys": [], + "horizon": 24 + } + } +``` + + +### General Genetic Optimization Algorithm Configuration + + +:::{table} optimization::genetic +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| generations | `Optional[int]` | `rw` | `400` | Number of generations to evolve [>= 10]. Defaults to 400. | +| individuals | `Optional[int]` | `rw` | `300` | Number of individuals (solutions) in the population [>= 10]. Defaults to 300. | +| penalties | `dict[str, Union[float, int, str]]` | `rw` | `required` | Penalty parameters used in fitness evaluation. | +| seed | `Optional[int]` | `rw` | `None` | Random seed for reproducibility. None = random. | +::: + + + +**Example Input/Output** + + + +```json + { + "optimization": { + "genetic": { + "individuals": 300, + "generations": 400, + "seed": null, + "penalties": { + "ev_soc_miss": 10 + } + } + } + } +``` + diff --git a/tests/testdata/docs/_generated/configprediction.md b/tests/testdata/docs/_generated/configprediction.md new file mode 100644 index 00000000..d5b4d796 --- /dev/null +++ b/tests/testdata/docs/_generated/configprediction.md @@ -0,0 +1,28 @@ +## General Prediction Configuration + + +:::{table} prediction +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| historic_hours | `EOS_PREDICTION__HISTORIC_HOURS` | `Optional[int]` | `rw` | `48` | Number of hours into the past for historical predictions data | +| hours | `EOS_PREDICTION__HOURS` | `Optional[int]` | `rw` | `48` | Number of hours into the future for predictions | +::: + + + +**Example Input/Output** + + + +```json + { + "prediction": { + "hours": 48, + "historic_hours": 48 + } + } +``` + diff --git a/tests/testdata/docs/_generated/configpvforecast.md b/tests/testdata/docs/_generated/configpvforecast.md new file mode 100644 index 00000000..ea78dbd2 --- /dev/null +++ b/tests/testdata/docs/_generated/configpvforecast.md @@ -0,0 +1,346 @@ +## PV Forecast Configuration + + +:::{table} pvforecast +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| max_planes | `EOS_PVFORECAST__MAX_PLANES` | `Optional[int]` | `rw` | `0` | Maximum number of planes that can be set | +| planes | `EOS_PVFORECAST__PLANES` | `Optional[list[akkudoktoreos.prediction.pvforecast.PVForecastPlaneSetting]]` | `rw` | `None` | Plane configuration. | +| planes_azimuth | | `List[float]` | `ro` | `N/A` | Compute a list of the azimuths per active planes. | +| planes_inverter_paco | | `Any` | `ro` | `N/A` | Compute a list of the maximum power rating of the inverter per active planes. | +| planes_peakpower | | `List[float]` | `ro` | `N/A` | Compute a list of the peak power per active planes. | +| planes_tilt | | `List[float]` | `ro` | `N/A` | Compute a list of the tilts per active planes. | +| planes_userhorizon | | `Any` | `ro` | `N/A` | Compute a list of the user horizon per active planes. | +| provider | `EOS_PVFORECAST__PROVIDER` | `Optional[str]` | `rw` | `None` | PVForecast provider id of provider to be used. | +| provider_settings | `EOS_PVFORECAST__PROVIDER_SETTINGS` | `PVForecastCommonProviderSettings` | `rw` | `required` | Provider settings | +| providers | | `list[str]` | `ro` | `N/A` | Available PVForecast provider ids. | +::: + + + +**Example Input** + + + +```json + { + "pvforecast": { + "provider": "PVForecastAkkudoktor", + "provider_settings": { + "PVForecastImport": null, + "PVForecastVrm": null + }, + "planes": [ + { + "surface_tilt": 10.0, + "surface_azimuth": 180.0, + "userhorizon": [ + 10.0, + 20.0, + 30.0 + ], + "peakpower": 5.0, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 0, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 6000, + "modules_per_string": 20, + "strings_per_inverter": 2 + }, + { + "surface_tilt": 20.0, + "surface_azimuth": 90.0, + "userhorizon": [ + 5.0, + 15.0, + 25.0 + ], + "peakpower": 3.5, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 1, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 4000, + "modules_per_string": 20, + "strings_per_inverter": 2 + } + ], + "max_planes": 1 + } + } +``` + + + +**Example Output** + + + +```json + { + "pvforecast": { + "provider": "PVForecastAkkudoktor", + "provider_settings": { + "PVForecastImport": null, + "PVForecastVrm": null + }, + "planes": [ + { + "surface_tilt": 10.0, + "surface_azimuth": 180.0, + "userhorizon": [ + 10.0, + 20.0, + 30.0 + ], + "peakpower": 5.0, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 0, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 6000, + "modules_per_string": 20, + "strings_per_inverter": 2 + }, + { + "surface_tilt": 20.0, + "surface_azimuth": 90.0, + "userhorizon": [ + 5.0, + 15.0, + 25.0 + ], + "peakpower": 3.5, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 1, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 4000, + "modules_per_string": 20, + "strings_per_inverter": 2 + } + ], + "max_planes": 1, + "providers": [ + "PVForecastAkkudoktor", + "PVForecastVrm", + "PVForecastImport" + ], + "planes_peakpower": [ + 5.0, + 3.5 + ], + "planes_azimuth": [ + 180.0, + 90.0 + ], + "planes_tilt": [ + 10.0, + 20.0 + ], + "planes_userhorizon": [ + [ + 10.0, + 20.0, + 30.0 + ], + [ + 5.0, + 15.0, + 25.0 + ] + ], + "planes_inverter_paco": [ + 6000.0, + 4000.0 + ] + } + } +``` + + +### Common settings for PV forecast VRM API + + +:::{table} pvforecast::provider_settings::PVForecastVrm +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| pvforecast_vrm_idsite | `int` | `rw` | `12345` | VRM-Installation-ID | +| pvforecast_vrm_token | `str` | `rw` | `your-token` | Token for Connecting VRM API | +::: + + + +**Example Input/Output** + + + +```json + { + "pvforecast": { + "provider_settings": { + "PVForecastVrm": { + "pvforecast_vrm_token": "your-token", + "pvforecast_vrm_idsite": 12345 + } + } + } + } +``` + + +### Common settings for pvforecast data import from file or JSON string + + +:::{table} pvforecast::provider_settings::PVForecastImport +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| import_file_path | `Union[str, pathlib.Path, NoneType]` | `rw` | `None` | Path to the file to import PV forecast data from. | +| import_json | `Optional[str]` | `rw` | `None` | JSON string, dictionary of PV forecast value lists. | +::: + + + +**Example Input/Output** + + + +```json + { + "pvforecast": { + "provider_settings": { + "PVForecastImport": { + "import_file_path": null, + "import_json": "{\"pvforecast_ac_power\": [0, 8.05, 352.91]}" + } + } + } + } +``` + + +### PV Forecast Provider Configuration + + +:::{table} pvforecast::provider_settings +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| PVForecastImport | `Optional[akkudoktoreos.prediction.pvforecastimport.PVForecastImportCommonSettings]` | `rw` | `None` | PVForecastImport settings | +| PVForecastVrm | `Optional[akkudoktoreos.prediction.pvforecastvrm.PVForecastVrmCommonSettings]` | `rw` | `None` | PVForecastVrm settings | +::: + + + +**Example Input/Output** + + + +```json + { + "pvforecast": { + "provider_settings": { + "PVForecastImport": null, + "PVForecastVrm": null + } + } + } +``` + + +### PV Forecast Plane Configuration + + +:::{table} pvforecast::planes::list +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| albedo | `Optional[float]` | `rw` | `None` | Proportion of the light hitting the ground that it reflects back. | +| inverter_model | `Optional[str]` | `rw` | `None` | Model of the inverter of this plane. | +| inverter_paco | `Optional[int]` | `rw` | `None` | AC power rating of the inverter [W]. | +| loss | `Optional[float]` | `rw` | `14.0` | Sum of PV system losses in percent | +| module_model | `Optional[str]` | `rw` | `None` | Model of the PV modules of this plane. | +| modules_per_string | `Optional[int]` | `rw` | `None` | Number of the PV modules of the strings of this plane. | +| mountingplace | `Optional[str]` | `rw` | `free` | Type of mounting for PV system. Options are 'free' for free-standing and 'building' for building-integrated. | +| optimal_surface_tilt | `Optional[bool]` | `rw` | `False` | Calculate the optimum tilt angle. Ignored for two-axis tracking. | +| optimalangles | `Optional[bool]` | `rw` | `False` | Calculate the optimum tilt and azimuth angles. Ignored for two-axis tracking. | +| peakpower | `Optional[float]` | `rw` | `None` | Nominal power of PV system in kW. | +| pvtechchoice | `Optional[str]` | `rw` | `crystSi` | PV technology. One of 'crystSi', 'CIS', 'CdTe', 'Unknown'. | +| strings_per_inverter | `Optional[int]` | `rw` | `None` | Number of the strings of the inverter of this plane. | +| surface_azimuth | `Optional[float]` | `rw` | `180.0` | Orientation (azimuth angle) of the (fixed) plane. Clockwise from north (north=0, east=90, south=180, west=270). | +| surface_tilt | `Optional[float]` | `rw` | `30.0` | Tilt angle from horizontal plane. Ignored for two-axis tracking. | +| trackingtype | `Optional[int]` | `rw` | `None` | Type of suntracking. 0=fixed, 1=single horizontal axis aligned north-south, 2=two-axis tracking, 3=vertical axis tracking, 4=single horizontal axis aligned east-west, 5=single inclined axis aligned north-south. | +| userhorizon | `Optional[List[float]]` | `rw` | `None` | Elevation of horizon in degrees, at equally spaced azimuth clockwise from north. | +::: + + + +**Example Input/Output** + + + +```json + { + "pvforecast": { + "planes": [ + { + "surface_tilt": 10.0, + "surface_azimuth": 180.0, + "userhorizon": [ + 10.0, + 20.0, + 30.0 + ], + "peakpower": 5.0, + "pvtechchoice": "crystSi", + "mountingplace": "free", + "loss": 14.0, + "trackingtype": 0, + "optimal_surface_tilt": false, + "optimalangles": false, + "albedo": null, + "module_model": null, + "inverter_model": null, + "inverter_paco": 6000, + "modules_per_string": 20, + "strings_per_inverter": 2 + } + ] + } + } +``` + diff --git a/tests/testdata/docs/_generated/configserver.md b/tests/testdata/docs/_generated/configserver.md new file mode 100644 index 00000000..57e537df --- /dev/null +++ b/tests/testdata/docs/_generated/configserver.md @@ -0,0 +1,42 @@ +## Server Configuration + + +:::{table} server +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| eosdash_host | `EOS_SERVER__EOSDASH_HOST` | `Optional[str]` | `rw` | `None` | EOSdash server IP address. Defaults to EOS server IP address. | +| eosdash_port | `EOS_SERVER__EOSDASH_PORT` | `Optional[int]` | `rw` | `None` | EOSdash server IP port number. Defaults to EOS server IP port number + 1. | +| eosdash_supervise_interval_sec | `EOS_SERVER__EOSDASH_SUPERVISE_INTERVAL_SEC` | `int` | `rw` | `10` | Supervision interval for EOS server to supervise EOSdash [seconds]. | +| host | `EOS_SERVER__HOST` | `Optional[str]` | `rw` | `127.0.0.1` | EOS server IP address. Defaults to 127.0.0.1. | +| port | `EOS_SERVER__PORT` | `Optional[int]` | `rw` | `8503` | EOS server IP port number. Defaults to 8503. | +| reload | `EOS_SERVER__RELOAD` | `Optional[bool]` | `rw` | `False` | Enable server auto-reload for debugging or development. Default is False. Monitors the package directory for changes and reloads the server. | +| run_as_user | `EOS_SERVER__RUN_AS_USER` | `Optional[str]` | `rw` | `None` | The name of the target user to switch to. If ``None`` (default), the current effective user is used and no privilege change is attempted. | +| startup_eosdash | `EOS_SERVER__STARTUP_EOSDASH` | `Optional[bool]` | `rw` | `True` | EOS server to start EOSdash server. Defaults to True. | +| verbose | `EOS_SERVER__VERBOSE` | `Optional[bool]` | `rw` | `False` | Enable debug output | +::: + + + +**Example Input/Output** + + + +```json + { + "server": { + "host": "127.0.0.1", + "port": 8503, + "verbose": false, + "startup_eosdash": true, + "eosdash_host": "127.0.0.1", + "eosdash_port": 8504, + "eosdash_supervise_interval_sec": 10, + "run_as_user": null, + "reload": true + } + } +``` + diff --git a/tests/testdata/docs/_generated/configutils.md b/tests/testdata/docs/_generated/configutils.md new file mode 100644 index 00000000..0f0536f7 --- /dev/null +++ b/tests/testdata/docs/_generated/configutils.md @@ -0,0 +1,23 @@ +## Utils Configuration + + +:::{table} utils +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +::: + + + +**Example Input/Output** + + + +```json + { + "utils": {} + } +``` + diff --git a/tests/testdata/docs/_generated/configweather.md b/tests/testdata/docs/_generated/configweather.md new file mode 100644 index 00000000..dee9ad63 --- /dev/null +++ b/tests/testdata/docs/_generated/configweather.md @@ -0,0 +1,116 @@ +## Weather Forecast Configuration + + +:::{table} weather +:widths: 10 20 10 5 5 30 +:align: left + +| Name | Environment Variable | Type | Read-Only | Default | Description | +| ---- | -------------------- | ---- | --------- | ------- | ----------- | +| provider | `EOS_WEATHER__PROVIDER` | `Optional[str]` | `rw` | `None` | Weather provider id of provider to be used. | +| provider_settings | `EOS_WEATHER__PROVIDER_SETTINGS` | `WeatherCommonProviderSettings` | `rw` | `required` | Provider settings | +| providers | | `list[str]` | `ro` | `N/A` | Available weather provider ids. | +::: + + + +**Example Input** + + + +```json + { + "weather": { + "provider": "WeatherImport", + "provider_settings": { + "WeatherImport": null + } + } + } +``` + + + +**Example Output** + + + +```json + { + "weather": { + "provider": "WeatherImport", + "provider_settings": { + "WeatherImport": null + }, + "providers": [ + "BrightSky", + "ClearOutside", + "OpenMeteo", + "WeatherImport" + ] + } + } +``` + + +### Common settings for weather data import from file or JSON string + + +:::{table} weather::provider_settings::WeatherImport +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| import_file_path | `Union[str, pathlib.Path, NoneType]` | `rw` | `None` | Path to the file to import weather data from. | +| import_json | `Optional[str]` | `rw` | `None` | JSON string, dictionary of weather forecast value lists. | +::: + + + +**Example Input/Output** + + + +```json + { + "weather": { + "provider_settings": { + "WeatherImport": { + "import_file_path": null, + "import_json": "{\"weather_temp_air\": [18.3, 17.8, 16.9]}" + } + } + } + } +``` + + +### Weather Forecast Provider Configuration + + +:::{table} weather::provider_settings +:widths: 10 10 5 5 30 +:align: left + +| Name | Type | Read-Only | Default | Description | +| ---- | ---- | --------- | ------- | ----------- | +| WeatherImport | `Optional[akkudoktoreos.prediction.weatherimport.WeatherImportCommonSettings]` | `rw` | `None` | WeatherImport settings | +::: + + + +**Example Input/Output** + + + +```json + { + "weather": { + "provider_settings": { + "WeatherImport": null + } + } + } +``` + diff --git a/tests/testdata/test_example_report.pdf b/tests/testdata/test_example_report.pdf index 59339b7631243dad796154eedf63f7c73c708433..20e6747f91ead137fb64e9f1e0bb72e5da69d443 100644 GIT binary patch literal 97354 zcmb@sV~}Of(*`)xoW`_m+qP{@+cs|7wr$(Crfu7{?e6X0AMw76i2bk|H{#Z*$|q6v zq4LR-hgeQXgqog)36i*C9k09%k{*u^&qm)Il8XzER>{@Y2#;1!&q2@9#srU6PS3>1 z9*^N0pn%8218HP!_z#fne<_IASUcb`{tG~>Dywg9WZ;0u`tPZ*i-V}5gPwyC9@D=U zPUwX%$WN49%=f@L2vM1=tuU8ad!;e7hAA`3}X%#Q~32-0C|7f&Ywx z{~5*cwEkB(^#At_{1f*-(F2UWqx*OGvt>E{rzE$n<*#8aXx3;$Vp3(n<``;(=kNCewCuL-9;$VtL{~vcE zW|rTZ#iJFm{5}C8BLf>lqkm$ycK{gaSwXsFTx3YwY_OmNacX=((s-igzI@QP8*@0y zOZgTsuL^U9|Fw%pu?z3Fx_df`$)kaFq&1s~-S5#gXRCE7RoN%>!pdU3JV0&s{s7p& zT`bt&MqdSuc{tkh+TK!nbmK?7LvG*&N_ux=2W1D3 zP37KlE45;@&6BHA+Fk>w9S+3%x%@qHJe;YHo~ z>F4o#!jD7&JkoI9Ww29kk=*6BLbL??S*_2xPx96a!EoAy=vQY@T)casV?wZdHgG|7 zpZjxstO@kxzNUf+m9sLU8I_Tg81c^Tp|y(_x26}wAGFp_cE1iU?sqX79?lr=__OLA z##kPEruA4J6D2p}020*mAE>=hHh!<5zmo^WF9a{3G7UdnxjQ1b)f2Pe_#|=h3&wDQ zj*w4@K$qqjjLL`xAVyg5n#l3U*%F5~BUQLvCO33~3H4IMI|(wZM5A}3Uf(b0j9|0w zARYO_?0sfILwI+NXV#}X`Xl}W&QJAkZT=T{CoJrLz(ME~Zz9nyy)y=SHE#x2gXK6} zN0-%owNkz78t{`%`9t)wj$ThEb>LI3fgE^4Ir>}2v9fG#wsKCaP@_rP?9Sf6QBQPF z$^nWIS^Ba5dKIlOx$*OW5&tfZ9MiKW;NzU5cl7E;$z)sst;@#!+}n>L8?rFhV?X8V zMEs+?C_0ogTl|u&lZ>E-8Z^kEy;tYV@k(};u@lSo_W^BmS$zFn8AFcN5`UOQ@oy9gc7Ev&j&a_N0@W5}6D}u) z&wipshx(*pn|r@qM?1ej-zS6y2rg`BIKL&YlszKSzXQfnre6p0bO8 zrwFsXBpXOhXhd+mXb33&qWEx*W8F-l;%T!PyP6}kgA~Ns)B}fa^I#mkeJF9wq)H7L zGP#ViE%r9+t1R z74CGCXh#&X?YGkdd&}$eFT(79#R=3^oW=mg}Ezx+8K+%YK)&jcl9?g zl6$2DD#>ujP%J##TBeHD_te^&3hG#-i)s=T;I}BfXLQa#i0*bQs9b|34S$I$wt2j= z7mXE@S!vyzU=xH$w9|oP{85Y2Cnh!Nt?>H zf}wkzThpBEE+UlK`y>CcHOn>2-DI$i9`351ld2bxU}pjGr&atHO9vxI2HL;Xm#U{GpeTlpT~7aN-5&B$YDek^ zmGaKFvf?a=1slkj;&6hbTMKhw*?NNQyg97^f)aIRarE$Y^^_Hg>SiVs73#O?g;q6m z7}wSG8sCcE%A|MHN%Fs5;6D_B?7y6_Q-3s>OO0mxnyr$! zbwy62Pdm-HF-}cuswB-zf#L&zz4QkHX0B14k|hYtbc9|PdtbJ19Q2TJ6hI(*kLmRpfxgiF4zc*@s9K}ghTBqS&i|Ke)e2km@+sKE3&KhDRdHS zv}(r`f|x&S0tQ<-oEbV>#yH;GY4KcKaAOdv=Dl z+p-zSCTTa3ay2>EcKENo5)^%oa4~et#31HdeQR?M9S%s+qM>AP0%W1$aWG1wdxfCX zP&@O0^~rT4fF0r#@9=j*GP;ewI+Fbao|+D7{TY|Qm?4TaQ7FW|A{RZ;>2w?jF7ZWF%tlCW-Or;jhS?s3AFzQ&*aQyPAg^5O@;i zDzn@x|JEF6bgpl4dbv_apa&qMzDkEtcOs4bgk{WFY3aJWEIdcM+Y|g0>t0eV4KX|L zS*>_}hxjo$G`g0c?b0w)N7&Fyjb7^5PQ%RJfAJGsM34${Pr9u1QG#VW3$Hdgj_=Wp zftek;?qX`3zshL#{1@O>QzVExY62ZzsOoZaJg3rV=X?bEjSfPyI$JTm@7LPqWX7I4 z*wnYSWIW$v8~4Ao-~ZA6I{#I7ZL2`Ga0a@t4>t_%k&8auID!BBEx~zl2s`_03-^`C z`#2uVq#-OQP1fr*9jzeJe+e>35KQ|JHRm?!;zXz_ok=f6qy z|B$^Wn1^FMU;|CM`cO4_ftz{?yA>-6M1(gi*dvbuMk8B&%$|7i@igwas1Jg=

4`}?K?h{P4AM2IhJ@HIcI#4 zCb65Y!3kXw(oxs`>A+OhP%#hl!gcb4RNd#Di@vjsK&pEBnAAiKY4q0LM({+Mr5kY^PduL&JbG?*T+@Wy-nz(t3(&al^ltk7}S}a*+5|K;Sw> zFJhH`n}tFYQeTfZ>VSzN0NdOA3mq2bTlha{0`u*)hTx6-M^iZ`&Q2{dD_@hU4+ge0 z+Z1+?Uqa*$!DU+ z-idD$KS}ikcSv`V#4v69JVe$RcqCgOmgRrfC&87DSK93zZ2}d5S!9_Nx%N`WvTO&v zM6`T|VLK<6h>jlsglR~-F4Ilhq8KxCc@d|TB6jgD1+r$(23<$OVhRE4>r=>m&~6D` z*LK+8cEZ+ZhA=z05PtUd30zong4$^F($Ty}qEa=8OsR#~a`mTo^f(xs+dB)tw5wxz z(>%<^^-aihh7<|L4;H4pXY68WKpCXlYyD^i5cuhX8OP(&hXPX9>Ufdkx+ZYHJnY>6 zgG*wVf)ynWu{c6dM9m4TnqU2j$rP<&>T zEJ|{zrGn$j%o~fo#OCkTm3w=jolZ8~`!mqs;oN?b+#Q<9$?`HHH~aV6$06I_x_bxt z$3rd)XWV_x^uW&wwZaT>h==BQL2q=7Yj||}$QRMAjxNl2Y4c8&cZ{VfQ*E-NoV=)X z)8R#ZRTdnNQP9$~0qj=D`X5??p3OK#3~i-?Bj~YN88Bp=t*v`pVs)L}Mt>@@6=&@S zyS|?jGTVt?OaCpyz$QNG!!PwktHyByo3~qfi!wSQa z4&8q7+!j&?+WbdktfdE~Y(3R8{vkSZsKSE?lAw$RqQLnpR+vxET?%H{SF}Vm{tBTDj+e%GvpXF8(?uRq?l93R4dICwI+u8fkJfO`4*-za3F+gRO>Fg zox_#h2}~60%@*5or=J1{XPg2oQ4+hH(k^Hb(VYB6N_=J!PT!B_Qm=i;<$uhd!!W=X9?_S|~{BCSn6sdaBa9CXtsd?hfL)C=#rxbT%52Ko+&9RlVJ zI3_5gN6a5~Ny370{godk`(6*J_bx9l5P_(8N+z;V23KKxOATuoLRItA9HAEj`)FEm zI8tvlU}uhCC5WUXyM8<64PBh(Is7yQ?zB%yaQ)daWWPAxBuESpv;J`=kD5Wg6)3il zU=kGYn}}6(Lbq{O=7VblzK8?yDS`s1UI?-mhe5%Oa$eZw1+t!!6+PSOK^c2bbk1pF zg$NA$gPn#xc#QoOKc6w|*dvQ)lX5gdF-%JFd_w7lSOoZ~#eGD&ofMV(PmMx2y}*|i zP7STV{lHwawj?R#ZSuO)$2lXYi_!^ZhN=1?gvT(uk)z2hNS(L_TD5CbPwsI^%uTts z)OiZ9{P02~A~Qq#qy&%f59yJ_51s)WlPnYS{%sKYVv&%na(MYW^MI3PmHg5*Li76y zpO-Il2NUKn5s@O!nI8g>28T=7LcIH7Ox>4;_nc%AW*W{Cf#jmMl051v?q*U^o>z}= z^JDP>A%OWYKo-qbD>p{{*xPF(J!38~<4-D5%MFaGI=1_tih1HeX-1)t*?&tMj1TUy zYMt_hJu=4A^BRjmHmVwN1|+I)%lIakTI1FsFqh-bh>p-8c6KN>Q*C)?6eZp z^k|rT7_tk|hKwECN$?>RPpbR1maT)6Bh&NPn;n#*5^-G|uFCkK)fhu73Xn}_XN}Uw zp!tQW!X*ccYjGekr#;g!Gs+E()!x%>$8w*`je(NmEJ4V~Lt6CwVE7BI9VwBVB@Egu zHxgd^JU<4XAid{6SjKO}txl zvD*RWMtS054#+tNz9D4qav`w%X&X2};1TBICQn^~a_x-zTy}jlvJ>xME6Tvb-U(-f z=wlI)gpcHrGmB=>(qvgxWy&U!rwAziY}l!wY+GU>iBD_DlpAr`8{zl zz8XTliLYhU6kcE=rz!`iL4H;F*F)c|zy%k`I2>o|$hX;mf6UHX>IRwRCs@lkY6M@O ztDB;M_f0}3jn50M0PTEVxxaoV>Iy@i#sxZ31{7&Q-K6C{{8&TDU9CosW6HGsyyT{g zquQvXmSNKwtq11u^6D`Owuh$SkdjJr@q!cI7fH9s%0~j5rW&qaLp#hZV;JuBpT;gX!3eGyy>>uu`Z_Ah^+Oq(Lau5{$7qrE(YoOD2tR$Q+ z+SwJpC8P^|J%SX7^~e=kQ-lf$(;5nOI}IF!CL__M<2t-Y`u<{DKMQR=;z-`M729wu zaX0cL>T=QYY@`^0>9g-Wl{!dXLR^h)T zZB-HD*4feYdL3+TGr6kndwRnXqMvW7DhMNRe1|`f@3+n(3hph$=X_3Xy&`K~CCZk)q>E2w^gh z_0sK04a7ER25Z91>Lxn3w-@#^Vr=+NPJObj^1dy1_v^K?ZP-xE@Tsc)5+^wyN5@D~ zM$d=T?~4pi6m;<1maylp@E2P=fldg};%P;iZOYPQ38h>b_|t)~6tCVN$nNB9x71WB zp_%Nj5`roAdzV4r3Dp@}E16z7>`a_`*AMg`Kv!WQ%90Wyb&PsHv%3^?7wCqWzu2w} zWQ^DLiUK<>EOuCoLgOq0y}cZ^>O;@|rhzU_Av5 z3ovm00{&?VlKKBoSpK(%z{teR@ZZ9c;hz@Ce~Y{SkA)>O9n=4*rNQu@Zq0uRONM_p zXa1+K{FnT{zW>wIVE9il`u|o~Ze&WuyNVS zAqj)>rNf4MZ=H=@ZfdE4KLM%u+1aQbhmuHK4fq1?qxr0ZSnN}Gjwz9?qPmBHQVredp@BEqP+&&M0=Oy9^>}CbbsAH zK1SPgKU;V{%f1e?+R!(OR!%A0cQWiUN`2agb!iC~?wd$QlvQ_p*4%tvw*-&8X76D> zR&ot0#y{`b;Cx-&ybaNQwynbQ8iX}QCb=!(7G{1eZS~d%bBK&GPYmIDmk00u?0&gs z#aUzB+|)0hf)}v9fEV>1ObQ~wt8BQfs6Sv*erYDL!GF{w#+{na1MAqgOOhwbo>~`x zI*Ci>v3Kh)EuB|8w8)TG4~rWzsU)1T$jC~IWSpyC$U0<}x$>1gGO3p41xScS)Ju;% zL^lS_(72*^N@`};AF|&PUv|on%w^{m&FJ)vqX;H>a2Z5)em}-w^o^U8>|wl4b*W!7iD`oOPRK5g`ab{}Y<8xra1SL6AMADnPr$*4n_%oQal2e)dvj zr?5EAoQ;;v9-UVG0QDL%pQ-O=4n&N5mYmIaRP`VuuUs8Gag%U1LO6xYY=o|hQQR?( z(y(Dz_bL{8f*}e^5h$^lMR*hg!zreISjt|o{2+;8o3*Q;h&z15xKFp6JNcZU4pGWQ z+%zx`nR%;_4oy_}Jg=nA76hT*k8n>$UTCuT32~87tbo9q&ZLKKS|eO>SIT}!RY^?F zwIIlmC1^pT=GmlVm|e{D$O5cr#&C;ucym&PUFK*^^TE#Koi4FpVxB5CWN!{H$~#b} z!IBB)qR-d2D9aaDPAxNfQ|f3_v`L}Wyi&N4k)@DoY@uH{`ztMd|0T6Ctuy16sloN3 zTrDWCyS-97@!*s-Rk~^lqtMsjxukGro!Gig5R79WH`pNVgFG?V37U}k5gs@`Axb}G zA&4B&5RJ+yYZai#xw0?#ItIE&gTWBtrE5NuLs*bIm3$wO+pNz)dG0eSXVl}RPW_ND zltD45VtBO5ae!9rr<5Ww%bO&pD!%WdEX?_8{98@dIZW_ZocJcFf1Rxc|A+F3AS~ev zT<>5^l-{g;(+qHS_Egn`p&$GWJgbge z+vH0bMq1)yR0ZWX7$WYR%SUI7;)FLCT27k{j7CCg?M#||!>kGYmWE-9n1va#pq9U? z;-yqZ!|}6jF@V|<9-4EDPm=+0O+^Vg8+n;*#;p$MZoLH!Cx!!_gNje9EAe&>u1BU; z5sEwkB+&WrM!)vd1@un)3-;>*jDbexknRiMp=^HdO?YP9Qg|3usxSGIdhF;aAS@e`a7!>|C^Y)yb(%@juH{a!jQKCS~l`wm5pkRZ#;07-U%`b&S) zeO21w#8@8)@R=B$HarZ?c{cEw5K_ILvyH6xFaD+h@ioK14G_^{B1{EAV~kQQKL8~` zO?zW~A)r4WM40an+}I1OVdG({&>&#o7Gh&y#hvpui&WOvCoULB#bhfndlmtp9aB3dm z#tk^{Vm*e#1n(CYZ)gF*28=Qnz*qsPYv$vt;QZZdEOW^%BiMxv7f7BW6qHt$oRWap z_^^v=OnF}7Us&wc?M^l6g#B|@fDt1?ctVE~zP~w{f%5D~=x8QL&FgZaw=*!dn*0)B zal$>zTqWW(N6KT%d~%Re)Fa7m;|VS(6MEKY<&NbK{e#Hy!94RO!82^Xk~RH9ylrA= z+lQsS0M;}DxAe886NSTFb!62qBMZRdn*-o#R7bTTS$R}8PquMqoRzNLA%7@i=)RiT zWcO!&HQqHrR}F(#)o_>`5~zGmCQH&JbI{uyy**SAa3f>0Rry#qu&YuGQBM=QsIbxy zw3?pk^0J-gHO>;ikGhoQHzMalJia7ij0mNvYK&thVx$g0BWb1U z*BzDTw8Be4WjQFxankv7Ka|FKx}5u0f6u@mEpCWRy_-?-ViW{<1%L zBTUjVi^MrEdpwU#VFO~kA$e4!w=o!bMkJwC;c=2yI%0#-!^#R@7eq)?=*K-pmXTHR z@y{HTC&MtBMcpBbb&jWHkRXD)GIuIMqL{WGGThhUnPat^H6Eon1BGprF&uUAJ9U$U zZ9;lAh95{z!_s3b0s}yv1xvO$^_}6+MlbpcAfP;PNSjUm^UuTeJ=z%ke?`k~i;PCu zo`#iSgd@fY3{fHc73%zIMJMA#lPsbeC}Cm;mI>@rOLZ9+sbi8=LuMaqaPzZnq1@AQ zddm>+SB#JwnoS^P&y)OWq7i2-(V;N4bLvdkDC~8=-|&U(&jv?cA&@Pb^<7y(ppLlL zR78$VDy^+{p08v;hMrw$Jr?htqYaXV6D^7w?rH!|-r|%@I4*2(zmfu0*M?>dvQy;T z33G#r<4y-?aQrCAwNSQjw>o!X)g^78gg-Cvm$vipA>{+X4N}TQX_!C}^$$}ex(~=w zZVqsfwKf4SEDDA|#;N!$PNRAQ@^mz^YTB~|uX(@Hen_(<*s#}@ijeM}{+%LTLYaTv zZ8Q&QR$b)Fi%ik^`wHW(Y_fiAYjxa7e}3i5uf!goQHwb1@)EV=xwb)7T1-IZTfr=G(h( z;A__3P^9@Z&wBJVInEDC?jZkV^bYgzyFOE2)8`!4rg+#3OTSi2euoc?`LDtr}M!GPvBrvnVlt>$9d0$DIAa1@NEkTD5alx_pYUhF$qPo z4(@MQe(GLaB*b&hogWbUb;PhNO45)nT_*~7epu3P{!~LSS94P+nzrx?jVH&#oe$-O zsm&6ilnfH)p$T!oY!b}vhZ`o`L$n{>NkF}`ThS~>JLMX%b&u^K#3m#DZNEWRLOz-d z3zXxzDa)=VhVICWBi~>o+d8Z?HnAKcrDvLCoI+O)fHT$eb5N6M)%4`lua1ZuRn)$< zHTDaD_;YPj6~SwWg`QV|&uM)fW3d*N0Pkcvnrj9D!(KzhV|zFPU`68fg7PxNQqr`- z+SQ*Ap0Qdm?(Y8sg49H~hTw!7f#8Z#IDJX4fg0F;pmmAQ6BS9(>^Tz1i?R|+IE^~S zb`W+8G@h2yJyIHJpM+25cZL!fvN@QnS;BS}D?c0I|Kk09wPK<5#6F&L9q2Aa?Dqvg zwie&SotYqtj}o$6q7dX6mXc`P3OnH=ckbT;hq^){?=PH@lM482!27Bnt5-K-U-Lj2 z4m917tK3s~NwUad@sN>ujR=d0;}Go#pj49|G?Yq-w5fT>L_Ehu7i^?aO)h1n7IXPc zgNWW9ug2zUlB`8GLLmiAbZUZS`RYlFW+q{b=R=%?Fr4MB7}Vb{!d5Y= zpJ2{+CzC;=W}4z+)F#azO>G6WycSlN@35)GkBx#^fY1iIfY3kWc{!rk$PbeH-Pm#> zA~%J;deH2l4~YMMP)ff<-eV;KH>_%;sMRV7?8CBvb7K=>Hw>0c=!r9SSqTlPY^ynv zL$3JmZZrEc)fT;zs@yMWc^_tHy4l>b{B(~%>>YLx?W&Q!gndOzn)8~)9c}w_vSlkm z7V}gXX5#9C=FyEWdc)C5#e)z6HrSSkkLW;cOktAzP$! z_i{Ses4jd&1xR;9rCf}&3y&V3%&vV8bvRvBXZ`tCpK0K(h>qBc@!CezMGPLvd1)4= zI2Dz7OirSBF2;qy+r30b!RX=$hqX{7acE3tPFq6x&&{QSBbE{!Y=+l2A;04_}@{C?KS zBNxs`gfOG;PX%mHs-hU0OMEnAa=0($0}A>1NN^o~m>UH!HRA;uN3hU|;?7S~(IYYq zgYz2>4R~m*>y!9Wp_kMRsV?s00o4o{GCWmTnc?qZ^f5#BYt^(2co?oc9M@aX@6H_X zF$@PLAXvQN;_o8(nlm;U5={=2pVNblXgHJ;C!0OCfcII3h;>ItXO7&3Xmo@(iZ8%t zUe&(9pGI^H-2xYjba{qndK9d;Zj^%nH7mM zDJQcrE4q?Y@k}jcc1fcPO%sio7_~yMjZM zj2UM|8yQX#*99I$t6V!mdYnG_fd~V(JLVE*L0yMFqUPxP^k8Ro#P6)xRPcqBtm*Y2 zHksIGbf8EhHZ)Kh_Bxb*1s6n)U&Wcmo73VvTKb|(6e=ZBlfbzv^@IW_wo(x+f4WeSr-XMt+Vo`DtzIotK(mb7M&}w|4-x|Vk zNWSE;(~dMZ!jL4G6`HQj(bSN{GN@Ql$y?`(4C|T)Es`IaC?wX01)!YXiy>=) zUYV3C&Oo5tCVMh{I29c71=J>O%KhHe)gP;cp52&l+c$+V4%2k!NzDPOeZyucD zA<-vR}tlW@g($QC?%tX@{_%hf}D&vp8OiMxqO<-=ZJ3 zRLpsx_-vSWtQFCVk9s&zPeJ|6F=|hu0w>}+Nyjt(8D}!~l=fA{eQ;*Ni4R(#{wOiW z*IGnalpZ};GFq2-SL(+Ev*epO7CYcJT8Q#7q>cpC36#u`J9YmYss%G@o7`d1CwS4X zUFtL8MH6N^g6wE28pI*vN+MBe?V%L;l{x!N{zX{^pIzdEZ%4%knLWNgX!`4m!~#t6Gm(9=*dICFfHIN91>8mP?1 z;%z(p?Dhb*uy@g#mPTwm>el6$F6$YJX{Q{ACTpr|22slMLAw!QhU0+P@Vtd;EHdi! zs@E!`^64O;B`H)xopr#`TU$7*LjW2r;-nY2wJcEd(Vjl>wIEhRk(1N*2T=EGtqySZ-)<$627?jkA z^C$3@Ey>@sF(l}ctG%mOhBGf9j>`MvgJd=3;1Y=3Guye$_*z9ZXMASs1V%9Xi}W}! zDMhKZ>N6T(MWrK?tK>_HvAZMJnhIzi35;zh1QsfRQix+GYYZd!Cp}?_4`S7LOEgM& z_%AiRSH5#DGUQONLUH_GnzXHanWn_?pmJhyG(%L?l!+z+yu#Sva(lxjH31^U2QAp# zRKwx6Qk*<ducO|n}ziNE7NZLlad4eKL>y3CO>AUq#g z6)T;Q5cu44h6m(#P^lR=BOXvSs6e3{J6Rim$JW}dtog)2Y*k1&V>tU>41Q^ujj$1`dUgBr@5jSCoR+gwuFLlLV5_ay$GTFU|&C zr-~UQxry9RE01*;-?H5^dPsn=kP>$jG?_29X(o=94o-k*X1)AUE>l$2WpCZ^`ujHz ztIo38k&?uUhlz<9GSR}s0)h)GKqlSda4lw(PjcP`-6Qpp!(dh>+N zaXb$gGJblC5q^I)t098@kieBUUL)#c`G|VuC>0)XGZV*3BFv(kv4ru0vP)d+1e1y- zI_FmBgl;jaqlVq8J#$c3&!>=XpuQGu5E(ayRf%+{TC`t_wZ0gupkPAGLNywzIzkEV z;xuVtr3>Ebh=nd`0g3J`W7TR)ZfuFAi6WzF5qOtak4w%**}xgK^(#?0`wj(h8Vn`G zrU}gtDI*AdMHGt(l6>^S-KRIUOz7wcWtz!p;s>|K51|c#we?lFMW4`!ntR4E6kU1R z83KgCc9)2@@L$mU72#V{YKD$bxRnY&#oCV(ptmfP8Wo~QQDFK-9;Rwj`4KU1?$4WOXXU7F2QqM*(gCfOd!1+zKoR_(o$io;Go( zV^Sr^49v@$E^=)I$;x63v(FncGtDG%>dQ+^JINkO4DjJm5Iwm52&S@&sUd9B{>(V5 zvbQpGZSsQ(> zISn;89dOjnGAQAM7pC7;0*R&GfXYZrXn-nzLK~^3wPSUQr!z3S zNhStsfY#AWWM2SLY*OSPYxrY%DgH3b>psB~`UrL6d8j!2?I-~GB@u5ms1|jc+AmlXC z!k6NL!6;6K5z`6izxlJi=yKu{fxKhfFe=~W_=7o_qMR9iN&{T=vB5Z(4|KN>*dv$V z+$^x#_KYuFwzfe1&avHbnruC@bK^$Y_rYLwg;&j}`wyQ%<@#|AhfdcYJ`M#*8NaA5 z@c5a$DxV!9mS#u>>779e{xc0wz?&bac?iRZTlwEnmPHHj&vl9r0fDKWkQg#_peYD}zRS##~P`FiAJ0E+5An*4-NkwF<( zDPg#zhzif?LRg;tg6m*D-gCaw5GX7nvYG3?2Z%>Ave<%j3cpVZ3ReMyFf4c4L;kvt zY0$=6Y^C`bPYol%!m6${7ZJ1680e0b#&wKmMWhIpSpIsYpSWIDM?j&MH5c?gu0U>N z@s`UkeMH9Ha~rOR3?D)@y8hh$ZPxDXn=qps=OPQfUEE4&xVyIp^>^=@5)6&A(dlje|LK ztYd|M+*0*=1mS!%LwPvw{DPUpCx5(6)YH18h%cCM7)btt!V&nfuhl9pNTQ~!&qu$` z!~4BCcLj|3;IQ%AK|!(68t586=bu8i@f!+%&o?mPh>zNSycnz@(_$l8*pSfbr)nGHpIct zg-x38wZbySYgb$gaZA*))-Rx<tcvokj{d$_rV)ii z!4WSK`Y|~H9c2$lQ0ab*%GY1R@}R^lZX}et7qmu7!3x78Ta09g-&FY$Lr$eE@)!L~KiH0;j+RM~1DuH=)c z-}%NHTE)lcKz4yK5w=DPbAkZ6DvE2*{_e?;Cdp)LYG4!Xl$?1k!}IhwuB*L&2mN1PVHf1lSpUY z1?Hy!_Wg|9gZebxgK&%skqNb|e-|NY{kpw=3ByZ2shBfZRN57usaFu!3v&4@Yu!88pp6yEE@P$j&)* ztWlh>&WJ8u(XXpAky!t{Ml5Od;$IwpU}^+`)R377e;+De`R!D)5`72HY#%R8OP`Do zl-HWgH*o@0XUss3D3yAjzr?A`IWt{eRL&W5|N*}yZVWO3A+23vk5$-rX)@q69isPPHG+wV3hS7xI! z-$TlA=x0ub&~@k%Z$%p|1kPOnY_eoH%f`m7tlTEYaw#|Kr04=GyS-#5k>bf~%_~n& z=+{mp?hB=tOf-u-M+&XkqH0*~vFP)y_s45bpc{K8$cy(Y?&sq3&BxPVviIG?n$Cw| z0J^7W=^ZU^Q}M~wCUw&Rp^5mAG`-U{TfJ1KHOH6$H9*6j1}3*}9?o^u5e?&olhdbS zYKFae_=3_w*8T`dw8Ru6qTwgPD+@*!ld|@~BQiy=Vf(G`Tk8{mPIfY}SaneKfY#tW z)}R^;W?`Ql8p?q)QFE zv2ESiy4R#_^XYI_$+{$W{P1|X)xO|lq`{lKcu92fXdIgSp4qrj+Iex=@vkR{(hI3; z$>u) zO5-rdmiFe->gK4q*z@YIj@iS*AM*FZ&!EJsuBYFWuluka$p`QKFOjfO2q{M+(0LP& z%`^r^$)*ki;=U1y3vn6ovR9clyTbz3>k#FMU#HG3Tl$rE&TSH${%>ICz+11d?@<8@ zA4gXgcT5SSFggc}D@`!hI!-;!ULjAsFGmZN*;j39TS|~ zrePpz)m8Zgu}zn!X?9lH%U)SBAy+T7$~5clD8-i_JPhMBgKw&wCLEtQ1oq;b z_?ltK0|EyojXLHlo2`#=TD$x$=&te#55M*a7^}jy&eFMaAgRqC9^`hIj(l41*B4~n zbC`=)ne#0=8)w`La-JNbt6weComL_m>8l?!g~4Q{`(=VF9&{>M8k>8>eH*pc!9;~E zZ!|qy?Goj$MrhuR%CajTXc zNAiI76mHnoDQ#dDqe$#;XH(9yriHF9x91x9GUP~nEIs^auq@VT30ir8W@>v6+DJWj zgT_z+nfF@nY`-UBYAb8y<=psyiX@qk&)XpBxaT-VS5t=JYTXPWu~$(=XG?ErQfP3| zHpB8f6W%MLYi}ENG-u-2jXG=BG!YFy9~KbIKLatNx6OAH*EkAE3et!^SAAYJBM?gH zSj6x&kubn4>=`(#rb#jv2YWd=<EhUboXHEYH^$=Tb#u*IZR!I$r#1_ z0jO!Mz~)Q@_Du6g@9HadIs?>=F(e1&jFK}~=}30@mH?vY4cp%UC#+>yknw@5MnIC@ z2)9bc*dLch`oCV@y}C~J0|VX-v(}Gy-U?E3nbvO?Rfu!^vR2#)lUOP^S|m%=(x+FO z0tt=`tvu3I4oFz4c-Gu<)A2$G7Z#jj&G6Zdr+XKnHjejkoGvQQ6#DjC^DAgNrzbFM zT<1b9yyg~n>lh~UTa8`kr!ja|k+Dx{MayQPhTAnwy=f?2g!16nP|JR5 zm0K0dCR>;9XxO)J=(4d7?5n$SI!L04`*bvYnm^4_jL&ES%!Z46T4Y(ky<7(?_b&R&^YA)unKNFV)9cOetNSc$r1!9;n z{#?bC+>seiZnQ1S?(1kbT4^*3v;gYFpVh-Wr>kJp+y`W1%{v8cLadF}iZ4aZMDKli zu8C$s!QFE)U01zT-KFtK#}utQW;?l)_w5JaG*`E0lHfS`ABF*E=AS(*7o!?GTsq+4 zl;P+YQmtnXV)kR)#EWuK_s$lA!}ajrfpQ`{k_xB?mc6CW7H z6MP9WVlZ$IrV>1un_W|fs|_|vO0But9?X9XnaOJL>K2oMoo3$9gbkPRe8bQwr01nT zuXsCjsRLF~tj{h>zH|AGRD$i_@@|y& zT5Wsxz}ej`W=c1jR;BS9UwZaC+2H#L?C0~0k*LN>E`#lL;Ogqzj(#sjqIM?tLVEu- z7{!LMP~}xd7kBA84JT|#45B_x=QTD3=A0J*God>|5J~H*jmsp93;7U#DE1KMqeQLY zRfY!pPRV1NfP3T804_6lJ4Ay^8G8tW3+2M~Jks2;CdvYLBKG<_%lRjysAq5{EyTL= zdCBcIHgD;2VKHUnAQ7mGrdJKZO^@%|br-nXwK+~zTuIh3Y!nS~`&l0-SRaihgYr8h z0T`;h9jMLi&tR9PPH`vBO7LcoGa)026{%~XxnFw4_T++i8>MaR{6pvv}- zAmmuO`%{z&H-EPF)vf5OIS##y|DE!-dVTzPd8 zd@r@orv4As-a06+r^^2oT)ef(Ew)2#}z`gS)#!aF^ij?(P!Y9m0d% za~^*0ymM!!YQDPl{go=7Q+;}O@4fcgYxn7}^Dp@Pt`x9$!t-kU-k*a|qaR)!yGfqa z5-rxSKyJD7Vu_Nv`(qXFm#4$uO)d63v2`|#4DowD3k49ZWDssyQ7)X~MCN)ASA5(5 zdK$o!8oPR^WzJrv65F{-=)nHfo|9OcB#nLYwKd)^`r32L0UsCBtco?n0-llRrcc7D z6A6D$(s>sW#4mHddlbms3=S_Q;PQ-Ua~<#ZW2 zkv5Oj1#Wl8cl!DibTl5VFMlS9ej~J{tg)37OcYQp9Ct4_ZgK7~xj>(=Jyq|Je~s-@ zmoUAm!d}npCO@btw|oHW-+;H4vx9tBXe!%XS+{UvljnfuQl7B*EimCOZaKqvNtc_E zeclWwve02h%Z&J77g9L4_Us)4r0iKwdZWmB@0@iz{oI+sec0s>BAD9np#ym0_SIo? zCt=8+izE*xlOy1rms6`M2erUEojiMD))oJtWDuMC-z-gK^l|go{UQ>oc9aXBy1 zo>mRMKj@BzTb|zTU8i}6a^NiXl~&XAEO`T24FmGq;@WJmIS8+r>ub#Hq0CF2F6ojt z`z)hHIUB4))_M*0Wq&uvbutAHynSmG5oUre$RDDXf{%9K|2F+<=H&vh#QIuzG#_ zUI?X__b9M<0T0N$@U;ah%>a|8fN(GVu1E#gD9}V4l*249{#4o zwv<)yqT_If;~0lOL6K&)C~^s}xYtiK3(KXs!QeAN6(rL5$+D&@8%OF)v-3%JgxgAD zHJgvgKwxfomq6|lY!scA2pDACuWU`P6e_giUwvkjW;ru%XUK;gM0urvV5WgAk>vG# zMi`Gm@-5+)Pagid#}nQ_IzAQrwf)mlfNAkcodMoIlAJ{;R6TuFv7Kl=i#>s>Hjh4(-2gP*s^$lSvvhihRaR-g2y2I6IKJDJSbhfw#_Fd_VteJ8l^ry=2A`?S_1dv`TZ-nhriY z4Jo-FyS;Vdrh6WZXDcc@6EDhGClNG>p9u5%;q>a^kaMGY#)L`c^;l6&&Y%3;)5LqN z4tT$vEL%UW_ehG!E32>~@WRm@&H=FF&J6vJZ@; zZ&GEoF0<%7E@wP-HF{7h=dmEJro1v(D8HyM)H*qva{pcAQcuZL^>-F26WNNgr)mD} zS}&(v>;3ajiG~LM9Rs20@`!=|hCo*EJA!|~(Es}<3-5pr?KwDE{;w|&C=Zgha=p|FY^-KTDHB8AW+)4z@uNcO5RH>8?5r-W4#suI83>9I% z(NMrLceUt79G|Ajg-1KDoVg&{3#D_K6}h;~-Kj+0U5`(Gb#<=k=1VT^HRh=Q9HBF= z@vXJQq^M@yw*p&w#G*Ip<9TDR4l&jp`oUF18PXix_Az@A77=%5rLJ2~(h9eSS6{wp zoNh(+!l>ogN0t%eDnlowcbp@daJ-E@L)lwF$)~2xo4*vskn)7!!&RFj_;0owN87lx z7b)B^TM==BnHwPTk@cbuc}df;eWmviF zAgRbsgg3S#nMc)dGVa=C?JcWJDTTta8&pN+NmlP#jScYG+N$SOJ-nWE;KNo}I*QH2 zy}QvEr=rNZ)K}O*%gu_q4kIX3mv4)pm(FRzBFk*nHU1G4^|w(@Jr3i|**k99o_yha zXP7)xS+U%6*TE>bb-yY&KYK+wErIYbAN_024@S%}ic?>E#^V&Mo{a{rYEkJsBRq8^ zY0(|&!zC;IBHGXoN33qyR_&$LoO&;oCz(j$m>y}rXKRp^9{XdW1modkZcS=NJkRov z%h#!~c|?=6<@1b`FP=Q1{GlXsXkLDv^;EN(I5#hxo#zJrhgkpDmq;A%*}z}HFk}7y z=h}<^G1UL>TQ6Au>#Y~8;P3x6)c-%;da9*_mfl8$-s|lJKvW$cdMiKs1M%;s_rH3 zsFj?N%{$16g`Cy{j)Vx95yre6vQ{0``Gm%fDIR!-@wlJs!Soq?2~DedY2M-bBxyYn zovTu|%_p~FZHW`jmZ|mR-s`|-b}D7H@AOdpF6;1T8HVHfFRKK?uuSHAE zd(KOi9^i}FN5U?5Znhi1*t7XW?78T-VEs;xq{Q4pu`_Sp2|reP8klLnVYcZhcg>21*YytUH4ajDRuuJ?TcT3q%_{B$^a`WOY0-0v!r?A0B?do`ZK47y- zpty%Mx?5T;pLz9FzJf_f@r>RmD-Ycn%VpD`p#LGt|5d+maB%)B%GtoI`>#;@e^2Pp zYZd=%O~dw2eezF6X9I6={Ll9)LVy3)yZ_1PZ2x?>_`hZJi94y`uf0|&2t#M8&O#<( zvdua^iUo;3f4&xfBkw5~tVU;t@R6PBoupgya`y*WpA7})rYY183Kog5_<5EpUvHiL z%KE!Ox4*s*HxIWrZhwc}ZjPGoi61s%n;q_Vj(~s9FLW;49{yrA-xu5;xjAQ*G*3R< ztuZRNK`fgyadYlBQZJh;)GkU+PKFw2|7J;^r^_G)*`?n@*Y^+NU2*FWg-22AlhxrS2i2e`K6} zsAUbdX|pHS;*@nBRQU?l<)thg?z$r1-d!Fv-^S|Pk5BVG7~SQ%eA_dl6s}dv&no>J z+sp_cZ5p5QAJ7fgJtvWP4mr$<-LwNF$7Zo z(Eje`=>AH_^;)O-Ugo|U0%?fqcbfV&lGjz3-gi`Sc%!yNZY-2&^m|@A+6KC4 zVaCve6l9!8;bupSP13J1lr7~3LUuQUMik3Ms8A$uJD9LJA$I>&Y{CtKMxQC2K+SV? zApt3S%f(>({+G58wQcm25`kOYTYSD7KPwu@FogfkT!(1KH)OkEV%5(WrLi+kHSb!^ zUhpg@RbHScvvSk=Q)(tLclw&*a;kg?ySVDBH=VrwsfT`3a(uriNF&8Gum95&cjQll zI4{afAJ&K!?(fh#Q(c9H1hY5UQtf`e{nsqb(~DFTzj7AyEuv)sze_1*2Y*PZ3i^-V zE%mb03^9#rvAgtsw z#}k7ZdU1y)Aq8L5Ri!9a^7l8V%8pgV&c}(6*d_Wt-o@sSu8H&62ybPfim{2sF&8g6-JY2s0z;%F z!{=?0)9+TByF|F|dxc23xCyckg(-&e#6;Xf=kg1fORkKdaZjP!9COIIftZcv?l)lv z9}PvpbON_$-GbMuc61ZWZEEDT$07f*WXS?FwbE0}{r{ebtwdi(yw;EIaMyjm5GkUN z+>%#LH0+g_5AY%04!pnqmN*O3OY|AKw^+_YGH*woCS!9E4r>YoIMCC_-iyg&x$ggP zty|8)l89p|KstC2xZue5yraCn!MPtqle9ww_cy(V3 zo$2k4l4ADMcGU$(A7HOudi%*X-yr+gSB0rP;)fsiG&@^4+^(V%G-FkopuO8ahk%v+ zvL2>1B=qCLR~+pgoT;{VIJq8NE#a^WxW`a?zZkjnQoWt7yoV7yED71pal#gjj*2gs zfcgn&Zxi3IaH}{jQRHbp;ecc4lF$V)P-EC)$_aJ-#}-C~&3$1>)3`rFs7pOm$5m3K zpx1$07yoC)CM0v;Ua3I-3a!IG%%I)b47UYEuw+Sg-By)^oC}HBXajzf0rSzD%e2sq zSKw-#2Z>dx?_*wroXG6I-tc8IOh1J^>(saB(UcVLJfYmiA<7OuYCb!DU+*Tqlz7uz z6Y=yo^V@5$1-UDl0R{xt7{;eo079g#0h>a@F0M5~V0!j4ocI>pltx5r6_@&;@c*zB zRV-6ZjHe=reC-}4y}|83154|hL_Dt6H*h`upzFCU@7SBy{jW(Re->~mBF+(tKu0y3a=Qk^}lT|`UGUZN72;H0p ztV(Un^=s}uEbjg!mr1NV?lE^Ftu7W(;F_F-6 zU-o6A)Ny^^S20LRGP>uw+Xk%XhojPQ?J`#KTJci*{SOi^?5zBlar`&_QgD={I7CO8 z0=d#)Q-bqYaMp%4@10bPt8ct&XhSy1S_viPGa_L69S=$SGIAR{@n^fQGrmpF1S}^3 zRt>#s$exg*h1pov+nVGlrvI{LX8&d+9CcsXB#V6rPsvH?O>m;v!pY7_+HKz8Y|I1u5%ICo7ve?eW$`Ew9)b;{w`cs7@WSh+A+~fl>ISkH3_9dq% zErO8&^tO+WA{A9J&-Yy%DmOh-d%2x_r&q(e`vS zVgC(ugwI~(hPgotd*BE(_HkkC2PAF40CZ1t>F^g*>s7blbT7~{Wq-sbP`V5UA5O<{ z?#PcjMkdE{IQ&7oQhx~;B8D7<&HNUkm30TF8~o+F?p1M0)N@w$k~USE|AZw8TR&euFU zErW>a39(UuGfPoDj2vQd4Jre&FwhNgq(?|r`7>b84m9`Kz4~Z--+zB}kg%|?o+O8f z`JDx-ycfgjXs69(YvM2Lw)4g{2Ey-siQ5aPKfaEd;IvBtHcRW6`cp~u$EA;RI$A~z zWp7cZZ^t50_k)-s2AXoMU6f4RY@>7JYjG^&`KVtQtE7?khmC$%#_TC!I3-N4TJqW2 zlmMJH{+K(vGA62ZxOGxf_{E+T3%LC2OxRP}gV!=LTazN%qgMLm7#^VQa66f0LwP6= zLfvo?a_eQR9VZ|PUXwM%%G(8T+(iNd=RfO@*HSD8f(EZm&U`TKs%z_&S!Jn}Bm?Ri z5CJM-^sKCt-z{kl80|m9%ue@aQT_ltluq&W-Uw@KzzJuE1}&z4!>(5vu-(0SlcQp9 zqW&^>^eNO7KH$l*&jRL+=+jkAk}>hYsg3;=)Nv@@tZ|4yT{tH>%in2dcO4=Z&uIO% zXg7ESZXv6%x=JPLDx`abM$El@zk3&=&IQbv{Jdi=`b*~(bxWT@+6P7ZCPO($OfsR; z%7%Y>_!1kh&eEDlo$#m5vNJNG8ESV_;1X*{b~X@%Ife)x3`v_3FD&v`lnU32AMS9q zjT9w66&4%#gE*1S5a0H;)9Q*sXH8j_y+R`(FvkJJi7`6GXY)a+XD>D`$o$V|X_P-=d}b6kbmC@{4hmj>D$>?yuRslgYZ96d8(m zZK(JjZyRtU<1{Scc;yeysHNrn4$}3{Tm>3FvAk;Q0*M)3@kl~()>2to# z*_U8hes)U17|~q^wS(x6(p@FubZKeAMtgBRf=+Ahg7pgUL;4<;*utpO3*(OMNg0aP zrs~Zyb$mtJ8dBDXnv}eg-G}LV&PVllZn+kg?30Zb$7T`c_+Q{qbA#6WYg!I z4wAQmZMtCyC(Jc8tUYs-Jb!yZbnw-PlZ?E$QMe`ermUi7WQZeo27*)t^LCo5MZK-X z6BU_soWPEUt`NXo(yt0_ZXy8qn{>DsFhg)G9PJzjB2)Hd0S>bl^(ALZKD{u{)Hc+$ z;?t{T1y^3I=GzHuXfh;hN-VPq-Pw!OZ?l=)<7z8QgvMp>W$%KJou+dx{eG3#S2g&y zi%awUAmD*fOo)qfkUQLalm^pYQ?Xyr*bqw{`i-eodTrdZJt?wD0Q=!kod<|I(J;HD zPv+Ije^E-&kt9AH=G%j~fE@Xnn6`(?~gYp+196%v_}yBesXREa4#j}G-@P-Cxd*Kf+PEi@9+N#S&c4X*yWgd_!VbW+jD5cjSZ1vD7Uj-*@iKa;rGWR@;n2Qt z+FO@l)Ilc4Xd~@ywFl;gUDlw9#;r>UPNqm9lZ;@l`GXJB1Yg*Hg()&lLj6oh=&OQ; zQyJ;YRKbF9>I?#KAz#>O;40%=#h1A-r1B{J=siY&rjQ}sD4jgk<t9* zA;#R~&LRi^zOLk*$~xt$L$=%kK;vQAkN$;l)CHzk{+Mt3cXmcyC@l%(g_qn+Zi=V_ zK_e>h1zC>u3~O+jgV=yMWMEQJCmrS;|7HDWYsC~*U_-JOungD?$LGOWd#}w~W^z_c zi(ernzb53C10tCa(o^^DU{QKV_3B3_zf{a2uxLazdFeJ0KiBTy0f*3tuluv(X##>J z67OdzPyO)WRV?LgYT4Gtgi3IEGztJE@yWgkt#HxXyskIR$Sjb%WB>|jI`tZzW;xjwFE_U7MsCFucT0B1}B&fIY zWH}QLl{_sNiS@K|=!HUTytIQwx!;lRG4SV1hSH?*%Gy&P&S`ZuY!NnxIbo#m^(c-e zzDFH7j2`4i^!~Q936pD8mY^Mxx8i~V+jh!$h0u;G%mbXRk=`l(A~@+5SuEE+s@U}= zt+?;cOUrc~Iyd`-fj^^S6eyaqZp?e*aXg_#ius(!5C?j#aZjyi>wP_TM}bq4cV7lh zEYUUq^D1E)XcA7%;1DI{^0oc1zCBqLU;dqOlht1-X*ytc1K@vpIkGs%bsGo_x{Fq1 zHWoO>3oQi^f+qCWv%o=p-5j4vd|g{UAGhZvlIa^k1#fMD%%ru#DOR{==LdmP8%_5^ zrF#t3-i8|nqnu?QUjkia$J{Ks>EmGl9eqt+W>gtpi=+w_un0v+?jwR;Ba4{v(zvaW zsKu?8TS~sR>zHFpf~z#+VInJ~7rs#|7MQRpBK^kd+TicxA}q_?BoWIPEP$7Q`Dwz7f2KN@(x+M?2I6%&G&}IU>;NaWBpUv zyKnq@ImPQaEp_37zQspHH`Ky>5{Dj*Z-`(#QK#pW?fwcnw$`15>2>6v^anGNkWS%y zu|>~(URV`K*I3@+BOey^D|^KxR~^V=u@W6elGqXa9Nc!+^GVOT{f*+H0rB)(1 z(q}-L(mNm2)UqsT-kWf!9*Ed(MVkW-<-q_dmY@1DvaTeb2lc zg45`@&Ep*}QEm(6R_7+_Ug#csc3NF14ZG(iTaM@i#E z%Wb;A>BIi>$w?d_=O?1aux&^pKUfda_r0Q%BDOn#fSZOiR)@lzuHVMJzA`%y za5?+j2Jc5Nzkf!OqI9TF}%1)ZTLK}KFeE5rbFsNY^GQsv)* z?rRURgY1VnZDV2fB>?~5V|N-7Cbyh6T6j{(+5XEQ!xJ9Z8PIIe?X1Gt zlXe%>vY2n`_G7-|gus3vlH1ex9QmGDl1!m9gl<%>Q*JmJtDrkGbm%%x=bc$udi5y| zyD$wxnJao;fcfHK(T`wNcEo6>K!#N?G$;5Kx(ZKRUp?TLm`(_q(>*c?tz!w+5*5G< z&9^GS?3omrDbTtR)%x&6zRUg-M)Qx2xg#c8B6K1S0R0oOb_?e^uq{=MHJ``G@-n^d z##2iwbhyu`{L!UX1lVCvOH(O<6`)XSS!5dv%FGi@F_?j`|#@{ z%wsY|OkU^j@4dm*oFtRiuB`b9mNv4I=}~o=2vY#CP5)Gm1zM=yj=fmU`({yTwJaxd( zL^JWE%zN79H+Gm0aQYhzr4yfQp*0%aDy54cc@rOSZOkT`vD7&1#0O~hJ};(`^`6hd z<9J0|GGa>9?hnn=9`9pPL^%?NNmV3Baoob)-V&r|btSugeKDb`5^ zUbmBu;@FvUBLDz^vgvj$8;(3UGuCx&6T3tU4%G&C)hyo#N839vqJ|KqRMkwU???a) zqft_;4M56u083uS-hnWf%!3giuEp&-G&P%fGT&Kk23rSY0*1dOT5GYmAvx=SDf|5)Ly{=JtTs~EXLSo9aDE|W>uM^c! z3xqaE;%5G>y~M7}8qs)N+|dL0L`7I!+J}DU>e@dAPKA~O<-}V2Ak#c&`W(9ue^&(- zv6VEX6A_1R0R1AF`Zmhq-AwxMQDAC;UuK5?S9Cv4zCz@)PFQMq#aJ($`%BGzlIMJ6s&y)f2DYqqnjZRBQJ z1F2@0T6thL=s;+O(c=m4my7#r+*P#V3V++Lf@83H+VDfHT)!pweYAKA@FwEP$a5Z! zI;}BlQPSgU&^)Aw&mqNH;EwhR7;B5+HxP8V*@pV^J4|e}K??*gYIB!E(Lxrq^;foW?mB}xp95i{4sR}Q9A}K|` zc~SEOPCNK@%u)mrP;g!CGd(wwl(8sUVi>G>s%C<7=UI1tL@jQ5MUydf8+EBn_xU ztd=C8uG#j6O7rULlHAj&0n$ws9(_*0Xl$F5G6b?E^OC>2>BDa-$Z}etR0U6Pq?Sq) zlIFtRDI__|4#o@zZAZ+v%2<1|EA7TJPQd^{0ZLdRFc6x3qQ(WWrnWRRC#W^G-`2)} z_K6`!kz$w)&3EE)keDIF^`2QPf(7?P*fv^%q+50UpMt~&m;qKsVtH%83F!wkf2b3) zmB#2gz}rA6KI_|xj+wLw3nwszpwsQ0FaCA}xGi1!*4T66xMO0^0&@97lbR5hlye^L zI!(j){uP31C)%Z-1?=snW0-`z?*Zco=l*1hy4dS>1umK>JYYK41CIks4;PVt+OXWU ziC?5KkUlX3)@G;+6=|PiHC~_CnRl^kRH_NlW!3qB4}bRJW#Soc&`SJ{qFPSW-n00E z6dI0Mp}3L3+z?F7%J(qQ1<-lWbJJ6#Z)w<2G%aeK;1_3V!#=gQ8wGtRvVG*jw5*w^ z6HqFYkk6x?Bn`zlnUtRAQDrdyJ~7ZGe=foAKR^B*tiyNOhGwUK5g&K25)n(*vW~X7sdcuQ=Z*pVWEhqj(yw1Lc$w6 z%D|(lTkAv10um0+Ld<@2vDE(R5n12K>r+aO5I_m^k=*2yIIu(B>^;vO_<`5Z?!A7e-U?(0~581GSUknav5OE|jx!u(@>yRgEvf?;A9Bf!R5RInoKyRnAA_aUAC(M%N#O zZvhx$OjWk2?NkdA&?3{9Sr?a}_y=%#rs43OvHAlnoyJox*yx}MD}aU7r$_~th`s}Uc$U1H+%?hXkQ7tR6>Z;KCp`=cJHex+>0Sq2&E7D$p zs_QK-%?gH(Ddvv_&yD~-iR5YhMt_O60}fRvr=~wJa8116Fu8kS;%zM)j$*P$RR5ue z2^f||B+tNdk42kT8Y3!1D_|-1O%O?4wNdzJ>u5YY@Q7FtNa96e1I9!pX?bdP@ zt{H@J@lX3D?J1WpcNX{yM!lnG>vci8fXXUwV_k7x2STStFy8s|NV8dxe_4t1ygW~u z=h%YNkZ~cY-2MltXG@J8o$U3y3Krw3U-{ZUP6ESu=ry^uMJ!nFP_Lq`ovyM0-oomE zxfAm}+qt+^BvEifIs0*3W3{YZ|855;-;hZ^&GhR7@YH2Egdxtt3U5&Z#TB_UTi2oR zPy@b7ey^c< zD+g5jn`yd&m~%O9sK2L>m2jpXpVZg&Bkdw{MJ^|E5gIqM{SGh3^-W0#|2r~;mrHTC zNGRtQf?pup06fB&(Pfly!;bt5JrWUoSfiHd>C?hyW4c!zqNlnC{M5rz} zGHLiV6M+Beu*r-LG3jwzU|N>5s~$xOduf!%-LItTUy&*MHunG?*swklSi#_ole+h9 zH1gg=i2Bh1Y5pdWk0KmeQy=gkMB~XdJ*L9fxUKRg9_O{FIL}&M#SeaZ{3{yFM}IWE zBRe%`+t5h)c+~LqZioQZ<{n=WOX?7H=>F9|aOt%Xxup*Ta?-gX0jO`D%)Te8pyU3p1u44x%aP zG|axOnIts=gi%n7aF(IMJRta4=#6?zta?I)2DFwp{j1hQ42G}0Wp8+zIpu(~y^dL4 zfyfouIInv$$vP}0Pkl24)p#P?P0M6Ijt|?`Y?H(KNY@ubvz3pwluV5s?-u&LNKM0b z&&mv-1Mi>9OEQ+T?bg#{1A4?8&FqCK=m@N^X_?l=_F_pu6p566P_kAA*xZUStU8{F zRW#og<^;BvK)83LdkB~`b=cXnsh2|60IBMY+M%oXFCd7h1&8snkg7niTggX^k|?%@ z>f{ql2AV7={Q&bP_?G2O5l;@4Pm^Eo(wzG`a)&%v*_(cWQBF<(BiVR_K1DOho|~@8 zaibE6R@F)djN~D7S9IS&{Rg0~7E^maPAK>u>q|aZ9&X5`yNE=rV~=W^N1RDe zZquUP2|xxhGu>rz6hVh(m%qn#(m4tl`e7S=5sdcq50rcE2srO)Wt~_AcB-n@PBQ7^ zF=Rqj4ywvG&yTu+LP`aLu4GATnHO;XWNc5hA=a^$a{v@!`mJkMN&$l<^rGV4o(1v7 zlFcN_J8W*&fpG~az zbsE@3sL7FSNuK_*cZ;Zj0IcfhP2y^eKl1p<3FBuw2dIHSX`IQgxcLpb85J!x<+ywE z>9?w?4pGNWBF7wU?$NJb&DIWGfGKqP2iwyu-FXR6k6`sqE|6q}^7x;V+j?XWm-j$R zFQ2KK9SRzp9dqS$em8o64$ThRTUQ3{0oVnMC`qQZ+(!Oj zhwY2m`~&sB?eV!-$?@Rf%9>XU-dtG%oaD0ry9){{uM~FU4g&nat50aE2E!fDg+9Y| zH5aDz*Arpu@SIbBJalgR2NBf*%{Hz+bu~L~DWxr(9a#)nMm=ZbgxAVR(`%g9KAEJ| zf$(EIm$9k}`DayYV3FJv-L!X^R4+36qb`lyD5bCu#W-C~`4?c}UMz}95vmZuR{-=tWl`ya$+P)pg};&Mu{4IKvW$R`6sD9U z{jv@S5d2a@eV>I3jX-D_KHe5!v?AdOE=EC;q#9c%0kp~=f)65>u3dQfN9XyJxQuS? zFR*q1nnSmmjW&u6d_HPpe+1L<0swU%0EGd8_yH`v?LGjXM=V8h15yq;v;vXd2!jwL4wzst-ogwva)gj($vZiD<{V zRnf~KtIZ1u*c9r`sH6bi4k4~D42>Oids{wH!KRZUwHxdNi6gM?==ov2V0M2Cd~b94 z4Sp3s6|1yYy3Z3>_ zAe9iw{SbG(3w2rG3ZnR#wI9_c%LR#=UVX{meIJ(x5$3bx*Q<=b0J{qs$(pUPzsPhR zMplwHJZJOC0n~k$DDq6hu3+fqMYd~{7hxAi4W}#1kgC9a_%Qn&gr--W;jQ+d@>x_7 zR@`^~c>|=4R2hMLbCv*Ucv7hGoO`wJ3PeFnX9jBbf^0x{==VUT?zeb^*Fq-8Aw$fD zc|9N>l)N1fmg4|gz92;FR#r+I{G7K)CitnW`X$^|6gp*DG}j^F->vqa0Epd|IrYp9 z{ikhN!C&Xryl3oa8=a`@a4k&%6dB_}?Gt7Z?x~5hR7MOH5(qg^J6MH=To@?CU5mRr zzX)y!97H^dU@)mHCSCsae>EL>e1hVgvwbE}~@y>n-{b zcH=CK$JGRI&%73z&p&%ng(!X60edzL3G__`Mu~LKb1s>e{skn=Jn!A@ltd^hgcDN0`MUwWS%@3)_3I;T6xjx3%R!Dy<{Ay^ z0gq5{HHl_Ma}Y+hBdb5rEj=kz7VOWu{*eI1N85b5WqN-X+hA2B^@6$-5FL$uW<{gc zn~2++-Bk;aXjD%|*}x8wR>{;Gv@=D%*g@VHL$QI*3#fj^X-HDFWdm)Kz;r==GW21P z0PZb|z^1)dwjBCLXDwP+QH?WlYm);I{V0wzU(qd2Ju7K76%{y@9`zmro*jfIA3v*~ zo>#`3)@9EiahTZNwFY{ILx_sJTt+ca)C?g}p9=uBnd<#-^`eADddP0Yjh#WPVZyt8 zEWi*UxPE1lua{r0>a<56is#BGUigE@9CF|gvism%82G+r6MhS<1FMdeY;S>bkq}F< zW1YqIk6uw49H3`E=N4M2W!Vrpk{$eE8>xiE5|*$BEe=$~%URBcuPxjnyl z8O5$s@O;wFWAqE|77qg$k4}(jB|0A4&E+jhp;BF2=AlC<6@8G;0i9U5;Ev+QVa(95 zw*%dbr040zu}`5KUQKz|H|@+$8EDw`wXIJ^(lUXb@-{3BBM`Z1xUxE|jlR_IqyJ+# zKC`9A-MX{h*=I$HvHheu3?(s1f&;Zuq`J*+nS%M8o|@3LbYz7y!ZSyuDRzb{})a>8mnS@ zvJ>1>8B}Xbyv@dC2C#x=hufi@eAF1(*&yjiH5OvWmMhWNyApv=oCcJ~UN!3gQlRHt z35VDCC^B)sEsW`qbLRE{D1>7ed6*^>7^8(FS;-qV%jVyo7w7Xk1g}Fb0QGBO0#g!C zsJy&!UK51PXB%0KVAl$08T*l6qRm6wJ*Ll$B-rdf$^lM@FUnxfz6LS)0<>#{j@qH^ znsFhSWR246!vO#iY7ipv2sgl0n~8rWN1?l;>{49Y`L)f^B(1m&)GiB2o_l1jpMn$% zr#84{;vFb+1C)-MN^!8l&JmEb`U4vT3o>dEL8MT6Mm}T`WjkMB{k+r>Q9U~u^kcv& zDAr`WUREQq0zLg(L<5X9o4={)@wf;!Iz6I>Q+4}(LE;CgQZIB+PHd6E zaV?%O$Q0Y?equJo&)&$LPYIs2GLJZzSV9HpL*i?%?#>Mk@H7n*oDrL-%lf=6(BU*2 zk}vn4q=YuFF;$2&8K7+##|>JaxU8c`t|+<9BkAi>J>6Bn(WHo*N2aO^=qJV+ROsDG zg4du9yYzRAqwBR+JxIM-pCwyR`TVdq>?#Osfaz1*)N}`nZ9wRlL^`|XonF>wa6gso zuLLCYquJ4md~zwpfHFWsj{|zxpT?r006?f1)8U`#>4IErLqQt6s{ba$N;eyx`*Uz~ zaTf9O0xZ~G;3!iuSVB~cL{u>BEnhQNXefly&3CH50$1LPa5U`2K{v0r3X9?MRxRK^ zyav-1NlZk?Pz|6v1*_5mr#kYR>$&7WfNV|(VMnYOqb#3YoI`<|s^~;Vk&+nsz;OF; zQw>S9nbGfxRJng~V?D~aD)-Pftte5g=KYed`I67&?ri!u5wS_eb%C^omh0V>+h2%I z(|t)v^R@joa3FMp*6mKL>{6$4>=r^?m_yubC$qJDf6DQ&N_>CxaCY>t%J*>4`~bOe zyY{|-gL59Fo%PwJ;0(AtYo5X3iwV(r*p9u2e4=_7r|~boZr0gw#q>AV{VKeU`VnwllJys?!fh{5pWzu5M^ux)JAyU;Oer$~0mD!JJ5;gR4g z)pJem_5lW5^O5MpIX$PVbj+;gD10#c8iy?Uon$fXzDh5|#;x78dT6P1xx{*|RDWq3 z9fRlcbe`bd1N*zeu}OW3fw<uwOP7m&bL?O3CA}*&9vl0OSx2#V)r4-34tcmm z(mAEo@ogH6GovomndPb*=0jzmJiy$FvNBx^&~(%LwY%olg@X`Z{&X9^qUru@+4b%o z-c4d4uZumd>9jPPoqqo9k$RW55Jv&oFNRUHV1I$J@=q8&dKrX6Nul$3@#O2aS1F!rhDSi_* z*_32)D2r7s(^r>TXqx!kQv*Unrn`sxdO~DJ=Xz}DMSxiFhcEqE%`{ZzE&TKL45MDE z4@%h~$$}QoR77(uW19F@EqkM0^V7Mn7;NF7aFr|Qz&Dz<*O6txq^F2=?>&oGj2d{~ zE-a#y_4M46H{QA?NaQ>2RnyU8n7Ij`%`Rl%uCl^g(i-zb*3?S4-j|`EaNv#4tl4JE zfi?cQdH%j)`sb2&c5_#7aAVzt~s#dYzphQY|3dcC>C(f@o-_ z(|;3|1;>E3&NZ;Z3)OEYmRiJVMg3R)4k;-GqE7;Y0t;6xGkfwxY2B}nm)$Sutt%>h zY7KUpzjE=s>v(3V(o7J3LSoiwWc7>rfq{pY!JTp`(1HUa!Zxt}jtcHZM8c(*1A~}M zB5>41QSV^O*3)Qupxlkl9QbcC+x*_-g~#QY@x3E;4(RAs?J^VZZn{m|6PE5AvygI1 zzA>v*_!zORnqHu2P}LlBD=j+aqQEgZw?mAXqG03_@6Zdh*K&30 zuKi{RKf!2tPe}Jd9jCZjauI)U&&$x*$m$IV}PsJmO^$CZRk(O(D$TIgjlTD!na4_8{6L;uCuk zix%0~${rG|cz=-L{`hIO0--L|CtJz_bB!!vtma>KdTBG0oW$pOmSd1PDG1B@T~Son zRBL;zoq^G4qU@#Lf7Jzk&n@#L(NT3)QN#7L+}Bs9WAHj2y~jiwxi%Ck;7 zBUe#0SeiN0Ce-+)JeN<6ux_wvk;VOG+GaxX2PcdjN61ktaCDrNi8uXxnkqOWQN$n!htlJv zzP^s>NOAF8@4N3cxV~-MMj7L;zOh`GiSp?4(?g8gY3!5Zu=^6u%q)e{u{W2$j9npN zsY4lvi5rTM1kgBlsP2QNd4!5{cA%j3O2Ch~`Ej0`=JZ8BO>wp0F4N^A6J5BH$# z6~j5IZ;j^#6jNhypGG8vqWgvh^4f$$h3`s75me{Kho}T>&t1m%FJ*Q6!)RFd?*(*h z{zy{LRj{v}G)|!ISnyTIx=v~5uv(|s8T61x_#`x@3VfG~Zu@JLpJucnL_t=4beGny zHUCB)U1Wuld>tu1zghKiR~jR143m?VKyCVu&bAW1-HVHhQ=7Fb<+Me&eUBH5&T~`K z|R#dNoK9@Q!M*A)8vqhi}*3M8~p&(K0KbT=(zRjFMY!k1}#XMfaWFZ|JGnQ?J zvomPIG$py5DACLOy}8JNb)q#l3LcY!EYW*xj2BJPBadK&rzZ|${+}f3QCQS0%JVWyF(p2hPh2Kykkw1A+rP41udRINUS;>Q5ogltJyJ%w0)tl8N(SOL-w%IqI!!WI zsPi1KDOC)(!imwYe7gG*GV%M<2mb7nohjVJxoJ{^f^}8OwV|HfEUa~i1(9krL}uQy{e>0dtU-6FV~M2gh(Z5z&#;@K8Z2}he7@*%OiNtS-K zm!vE)E0M$39OWiMddl7RPAp+0OnjPAY&D!P^rje3aU0%eryEz|CZ3P&d%jMbt}OZ0 z{p}kbR=pCExy)j`9zP*1+=EZozdi>@LvC@(u`~YKl z9X*T>@Jht0#ZvLPA&l~4Yu|Wb{qCzZ?n@1Hqgeyj#CCS0S-K8nhsDn2@Kc8a6kJy!u+~b zQj^e>Wy6@YexdHBCm#dW7`f8xzkoK#O@TIOfpx-1xJug2l0|`iV0#v zFQ-&AA2*IYZM)7F|1(Z=d^UvRX15Tkh`>S`JL{CQVq(sQ`ncGM+sEbTz^pd$GH?`` zTm@PM9-e0ns#>-^DADb#5#LOw>!Q1h>TRf$!x+#s*9cW^2X8C-kNdfLT8NvD8pjH{ zYmG1cy1yOJacI<$3D490q2;!)Kf8_3prLMwS7CN}&UpPnI-G1H1S`GIIMUKeNVLYZ zNcB_J5W%^6T|LDXk!ycR+4&UEQLN@Z-PpouIr;ev1tyX*fj)p7E)L?HWBpL;*zG?mz*IDmT)52R^j!emzY-nKqgN_yv2v{~gG&+!9Am zxDVJfmp}*0^SrPKI{CCKi$(V3qI#ethvJ4z1y(AIcF%{(=tJrm!CNn<(*{O+;V3Bi zTeeKpI+e!i#R@kB*_dtf3>=q-S}ewxlhIw~ zxiu$%%J=)&y$PyTXWq^KOb)h)P}l}ue03Mt5Xd+6g9IpE+9*b)Bod1vd^2vNg(~SK zdxn_pDft|7V2Vf{>9ObQa<7Lhq5Y*s%TKiv_gS1=6m7E(sWalAlH^=M(G0}|Ii~lM zmHloPaK4e90$+2t_cr~S8%8wQh#564@;IfB4*E2&^T+O%woH-)OjrJ5`PtMI-CSih~nJ!+=d?~%7RPJa9 zh8y6yHbOsXF9dMTq7%E1>1NJ%WB)mQ@u_wz_~6bNQ|`&-P2iVuXG; z+{+JHa+}hFr=_Rm#F%!`i{~Pl*kvQ&wJt6ulS?2iTDu4q5Em;BWqXHiCg7}erYN5Y z)`MM$rKEGYYyY%@cPBKV4PxCuA#q)^WI~jdw_St`*L|Z*A(M>B!fuH$4$mM1(Vey) zy;HL-D2Lg`X0T*GY&jZZPULQo)1)sZNOmyLB>6o(Mdl{PX9r&v2`iuJjim$1f4)<8 zLBq?}=B}wvGq6Jj2DdzA(P6GK{*x+mWt#qT+`kaeu|G5*KO7>-x-q@mk`4zP@|7>z zevu)mSnan5+aAhFLA??&Y7kJ6NZ*{c3p7;Kf}!`1@)C6j;*GZua4n#r6Tqzsn78NM zOBSSHUBCT(rg>eIjY)U4K1Y??nl4`zY+r7Br56tPsGB&YiyDQH*rhV5i_3o^=}@?_ zYgvoC?w1D6#M2@|6$fFno)mC#)1<`*G&NTR>F6#dGvC^-c9az^ z7!?-w-h1C`#WN!6>Db0-26V4eIBQ z6U#rw%8K9VO-Zi@J9z&KOb8&K6;&^c!`4LK4ya@4Jb>6qsE^{e2s8xOOaz!sqS^Th zH`_}&xjtK%VV3MHV*b~0c*EC_Xu5j_eQv8<7P zhj1L(#>$b|3uSQd*3xW3JtR3`4&xGzNPdKLN^5VIz?FER(pk!rzIy;$4FomtnVMa9 zNhgjm6F!z@88J-Nz9&r3b9(0d#(b|q`1`$Aoi?HAFm9GvswB8ozL#}GnNQPr5Po(F z`SQ56zUWIiO33=Rvugakao+*Nzu|doR|HmHhA=lCgPY{#C0hx8_~J(BuCP!^CBGf) zHPnp=p7>)JDD4o|X06}DkVAQ?(z_jl1;%r6*c}5se)&1wmQp+nD#UV9#vaMmon1>p zHy<>avOo6}&&5c~u_O7dYcIn1%X+q%Owy|v1BFY`W4PR>^r73W0^nIw@Oh_V0}^y+q|Hd9xV*36P7CiDl5wL*<*s zE8?)wkxc1(cBZA!y8J|sUlXN`peBPMCNk+>#mVQY=efht1I53gz8B+{5k?t+#mu&f zR6x)YM(9G4BJbXH9LiALl8QazOy1Z+*;K}=ps#%z^h(EdF4OuQ@sWAyd5yS+MRG0V zT;I(9YLl9>Ra2{k^_psa-%RnN$~VTHORxFn^4k{A)JIUS^k_c}3j@O3uop-%Av_&m zr*!wec5#SAijtX9m-M5M%9_+FoJ5(>GJm*x>jmQ%X-1`8Lj~8^`d5*9detXypfSQrfABY6Ny< z!EC6nC@6WJVbPHvf8gcjWlJ*X@JI_pkD!WqI}!vqibXVzzVk{JS#0 zirLEun!kn}T6idA{#W0RD%Edc zhNp=GX77(7QRSKq9_%!Z7nurqXZ4shN?tZS=apSFKIJ6G0F*9nV*2Htn9=)$*~tn^ zzHzgf@*{O|m-&1-!PT@(RG_+pPXs{I-&@_0%*+rINQ6(*J{a-3kc z6Ysdz!8FSb*G=3I#?;hL@r7p31dV0^nSIjZ^!XoamNM1T)v3n6jE*(w`G3ogqC=4J z>(HEn%lo$0m-UA@=004Yjc*hSpUKHP+xwp&eE)-#i%cXfj(%xEe{f)I2P)rah)e4I z@qOk`7QzKaX14oZhoW=1dE}5YvGQz&p4gdqVKnvR*CF&3gut5Sr0~#g4uS0|d&Xik z!lb#tEY?mK5Iq=Sa^{Q2h@N?l`~fBFO{u+uaGT}BBJ7uR;)c#J-{2o@BU9E2^K2Uq z@c5Z;GB@&F>@}2Z9l89jYsxqP|1>E@x3AbF$LZfOht>-Kt(|V4v$*OZA2ZgRCSb18 zGlGanoL)0`*HMy4;dE-Qo1c0M?ht|92I&>9M-e|Xg@?%vWiMZ?jA;%qal%4U;aS7m z1Dl&^gb~>3LoLbbRDd~KTMYbHpMPOYrs3OnsR2K4E{<5|N}J8EN(KcUNEWQWe~%Hr zH7nKxR(PUq1Y+gL!2J=wO2cacmM^g&v2!AhiUi}mj?cgYYFKQff^!bYHd4a2Ls(UW zr*jl}*xB$~|5LQh#*X#kK%Kr4z|bKXA8Q8BPKwann&YlwQ1iy(IIZ1SPnfmqD}8v+ z8Q`;(DB@1*rcFjO9i}|_)oreE2FZTwNHw(;G@X^V$ygBj!stY<8zWbA5A9DmPw^)? zd8y&>?1^tCEtTkDjWAbRiC?!ww*{g8Km_X{YdoX0F_oZ5&$__6HM~A4CRdJXKpD)U zJasyIjJz@VCX|u2Hh#1OI7ZivhTCt1gqc>Li5{ zC;b8AgHAevx{RXaa?Kr%E%A}^oXuhP7b3gyxQCqXv}d5>Dv>=l?B@xyYoWA~O>XYM zE%81lY^A9QS}8jtBymQ#{)s1?Q)feBay>;i1~vQnK4q=WjBWCwn9))KE#VW zU&{&|u%hX{tAU(_2BDaSRQ|U{_$^ z3$^pFSB!a~K#opz=?5fF+}81gRUg>`m$>rVvg~bX3zWC$Fxt1M9ugF9Vg4%X@b|C4 zIV}G4&e|;ym!1mkWk?fd^C!PH-lS0Nw#c=dDyncu3oq%62QogHEK6to?un5AV3!dP zaGS*RAktzJ;&RWxWSv`%x=+gl^m2Kyrr9%xezw4KA;s_hQpnzPr@}N{np^4+WW(=_ zcbZxM*{SXGZ+0;NncJoqEu#vd+T{od6Be zhCVp$P~OTq`j7Y7j!UjxjNGJ4x!`HVu#2^r7MEH6fyhiQB6pk5A#+GxNweqAR&#GJ z2ETi>vsSD9K2Vr^Y@I35EKo#>i;qmmR(ExMjdh2;v!$Lp zijxcDW5S@wPEb0Hj*+_MSVAe0Y^p=7f=oe5JMin?E~z^}+gWRBjg#Z3>pp7!F=L>rv1P3fA|9mPm9-a=+$ zb{R)?Km70beQU^N1|Hbr_$7EJs~-BZpBWu>oUAriPX7pU-Y4{m`Kv9#mctoW?-)1Q z-fZ#Tb~z-^x#A>3>zz+BI)26 zpW`-?uP<-%h=Fm3Jr(_-CqJ_^=~%iw&elf25@rXwm@7#SbBcw~`SF#joHMl|DUN|PN$?q!M zKLppwwm@~p;|=w_rYnq()D8$|UIi)2`p-~MZhM>KXjjZ`byH@rTke2l;OZ&*R2*5A zAL8Y5SJLMy=8>q_dF)a6u##^nbx<)Xj4ZQnuzYdG>l8GT{)Q=;a)&3$tu%HZ5NSx zmVHObHNzq~sKs;ARHvaeZ9zj3dFO+PY{x#Vz7=co-RE!Pbm#4R9#W0j4A+(pIK`=G z+pw6_)7?B%g0A>6;_Bh1Xzx&UGf235sT(kI!XR{WmFX$=c0kFGy!OtV*$AI<1G zPCI+f%)|7iz777BxOYN84Z(i3U93x3Qx$g4{lJ(oqx6eo$r<713n!8uvZNBG-nG}7 zgnrnK;;xmg0A5fPG3O1BKk5>9XlxxHKX!X=P09DoPsBB%LK&KSvfXGz!82GoW~ay@ zg%k3ILy042bR5BV_J0YaTCt7bc-JG0=I4`!$60$GF&o|jp)!#oZRBfZEw{^BEnwAj ze~qx#bjfZl3#f*(*wR0v!P8}%d>qKDus#H1?jV0{9HZWk1~xAem&J~KK08cD1J0k- znT%3f444@Ft*_g_Myi-fIEApV+e|-;8zrmmq#*i)G4-jEC@OA zAXa-^uO|sUU_1y^Z5C^BFQP`JaGA^oZxDn!2CS%|rq@T#Z8hPK`rWF`mL9M8??1Pi z%(nU4m#w?&pw)fxv#1&0U->gD`;S9H%%;>SmuS zf6Sd~??n;sYX*WG{!@a)Wyl~?mJiQe^VC;#(Ky|ZrhV#tKsBE%Zu?8VqBUEJIkmWR z@m(*Q(}@objMfYxd8n?BpX=o?0HuWRTg>cvBb;WH)>*fup;nUu*OtD`t2Cn#Tx*Bm zsoK5Q_Xj=M={SGjNz)iY^(GK;`vDHaFq$fxN?m{ZDZFF3vwCsV5wI{vEo zma8_ZQ>5>4@5NR84vtNv-!<970bDgkY>9DxF}!0GRPvfCrF>3!gm&;pd|{j4Q7#5SE=Q{R zp<1gPXrrH!2Hjk>hw18vk0eTNYbnUY;o2VxXz;)JtV|XCrMw*yYhxV}ve?w+q}d%l z9?adempXHPbyzySs+}y0e?M4fpsKMgopXa zx`iU#f+y&!`@8hTandCBl%f;<3QOzmO>@+VxB4%(SMJ(&DMhq_l`D|DYDevjFc+nR z0}?LZePU)DNR;2+^EpEcLd@^z{w=;#_%1@sbl`FiPc)ymeR-~tw*rZunbG3SswZ-ATKPAQX=|T+h~dN zuV?|He;yWD{zgt~AjV#^=oVJB+`#5K^3Xe<cj$PbZ#)uo>dbracUp@>V71|(>#8t8pq`BR_bB}ykkuJ$2P z&?({Z(JsxsW!@sC2ciqDO~PR7*x}5%@l5jNc^PWk_DRMe)+-I%YQ07b#w>DKo3z&P9 zZLBf*?w(~frLngEQs??n!?D3_&pjKpZF*Xj)KfbDiRI9pbX@$>W$TPy*Pf+caaq40 z|9Bd8hI}BYbYtc7>Cy46r&U2$=<^AOepQ zhL1f>ge0kvVhjmX*l-u^!X-+lIZZIk`N^a=q%uwwN97a|kX7zirHz$*bqbr(KbQr= z*@r`+7nPgeNH^gmcuT(LEDtSTM|L0khPv4h`e`f^o*T?O6rIPzdTl$&6@Vjb+VI~! zm24;`9Uz;)!PPNEJQ)Lb6qc-t)9td*Viw9X{h72C6Qc`JesY>f_XDWP)pr*Cg$L+R z6)jR)@~205U)qJQl~N35HIm-UV7V-}6W5@RRnBJ z1}6*SYI(7?Dg!mS!t->PWcZj!@$hZ{<}8AYX3a864MiAx*ck59x@c1EWb}nwo!EpEV3D1S6hysQY z`HH`^!ufE(&-A>j5D%ppN2{F0(C#_w&XIS#y4TOL;W+*tm0Kx>h}X3hbyc7ZLUSyH z>bmV6MJt8g?f zLbfgnBD$~BaA&+iIVbEnY~Q6LxDS4AD%nrChwk+E7{xDITP0E1UB?KQAb)VE>jJ zVW^#BPXh-&gpL}n>B1@tkrdxEr^+gIJEo2iQ zNnnW?i%*~x#(<7NK{-hB;?vYh`lM=d%^_Au)@hrFva@T^l)6rexU8LV=jQZ3BU|@Z z1OFIcH`L#B1$}zuG;CetKJWa!oA;Q>srxK5(B>$7ZMMidqWJm@{CLm(>-F)D@^@!P z@b_;sHg_%|^g*Q@)Q;;;aO}dbbhctg^qiE|>cmA6Sc`6q$qvK3PLJ$U3?((bz|QQO zNSj~l`kY#FYO9h!`L?DxdZ2xGwF_ z@P^Cp^usgmfn+X*7vWy|c>)+BY-b47hOVODVqSSVbS~~SljG~E`F?4H8`LToP=6<$ z0bDOvFd9&k6Cmt=#-v+hGdD58D^S!idkkV-e`doWbK=_=MHSU zK-d{b?b3>u@oqcCV6cU#@M8@?+mH4X%E65PQGD*MeLP zR!w6l7)&0-uR6@b=!ESWB27Jeh+)kPQ1klVWV@f4{}e&G;Q2IVYUPq8>EC)A$k0;2 zG2xwqGd7;ePP=`C`EWcBW(si5zQl)%7e=41oEMG~ z80=idHC1~;k}db3!RRft+s~Oi-sK^Gaenr>eC|fhv=;fHu~tuIiY-HX1Xgb)se6l5 zKW5qg_`$^ZEhl}LZ#11~tGBYULEMx;di5D&{X^9U`fjCKY0(qlfrs&A_v1xNfScnh z`8dh(wA39n>0G#P*N#4!iH}xg&e5RWF$>GwB}HGSK5ZPQ59 zu322u_a}v9kKw;GfWX;~4TTFtc&djC%h=x$J?~*XviV1>`WyNRovOD}`>A+F3aBzT z3CUi~GTH@8SxmlV&b&Q-yMK3?_|!^pl9=TSjOU}2fZy(=AjxhlD)*SBBe9!~U~YaP z4J>v02i?_%hM#3FlO^`Iy5S?$D>UWttL?%&ITA(I`_nz0RG5!y^P=fXNM{cIVgT<0 zsJ=fu-)R9oOtJk2E`#*56dZ~4{@4T1#A)RmvP#T%KlD5#<-J;3ErPcNjE^HQ!giTz zkKm&dGPzmw77b<0=;zuyslCjg@zLIIOH7^8e!M2eCMN8N?MqI?IDzY7@*J%xX zFrEvNcBE>DmOZ`HxHQu&l0juR#s0~>N(5X$JA=~29)nSbD}cDc*WlJ`CB>jrrfIKu zbyd;$(+0!Xs`RB_j-{7ahIXivZNvC_w`rNqYP9U!^}W_T*~IGUqhGg+UTB?+Mb=$A z@VoOZ8Z?t;jYqBBDcI?e_S(Vyr>-7#UP0QuQSPkxaAyC&etRa4=nYfugGgz(+(2s0 z>;yVTT|5^2S1W$FqWfxM+#}{v#0uPoL7&$zhEt??Ll3w?Nc+lzmaqtX>FlU$3p*xH zhaoN1WN*7n<`b8t};H*3Ax(jA7GkNTc#F14UR zbAqVMJY|{6olJ6~fr}nsHY)l2=DDP^C5kw%IqeQhPSu2!AuJMn{c_w~MoXRjjF6Kt z9iu(WQ&T2Pc8!`2BpTB)`Q|*J6LXT`rk${<6~WSUzthtu2}TwyNqi!mzFg-|Gy%3R zn7*B%zo1Vs;BH% zbN7UcRdHJ!EV|@h)mP12LR-G4k9PCIL#TN6EBM>igT)luD&*c`nZy}PvhV}lo;@kh zTKI&p7f)u?TFWIND}j6WwTtm$uIU9lt0Bw2W!f5+xvQ&*OPnU?25SdMCp1#)E|zP5 z%IMQ6a-bddj;Fe0FOVj~V52sj%`B$AjluWseN=>)n#xc9>^o?w(iF&XomlmSP|Ou2 zjOC71NS&xeqIKoc`N7S&yd+~6f)c%mJ!hTDD`$Hc;ODAJmtIsaF zMas!jl~`R=R0lg_X$FKGx5I1}_`boH$gjnzi^0(&EnCY}h%}-Y{r0WYP^z(c8JoJw z;D}9N7@{0TirnMr{F(D%xmKF85U1lO=Ms^HDe3(AZ{~rVuT`|2pM85y-|YD)9cz#6 z{fjRe~(NjYB%}2AFk{JG~O-3ik>P6&KTM@CM zhfDrR?EuwNC|Iq&WdQT5hOpt1^*o~fM;xRay+j4&7U~0kdUaB=$AK!EnF-*CX{+5d zY}{7ykZK0M;K#R$^_oVNdIPCG)mGrG+1^N}{-*BY#M;o4umA0|W>HL``xpMWR2)IK zA-5}`^!tcj;C zKnLw#$;3naH)V<)84EzcnPd{Tnv(R?thax4t8DJR|b#svoMy!mdz_b80;c9R!YRfd(;8ng4(&QWvW za?#D?M)&s2Mqa& zrXRLa(pf#vMvJW7%44o&zub7%Wowl;?w{)QGms2w{HEh;*mBZU;8W|=6|U*as`*6m z%8aDUvIm!8T!vHX;=hbiIn4`j0<{Xap~wM0P}sm7+qWV)w=tlP@#CoNTnu50^!=o( zCH$d#)IW|QX}`~`=gjX!Wolkn0vRAa-YJ#m*bhkRe1-_9MkNh=JBoupZCPg`$nyma zw4}sZ6;1+g4o|YUt1m)VdJkgIHU!MkDC+7;u#kd&soU;Cb)ZF+^`*-PDRP7C4#InP z++=W!4EroUTWRnsOisF!5rvrxA=4p)?NCYrdO zc-S`0bl7n|Uf0!= zrwTC&CBCTvN*FA4qE#O5F4lyHtm;)oMo$!aY~Deu8SDN_ps#Ww{_V{5$P9+^CS+)n z>8mz_+KfW|>ZINr78xqBOBCSexj;?y9L8`k({>T zLAGm%SK$P9V5WUf0KbETbECR-yb+LR_!Yl7@pxem;`KpG* zp3i3t-bU!Jyq8d1G}JUO)euT(G>Eh;RPhk>+{2sZK7Wv>2LDyhAs9HENM?1lVxiw` zKwXcfwHFCiWt>K64;=g6UW^C}9Hh`W6x{Z6czqs;JFwWRCw$x^9Wun-zsix3D& z(z1R+{H?|S@eWImJYPW~YL^x;vOPT-DWg1qpn~N2{<_?n;jSGUh><{8HaJhS3mIG% z#2HbY7{|Gl{7RUYVr2UB}Uu^506MPOx^|G zNcJ>^Oi*M+x)>?fIgk^hs__%;8#?3^cUWCG%F#oTB?#^*(5^B>%>T)Iya{p2E7`!Y$mY8+|_EH|!U8z`iAwPFP zW^*5p3i3A#JsAVu2Hh$SHM85yp)syAR&jp6hBDCX!~A=k#b6XHi3o?gD!mH`=8UUr zNHC3j_;gDrXi+j+xSbd2*UxritvFWPy5h9ySlCkS^%2F7kl#E3T-(hgjg^!H;=3&+ zvKx$!<`jIJ!bP5&A-ZOE)9K=fI85p7gHgs3wx{fg?UP7wx~>NHzR{u%D4Ii@sqE@u z&@16SdpE~F$B2>7m^vE~u@TVl`ATd2!*H~fT9Tz&)wQ#VJsjdhX-Vb6q+#5;7)#-6 z@RmS~sdA`=@C}bgfW)~gEU+IZYkqF`)m-3Le^HjsvclcQiN5kiw3|Oq=im`iy~Q;xiJnI*M*MkMzyw9x$x%+_sfOU_amnKnuZpPFcH5Oio4>IX z8p!EA!m3Qdn7Bn|nA5qXU~AF0cwzg7nJ9q0(Z>?z8VAv!7mdWaDM!4bF^ha7q^CCr z4>LKHjb}_7a|!X`+tU1VE$0g6wrZQUQ8X917HBI#cv0(t(1ZJmzAwTNKRvktgOGl( z7h^5$QJ2+5UP=(mCL$vW>J<*2I$T$;LD3%)rm9}p5JgyjTdY>)pKy*uzLii+B7Boe zptp@M*i{b;go-!YfW9+h#xUlD)-Mt+>e%QvUS+~uoaYo>G!xCDo*Sib@4ZTA6i3DK zC04z~|Ml45x?eIIT@>G_KuQbzm_2ok?o!SSeW*qikVL)(10_dRF z*}$9d^dpZ)A7I2@qJ}czaS)OKwg`E~p~Rp%$Ps72^KsIKpQ}(|;hmj3WseK8iV_&P z1%r!$9BPSwhqnw2i!FoXTwE%Pn5CIZ;PRyFJj|0rc}>IVfF1G#e6-|W&dq8aI;vqI zC6K+=NF}>u4P*u+{casB?UVcI4VdCCGfN2J80uQiYo7j+KRuG%&YvaGeFl&gBiqJg z#Pov5gBMuE@_gflXOw?@8`~T#gthOshZOaV!wO+sI-9kN4yCQQg))%}CJ_vT{p6&J zY8^!q#Z1Q-CRGbM{2t_%L4nmZEQ+X&)|h7*k$aIGc@6hhog`G9ZV}&GkN-6L;A30A z7JC~9Z1i$L!}Cj@iK*tHW928AF{Uv9<|wbs>NF3ojEJH%^~N}Hrdv*DKrwPtJa7->8cOlmpZ0QK;C+(*O zgwpI;QHkrd`ome%qVTw>zrnmDFZb2$LUY(Fk|4ct^cjr}7_#c&rvauu-`EhgKnN*` zP|2dwb_eBhE+5A0C%P&;w{!X>ewHwH^rsB!5U4};gK3_gjQ0s+Xr}Q9bIFjnj#ybF z{z=Ov|B8tXl(kRi-T*`=4A?Zub8s+0XoU`|jo_~&?#0TmkyTJg_1@FzuAW(GZDU?#vc1O(|H>82Q4_)Tq_3G z*$g^>k?lcsF5Qu4LF{*$i4%!hEt+@(YYeX4 zra0AXbU{^S56BF-qT_Tiv5v@t%5|tlOEF$GY%=D^WC{zy-6piO#LRKo=U8K#l#clm zHWEbGeq9G276WkN*SI18?=)kcrazq_i=$8Kev!cBQA7ON_g>VMfK!@qVi3ho+u66F z-LPN(KIEGCS6Ysm-u_XI{>ZP6?@SP$J+azIq)sdqn|YYb(R&CD2HSf~_d$^P7kDzg zTTbOMW~e9o_bE3_6Xib`(%m_W>`#a0yDr+R3BSZr*e-`tYdrhqR7>2T36Aa6y)m~U z?Mn0_0^~QTgsG6@^`M=?tG|&1&AyMri#uc3wWzllM6Knpj5`>jN5Gd2NoFVCnev))+4J z!>7w;Ux6xcm;F+vi_m@d4Hueo+VNUVbx{h6A_2qq!xt>XLhZt`K-ohFALsHcB0k3= z?#F@hex4{)B@h4STAK8^Xoafew~^1(;F9>Vwm59LaQKJxIGnINIKiEQ(OG3VTCrKU zdxQ@xc)Mayv3HavHI#@2?0F7pV=$CXf6c(419r$DfF%=RI>iLJJq_}fJ+2rz0%OBo z`G=ERAcAp&aq>LvS)De479{s=7G1;N4{gY>VhBc~@Tx4ZV({Nll-8`#;;sW?X)y=* za7MW2i$4HN=XFR;j1Blb>QL-5&_ayVs^#NMfN$1b8?AXRym-4%vRMgFnBv~3a8Sv; z>9lMViw+Rnw2lIWT?E)-#jFJ3LY+LfxWaVwVPg&?7JZcia6|03teEX~6iIZ70Sl2= zc5rhDt`P~R_B!xl1gYK*nYf8=3@9*o>#-!HV#v11GFUvqgmj=HSb8yGI=+Kmf1V5x zobYYPkW_0D$Af-8skMHVT(U;6zBq|20ZGb9ab8$lHxuvJ-KlL${$4XsLmnSMY`wCf zid&m4CVWB9dYG>f!R+c-1Fr%BrAF%=3{)V=DnTw_wS*NX1-uG%?5Oz1? zvz9I8utfx!Jz#It>+=GdYXrmi76Vla}6yr2pC3uCLb$b(X|kH z+;&#$2lwk^z!Q=(0o(KkAay-~HJZYBUM44%Q7}GaJi~hQpjH&~;MYco%s1!* z_~h?Ughw;_hs%j-Y?!Ru*Nao|YCIO4RUWBk-b9c$sj|x2^}+G64bY-nm%57Lx~^%Z zkCYW!?K;kzv<TJGQ92I$EoNa+QFEGm?)aw5;eY(G+cY4ru!i%-m5mQGZciuE& zKFbN$o~u7U!2uE6p2fx_|H7))r1q=8+BEARxLODpniQ==v8Ja4pOIu*`om4G%}+mx zY8A}Mc<&6|Gz#J+FQtR77=2KcfnBsu*o+u>*nxtO4ArVP1;%j9h?+e%KqBB1<;+Cc zvqnPl!RL*e=A1JI4r5Kght>R8s_Y-{$DU?*6_hn*6O(}!?mnmHF9`&l5N1zC!VG-L zTCA9|bU|qJeH#ti1CdQD6bf|@l6=LiPud8wL)(Jjz+mmIl!U{XqMT2Hw)quf`=t@b zYjfu4+W1EieW@w5SrZ|}3Ea4_m6c&x^oJ732C%d>*xGfcqJO&T;B4Bi?jZ)UOZa=k zJc2~Rhvc%vBdDPUnHMzSG$JS>xyAQ4>-h)1V;iE8(=)(-m=~&+i6$rUz4}vjr>fO^>%Y>{~AO{a6iFtd6bD_0bbE%@@U1MiXW)hVy4Oxy&sraZ2$}x#XAbSQgq4)&*nl*BD~tHR^F7F z;aFCng!tq3C*0(6jqTr4bfZ45>IDo7ya1auzITCCXNH-`A zgt<$@7*7H!Lw}2)d5-Q?JB@!!NQq@}QMp(SXmHO$_rY0I>ff4Tnb=iP_)rW`@DX|y(Le>}tF*b=zd@zlLShb^K(D8(IWRx|yZJ`9>y zF{`7kd&z?1D0L8#|J~>u9BDjK@CW*n0CX*r>#P1S3@+HAZxf|4tOwbccYP?q=`k2J zmxM^oY_K~SwvOW41hG9zv1Ot~ZxW`wJ7$6yPUK;05H$NXEMy(3+v(iB$7gYh1twcu zY$gcfelPB)PWB@>s*Ag9{$@c1|Eq&gai14qP;>_sm#27iP&iVj06oF(x%3Y*Yl=gO za_10&-m?|1b-Wy~nN=znM<-#uBq`Zx?qA<0MQX`Q$Z74u@jw3DDFYqbc1)QO{-tFb ziT`Gqn^9gb&tK`4!@=n*0j=H)x9|n4{6Jryuc^1l2kJh?6DOl}sjmn7y_vVn2dz1f zc7(sfN#BN-zuWVTfdAY2Tk1~FpSQ01f3VN>ij#Vy{8MW8weG2*ion1R)ghaVhd#v$ zp${^m-ts<E*?uvp?G~y#^;QUw@s12Epeqx^8#_IzL}PJ3an4punHr5A~mpO`L=u z?~83?f@}8)*q4a`56l(B{twKT0oN0hn-}hytCA`!L{K&6z>f+gqddVkG2RpRfV&rr zzR&&KMXeL7Kwrh~=sxDpKbJdS802%gMt;vzZlG)I!oq;RgG73KffxAZfBims^7n_s z{EG|m3HaXjB6FD;UyR=0-?B^1srWluem7^1z4)%WA{zQUy)xF{WDG9)KA5MjAGPuO zyyQsK_vROF*amj&(7z_?>qf~WN~)vZX!r)gjZE~h9lAMXI=^hB%F$@osb|IfL!dF4 zG%+gXcX$i-0m+&fEyo%oZZzFLHWuk~;EEsD??a|l^i_I|R1mPOaS7@OJtx`_y8W&u zpfrcg?bH1{(w2syGCsZ^c?aXmpnN3cTKT?gEF$Pv!aITY2?)CS)9V|NyGy956j~~xsMxQk*$Ljk z@?_*O&JR)lp6MH0ZIm1-K0g1mxV>MB{?{HjH9TONe@kG$Yqdas7f=u1Qq{#bD}{e- z9E^aIQ}*-i80|kg!LdW&kqA7Q-24y0g$|`18KEJliSRkjG zrL_-Zi#pMu;Pc8eP%&{uj=|NpS&!(4Zjh&V(qEx@^1Y4R{yUZN6}InQJHnS&k-w1> zpPQw+-1_Ph3kE)gwy!tO0`zuvo<58|kDtNvZ+?#-JUm$=^>zHKtw;_0{PP%G@Bj2V z^J4e&QKwHpXzT0s^{3$1Rp3u!lL{-L9md16_@7S6FGe3i+XY@;d$EgwVD$weO#eGQ z`G1j#|3fQgXW?Px`7eGl>wg%>|FP>|M1NJhmDQn|6uzs!uo$(%l;q!@&8!+ zzxe3?!}h;m%KyX0`CnQD1W1_WEgY@ftx4Ee*+`iGZ!qN_%Dm_htgsiK@IQj5jQgzE zvoRji|1dEbpp*K^D7m>i$o&3P=%r|L}$nytb#%79+^f_8zm&PBE<7XiJS> zZ3(gsZuDxhd%3eIR=hgjDGq9-TR-l#s4>dlpe}2EGbdii`~UCZIsX)J{P!kb+ncA^!g34gzG<7X4B8Ri6JY&;`97DgyIn;3SwI z8o1`Z`z1RTdFb!wa8cly6k3ep2`w)BEL6h!JGk`zOo~-Yz#mD&e)&JB`mdMmzo}wl z<@o>MROJq&Ar{0-&JjjbVSsc@&X`Ej=ttYUilWKwx2=oE1Y|w$XmY>*hrO?kit3C0 z1?djya*%F@4(Vz>~FRJA}Zoa5NGJdSGf(y>X z-29WKWF>;voTsyWM+WpdB*1VohK{T92#_!<@z?5!0FPg zg7LNkWx)(mSP6dASdT+COoU@k?Y%OUBCE#pd-C)6>WPhIDA7~pw&@=R>{YJGjV)`9 zQ`Pwo4?w(1N$Q*so7p7#w8~a1-`-yyj}V*p+47sfHlkSYvZnVeTYVoEzaVywAzB1u zHSqmDTyCB#43{^hF`f>LDg8Ym3=OkQ^ZwG;@OF}>uT7_A-DFt!w_>%6U34|+? zr>usn?7V*8z!fuh$g10ks`quLqRWa`mSS{7I_=v#PLuNr3`&&geNR2DGdtZ@J)UIh zKa%TATR%%mo^|D8}ep==fv` za-(J^Ds$DkV!bhjW_c}ZpVitoP-d;lsyesmdyH{%Tl!egO7%}Gb?Uc{dXKn7)v)5W zt9@$OpOuc>Ci(bIK5YGz;+BnF+t`u(bmMd=R)$()(xAVmL_6rXdF(_8t&%%I^6r%3 z@v}IK(;SmCo!jm731$3BnD#}VuCdEbGzSJA$Z>`~zPNZjd~kYphJacc@&U#a{o7oHiRNav`EN^Df!12(UcuI5EHi@6XVd-_^ zLPnrS9=ZnB-rL4hdCZDXLcX;FJ6-xQBB$z*g{h<96?S*t={d5KZ|@8-c{Q- zrf8-rOBH=!Nwj%~Y=80;VmAk|rC;B_OI&1tkNGH|=o_0XN6GUp7eb#K-wje_moveZ z*r=Z?wxU z-Sf=~Kcmi@^To&f-`JnqY%@ku?ojnt^>it6=yGogMk;dpaz_yuMEVzfc9P>5E3G3f z%y`;d!DDC;cPzQ-G)~wi)LkK4_{y;dAD_BkvP(U%7M&rq*Y>Sq6U1H(>lW5du(Soz z7gM2f|GtvhdKa1|qk0sA4TqP$;7PUKcbQy{h{w_p4qXG)vx`#Op!g|{Wl3!Z7(^5N z{aoP!WVJtvTfB+=&fT1t;X9dO7-x5m>Qek61R_sQO_gTR%)}$Ck{`~}BXjN5c3{d4 zhjIwl(R=Wnr$Y{o{oWXsJA9dZMJv!Fn-GRlz@R!WS94?^!9@ST>G48zq9fDeJ9{&2 zIiIgrJuzo5fLEo}U+sfLonMMBP7jbcanIj9_Kasvj!?qBA*z4@5Ib0HT3;GMLvBx#7nj8BB|Vk1N#@uf%<&A_cv7-Pt7b#C_3&fUe#cgX z`z!UAmNYL}p4?EQFX5r-*{ux8dsmd9UvQD#+&#Q&mVnUmFvif6Kssz1NvA%OJ z>w1>)zHG_!COhQ9&6%0|!XD>UHeBo9;a?V#KY`IX7$Pk{_n$k*%?E}`+HiQXCF$s& zY%T?DB6{=^`cHtM4Fcn`pbT)qjfpccBH<0gfhA@^u@bB?7Ti*vF+u_eH#6GPkl|?Q ziMn8P2!@c#C2)l$CzGP&(aDJ_eJq3|hr9InKp-uvYx#VRM?TYO-nnLxdO(v{MX}wo zDM@gg7-vq|K~?OvzWgz5hDx{UwmM#b?0^E0zs1`L(&-c|yC6@W$~106myMBGDG|%} zR?+sUqAm;ny2#Csg)EiEW7pa>TC!x|7&O<`MEf4bA>F)(sC|=D?Kdg~cx{(S9q?+}Zl9~pV8e!c#hvQ_weXJU?SW2eH^idS<>q2iUV+KkrY5b{u$R$8>l8XUulaHswVG-_BFh~qT;sy(H{&~`6)RiDe z5}32cOahXkMPjR@{T}(p?-gfD*-P>tMJ2J~koat5Dol7af3?fXJb5{=E2wvhbB_aG znx*_B*44C2p?a#3hfM z7)!-m4-4jtFL>V?r?V((t6IKdbkU7{6Z_cg^!Ri=LRtpIq@|`}63YDS=~!gIjnIU}S=k&Kzlu=CYQ;L?8_v`aL8IWy2zeU(j5c1OKwcDn)MU=hm z><018Y{{tY8R4vM;xfjs>e|}*IIFxthJ|f%vM=5Vho&-XOjd}@MaK+TaiaOE@>OuM zX=k3lUnP6Eoz6ST7R0YuGphGT=c-=_I|aWmy!ADq=gc_Ul-KJZ~HGD}McPlJ{f4D!H`#Cfm4FuD0S zHK97VHYb3h;^gAd;^gO3mj^FjIvce|TB76X!(jWS`??d0BA_5#q9Ckd6 z4#1Ez!F*Ra3Dkw_pcax~bV~TR2+ELB7@0h>s+8S}Y)$C4g z!Ewf~8k#Og?uF7Tz2xw@c;KL5=kt4%YNBt9A}aA~hB1eY1`iV!rPZF-xYs<^rOW)_ za?e!#%R9LX@J3CVAeZlSgo7$3W1krC{RD;jRVm}^DBR8g6f%3=w;;tU52D?P*eABf z%KBD`x4%r~aX3s0+=MYy7$V(O>?8HU^g=DvbsAeEdw+=G=3ir&p+3MmcN@daA>)@c!4-1%{{Gul;q`A zbXD|OqyaOJj<=?{izl0;tCJPh1X>p(3ZH-IaJM^fd>D_27pQ=C0+eMh9EXDCqE~MjR&Y3EXWW1 z0rT?$CdbQmRLs3R?4b{FvU74mMFZgHuaYr{1KLG(4_8Z15D#=!K+rj4Tr6F!>|Ja@ zP=Wu)pnu&$=a9Cyv9b2BcCobf1nT4lwuh&ixh2p|Yd=dTb7v4YU^a92cJ;FM1Zr@y z2Jry~A88QKI}TY8Kd=rILHxiSRS<_7h(jGD0I+^dkRU%ghdD?Pc+(QZ0g%ob1m*+; z7JJ|WxaSPwZ~<|+0)qlx_W(Ws0(t?X6BR}0(6zVnvIBY!Jz4<*YPo8=*aMxn2Jt}G zAp`_GW9?!Ewd`He_?L9S4#%boFS@koO%ZRx;rlLcKGj)9UQ2+&e4m!K+$7b;N# z_tZeVP{9W%se^cbE>KMnFh{^W=z0g3+vR^PLA;k654!Yue=J*I2VAN`z`H-U6(4lx z0+;q6fFoU&oI!k-4WR?OgYRcUt{}dj4gG5i{ybhh0E+y0_s5n6w&rC`ULd~9W}!Rk z*KUDvjB^8Sg+HpV#wjX}fRQw6=PRL*Pjt$s{(RhP#ZAjKIO$tFK5mx9h(vD$)~PRU zr!0AMO3@y!rO_BB3=djbMk6ihig~lKkKj!D<&bb`$escNhMNg`CrFysc}|WmWBFYI7p? z$4}%U^kR?W^UrFY1Tvp$HdkX)rwPm$Y|w|-k5dLXSCeR?81QO|R0>Htp0A`ju{0e@ zJggH^aX2Z0p!Lul5nC$`-KW;%>hqLzB1yI_A}POfA2cYBQiMdEzl4k1XS#1fp@C08 ziczN`o|jZm(#Z)KOR-2}zSdV{vXadr?Pk>uMokFAEEEOR-+hi9lk!NJ$?WyD{W!m2 zuAKU1&oPyA6H-&=);p)2FSCz%SpDg_=<_le)^dGlM+p*@ z8!DxghbcB9_SH(8jdmjh?kj{1sR$_=w1B5jH^cEpKaZ#w$^?Zzj6Oma-LEb4%inYr zX-Wjrn+&X=(68>8R%Q87vH>HW;P{eD31z$XK**J*k=lK`bfKjyi$~Vs~cNSC?P2=Yht~+@V2mI z(h5q{wQy;@dsC0*0n<^RL`&{oQ&E2rU&WV+zSr2vpPY*mwc6c=l-n9c`39J-J$nsl z&LCZ6%W}+~aU-+B6&q=DXj!W2E2ZX&VCMWRvczk&M#h&BwTjc6wB&XkImK{u{QMlj zxYV`|208yJi2NewmF!6xjn0T*%mM8#9y9Txlkz-XOcbAs@@OLgfh3l}$M+1UciW-` zIw9@ceWCZn|0`x~7-!If|-NL3fv)kHndTe*cLRL z`wM#)2bXut4vZ>G%}uWa5)9-HTO5vNI@P~dLDP}-k?_7_p?5>!+2=i?4;C`=qix8P zlZj)8^O5OFjH(@Zbo+U>qGObus) zO-%h71$JPJ7RI3s6|nx~;Z+$N#A*YfB991ba)$L?h^mZ-zq>Ga8h>;xF$niQ84>32 z%?&%P-Ljq&Anu`Y2A^k(%!#hEGfq z=-3dIfWOL4$Zj8}Qm|(dqG8>dX?hw#UWLMSo#Y%ww_upcxBzDVXOd|sxO8%2Hmwq# zwZTogv@&!(ivDsj1woJfViq$h8LOymMvc#bwhX>cF1j3hJ!$2H%F6M>rpLjB9fn5Y zS^hlM&BZW+a6WHpchmeZ-F79W6}NPug^WsyB1jSfS{L@GW2%9-6NTZ{&##~G)sPvr z2#=+GrKn9#~0=Rw!FPZxHY)*zh|zm#R4y>MNB$o2_O?3;MPr5yiqtGgoK zb4o0tM)64@WiJ@UQgS5}>lI|QJwCq*bK*+U@Ms^W7kx!$KZYl~+%wSgF^*LZTy#`6 zFuwp{n4?K62OoD!=wM7j_H#ikbL1i+`W>50x>0|5nf_}d&EX0?Pgk??e(s&O_@yNu9#elsr=P|lCO?VqdVLHx~7 zPL;-;K};5M;S-tHmC;LkH#jvNal>7vG${g?Bxi>Ei3w%daz0+Md%3MQ0wc7<|41Wf#LR0D?xufa9$X;&jyrv7*jekKgNUj*t)AE7 zT5Lj#$KLdV**!An&v$DgUBS6g<67OXmqhE?M+iAHYewC9JwwYoIu5cUMUfe-@=E<* zPqC|DexNm$M1cCUW2RcW>>s9Mm40UJ1kW&0XGWUaG#`sAtK^r)qz5H)G}byKWqwMK z=vOKvlqDV%_DZYN4)>b^33mlpe6Kq2vS>NR8Ga?I3uDAEg^Vi*mLf?KDS!nGVdnr{ zq#YUlOQ4M-OZ4)^6rpNeMsfv-bbliPLg0F;KI4s(OkxEAtm=EDZa8jqK`x!B)2j2j zdL4au_dS4cq~l52734hKWdTK`VePFoPBnFfbc`xm!-<3p_JsV$>~F1h*MbL)--sk` zHcJH-IrYC7Oe}1e)j;G`c^dh|!GOJpOs3*Bm20lbT*5mVqOy_?NEOBOnz>zF#r&vm zjpCI~8GAtf1R;Z|fzJq{c3IrTlTa z3z;%qiRxEDjp*y}$$Zjf>a+bvu!e8OP?Atj5X`t;ZvQV6sNNkRNSkO&WowPLZ zuKHDUr6~b=vi<>$wGQyT92T^7C(`rr|e9t1Zrmmava%K3Pe|x zbTZK&C&X5>+0&_N?j*|~Uv$nmhzeyF(|B_!Oev)(jecFchg&+PG%8r#;f_LN(V!uJ zoIE9i!YaejeCKoS6LzXcOqN>3Z>J^IUS@no=6;=CRZ;5Wj2qDRa%YuCtIc-^MG>o} z%Wmw3Nc`JmceXa}NxY}dad)n7lW!j6E@6)9SIiyR^nCv!>{KUkl0B3m)pue$^)4!q zPifv;F`!iVPRzn%F&bU3Lnj~7ar5kZZ?;BD3$x7&?tMpU(%*hqqH|#=&TcLD@M1^g zcK5LuhW~8{j2VI<6M+GJ`;RlEqSlTDEa_8QbPjbi)QI_8y4t};#}|{dy5$gDY(C^V zazxL?{Ll{wGe~OA+{r$R{p!nRD(}+^yHAwUNt60FRIc*?OX*-YT31 zHLbL-TjRdM(#?CTVV6ir71A^2TBM++SwlPe6&7iqRN$EG_rah$!Ud~1EIfb)*ST_67ej1Iw& ztgfP>ip2lwkQpK;^4Y1h2D%$uoIMO;nnh1pIcw>ubQpanxX$YFNOz4-NMG4< z`!3DaZ0=_HVH=_)-r4K7<(*}I7qcGK!H4siWN0N)E&8ydO*Dn(mYiN{S$r^Fv@xr` z+9VqXS5B~QVTKS-;p3^0(&{}C0?@#^fm2<{g{YroC#Dy%t;StSjD5mo9G#&Lpa-#8 zi9TkBH*I=v4@mm9@0*yNs6HoCWTwp;^Jm~T*bNL&58dro@=>M|#eZcP(}E?t68%nc zR^l5yzZK4{Y3}Dw1!ASL1Nl9wzUqD;IvBzExUNAPm7&$Ys&IdngxWLZa_cSTC> zGhKtxEAqx0XgOzP()S~r8)X??9(Rrs3Ae{qO>rLL>fgfT4m@7#)zLUqPfPRMmE_4t z%FldP__7Q+O(e3QSOjT>i~>WP*sr)UTD6d14b`*P^}b`!QBs*WYxbI;A2@?2PBrz= z;eh#Ws-uDhd7fG%SEG0>(F^Kw@{Pk3e<3rMeYGiD`s5CHQgqQk<~jO054MDNa%$!> z?7_If%@y&Kv)4vz(!ab5ocm%jcRw(vwc|$f>G=)tLHyZ6vG=t`?J#HuL#X89zJf!Q zO`|s+U^{?qg^%-&i|~BG}UeIIkP;%37~liPkm*Epz==Ph?l8XO-82D0Q6m(G~W2)1)Xj)g{J&MK4BM{bYA z`4_$nBrDBvVYwfV9=d;ji`!VezdwM+*-;Ge^2IpR!qvMUuvsRYSRl3pWqLN#q$#*m8xUSU;Ell1- z&ar~D19 zGUGWpujL9K-dIkR1$sVOmyj?zGa*vOhG*>1x34I<5=N*ns0K?wy;6LTx?lp-h5jA= zHt3GG{(hhho>TJo5c0lMCoqT_O9i5I?K+-cQYY8#up6Yc1TTrtU5H|*aNnB}A=X>N_f9P&eS;#U?q{Nn{0uALtg@T_Cy zs)LCRRP}^L%^PsW^7gV6z$B*Jjo%&fGG;Ut$-J~r5n1suSD#*25ty37{T3;BU(jMy zehYN(^Q{3UEXxYxiO*`ERHu*Yl!G|@;!VFSE2m8z61aS|ReuxVQkhjsKwgFQakUGw zfuhgsS~8jOpxNjB(vU*8Ne6y<{PW?Xos)vo11u*d10xtjgeA0Ii8n)3hA6?PbHhw8 z#EW;cmv2>nn^4(lt;BMDNzzW+O_H;pLicPXqr{4G=EJ_>jVT%`x{w=lvzh)eAM2lW zO{EPN+TP8@m1*qCeUZZ7^HeKFea^VB-%Tn#9wCY7WCZEMfgfEe9P)GXV~oTn^f5PE zymN?h^O^7n+Ov{n@X_fQB3lcC^28FE)K|?|Wp1>ZVNG1iZ=kh6@E&!^p!h6RDJ7JU zOmOa1;?B;h3X z^V`b&L}7~L0`!tc8Jl8XpU;iDRBfge=CP?if3`Si8sEDV5NNRVw+V-gAG|*QsoCPRW z#%gG=Byww;wjpDw>-W^R`!){oZhZcK1|ueR|w^WPoE;aVBPpw9}lI* zl!50s-o5m9oh<&V&X35qPgl;%LJXIqQzz0HQuTPtFE+w^39R8ZT^=m>pU;|n4WuL^oN4pPbqULMJIrc3RUR29Kn~Wz%IK zs!FV%<1-LDa2nWxdgvb494(&iUw^&udMs60^}A(WwsO~^VgAdzXx`LB+e$Pa=tf#% zJ~Q5E=IAH+L~vv@zQf~Tnh$RoV#wvMK6o0T9A-1)9#kZ$=et3~8mmE(Wt+6|#ZaFT z0utU;q>2&7z@4n)2W5B3W}2~3J4(b^Vx&YmkP5cY1Zb#BlIFJ2Zm2w&vvW8ya@0L! z9U4TGdmFBnVBU3h#%;4MOeAV(p0L5(dJx0Ih2bkixZQ(|HI|wCwgIx5ptr*|vST}g zeQvTJ9BlkhtV6~q4;CF^$>3Mudn)k~*nmA4ai`hhA?T6##W;`1+C{atqa_EYIp33V zYHGO~bR<=145-22bb~0swub#*~q*p4sQQKdaJ@)N*%Q^we zLUewEGQxq6iQO86@ z3k>K>Qf_p1>P77>-!Im+im9Y-jlTuCsTD-yApfFobzd95zSC3w3BiroJx~1rn(@!* z#+zHC{(3w*^;{mkwc%Uo~iKVxDcmIW?KA z92zm)(_xEI@4Ed3xn>cL2Bn8>W@E%;6Hf1o&p?>u@87 zk8WBo?N1=e2AdbF#NJ|;;x`E+Y&B}6cIaTRk~DXhhM!yV)OdpV;R#_|t*h1R%Et)v zZgF3V;FDHvEyt0RqquuX(mpFvi?y!&AR^%?&fE1SZc0|RT~2H~Bw{njt^R$8K1IX= ztm%Ovs)Mg$OqEe@!~~`lrbe3>hDf;i6S(9clNRJAo?h$zaq%*VK6>ez>Tj}oB&Zc z-ZB&}o7t|U*cx9DquCJp4r{b9gloWk|Hs3Jn~z%ykQRXeDK0)8PCi8Q>yZRf?AMhz zSu*;kM^Fd);Opb!Bv>EO=D;WSL!;0TN3p@M+J_~rUcm;(&GlRz8V3iDA4OF1D@0cO zRr%dUs7W0u0m+0N$a<^gEwEf zqYs+JSC#1G25k6^d033FT81U=UB!niqam1rNR8R=ANg>R+)vHbS%vfT!pnM`I~SCO zlwNb}WicCP`5}(rTQXkXa@8S!M|==dR|KPCucW!;l)M!011{n`uu5{b<3LlLn;1n5 zReLe(3x6iZ)Oy~mXVfhLUw8Gq7dKa{lcl>&p^XT+{P{}Aftv&ZM0Z?ccoWyQQ8DizN56A$ZhDFY>l1PyB*9Gf`Z_$05N zeb5%4_sY_PdYbd2?TXLN1zGjpDbAsp%`z-{!jSyHS8<2R0Pav|a9*Hlt=ve)uzyZ( zk=yoro5(lr@TVi_3L_#Qy$&`@>ZYaQ$`Rw;ceDbA%Y`lFD{jc_?>+#8qYV z!B2>7Rm&n4KQCrLP2r~nKoS@X{|&2G8t^KNL-o(aHO+LBPa;L#C)-$S`ETM6O=~Jf zTQR*t)WevaC@1aPP2>DHbhqZ{ePQnF0`cYMs1~7AiA+=>^Wl&2tX@i!?iYZR`sy4` zBQj77#&rCJhW;mx2nhB1<3S>#uHXVFZ%xB2C!ug9L~{puXt5^WE*xb@dll-W-S>*j zLDxBK(&BSEvojL!PKUPkJCqGMfo#H_Q#*ZV6hOF*!-+EQt7}*l|577WlGFAb?s8T9 zUN=1}aft-8c3waCH{YS2wN8=)V}+(jlg|355R8OHEwknp}i;+{s2w&%jna(WJQDcO_j7y z?{6>)&op(ut0KOZLS_SN24To)TtGzPA7@ZST@XOxU8>(^a7yVx&9-46nJ*X4@`K0< zbh8EiehJcV)?YN=Sms;i3DV>4*zzM>^!V76d%mT{v7BR^Qily`OkHgh<8FRR`ed!a zp5>cbo#bIuA4b`AbM(5JOLO-7t;a}jSB{u@kW+VgrbR7un6pS-S3A`&a;;=ko*75p zqKay2eLvj`Nlw$^eW)5K^fA>u^QhZNA)dKHb<}=^>qw-IP95Eaa5JB!46#De^rW)D z;GvlKnDG%T+QAZ{uEIt|{&qyTK`2Evc96-;--Sv z`M$p~qIICF!e=NlW1E;)Nzt&@9zc6`%kQBuEB@B5hnLm#hO)P^xz{S@o1KvCg~M;x z-m`K}!J0o9V%b%6Qf|JdfLO-K!-}hvtiv{S>JPqXPBVQ3%~kD0 z#N%ui)f+I;SbkKHV?CS;zdN-Wz!K7lX7!_ov=#*i^{7~qo-pfAM>YM3jaU(Rzx@Q%gLQ5

qhkg64}Ps;G%M9f&E+w__4LX8>W%QG zy9}N3`OHo`33(ff`PW#{c^QU0S9D4$<0{xbJB4c5Q`vS{T^M%AP^UC^l|ybnCBinL z!#XIp%VA_+*em{4rNP-!k)sG!&P($TbTgu9>z#Cxs8#sz^xik6B75ufrY=(cfO?vs z?mRZ?Lk$@L-#+ynZ_EvKNC#?bH4AU&u;o}!G(?Pd`m|JGc7(Jx z+Xf|8iz!<1EH)m7)j|A z>{1w(oX!?bML^gGvXwpv40;LO7wOmWVfa14K+MW-`+}S63a}ww$PbnTL~sE&^QLxO`rch6V`jgUTr)L4zkxZRgAV+6CEM9fzNb++aLEVb%s*^{e(v*bff@wflpobvo@m<|}bxur_jKSmOQ<`0` z>U50MFU3rqwGMsp z@hVzX{9|hZH(%H$$Y>&m+)O^W?$7##SxGnYWA9Fx(KnUHRu?}1QfV6dWEt+?rvgln(3 zk&4?tD4?jq1TaY7_^`=f^#O(qbv1Ysa900>H&K2`A9vjW7Q;4R=hk)#X}9TSG4GLkScw1W8Fj@9D}8k6>K}jb06d|orei4QwpEF z)Fqo2*ou9d@6T3;yz5eeWV24CV{TU~vG+(WA@~c){av?lPg(*4<%rl9&;1xhF$G14 znw-cFsR-6EypF;tvhkCpak&}viKEWraVW2EBz76?BM9}#$cB2Gt<=qwNuk$M=spS3 zoA9ciO++DPT(CZPsH-Wj{vs+sF&`U+^&`ohsF5VX41@Ln{zk0F?L@owSA#9+Z|Y!1$5jiMq>4 zidlxAFU(}l~ttj%C;D-uijIZtTwNrU=#^X0S}%crlovG5&1ZY z;*6P5JH9yIj86@JtkC*>e5TwrrP`B4HcW>BpKT0_v zionTH=?;O+xTfUbF5th*&z;4R)5MB{j~5s68cr$9tgcrPefnAkG^X!6N6o&=$$n7m zP$*QiIpc8*Db?B0z{XR?ogcfN#Yv_&H^{)3_q9u9E$RsJlE9Q!P<(Zu<1wqTc=Qod zz{^5`*-}z0IiK$R5&gG4gmW3P7Nd=fg)>{KDle(tIB>Gyq@{>J3>X1 z#%)-nfgx&MDQpVJQ1v4^w;dI&6ZcDbg=ot6R+*T@;5sLo2o$BFw)?d?4;6%B*;OP) zsnI^RTu=9xa}HL{Y~d5fg&n!7VDPfUxyd1+4=&~D;LZvcybk< zB;LkNst?$`nbV+xwBr*Dhf4VwRu5rFT;MA?YlbfozKB3lvIiKT58&>TTjd4pW2df; z__Oa)#Vl(;WF%!y@Z{*52^j=*8Y;U?m~w=qk8Bm3!h-38mKclZEhQy&WIOm0miE?O z(p4~NoZFwP2hpx2HGb;UaWibM3{QFEamS&oF{iQ_x3PNbwiEJb0|~;-$1~A8Sy(RL z&%M%o1zX=baZtk;77#x3+wpLv{CK?Tjm|QF0{fo8U=bnRd**-^bFVSPz&?S>C4fl)K-^G}HHQp97Z8UW02v2CGx{a`8VH7T`Wu1G;r0uJ&EW~fU~_nbID9}H zz5vMqON2iNm|5VSl|6tD_q6x?H$)pifM3q=-wf)$xA;oh=9))c+XGuLA`DX9LQ|KY_^qEpr3myU^c%-UkxiT)xK*fc*FX z>j&_Q{`2$K^>3e-&jQgC|E%Y)vH%Z&a|Y@W6aew_10ZMsy3Y%UKm5>NVBY?E4tSpz zfF?t)`2i>t|0Op1CpcRW09bSX09o?`@ACn50)zewQ2oz3ek+54I)Hb10r2!?9bjG% zH?&=7eL%1Z*AKAvWnIv+Ah0~3_xYgs?4ND&0LF1%U^cmdXeH?Xc!5tK-Rfmuq2>Ry zd9{B1PxSit{r_Lrzv=N`TL1U@|1G_Mwec5r|MFn}b*e}JYyHPLBndqtFHa;XU=RE# z0eLE*;ummA%76fn)@4Z+bctOA?#Tgb4SEk9I9`DSQJ4Q!hGww1EUAKk&2U*#gPyRL zB_O}W&o&?+U>jZD1J2*eHqe1H7&tF3@9BUpIk~Bg6`5?~>VoFVlhm z_1FUc_}Qo(Q2g2IPxb|5OZZU(G%d%^0XhN2pDjBB#mlCl`3S&36#Zpum)r`P#pCh{ zn#1EK8vxiIaPf1jP_}Z}F8V*XC>S^}F5kLjtUtKv4>tqQ@1Lv`3>+et&-_RW0u<2! z&IDvX`EmCL=Y(d=y?hADO@9~+ff{}=VeqB73VL~L6E-Uc@SlUlY{9wy~Q1PP@ zD60lvCTs@Y@&<}Od+;~g19M%H1RY?FV89Z2*))_z{_KT6Q2a@cO9=pcDS7^j4?+q3 zYsLP=dh-D7|50Hx*1Sx<(6Xv~GEFHv+g+>67+Gr^FpaY5Lnd6*V07*A_4&h*5V79j zO|#Ht>fV4InVzH9U);%072a0cy9sA$tre2s z_Q+a$5pP~L>8Rc6zQ6qWrM8;mwpHN5j^EBvye~+ z4DF$Pcp7#U(%evM1Z0GM2~raDf*J-g_rUrfjvkJQgnPyq8rizo0%dC^~jT6#tshc>f&k z7UZMC@6t6GVDG<8IFvX3al%y~5-vcTOVdGY&?9^_g*(T}+DA3tFP{0BJ$B%!W58_p z=}8{j3$l|RU>}bTa-bO$TrSmHPS?edtHeRmR>)rJr+c?-Po;@pZKzn%KV7(MkFcq| zKE0{Ly2p?ZkcE|@M_;mriZ`hel=VHq1S5*(~L|TmCv*IM0P4t!k={L%2e&wCJD$zdrq=91aPuX^4jQQ*x?`RaaAfSikw`-43 zu+l>ib4JT7H?*gT;%Ns(33li|&@Sy3pqQg*$A@SY($8n=_iA=o5A8F|)PHC6-F93$ z#ZOJIR{6q<&34TGUd#qZ0tWOI%nd!P{xi<2Bx9@RZ1~~r)aK;khh}1faPskL1KHTP z00$_yXQNq|pjmoyBVH0xicAft4x0L0>LU&oEJ{Pq$p7?!3t&Ni-V@2fK<;yBtkMS> zv+$7B8Aiq0#t`y3Cq%4+A_<#YF@j#8SgEFP#b{!_Eu9lKe9ITvoy9VrTVm*Ld7Ihb z;=Ldne(@T?4;9s)Nw_qXhj(@QmYoCSWp=}Xof)qr*qK){_i^$-PJB_7Y%H_> zwx7C+a$g+x!a&^k=iZ*OwB$(U*s=YB;>zWgLT%gbdJi z|CNLc7?iWUi#MQ>+Smi4n>SRg_I9zCl>|gqW2i>;?GYagMp-P|0N=ntiLM!m9OS1@IGVpc85+7Zp$g5=e!6j zJ<+vD(ETmCXlJGIJVhMYHy|G7Zi@Y5%KAB0FSAbmot6*avw0EU1?xpurZp;Vy;YSC zK~@DCOaY-`2l(#wAgQ^>41}Z%o~=Wqr9*RVvqUGt$)2h#6nV-dyFGAsTIs`kzv5v( zTs;@s$RG)ElYE$O0SS{(+GJ;bRfZPo`bpC^#@AmYd8_I~k0xx|EjYd?yj_qvUg0aA zzM+PK9b1~!+Mw9^C$i7%4DLsu50428h>)~Yq`u3WYA^*q0&$ z`P|XjTuFJ=UCa)%_J!ibwNEeop267VfYInT;sH9qACEqf_`mZz!OfX{0rEQmmdJBj z`$Ywmoj5|wb<3wFA0pjR34!meny3AsjzO#K!VzpgCS!n91!lkJteAB(?}qX5B=%i} zr4^c$w8~Fsyqf-^Y#|hWuzCqYYPnMG4{pw%xj#yfyZ?u10Of%@td^PxwCRyxE&L%) zxG^(?b^^WWo(bIcJ8MXa?WDNm(4c_32;}~-dIp0_LydxeCZPIXF0;jlV^Eh_^XK=- z_k)>O#clE6G%Ro@@}wxX!Z=!*38_Ng_xrZD;JsW-M_n~0J0>Kpd4^|P<3;<%+x5J> z@M~jOJ&LuIZjzu{VOB=F!CL-={Ll+hZzR6kb!IClJ4&Q3)2lV39~Uv4vpZ)^KYgtk z5r2HMPDA9I!q$E~k4)8Vb-55db(VEJ)mJBbM@|ovUp{=78sA(wKQ-J7shu9_eV#}0 zo$I;4faeW1)2K%{ypfd1u6?SRPP@JCdT5_yTpDU0=)D=tEG*$o{U~^fy>P)Ks(knz z#mHRe9xRf$uXj=ym)RH(#J2ZOde0HC{uM5X93NJ|)nx|9|C}J)P($mV+*Z6P z!q8wtsZ+s7cHFjaGkFJ2xJA?Rt%HRO_BuxP%8;k4sN{V8qRo}ThHB5%TIwyh=Va4#IG~>s!gW3V#xdlv%`)nrUt;j>cB>!5;9cu%RW#a8%*sxj*BBe`_xWD#r|h|;H0i}~LIH?Bl_ zN||Ww?I>2=71#@6S2qYD{<;Kdy$vsKA94s|y}ig8XGaFeoCFQ#J%z^I^ru*# zif9suC|`uIF)5EpiN~|vRjR}B%4zGTw-M0s3DUneP13(-nD$2eB<$hcMY9(l9Bxx2 zqpd1_zB9+%oW-Xdzs3u8@+>{*@{`b2K4@m1;4#%YxFDw*Y-M$uOC;S zZVOpXeI)(f~S9-~*VX!wgW|1%)^DzUyIkY6}AW6+)3>01ZQ z1#3wQ00~hj@wxt2d0!q+W|gg!r1U+*k#!2CW>6SKf(-m8XR%57%T6 z(Yr3^7C%qihtGGTR=dECJ-76%7SYam?P^gou9e!r^VP#&I(=N!a_g{-otxub;@z08 z$j_;zmH8roPnX|*mzUb}HF~_z7?Oy7dh^IK@4x4+V z@;jF|>fL{-Prerr9d}Tk&L@fU4pzj*hso|rpZVG--T5Wt!0ggrq5i(C8W~b)8J3Yh z;&<1~X!Fwt0bU@A7A`&^D{IYFsiKkJ8h6GA2g7HS{87!FuE@$wZnQJCPn^$O0{>-J3~!VKDWZJTJqLL9hV<-2Vy?h%#x;p z<7Y$@CtjuKiz*KOus3)}s()E#fcR+0Rl8wqpn&3?yCR5i^5stdhc|Tmg+xqzOAPL8 z?U&IssOM;%ZGD!sSxgaou}sPLkTrhWvVmpl)j2`_=28Y9BMivwj~b?8)>*jU%O4N) z$f-5zLp5wWE*GAxl#=$7?~U=q6~gXf!iQU88OyYU9PHl6hqh*jazO7MOL^0d_Z@*pclv zgKV7FYts(s5sScvd(>s{z~D`bP`SY**O6ADs4Yq6I%Y>(V%f3h{oF53>CQei8&Y(9 zz3p~}(fieYvOO>TP74uJAtEXEW0ph==h4Wf#XmRp?EYRfyS4y&IxVp*hK?yc9jGRTUQb^G$&u)e9Tyli|@>MT0 zE>p)JR-ZgIdt@eM_^lV1znmo7I#%t*Ld*Ok@`ky7L|$H!ac8F?_bEeK$OywbM*xun z6r9aP7Y5NcU~%jhbS^+#4a+*P=&09At6||hg>PKYn`I#KhK9RBKUfutg@XAfjYX{xNOOm@7+^V!e`GZzj-g68DtNa63fdi+^2W;1)uek6ddM5snvztt}e`E ziN_Dp-Mq%VT?S*zPXFW=V^t&;3LG%H{C<34EZZozba=j72>O7C=Y%&?l8;+X&3?!T z=6TE~VIqaGakf3xe%Hk0u+lT}4cwan*2h z`)=d$Sl(Pg=gN_5B97U)BTh)MpP`2`t9^x1D)lzNc5**X9e+KTH4<*HVzm79NyX!? z#sV%CY;Tj>^f{YGrA7M>NUs1fLhr;NA)fsPM zV2$5yA8}A$r?>8&%s>DYSvw`9UAZyGH1^Vy+q>{mnCu>nOX+pNZk_dyj^@1^@(h0X zUE6W^(W4o3c=#FwNBoOD&v>4CcG~Aj+E^i$kwpA*Vr z#xKVQ4<=ibAB8?A|2m&RBm(o`5PK^qCMSE8&Qd z!rq3pQgX~qQA$D)uPv>Vp3JPTep%J&y59Jd(`Z%BzoH@``!6ZRkwu`i=U08u6=MJ#`cO+ zw?c6+cFF|%Aev($i9MTK{qW2Q!?LG1%lwnrtXH|u2>Wtw?TR+4dCkisS&`IYNT3Kk zBH4J{3`7@&PTj$u70yu|%s{QwQor_zw0-HA?tv?bqV-nCUcc%v-}XMh+sVr76zc1- zmTRZ?j(so!Y-M4yY>k(k-m_{)7D^%NPcT*nz#$iK`%D0elN(Dc#6AS^`2vUBy!8Wl zwxU}eh?XQ=JHwORd#;4Mg*Q`Gz31`_>0__l1*uTu35n`>k6e;bYL+N|MGi^6;nLgC zFzWZ_dUfA-W0Y!fCHua!XCK_;O+Q+7%V&DSr=2x?5n<*>IR)LLG(47$vt79udZkxx zg&G@|w$=F;p>S6L>ovz=reRvV;*Ok_Qs%WOs4>A^qkeU%4mM%?PXgfiXk zlQ=4QJ65{qe1p`W({Q2){^Djs1ww1qPcHT-qF$IvzUSZ@_Eu9Xy;Fucd;HmF*tN$d zZpd2i3H+e5WLXpY?zPW!gK`GhT5SVwr9aP(mE<^r%jTex`94RJI;uFsk4{H^J6AKP z&sli2jq-Hp>ht~p-BjPcp#j12u1&?^uIQ*yyNlvK^hbY8TO#bgbsn`SX`21?-t5;& zSeV$;#!iETAXfduLZ3tcUiIJZ;~<#LT73zwC;O8{mr9&D-U65k9(@m3`PY>$kHn}} zzBrhM>W600Xs3W--kninuF(}%vkvVE=QvJLOdL$p`0M)D26A`slDOZ#c;>IRFW34e zqDWIm(c*B#rj-W>zB3!7O&lX-T2?E(41II(&S#QYX-vN{Dz8FEv(fW~X?|d(lB02s zXHV0R+#ccAlkFHmmF*@T*QzbmUGhKJ@7aIx`^t~*YJ;pQ%tE>QBZvNZMG9w61H>aP z*ut@BE80~6ZYj75FilmuaaI(5w^g{AtxeK_uOD*g*X3Bfj8z#}D0AS-{P%56sapg5 zH1+pe+~j)vL&H*KOmeu9H}~bPE_I!Z=SyPCv|@!#*PI{z#>al@&v(wJ#*~Srw6men)eQ`klGLfC>Ow2tdJC z;CA>%%OOH%N-?zj)lf@IT<@^emk6_8a?oSTPD)s{HRi9T?+!|}*ayS{OL@{QFL&ws zET>$_4Uh1R&weGR4Tq1bH>(wKxSI>N+*_Fv;GWtd5z}Ajxm;-9poPMfiIcxBk9~lE zSA>r;`K+qILe-+Vbp5UhW>#C)ErW(OHx;Ix{mFF44hSyIyK5|TByH)k^i-7xZ5ZwM zFJoTcG`p0^(R;cKLD=BE&W?CGEw87K%pqrZr*PHa)`~DpUPapUnL3r4$-cqkzTZZI z3`UO%7MceHj#Asd=|}x?cjcrmHxKSeKe5b_?2K0uw0R;|(sonh%S*f4#qTRzKWFi?c_{pOJGB>ZI5fg9wg z?IO-!4Q0rj^(`Ab?Y!;;ZP!jDUp4>PnR~3_s9bKQQcc_nrH@X-hd=3T_G~lh^es~w zq&!bZ$Eb$z>Jx3fJg;e4{j)-R7FS~tpA)Ja^AY<`LQ6x~Pg+D>fc?Bt@KIw`G`hc$ z_ppgV{<-~`*}2Hq61uL&>3SB9ck~)gXl>cVS%%!lKdKPSe=R2~(^)sehI$+IhWWcf*zEeS}EzBTdft8u1~rY+OYyh3x3l>r5>L^5uRN z1XoUXB)`n}w2K+6s=E|-wErw0zVFASziw(C)#>?pgKzj`(n`K407e7eof3Fx?PH!c{N_IDwkt-ynVX6OAa60ZcBb{KH%rd>9gjM!q$<~Q>Qr0 z5>keTlWpAl6IVYtXcSobY3X#m$QBlbqj_aAT1Wnf039H@ff;Y&3wNZ;(y5H*XFgZ%XZb6y`mqonpdU3%dpBJw25%os9{qd6Hzlt7H-VpEe z+1PHWzCq0yytPj4wcDja)a`TEhHoiu;Hd4&!ts-sR+U397qxC*-do1dlJ$hy7k4?rW>MD2Dis8_gp2`W|lVkUi>1fvm!|qOtoG?0#c{i9Znv&=e$1)`S?Ftja(I}xGc24><7JL;19m}hK6Tlt zB4i4ug)@W^gn853Ag-9V0lZIHWzIqw0WNI69b0Cpv~wtp;Yp_`>zlZ{#mEZB?PM)$_ z9WUhIweg|K_wz5}+Q(%)itUSU^$nj}MV8d$;nUvG9dMrCF;|jLu>RzAeOb|udsbA` zaNWEAW4h|K{s(c{OZZk+m1Uuu{2^Sq%?d;})c?RA$6wgKTXiZT`T3HsJN97%Gfvkj zw|!I!wL+NftI6m|+a~kjua~cS4@X#Jmwo)=QRXo}%?T7`KlV_wdc3<= zg+kpwmHH;=LD-Hw+&(kW(v7`g2#18cGRwP}(Joh-OqXx8IzyFpt;j zqhX!3!3kYYLWOol;Zl_MCY@81J+=FJNONm*4)^Qt{k$D7ZP?w=&8N!lWZg7-#^ZB6 z`A%}f>o;rHOx11*)U7WJR8bt44*#hVlU$@OyxTj%VNak~LUlu6ZC2tw?ZgioJ=5?1 z?2GG77WV7xu~rZo{DJ3L+uYMCJbr(LzCidrTcwSSVP}6KT$Cby&5|p}K7E~88eI~> zivNY?>dCOU{Sgcl(p(o{pcTu^S#>%KRq2mlpavk!9TX)UV5*E-53}D#*9;Sx8$;hu_gvxP@;q;2DM;SN>G6H$dda#PdBa0UFo!KV6t=49cpj zZ`>S;V8!e~^MGY&DkKPG_1hV%oPIY5oI3SCoR5}HJXn^0&m)-lt?^Y--SWIDF1;nr zcRh@kS3ckk{Muu5rESmG{np>3pDs;N>yQsQn_t{r{5|m;Ur0WApXNg&(V(q@)9bD% z=bN>pWZqiA?OJ6{>@Yo>HLiTqROR4SVY12_j%lKEpSk?qiX#ewyIwS`6&kV$;tTND z_TYy5jr!^F6G8maw%tRqEzNJ23LGN6>ABWw-@}=Se)qP#JnFIo`xE5*yJh&RB+IG< z?BLVy(_e&_J2zzXYZSaO@Zu{A$v`bb87g0snH@^alu!MvRhN72eSegnkJ6s=k8UdT zG|V1)b)+*OLQ}hQT6<@n6N?C+2%7saLo@yn!IBwp4+Fa2vJd;>v@PZ1b)IQD*=b&! zZCtml*Y7&#ivqR!#C0wzmh5_x{4IPdd=qkS z3Q3iY3YS;t4psAuthg_C&`O6_$qF7;KK{OBe8A_&X~%(n3w@FDtm;?xjl7qfUv!@1 zzoL+7)p!pV&-^qcwe;x*2~REJg$6Z`*Y#WshK6A?f7t3vw($a^Cw@h2B~}J!JJ( zDvx=hos3c6iZEl{LqBhvRQ}f8kZ6|~Hct6;CZhjqVw#<%T~*0t>z*GSA9wl+itMOR zXkFdcdz^pohK)wwE1yo!3=O~g!qGU6O=ne8R@%oOU={zSDmoRmNhcLCuh{Z^+#Pm0 z6aRuEHBsY)n6p?#TB5+l&+U#SJB`;ScW?Z%bj<$r9o2I^g(tTo0Uz|BxBAkl0by27 znlzVuh8{oxZ}#6VW*9316*`anO#Z5~WGZcQqio7;?AL>JC|iRRO9yc+er z!_B;PD?c4M?WbiVlG;nE***DU^Nb*0lMk+CCh&8K$`qe^^J% zmpT)!^q|d_D~@lhIU_Op@+ zg_W*^Jey%x0sfwd{vg}FYuzU{#xD7^j|<~;=xb?9Fwe!+Vg$P-MHW)sg4V&@KhG9r zN6s!i!gYYvh6~J;{~QEadnf8y*jPK5F7i$^3-AZ-dW)fgi-V=o*xCS&Z;|sIgf*o( zqtQDlE<{WHw+t4=>%@Tg56U0{5eO2A2T%pznEyWxm5KnaY@iVThj2mQ<2V;1)x*sf z29Yhe2l)8`7#;&q5ITqb4H5)+JAgdE0H;kFG!P15u^0ph`RW)$uoQst1E3Rt?gKy) zRkvVQ_uZak8Y&TTAe;Yv4opG7a03rM56|Eb_kds;aD)L02q8ddSireZtRTmK#{z;+ zL0G_fAfRgaTwb2EoFR+^Eo*>(go2*Zg>FTFJ1SatF7n9)O~T+#NbB>3DgDpLS~M(Z zzN~X_(SI8PbMU}DN% z{uAH&$NQBwnHNTvE(}5QzXblR#^4cB^uePwc!Ugl;L#pDLKZLZ=n5X^Twei%7BLqV zmV0if&yD%^nym)dV=7Pq~y$fdSJlts^fUG&QdyMXeV3o1*S@pDmSxxuW5 zbXZ#M0Of_9ZxMjE1z!#(;>xHFtH%>tf4ad3Y@ad4X`Hd|Kmu?r7u zSv3^zsm`n3x=v8o!9zc2)z|44TY|;l%eq%;$=p8N_3rM}7l%`Qmv+3>$ln#ZPob zu{zP-ulXa4Ag?xbWEAIo=Ir}2f+(rDlTKH0eHI^lD)q$OG)Dt}G%vP8jM%P*oXE_-9;63`ZYwP9Lk zI86w>gg2g3S8jbmp_JS*PyO|+C%FPj!~L4}DmZ0(33}gJUh5R)Ruy5J9<2FF(6@Pe z!dPfnzU>8K+SKwA?uRB_^}3%p?`;Sd^hUX8-5d~zkuE3pE-j13h;^YWZ$F(C-w+)d zxubE)UNInlqW-Mt5kG~`zZ{i5Ma%v8wZ!q+APYC<#ilAl6bSn~H)0Htu!VtXhyV`o z^CQy`&fFsxIbVlg?!hS3!YqF50dZ-EI9dITo2f{t+&| z{4hqq7C@9{aL|TxEo%mvb8dfEHA8|mYF>(hEkH5N&;a-jNx4A5VAkA|?rAKUXi%0rLUWK}*cXTSs9)@O6EFqYbv9VLzR~dJnXr zFi*hHu-S8ux4WmckshElKyl0c9FvwvqkJFRB%Yn3rxUMkl`=IdwwcZT!>H)Q=ms^L zlsBUyM2QgT7^-+tqLxY0(-*=9>|2Lk2*WZ{2VSf_D+T3FL3&aVm-fTGsj9oFDm4+xK~$ycNX1a9!d@y#zVaYR?f_Nx zT$IeYDCvr5sRZh#vKYzJaT29*u#@rPMF|^EB!QYyHziSx6vFY2shibfH><^LUTUjN zOcKvcA&U5JN~9`;Q4LA{&NHth{(1%b{Awdnz-#Q~`q7u`#g8UGe<@6o-%piHp*()R zzVEq6&+`pNq+plUXE5Bh)Z5P_#E&Htg?%<9Q?3n@L8J14^mYEsF)oT;s!sLb{9?aWD=>>eOs%CzGkfb9WOoY zILl@q(I@q zL$FW9JEY=j!>}Y01+$%kzPu0RMnPT})eW8YE zN$&h1*+W#BXsYztXsPlT@q$FjGqDn<;$S85;>8IYk0(i_ryB8l%0y9>f~Z7TkO7CI zav;@k<#zGHM6?Sh*+=ZApB$lQ+!DP)}a^f=KMei)WY8?w@}#C0O+guN!itWUgqwtI3p2E~6Hr*uX;Ln)CpaQt{3ROhY<)uExJ*Zu zZA~mH&L_j?GTg|zPrzq|>X!0Pu>(zt8Rh3KcZ4}xUzaDr%7)j9a8soE)b;p_%1XXj z))K7`D0T3w9I-9nQjXgq)yGzpJ$ba`nrSV``hfgJ{&ic*72`acxQ#IUbicXe>39)pxTAmrS{WnV$LCs5vQ6%yVELBvaJ>yhQ6)Yq z7mwKP;Hu=VG%k;7a%>vQc;ir19eBA!ZJV4#*)O3}+?~edLUF5OT{C<#0x!TFtu>{7 zD3(#zDsagfm#=728PPTVayjr?$$&)G+GOMMq6^|;>IhbB=d^JOohh(rTF4XVk+uac zoHPDyzcLI6_5-s(FCYx&1XBi<3Otxzz(fI%o1Xq&WT3L65CC}T?MwCy2Ghx9YS|CXhd55%9u1Sy)h%T zVvBjr%kbW`o@=J5!ms)+$h8z&DL=d)ia*oY73fY*sTMf(;Nf`K$-LLt;Y{ssRgMyd z+M=r}WB0Gqmm$d`I1`?(RDE)5c1`~F2RFovDUt5m^nbFj7cSP5{(H4wq(iCx=W0*K z3j3d{Jx!hc&(3?A2LGR}J?+r)KdU_&ri_L`fByep^O-LMEWv3k1g5&h+am<*Z}SX5 zK<|SNdGlO4^Yl1#OaL$%6@@iy{?rY$PyK*UpmO3C9ZwN7!li{q)(2`Kq+-ee^1nde z0J5*Q8*C3s1%Xh3E5UY=$$=qis;Yi}|DQ@gu$Mdt@*EuC5ejT{bgu<^?1Z^NCUk#~ zg-?K_v<$oh{5L}EFeop?d`e83v-bW02t?<{AbWnAHq290~&&b?MUKfUh27 z-Eg2unbM)r(9Hw=yBOMS2%QXr$3r$v1{qj57-a+$a~~0~5VIWpy97LRMMEb8JeN%U zhK1AaU>MS&nf4DXKw&U{4+q_zFuY5E_F;M%AjM-t^Ec5uo6PU(5mH>w?`048b7Hy{SLVy@-8PnmJ)-@a)2`P_s-^0OK z%yT#zx;SM>hh@@1aDdN}v5#=T4~4PJIItWrwhInHfUcM6^8hLmV_)KsNT&T4hh%P3 zpfWJ+gP_}(#x4$vgBU02^97{6%rfXclOY`j%QRkaSUA((g99YOO!Wohlu?Gq5g-OB zy0YLwMHtHhG#{pUjtAY#*q3-bo_QSy-(wsbU{!-iRO#}?;o;DoK7$Me-M=%)&`i1> z_#Bh&gCiiB>V_j=nC1+aM$GL3)HS9yfI|`W=*xnKV<1ley7J-SU_>*?@XUFD&|ysP z0%?qI4L3eR*7z@rFE zYXlyRWNuS5^S*!wQ=h3_(0HafjmI$Wd3X#8;;W_a4-C*enPhk-y$%oN5##s+J3Q0b z4-ZJ%nA!l#toz`xIA(p9MgIcrZ*e=5gI&FSJ%hO+>KPmFaNsNgd5u{I1ON|wr$(Ssduln?%wC#+uHeXZ<``hScz=WM^;$rVM#>6sbWV92Z1i7MM+n2DH(9E`1DczKB!RXrWeh!{nTT#RfT zEQlBtjV#QZiCF$RC=>DX!|Kah|I2|puAm;vmG*#8;AFsfJ@nOfOf5OMt53UDw{F>@i(`I}Z${I3)<4;Lav zDZ9TCg#UdP`S)FlNbi5hVg7$^;2+)ps0Wz+RrfFXjLK%t4z2)`zx(+|zcLY{oSCVW zk+6fu-;7LuFJ>ktE+S4ACL%o;Mv=d}`Wxy@#QiT<{Ql=j{A2#l(aDKRW#!*TaCsuH|kw=Ztsj}FOnMs*R#Tp2d;#%$o*am8q z-90@l96m*9>~A@Q-MP3vFgf00=MX}Z=I*>?I|4wb;=qF|e+9B0ZvRQU?u)!3kNWb% z2}SIGK~#JJtC) z5{f1y8lIY_KPGXVmJ*M#SjRQb4pP{RfO-J;U%Ynn5b2%_f-@EmLSj!oZ%$cM;s^Rd z#&;tc>~NAYVmIyUDwlROWlEY*n}}NiTv99VBJ`?sl}o=@mo{RH0AIt^8G z6TTAM1rWbHzbCMcic6*H>uw++dAxYtz1+q*I50X8>Tl|9`FRu7UJd`+nLz7n+(iMq zKDL{`Y52a{AdH>8jl*NAi;gccph*G*1EJL!@ zbGvDvd^ZbW0r`Xda8Wp=LEt+9FK5HA`|$Jkd^cXFd2Ht6%n@x|_fBNwfBW{sprKde+3>TGh$&eIxkA3>U@H)5k7+<{Q2>@ zh4EhM?sF3?sF7WanaaUvcl%{0&YxrmY^x$A9fGp5Y>5(7k9{6=0%^^PB8^G86k6WJ z4a*X)=+byxM@p19)%jlVGf{#J`IxZ40TsuQan-hlyBYxjHX_Lm@mA|nrWOPcU^Q*M zP3sdEh37|X<0o%Gh?Ecv0rj)$yb#ElDBR!BkRbSZ>SdAA(G`n$qfAnlw+U z)U5)EjI=@`i|EL@RW!$&UMecs4}f@UvClZX{<>Z_rGMA+-JF5NP=6?!P0++nm0cFH zI$|^psy|()Nrs!eBt7rTN6`LW9bey`w-fKO*)KcUr^XP@(u5zugLxLM0!8b&Z%(qm zy#2qJ5q3@K&)s+_!1muY#PzrA&YWs@lI`JB%>AlV&){=vjDwgI7R=X91#`4HHg|vJ zUkZ;|Dej_V#wW$CDETJW8{y!dSOM9?Kd^1fFjKj?KYx0@uj!wBU?T?zP6{Yq^*^M+ z=bX-{4AhWC)$C)%gpfI8#Z7Y`4oOhG1j+2DlZe$i`SE$i+_7Y4N>-Z$m(L_U)hx5V z=U1(5yeDQl#urtki-IXCqW3O%CI`BH`-NAF6{61L72?qP3iRqdm zJ1Y`%(M#Ja_2(jkJQe0>Th@$t3~jy)x;mIQ?~gp+L?50QcNGF>M?MrBjRR#;1SDIV z{2!74N;A2|Q!9)kw5jLnKHC^$uJTTB1dJ$0mTVk6(sIiitu)^wwQa#oV8_DG0ynZo z(Y$&)Jy2q-($l;XuXb#bOFj&-4Db zC2h*>8cY;8oynf&C*5eeuS#!Yub%5P-(wl)%6*51*dOROtLDSHv=EY`XTM_{JNU;( zO4Z9BxnQfw1;*WE0r6!w!EzDcob=|^sY_R&R5(4Q>a17`g=z`xn0_U}m38ko{a?s1)pS?NH=QnTPNSpq81Qxur;EX^rSL4O6Q|rqc?bsG~ z#z@Q+#vGsukFD~Su75+?|CzX?oY#0z0RQZUrd(P%+SMA-A_gD8h*{NR#XzZkF6fC^Oerm{$tSxgn^ zt4Px98W#(4cjvy&H@NF17jM6DWCR0YTee?K?s6ZI$xof$wjMvOCdQt&xyk1%i(dK{ zFo3C`re>nLJozxyN3=E@GOU zVJ1Pjuws)SLPkYE5tgEVd|aM=eiq>cvwW{ECVMwU(`q7l?eoG4V|FaE=vlBbCIL4J ztYcZ7R`jq}J1D)-($BbeQbZ)xUvQReCJpL-jDAYwHZC`Wy@5C?ebSZc^FiJ34iKfq zGUUS#7oNg{7bBi*7;RPbNDb%ZL#FjbejX8Ru!KaeDkW7u&LA}w%1u%o{>RGCjSTq< zgrRA=fig(+K@^5BL9*N;G(6}*8-}tPKExtoc@8xpGJxiPk0&v*sz*^zQ}l=p=lXzy zQQ<~jcO+#=pwkwDI6DbP^EwR2qGBj>q(I9d2fSu1giS4abU61Uiv4$jP^wfMGjzni z35HpI-bChjq#*FB@qZ>vBLHO*WNo}YTFpY42zmHc!4n?)3+0^{=;7)UwFxwORPT1D z6bK0vD0LxZK?eJNBCD|VuaAw@zI71FbO{#WY9~8Q+p}|4(n7;SaP_t+$4~I{0a%vX52;08qm77k?6Ri?nJwqIt>$!HaV;!f)pq2dH2fF} zb+6a-zF`C^r;*1Jgsa*n@L!Qik}Qx32y3R zuAqS1#6{>((BsW`BwquPJJ;=M5SVy@VPDOD1D3n3`RdR9PV3t<%VzdYmuYTo1`c(B zimUG2j=_V+B$95LBoMF#+rCfSrY>`m&}u)G^+FU+N{@ObS>B7A=LjmS4Lj+`-PlFd zk0#Ifd08rK#~0vLt%rPyAe13}&1LWd8A{@abA&Nrpti zAzE3&a5$rCf!Gt@`M9vc-NGZ!#E%dYt3X*REpgyv4@aGP_a>{s#zkYFcX}=5W#Mp= z1hg7p#-fqIM#TSW8+?pMA`Cjgt#LD z8+T8cUpZB7(*44~we#P0 zoLghj^>i$QJ=BL$xWw-n%uh0HxiDI~5WUc(P7re=`Q$!zy&s6FZ;eE%t2E;IAgN$U zHRo!<`}uK@&5}jok@ESBsJf`NN{7b!jv@HrBwJ4OqLA`5ixHi9jB_@OMJzO9B&Iht z@fKgxPed4dqP0F@zKd{ajmaHXX>7gTm71#${o!h)yL!Z-jzySJN={Gf?#SNEXr4RK zDf0R*H){Z}U22|F?@j(Mmo|x@vx^JYw;kljOpcDR{>Qa%xi?)_=4pXG^UQ$XL_@y1 zRP8mJW^yZOG^eO791T>`&-%)J zor>NvU%}O35|9>xUck+)%N_}nXXitM_cJo7DGztu1wD2!#GA;3Vg zQe1uevAbbjb*YV4$N5oDpl-YH3sC8YhW?;egm|A|Di`XlK4TiHLoHGknlL1SSUv1m zX>|HEADzb;oX}m)Gu2=SpO4L+#XIs6~&zyd~6sB0I|H{nijmREs`DlVfE;y2Cz+HZ8d<|L*)ZxGms% zSSlyDi%zTGQhkPubPNqy*16PoTdjXMu`t?&pk)xL{53kDU`!|6zd-Qi^3#vi!+n3B zSPUWaExZw~wXu~2h_zH!1lc69J3|%RAI)g#)dO-UtaFz+GR%C6y}~>h%Soful&LlQ z)8ipCRg?KxX6$I9f5~YQ8paqRNuRa1slkVjr3%%V)`g)jDYP{+O*8$y7=&K-lsR7= zZ2a~Cq+TAbZn_O>T9d?P)6?ix18jes(A$Nvo+v=F8^0t!@IM#RyOg9!yZ2 z<=(~4fPViI5|Xle?80Yp_(=%!7wdo!d5F6CHV8$<>-aN{(3k15k_g#>!(w1_;^6tc4VKB6r8s7c$0Sl^Md z6li#VcXSDyGq$vZty25Agch9Wo$tW0vsxTjv(7EwEQLhay`=Db(_tbo5D1CW9+dbh z5$arBw|`6vn;3e$d~~=la`w0U?SPoO>hSC1gCinxy6{B+WE?&9|E0M3PxqFUiIe3& z#SP0p&FTLX82=xO8+InP|E-zK^6x(Rzl$4|f3?N`ySVu;`+su(yPeGP?}F(6RNO3N z%QTV=ITb%WY0M|;)*)RDoe%6lnQKRo$TnjAf{d{i%5w!L0g3cCIJyfT9}Qotg_VrK zFhy&}*GkABxEzY}_7L^E>CpJPnaI`um`nTJ^>HP@>wELOmh0zn+ahr0Z?Ls~G4;B& zW&inD{Q2~dG`F?M>v@#m{&IL(Kals7emmVroA~B!&Ctwy(2WjH=Gl5wvqiA? ztXi4IwOL6xXQ)ib$7y%6X*``rj4pizjEPlFFEa;c4{RfXIvqSH|!jLPjx~nyjaeYj+|`&g20%O zTA{6$1(N#rSr_2MD4Ov8z4(s{to)UlO8Ts_F1?s?m)a6<6Ls7OYe+BTibADl^yO6- zkVp-32o?jgKiQ(IfPlWWNGf7Rp+0RixX7!NgcXm+?Fq$aBD%o6qB(`_U^e=mQlxz> z=-p#@{VLNe&BNtm=B!xmPuqrBzE{7_wY>IeaT;~nq}kQqlau23HB9@Ll9$ekr4#k# z^e@?69F|yr@qbql9_&o~4Bq!D-^`>{xiqna_nd*nSd#st<+k;swkB;Zd4PG3do#&g z_mj|PF1i(E==xcZ?s)dBS^8qrCF`ThI-bmP+NSwDti#ORf!$nA5vqM4M74BzM9*~& z=@JmlGf=+eo1h}~1Uk3yz~MDZdYp#e=15g~EIT4Kdg6sSNS&@&Q?jpmSxoV1qZ z0qu_23#n;XxIn++8L-I;RnuC!Sq|h6h_!$#7v=Mw{qeB-efmaT43!d}vX(od{AW41 zmeqn7F=TL(h#3qRBeksD!q|lc8=iSF@&a&5~mv|fQLzL6mht6 zbBdZt&Qon&O%k?v8Yw1>9tbvM1w;w@0k*O$BRCG$wwG2 zdryZ`Do#KTSCzzjU|6EXF3>O%=$goxBjHuaqVi|k?I)_7F@%f_a^=Y#z(_mcgLKgb z{bdkYfelg5e}<2Zvi_M=dxQ;MY=918vVc$}2d(mcTxh<)W;1qM7rpx$jTuTyRoY@& zy{KmRAjvDi(Gd!?R)q-;zF{}(2Uno}j0q3?5o+Fj@G+ID*dVogeoDbuGNqW5(a(~U z<%M15?&sWoQ!xf>v;YKpTEH;EnaqBJrgyEGkJyB^uID2!i*FlWV>7=%c)sN9R9d8ZLE}CxJ>Y(!nGC}>b-})d$}>%AM>y*2 zR3GLGQddV;4-i+8AIS5>=vm_N@W^dLi!zmCfdW!H-s8fX5#I;9v)Hdg+}ua=+<2^oq=B=1CJ| z3XHpJ!gTupQAQ!es4_6USX89s1T>m_kR7aR9&@8##oA9Q@3%9<_uL7h*`KD+i#Yfiux-&2B=a%UEqS4E~-n z&IRjE_K}3#k!@p*(vc)J81pMgOU+7qy|pS@2;63o^HL^N8{2hnbb(+Jm7-=GhmN*y zAUk|4HRX3(1FE~Nz=8S8aLTa^Lkg{c{vB9bDqd&rbQ+s*O<9k{xVzMk{b{p+^Tc8( zjCK%QCcTDC>7S{XPxP=ArnQ?oJb;PJ0Cus_PI9e)LQN2gRx&d|vy?Obu^my46yn`! zo5>|f%nP7QC{(WX2+9z7{(xx+udjljS;J^o?>oB@WE0Q(SLTNFS`e{b{Xw5vy?ig* zWt($bFfrdn8xTcQto)jeGtBDHocl9zj87!-;h0-+C@e zSJJmcTc~|x@HKt|bo--NFo)*OMdf!C z3Jtc91lV@<*X)AWo{^D&VIhfP8P`$Bd*cUABFT8=@pa*Ei)eJk--J>DSMhtvu)!X( zparUn@2z$L9?EoRRo05e4@k5KnNBb{At+Ge5){iekYhBcL1 zxV(g(L4Op!FiA2_|7aG)fJ>l>gbh+Ha0RRxN-$<_w9;OyNeu*+7lbjGjgks2PBYN_ z40W3Gsa00B3C)oASnW&xZcREvgu!F=j)r9qnAe6?Du`k%Z91 zK|6e`wQ>xPtR758rNsbC#@Hd zdzEE-i&ftG?${deF^}+QY=^)UlW?Vm_*5CEsI$#V=T+(DI z6u9bigxG|}AB%5V&^UkUcfzDwc-X&#*2-G)$G*2gfZNHe9^7`xKVf-JQ) zXN$*4$*2jhS6V&r3(Bi)($gMS%LI4!^mgZB4vEmt_uIBwJbmf0 zd;03!E=!CL(9~{|t11mqdEU5?Z|6AS>Lzxy-!gfzNaSg){;XLY)(~&; z;l|EUCzYYFBHI{GCtjl^*H)K2+qPi+nz)oqi)`SSG+jYN^UP>eVafe*I*gq zxj1bRQ>?}&ccJr>za%#pG3$qv2COJ4YC8QCZFGZ4r9DTJaSL2TFB$fRd{sTd&@3Fq zoEiNVyj#;g45JHco0v0X4Ot0kPCW%JKazy02|PEj(%n$(q!}3L>;LwOLd#?F!zO;Z zk=nBv=YYIO(=|5QHA=8xJTj2^963sQs4ejp$`Ku>ymAi%i5yd{F|70F)F_Ii<5W=f zeuTFEN|3V|^njL=H}oO8yJX1=`7><{n%2|)C#)7xmuzgskTe6CLJ_-hPB11Du^8} zdedkJn3h3?NpVkt-eyEeiTY@C-AhQiI*n=$o-KJpb2=jBZ)5|Q-dz;B90N}l*F7}7 z6;0RT`?+$lD!NW%EOyo~Seq6SAmRD@=q+t!KXj3P5}2pXqF#TFsG?0;+ydW8&jXu7 z=R#tUs`aF66P|8TrZo`5cnv_Li@tf*l2x8=Bx(S}iz-Gi(^t1@KQw&?rl0}QCP#H5 zlt(ge>lIN8)l_P*3(#XBw5(*P>^`KDnvZh->c3%aUL>K=xp zXD8V`DNvwhrB@Q|w3U-%6sI*4oiUI@l#OH#V2Od$W*f@>U>GuJR+VVu&*uFGu&{J7 zG`bMK(Mh0ky%yRr8LDW>pKhzBg_dguuB(}5ukvuF+Ax%P9`u*{)ICm;Yaj=`^S?1N z#IpI*p9eH#wpYWEMV;mg(BiO5I|x3gn|kR~pWanfis^t@b0H%)ZEW(2EqcIIa;x00 zitgH1fgqtK(UB-H^GXg5DJLEN2?969Ml%%q(pQ>SP@(gsO8C8^{gdRL8R(~l4t4Mi z%-4b^Zr;MQtf6xduiZQ}QnQsB0dQt0M1^6B4BOQc9;-$54PT>lbHw$BnU&26jxjN+ z+AYJ@aq?}6rus6oGUlafGJQ2xN;SD2^5dMP=1tqOFme-cHLpS z;BOrqnfzgNc@RdcD6idJj4&Z~z)D>ewKjjCuB6_ZBY&|8xK+V%P`EfYKV8LI3>C4u&qUL`nfm9c zCNOe7J`0HoVf&g7w^yj<#4vxZ{rkb;;hNJlLY!MwD&Yq)2FomgDeZ-eSjH?BFnr~t z1!!IrK`>}2hD+`V?f{<8EZ63cl8EsP*HymvI*ekBM^#JK-@^4rSvzMV59X??nHsTb zB5ql@qqi<3Txc}0i{4~k)0t2k>q27N2(yAp2OCJ_iBenlO)5hyfjpI(^E7~0rQOj_ z_sUAWrxALj-TlsU^nf56CQ#S|RXKzjq0fAjfXs;Q5rLjd`If&Tr@I~L70rU%AFC^- z0EvMh?K@^VZMbaTh%BtoPR1~%Epoh8U3vsB@_ajT+2PsovLFNeH;gi^q^d`W{g@I^ z7H>;jKE{3wP4|Vh*&XB}Grph%)qTZ7{^NomL+ZU(={i^-3cOpIr;bfI9WVb~9%suq zR$I=mL8!`JGogrbq0J35RaJ|!qRb5Ww(Hf^qU~jAy)b47990%bgi{5t-T6J@w8G_v z?&_LQKmDtM8+jw3>_CTO>|Rncsz3mamLohe}zyx9O9lA zg@h+0_N1Z|ql1`Qn87{7e-QMD zBdIj8Ea3~X7I>CC!=O^BV5fa}u3i1JdeJ${tu_n@=0Q5xo_Tgp%lK)_r+klg7K7-I zDn$VTp|hpnqzP81SXQS~P&0g5naf1tyXNK0nYt849iU7Sl%t&KQq^XSI^Td6+Uxc! zi1@oZ(Z`|Nk+ZrLN8W<1M60b!(rU)$9bMcxnwrMmKm;?U)1@L}ZBhoiURP0FhI zSn4IZHzY*gw!FA^DNR%)RZs?3y~ZUv$t{-phc#iUHCcJ*Vu==T-&M~da+C;)H*9D3 zf0QfK80I86$bwbw^`oFUmk|!H@N4HllBmL`-9+f!%IjNuX34q$d7f_XQRY4jMjsp0 z?`6TJ)i^VFOQT9`Gs)PDp{?dR(c#4)cR7?2ITt(cIO^jqRwB3HLCxu4%gtEvA_HDc zHqWEa1-Y{C%+t^|$`b6%tBQUm>aR>};@m)P{7OUmEo0fp`{77~!o*Gd+`^TQ(k7X~ z>L70ub&nsfo1G;O6{ty%OEul5QOHTv9t0KRD22ylmo7q^0|oo?2Bx%~F5J<4tVbNP z$rGX#i3JOh7+*Y7;UgxieLO&6N4+0eyC^L2;?{5CWuR?uHRLb~UNhLf=Ua>`G7_-W z7u+S|4zzcLr`*GL8eo5CiZwRh=tqBgIE=9IC|qDP*P+(#+o7VMfU%W%o`0h$!5H2FuWw9l6+g+?dUYK2;Hc1^}p zza(kf6{qps<6)_w!7|DU!BuKV5m?HWncS$v^EsOg;BV+>2|CeLE+1$3^y_(EK3a|N_$o-ni}agT2xAVLk~1G|2zHnFZnA1 zusK=@>zroz0t^2g$8^8Vt|?ppc($tD$?OGqMWv-M%T8T4cs{IK%?+{OXU*z2#;u7z zO{$En4M2-zZ9u4G)pf>sod$PnG8b%Wbk|O4iKf;uu@~w#)tr}4ju%=@iymLS|MA>x z?wDK|Wv>N{j@tn3RkLPYf?%F=$zDNhI836mjOYoW|0oJ6fds6w>ouG4z^}C2r>bZ4 zSg&WogUd<7d*=P9kfi}uWLSp=Th%O% zJ*}>77@?|w+9D0{OLRF0bEzHDmn)faO#2{S6Ha)?r#iwkGLy#Yd2?kMu%D{#7Nint(=hlR703mo9≦tu3(N{)?42llcID+~_^ZZSw-RRl1 zArqpT>_cbErfTS*X4N)|IsDHe?_~TzDxN*2L-s)1qxI2Yc{3Fwuq?Z+N;0 z6n9umQksd)>3jqyZOoDyjzWZRu^Db8AxcGAy6(c+i%us`ZXS-;(!0^lyQ>MWG|O0v zIo?!hx>Lj*Ief;3$&KpJ(1tcVq>KXwb}q8XUUyR4xsY+-bTa#Kh589;g6nK)^h%C< z7e;Hp=X8nA^v(XBD51k5mA@+MGD^WL64DE3k~=ur$>+$DR59xk*r6gx9|AcVsd9ANd`wQtn1hd^Hvm{i);c1RctLY zo{S{etLbQuk&o8VObry76}bRN1x|OUY2Ea=Gg6(CLHWTRjtFXX-a4=E7Xkp=y$u#< zU>T6@wNW=y9`?uhLX%*Bk$x|Lcwaf{Vb*N+uq`NP{l%Pb2V%Pdgtx{)Tv1DgaehKs z;Um5g>cY+H!<^TUC%Fm&mhHBD4A zF~cByB88~+W~WwaFh88Ms!`VB7%*gy!h%gv4W9{u_51i zeaJBl+&2$Z?y9yQXVi-GswF>O`zB};L3(s1M+klaUNQKRQwu1TOmYxt)-Yr0ega{x zOLbX?x{{A0V;d{@>e%Wkh~p@F<v1Q*^U``m zJR|l^$EgoEl;R=ABxn(@p-u!PSmqO}vxrn|^5xdHU6wK1E8>XQ zn>$l6Gn(f_Md&h$$9fv!HKSn^;qTfXOG@t;l4?i#G}7J_rfZt9aKnWe9;akEnE<2* zJ)fDdR3rY~PO+++X7Zr)^kQq_2(H1MliCO7Vk+V47j19(1&TytEMVl&6L;B=lS(Me+JC3!37TurQp ze?zm%LyedwN(__gr|SS~Htb}p9af?nY!hoQg(fUzhIJDeYd|fFK}2iHEf5nR_D69} z$^N+m$_E>i7Bn!HuPF4hK$+BVMxFYTBGsOGLY0Uqx%8Hd5NW12AvXrgF1W$a#2XBf zy-VEdvB%Utw!8EJGD(X)*+nhf)ijc2UoJ4QF-$GE2F7BnTjA|~`r9POK&PC=E@O!B zFwaL*66P7W-URtdYzCU!opCjDeUvNf(11NOtXSoX^1jPF2Zyu8Ns@R;r)OAo^62l? zQ`tq{ElX2v-?*lR$O_w7yU%cQzhn0teiTJVHjd zwH07|objFck9W)lV?M6ICv`xuZg05thHxkqtgCb|uBY+4vD#UR?F}!BkMuY!cVpMJ zE8d(VxwG43Y)j%z%3B6b1@m4A&Fs4h9S;5YY;5@m8kPo$wryo;*MhL_i0&=dks)z= zV&htPi*`8k5h}#GA?a*#oJoDKFve~S^SJLHSfx_Gl%W|y*!q?`ltkt|a%8G`-NNj% zIx+;bF*f;Foq}E^u2D8y_SejGMX16N28ReZzTR#m*`+=^jo zG&-S~R!1zMb>>ZI!!C_EK)7XjymB)Rc0Lqy>1ZIP-&pO5PsMrh@~xA#=AvGE$(3rc zCXtLzy8}77L-TAg4^ay;QG;r|Ix3fYF>kx}DEpg0nvtRQ4fAo_K=Yvc2AV9*PB%=jbDU=!%P1NC~IR&|tbk2&LMxi&|5 z&V2ZM`S@V+t`ySY`w^I5M{NFZ(xRY`_<;H0@BS9k;n~Pu@yT8{NXjs%91^{+;E?0@ zwlIh3uMlzmxbS;b$<^RvZ0@V$<8htxA^XqKw>$mR%qQcivW$D(} zQ^goUup(QEa;$r2xPDR16XJut>xvrZv5xj;ea3I6MB`I&i81Wt55`Rjnu7{GCAGju4Bu#VC7Z&2W0hG9oAo?vL z3z!S3N2c%baPD>;5$Qh;#90U7Lvs9|=YIVXf1bF~|9FY}dS4QFzgZ*nyF2{l@b%h% z;l8!u_eDJ~eVv89Q?L~EzK7lc$HCSSdnenR-6e^I*4s{=xmtiS7B^E3$eq${X3Fc6 zUr=i-eTIzef@6R1!av}%F7{CV6X;x^a7;R%Dc~J-5vUS&TC=F~4%Vz6nrYn?drQy^ z20hs#%6w0njKlA))13X9&*5}0R*CsCROyOsd+dsT#C_y~7mslPA+xxMZxK*0aJA8E zDYU4NpD*29>Imqs$Pasm#A!$1U~h`PrAe0>F zV`4zo!Le1H{~94j2y!K;E}S{RF{z9rWyr$Ug&1*+aL;L9! zKzX3x_W?wNntxi&vzWh3%X`J|Hmug;OI@P(g9jkN*VptKU~#pg<-xKsY%|rasB&{s z?-ybUzMXR=FqNS>tgi;W&p_HDu5a^kI<&c|9XPV8IoU8xgQ@Vc!&{m zN`o0aGO^lIB%Auowt(V1m_nXJ2-=2NgWr*VVf`6pvO;+Tze8|q0hRW_ryN;>+sC)> zElPICU$ApW#YB|* zl0hZUI9up@7KeQr7a)tx1pu2509AQyEW>i+S=BahZNa4rm2&u zrL&sN{ulE-ha(x?8iU`ZeYRO&H$2;nn=U$9Z3@|ay*vF2uE5Ru+~=OIQ`-|Xkv+p0 zw4nkS*Q_ZFZkqrV(X2KoWA#Q9z1oRM)QiPzUHpwTpZdS}nm)`ct-E}XM|jG(;lwh@ z*nUA^oeGUEPxY`EvUvhAyr*e(AXODpj~_m&Wbd2hsg1Li2NoxDFcOPW zOTn1>V|nm?(0cD|k-R z4z!A})?j&ax}x#ZGuHCQW-e)fS3TeDt+P8#0HlxCNdmZR*tdpkyq^5u1!r!?=Zc^c zNDY+bON};1szMcPYA$O4Bn8N#dgk&tx$O^CK~qBY!z5}S!Cs&YI*-SXBiS{uk{{?j z%Is0@n{GY7CdC~;=eFF>-|s^w8tyWx5EAIyDtKn;fEi~C<8QqdaUIK+75m1u*{7{W+KE9Bv3wzHu|CdJPiWKL|o$n za%JNf7UFdMg;5!|(r;WJt*hcJK*cPW?JR8gQGRpnW_ftkA51Mz_0{eCYej`ZJr(djVps`VTZaA8ob>Ti9= z;CLjMSu0Vc#F8=Pd%dKxxBPkQ?s+kB3u+=fKV*U4UQez;c!* z)d0oa4-QLPw{;#qSw`g>L=4~vGuP%g1Ba^|ssQUKPjp6P64mc=_0m~6PzC+Csji1j zYM+1(9Zu5Z0K~@nm*s)GHu+8Gc@-EHCskg6kA_TD_?EGTy$*~OY?2v6-$jG0! zT&uBMe$9)fuTUP~q0sB!)UZkH;s>{ARD6h8W0f%En(JuqCY7_c?T-2--?MH%6kN?C zYimkJMp`*}VT-NMzse(FD*-nUWG8<>fk$4B^n10(_On>yy$KUXvf`W6FT0t{xsK*G zMxdmO;%F8CxXPeIr&qvf!S0FxW@w?ip&RT+r(yU0Ck0S{#ah(aMj&Sur0hmOS0jz8 z-wF?JdLj8xxFHFgZXn-k3%_uC7_;>-a5M0u~V01uq#Vk6x}1#wMKE?F@GDb z@(VWUwzmAm4|dQM^vm*cB-1EAb?OSs;uLtZg*u$TsGCA6rYE4UK(95QEH7@6lahoO|La5noyqi!F zFw7Uu?|x{W2oK=Ex3y>4J!|YPhT60Nk-N;RlZ^w+4nkr?69^(7x8q|P8Mz(Vt9%-n z!96WXtLyQ}*FN81GwiCG0bpSUBMy5$i!*TDjtAoEQr-Z)Hdhr#_l%tVlqhFD{Lsgf z5m~NviUFTCVs3KWiWs>OBr+W(R53|prt?`a zu5wm5loYY3l4)?^+|}o7x~L_g8N*L{`lwanZ@(KwI8%RC1-^>4>FA(xMQgk9xKHE? zExl z88mK>Vk%e$D%dRQ4o%u>>!s0ktV)C&t~eaP{0vH+lp?pmU_2a}uf_I1*w#WQLa+$| zL32NUi;lAf3wH?P9B95ZL?(&u->ylCv^F50>nLh&P>S-!3YI==MpAy6rZLEcmRTuM z*4-bDYCzMXk&vM^ig&9e$0jj7_#tOSza1~+vIeYA_NM!6knyEQxJW6^)q?dtc{L(O zF!wGj^l(S+D$}|85b5gBK?LZd$yMS$y=PQ6E+zP~l$m+Ey-IC(y#aPh(-UQUo8n!} zrKKh09*AFBr96iWSD5Tjf7=0A#9R-^eMVxV${Ao&d6TzIv_1UXz^5n|xqg8sXRGRL z;nMsL*RQ%WYmJ&+ViqWJCA`3O#J{$k@@Cy!6n+tqyRw}&!rd`>*e3p8ti5Ga988lo zj06bo7Cd-x2`&i|++7BD5AG0x1b252?jg8C1`Y1+?r!f4$^Go^vwO~dXV3SGLr>FP zU3Jw})ji#XTh&y>S3q2A*bPoP-Ef}w4xw{g(GRMK8XsM$Nv1IL->#Ge`9VQ@#5sxk zM?4EEZUadq6FTcPZjx}J>G@9?vQfI~9#fB9aN)#(poOV>4A|iwmKl1BdrX;6AZ~v- z@e03YFL~^ot2vy{QGP_vV$8v{w8cNlX>l4vHIEFx4-n3kzc~*)54u|8)r_)V@hDdQ z6vXhH$o}?mBbi(6AXC7mj)VwvoDr-zcUf8<)Z7MbHr_Rb^GozVACf7VY8;VT-cIB%j=i!Sl4L+V?h1;w`T2@ z=bfo7ZXcf?zdoYuzc&@`JC7FlJ`tyB^PGb}sY%UK=&P z`{iND@$=1kiBE&q_2F~lHwSMvr!l3Lg68O>& ziDfB0M7}&eZZXw}7n(&64s6zV;ndlmcW{wqpWp_i% z$>75{ZQe*ln3sNuL*vbdM5`o`fpBz9B^?NAX3=HEygCe6avZ?SE6RSh5#X>?qq-ZD zS+7^e^Ez)UPahZJS!6ATX5mS#A{vJ;5>DDQ`y+v7Sr6e*U-=BBPdWORqgcy3#0P$P?#3e*Bsci-1#yzDm~m{{n1MgcZVnRw+BK89>b;hDP;GS5ge|Px znG)!NV7fbnJX%34DNI-GEzWNi469LV%q~w8U@@gDT7$I%wXcN^1K+ll{l5o%&)GdxPiKfe z$gJ^+W>H^#cb0H*AsNxE`1v!5C|@us1;>UGSGrfM?&!!Z^aEv!D^>)eFEx6547N2+ z7J}((`E6>*lm{}icVXKlV*tGSga4o;ypB+7%E%5G>s-^!%j>9)+7S*Y5 zY$hhhFDYoYEmK8z#mUq|3^r77v}~$ zJU1KLKLo@Eek$<~3H;w*WpIGrvvYE@{+CA?Y=7(Z{N*3EKW{YtgMVH=|7YsIk22U^ zDj0!k`}{98egYIaZ04DfTK4IFD#Z$8OC->H)SGBPWAR&+dAxC!x1LkgD=9Ii7HsR< zaTNZVqXEOGfc3id`qw1vb!&Ii;hWZlnn_F5PgMG1W*^!k>>%K^gt}fC^GoID%n;=c zVYhTI&#woClU2t(h@qLon}$O;C(oW0I1ONP=D! z$#I!rsNc`5L9}Uy5eK1Xd+)tL!tNbWFY;jd#;4it%d+{X6SG{BI@%(cy9B2Qf1E1Hf#AYi~IVU%UfdEH&SV0Bh6r z4B0|!6%=CHFPCm-->&26<}MMg^;svPz|(4{MRM5ww+o4fD}CsmzWw75I~7)^Y0o0( z3n4~2Emin(8*op@e7BiqaJ6t0;p~86xxS7kh!*}z=l=m+|Ge*EZk-$uuWJ!S(G+w zq`_=j)>YW|$GPE$vHN@Z8EP)$bT@WPRL^SXvZST7iCKCww+OG5 z`t#*$=MW-OdWV<;znd34h0~E`m{sKsS@x^VapY@q%E`Vql*O$uO7!TQDoc*FHPMtxm}j$Cx#lA5>GYSddl%v#{iIi!@DJ=-}!PU;+3;jm;Il_F23+V!5_n?s3H)kf_TBSK?O>2?AQ09?$vvIZqP-i=* z-Mp?I7p>r7q@h#%gIxQqP!X3ht^zC(awv~(K8MSzyLK{I)={%Qva1f-eb1=gpRq^n zAF*zyz2#k%j^C*&h`leso*UgkEG|60FIYkr;_6w|V#u8=HS~JWN41PYnyUH~9)m5l zYsKB4L$2YIm!PKNW?n=j_H{(5JV8cFt>yc#V2j-N9r}5zuipt(gIOD6-tR;=)m$pT zMtb5}m!#dj%SpAT(!k9-caD<~bssJ%IAmyic$ElHbKyR`;q1|*4oA(sE2((OeqB3u zB(w@+(9uC4Tv21tV&F9<*{dga=jc!N1a*&W?)^U?_n#z;o%3Ib89SJ~{(;Z`oflt9 zpZ$Ac#{M_S`b&)L;3{kXx%S!1=l{(8ml)aqzHs~Bh_SgvX2f_%rGR@x_j91iSERn{ zJ~U31gs4W!b_{4`a>E5uVIar5C-3FykXX03`hvV`+>URUy&YPpT@HxQ_dUuy9@m(B z?tPwiIolpnbsx`8eI5@Jb(^0qQU$%>BYmcPysuBdKb{`ZiQC3NzxC|PJ)cp!E?=Uy zWj@P$P6^&^`#iVlf+}4k>OKT`KTtnzsObuoMk;C#-+6R#;%55XJ)X;PQ>(ol8AEM* z?#MU`>f2|kC%#ypXc`akc{p4@CVsxV>CqKY4UTLZYue=pOsBf51%d24Rg>_Hni(Iu zpF73X+BPn9*Ls)O_INfODd_3ZIxRpyGA;Y9+D#eW<|JI#yU6y8))BLhn_QwkNUqUESxd#5SLY`W?gDy{_q~r~7pu ze(y&$-KV~awuicwmFMGmrIJvx+9aLji??1j9yTOSHT*>l@(v?quwMP9Kd7HC4tG@u zhM{@H!ujew~=DUc2x9YeWsA1zyC-0|Nb?mEmn+<#9! z4DP^o+ies)qXDTqJ49^fAlb8+A`?6v0}mFCaK8Ma~^Ydkc&cSFw_ zh8o;{_h~74I`U7(*dN|vO7dH{c;8LM?R4(gJL+VDVfIG}EDiD-g_aJUPZL)Uj;1;6 zi+fRdEdLpYniSJ>21rHTLPeGy-i>Sufi}(uPVmF@|FSbxVdQT1#fZB9Tzck_+(cGr zx{{Uw8{)1~lf(1()WzM|+iVxY{74Wi&W(;lwI8v9p3If zi4Zl;^3lbfqv24L7R2_nmVzUO4fld~Rq1Y;GKx{MK$c-{1@ z_tf$RG=@~;ENvmBKuy#O*za6DQkZNm66zvv+1w04>)=dDB~EZG!WGm5f$5LCRyF3K zmgoPtpv~oeaLjFZzi0*DcY)x|_>Y>X)zpMbjPp*D7d}6*O_bJVr!EFQ{1;o7+2qY^ zwDig)^9!~a5=WAD+wH`%i|kZDh^`R(zmQzCh|LtsTzi&0@%aJOnC?!NU!7pX-pbG| zG(OHmV_$R^8~$L0H-H_)A-OcVLfcY-n%De0>uOtA_Hr&$MER?va^y#fvv-=ri@0()JM3x9uIkgg0Tnei3#&htWi~^15CgS6qYu#Ja`QZ+_ zuoBXB#W_ntrzxr&O3CALCo8xZO>w;}q^j0IGXpe*F<|TF#(;Z*s$<{@s9I`XDX5M0 zl_y9P~}j=qe$(Xyru7x+nH;fe~hUGP9{yx1_lv2NXKVCclyqSc3Mx;;~ zO4>j&QWsh3o|_3|02}GNDujGi(~Hqq^xI3<4+ z?W78bi*Q_^Y@6Ql+|`#rEX|8lD;zYtU{unDI=1$%=t}+`@kf!so(bcC_;+9T0N%Ev z5=G%EU5t6DlcBjtC51C1B8%8f^#>Luv`hqTQ8qZojOMX>c&uNSOk9u-^b{cLuRky|cUA z`5NmS8=3i#?A@Ywjduc*Pm>r9!dWQwbP4_K0qS8fo%8f$4Ab|fCMz%wT6o;#*RGt+ z)BsgY>_wg(nm8~`F(92pIL61Nc`mxl^22&|uy%qW+*cB`Qk_P$Tdz6nKZM;R=J^qr z@JFw+z$Fq4$;C|gR66I#8^4B#T97L&mg=OJ0x916Sf~T4Ji2iw9yOf3$reimv`;kr zsEELNB|e2wD&HS&6DO%4{#t^ilLGh1(MIjrF_Xx51%fU=Z_y2X1}{k0xBRP!7`ED~ zKG8O+73JoM7mr|bfxF5l4nM~`*4)|IXKb$RFQpg#r1O?LgUNsQS7arRy-gX3aep>! zm!9Ly|x$c7;|EC#woOzR{Z{pE`WFq*R&b%vvjH>Kfege0;_L8 z>~LeJF*Us-a7ljY4BW#Ply9A>zH#aAv)o=Wy85+l7KAfNnTy-p&Y?Ql2tv~cBkCjl zhH&U!%qXHxR@`~@F&H0Po)%w!_a=)v0e4`SD$8LfL+xv>?DFd<@{hWbEt(9@{ZMHN zYzEfAT2>MvvXH`feJEWAsB-Ipqjys&1rKpxQDlGUBw)(&l}@$d#F z^0n7WUgK_>93Hrf+wX6Rwxs)pB0n?jVJ7r|@s%pY%2D^z`;McK^P%l`*WE6d!@8W3 zQzaA&GK+rD>DZn<_wKRyPowIb^>k@jYPWl7@y*Tmb^nV zl!)wMCIP6vZ9m5xV7zYyDVPbx6T77&nbHZk4c%7}|C7{_J}eJ}fOYlKP( zBZzWHmdpn>M+#$>Wt$=W(fE4g|8UOe`-a+yi2Hg>Tj2T)_TlRFB-sCYg5Sqr+qI;- z#-c>8t9c&&C3=Zp3WjzTG5}_eGcND5y9AW`fQ!{Vpa=CNosR&qrjpu02495kp4;N> zi^PG`A;yF+txyrVAyTv3ARMRb7V6}!^vws?W7$0!@iD%z4yYjXY3j!5Y@u$Y2UJ+O zSUb8-D990*h@0sYK4Axln%*7yEhEJ$DDQ4~Zy-`keG%wqa%>8`O( zpY!sk!$A&(BgeSb9_DVQ46yg;e8ya-2Xie*%R?EKJnY4c4g9Ombz3hSQsROOCz~tC zUcxv_WY;tLpbqk9aKI}soa)?ZZAo{Lf=8{~YHcTH0O2Rr@Ut$Z{nBugL{>jEz1L&~ z7{hCyqWtAHT9-_&!%)=?{_w#PfYF|V!Wsq4Od}v%Z-H0PS7LfxTlsMB4kD&Ak6ySz z?>_@>D_K=9YrpW76z z!1eyd{rMx{;&hLa3n1;rJ9o#1?9y|jlA#B0{C8ZVp?;{~AQO;s)!h8LA{$?VWD5_P z#D-C3vP$BD&gN^X3awVKbI~sWazAoSD%I?T%+qCL5m(zhgo1{C&xP;Iljmrr3esa2 zCZWDLrltfabasgjGoVO-me&AAO8G2vC$*m>TFtS8gin*dP4;;bVT$*$?v^i8&qO04pZ z61o-yg$h|o`2|&{E87|mQyHSgKapwGv!YdU1T>f0!P4hf#zJMh&)_TtRkppI-MbhYw=ZbN}cH z$)WRx`^_x|ST(-H1?3N7c4gGZk6iONAX5^H$^QUau7g}_ZI-g+ttCjg*iEKPa?F91 zJJTS_c6Umz)gAt)Ro>4yIIIGMo1j>3+Gl~Z6cyY9#=vN5u1|mBn$vG2bJl#!>%r1S zgKNT*SJOIq0roY{920kO1NxNTfc;zw>*_HlI5#eA5~>Ei|@!yoEq zoT3V9xMoB50PbNgN1~j8nuY5*+|29nfi!a+gbyUS^8JHUwVN zjaGnAyI4vLNQ}xp-XBb9+_(xn7TfN!yaYQ#)h<`9L0-L*ZNutAepT@lxXmHg;<-+0 z{cu9dhRcU`@(-tQ9)~1a^7|n(y2dUO4C`7M{~_&-<}eb)NYkqc$~KoY!Lz-JNpN=C zvpf7A13ky}<$yF@J%k5wo7XeUa5%&Nbq=zIpQtG}i+Ze$7j+)sG_Gc-`Q&@+?u# z0AgPO9Z%Qku9MeuUCqnb^{9LwrZis&LXI&$|2(1DWGvq3Z%1Y3J}*I2@4V!o&z!w%X!;ZN>$TrU3mLLa}A^S0&H zLc4N8;8i?``N+5$Y`^6Cq%I;t=+Z`rwt%qS6I)`x(rx=fsA7a2No&=oApUpSs`})n z@btU1(Z~g&Rme=r=?OejlWGoL4H1|PQQFnu_kZ?*2W9!ml_Z)fHw20ba`rGmBZzun zsrVZ*YDKGsmbRTwT8u>L7uD!MnMOZzD<4g@>wE$eyd0Wd%^y%51r8S!x~E(>t%16F zR=6$RfC);;&h>Dp@L8y!#rHVA2-Uj`%nWj}x;nvgH}wNMlD(K#Zo8(sAmV1z{pmqq ze*)4Vb!8$M7#nT{p~uw78A0>; zMWrh(Ao9@xtI6x<1i3Nkann(LKl%aLmD9Q|D3rieG6s?ntW129m7kiwUt{AZNheg9;iY*M$NY7;@~l+E&GovEB~lPp6H2LjA=6*3B=gG}L7D=s z-%@4Ri>J1T(amcGHldQe4y@8tnP+SMn-gc0a)?9hso^$IRJgKZefL`3|H>J*9O8g- z+)kpS44e#(QnK(6#4QbXsK7S&Q)nsggYm@s_^H*)QxHZC@=9_bsO>c;B4=W1t2)(k z@_VPAEeuL~h}`4-Ii|6o5;eU&AuF;8VavUcA-XIb=`$Tp@cxXyQ87om&k>vwkhrnT_uLSN` zD7J2va(=nIINFEm+Lqpn8O-fFDbt@5)qCe<%CSE%EuW8#gU!tpaf(G7rng6>ay~ZN z8Y7@Xl*Gd=`++|robKnwwcj~sjxK1Q3^E({>~HnvAP&$j^6P%UaowMM?xbDr7Ny!kgpkN>2Aj^0>*+p-F?RaFZLyOC;e zC5&;MNo&_|A5|b)9j(=nn_lQi=(7m?!=J{GqwBvX4&D1_`(4=pkXS^5%(waZFF+uI zm@_1NE3W;S_eSUJ0j#cz)44%=OM4PZ$0ZVIN-^Yv`_{c_a3r$l*PGgPZccYCS@8f~ z>hR2gqsb(&I40|I^^rv$+p}VA9|K(ngjyG|QIn}jA0H%TqBH2A>k@Bpghnv{oW&&& z&Z(#^EVWuZ4jD6nH?VBOxUA>vteodkzPshpR~w z6ToJ?v#aJwo#+5JA0vBK&;uHeK#BC~-kGlUF8|DjU-f0TI47_6)iqxvn+=#TdUd1J zwCA@>0HLdp^EaM4tsuuufcFmGwvf7&wW4j{L@NkX;`OSbSrE1u@?=x$>F&jdI2Gl6+D6L%1SILt3YFkRtkRh5 zG5pgABVqTlqjDnMZ%+GuFn({C^#(#0V@gU2%#{55x}t7aWq$;S7uLZ#05!Jo-2N2! z`grSnRhAF+b8)i}DeQ788b0EcuYvl7QLl9XVaLW5ycP%-;;)PYyknZ%e^XvC_G<&hF7{KY1 zS99Z3Xw0{>8x-UcDQ~#dP^iT}K%A`;>aY=b4?2!&SwWbp*6S?=(`u-2F_ph?*Ivvr zV!0l^lBEiWW{?*+|J%g?SQyKT+&rPtoW8@maZ-8N=pDeaghrj+ z3unE#(fLSqTYU&t^_D%*1PFG6L6lZC4LE7WxYW8Jf-tn{r;Ks0rCpG zr!Is(?6K>2dO3F3(j;;pb$(OSxONeNIy0^RGU~^Y^RLl4h0(=yI&gCX_4$4_*$bsa z>3A{`UVQ9@jNC+e6$@iyGcO!1OFZ>UKP^Q`EluDAVxOn@#Wfecv6_1DgSxsA8C9sO z5j=^4lIK-GdEMyd{nM-O9Ci>=I?2}HIG3ian&NwRkD0~uQ>R)a@y8#|^L+-|5*^F= z?vZ)`p|$5;oMO=p;MPmnMic@1?oh7PMGFTMk+qw@D&w-{ZlMen(W!zcOG2|39REQ( z@>rAAAKTYrADk*%F3_d4UUK@MTa@dF!kA0OS13OcQLPoqz)(M`J+PrAfys*_&TY@v9ZxsgR z$CfWUQ6)QXW#K_LpEt^KTwjcdCct))lQj^yQW#EoVw~k)hsJ4p;Xzj?B!zg?k?=HL>)bPDcu*nHpYFJL0)|{mp>DM=pgyq*tEplNSU}i+$JI0FxANeirt{~|8%0Ai6&>-C`x*k_%79akF5Cli$OsKkS5sV1xUy3at8BRF zwQ{xoasNqRthi!#Vc;Zo8}xbnJkR$R{?LZ#*D9EtCsk=#MUa#$B>?jWSG!XcFfQ`) z@iRzNrWCE0aUFZo_F`_S?+zf^!sTqXmvW{Y<$pue&YI$GMHH3cJp$S?e$Yc zE@C9e8Fa`3nyCi$4=*o$-A`xHUiFG97O;a+(!2xPe{B}%b7^mQc0ZoQ!>rp)y!ZDk zqSg{dckfT*`#_a8Mv(S{=;AJ!TXqgFwBWX+igQX|n2x92DS2X>tF{DmY5{Q`|HuPE zFgSsXMrHWo3O?(;KhgZ)QUB*0!h!5gAxRHZyJ-<(i!SU3L?4)Jf=VW#OTFGy-hEk| zhE?r!&{X!}eI=jr!SMEm`}t~K@Nxbwug&{m_0(^bG#8zuD3^}-KYfSa-FN(S@eTAf zzQyyyNx9$?;JJ;}y!CGCd|ge~`xNx$z2Ngj>hjZ2h4<~vP-g21AJr=s-D&5%-Rx!f zxyZIU<4n}6$mKFr?+1i_AJ0z^Ui(3&4~6{5JkEkKt*vjyYG$}$@NGD$_Z~70=^VQq zp{`Hn!U7yyi3IsfZGJ_LG|a|CCBf3tHRca7xSWnuQL`9@X5Tz`$>EIMW;~pvHBbu* zY@5g#oD6aXqbt0(xu$s5xDpAeR5c&Cn~TNW#q_8a*amgt+qRK2`VNFBcr6uk>MXaH zX((p|!;5fuf742hxa%qB?K->=OOokOLp^vDt8VN}l5XqbG4@R85n~vwm9`Uf_v7EX zNq0x>p+E%nxfa$h_{eMXZPUpS4o1<6ljCp6My$5z-7(QI66GA)eA84((4f<$SK)px z++FrnTgt^oc^s-B7PNUt^p%p1A+Id`@uj4zcXG!uUQ!~GG`1o?iXetsT7=f1cF-q* zdn_iJE&iav2~#&Y;qQ7v;2#xWGzEK0dpZ*RnRO1i7jrclde-=Wv2NOb=b&)qNhjSqlShC|$wJcXj zRe;H?#SD8W!s4DKPNCMAW(cPyr0Vzq9avy&JxX0(!z%SM3jpvxuf+0cTC&Vx{oMJo z_})D$ipLS)Fve528YmQ5zBjymrANh|FYNTRR2l3ytq?E$-YQj#9=@EPnuMcvmKHyB zuZjnTN!Wh)rnD4wmNc-so1aQRBj$->ppLpE{$f;tlX56?zu&kK?JcR5Vbqi_l4=wO z6aR_qZdqCzF-o_$n^ai-xC&(sS>qF0rZ5sj(Ih&SmQ&7?h%o=kx@V+Z^#+nTRgV|}bXvPT3hxEZP!I(CKV7(yfnpU&> zB>ik*h7P{DWuM?R0R9b+JdlSm{K$cs=Qr2goW=eYafh})aZxWNHfCu>-PN|kjU%lI z>|3&sp4?9yNd%9HDDoItlCaY51*HjT2w|}_z8vv?s>E zJWANa_3g!zNOMgsi6`I#I#oaYXk7?w|Kh)x;b^z^h6K9@{!D(W4+Q~$Pt=nqbf}c! zn5;i2cIm)XJyVd(_YG6@Q{ocn)dSV7RRzh#Zv_c;JDLV9cVc(@>Y!Zbkok0Z*++@# zXmN9i_6s~G(XbR+{>d)lmCk*lM^Eu+*9k_GR7@HAL8}6;&j3IFS1B>kBHjYcRbTFs z;EuH%$8)Uxj>^?jn6|Mf4Cl|eKh$1eTq>;9DJBO#c*cGVZ4CkG4Lksw5Ro68iW^kD|Ly^hS$NPIiF53 z`-Va<(7IY?x2{Xl{ukn`ag9TR+=$Z`^k~H?wP@tfd>Lm$*#>_eWM)!}{d;MsUigO& z{2bSxBIizT3J6Axk80I9W^_8GPp|`JKX>e3ZfU($b_z(9;e;mgu}KjS)eV!EsdW$O zlj+(?v2=O2d*z!E^Mz%^+3&M!TXg= zku1u{pG~t6b9egAX-z2ivq+eKhy;>LBz`48HbSU zG3w?Wx0_XwyN%L#fNt`3NBUdhE41V=n-93vp^o`U*c?gNRPLwBQz{Q>U*`0F){H_U z$k-MWgjQv1nT~M&LP#!ZT)`+N$5yI|>Yzsfmdg?vzCmpkwv2J;2}7Hu_tt@_q=qVe zn=2$q=H=nu;ZM;XY!hAednmok45_;5X`dk4Gg0E|ugvyu0=8O4N;uHpu*&R^ro6ZK9mZ&0 zoGZ+zcPzfMC1F}YamOq&nAU&(8`%<)h7yz4H&{+3D-~<&6%4#nqh1#u!)(!<)?$8; z750HTn~!68gbC1W-4{PK(fWMVkN)G^Mx3#F*zv7Vc@#&cYJhR1U2XjYHLHUf^_T2i zm*~$Xjv8;)zVU2@?Z3Ol&U9R4DqTn_7-lKa4y1jrs6q95>hwFy)$K#S>-UbuIQ39z z=#)0j4anKB@psbz;`2vJ8FEfOi0>3mwynM_El5X%hb)8rsf7AyP{E%3pU`5rsl`%5 znQ|+W;bL|x*sF~~5+P|}dy0$6hWq5`AtaQF-VDZ^D8G-9=?*{jc-t4dk0myCh=H=U zVBS8(VqIhgc>Ja+Art)3qG>Evptfz6*f!$JqI56(wO{qt0uXCEWMxO}s&e^)t-$a;UUIU|V2SgO*d4`G( zmB5tYI<|4=1c9kl0{#{5y=^lDG|`6#?hOAVxKY;9jpd(YLr14(C@7jt(Li5Di4dlR z0>3D9OGyITpn-s|!siOTJZ)5gqhv*FDUfJb}C|0=+C*`cL-~;pr<_g zTV>^V?nF@E(*YK^+lrNh6@SVGv#;|NQ{-ZuE>`vBVc>-t31+6x;!_9dtFW5)=1D~r zd>b*9R8NRxco$FpL;Tw-e)NGUbqxPkL-$Oxc7A1*O9e)?J``bzNJL5P>qMr1V<7cqdrfFU*ep? z_3GvcgX@Z;2Az9#(&R$=EWARx=;AsP+bkY<-mv zYJESf6kR8Y1uZ4U%f-A&-C_rQ7q&L!!Io;{#9PP4XFd4AR~FidD*Uk>tKwVd?d#do za3)SmiA-jVDCY21RBGx@+_Kah$!fhG33g)2JlA@{gdM!~>R^^AxUx3_!_aA7m94ml zB=4WC`;6vxoDHEk&MgW9C2Bsu&+Ti{hy=33H|0Ger78j;-4WH2RHIi&T(L3Ixpo{> za%?%WzLZHJRaeOI)Bv2QqnggTLox>>TO~upDUvpen(()(r=;E#wPHq6d*bs_YZN1W zyIyQYm}}ZT`lZ;@X*H5*D(JS5B4o^#vYz~UEr8l>#|HI$(%2%8!za9^$dUt)|5k@n zi)v^~me3t#+$Yp9wSO{nb zbd4+X?tGL7`YU3lKBDa8FLo`H#?N?onKyyANgv%gQ|8InH463Rq5~v*-`n-PDbQx| zG3ZrerH*_g|M-AK9Mi9eqtXcxK*NAfJxyBX^f_}XT>X9Wb95=B;Khi)2dFWIMy;W`>Ij)F%=ej!{z>125yH+u4O6i$0uT zy)>bC<%Easrf5(Iqn;B(SB_J8cMMx_Vo#Cj*iet9AuaXwh!zoA9U@+0OS~JlyZ@qT z{Gdd*Ztj?}gMlLnEwwSwl_ld^Las;nRnScGtCn#861UdEx?h((N`>9;wq`^4wqC5j0I3zlDy@F?Oy8cXN`*M)il^cy0OC1!Ob-10&KDPKU-y2wMnVR^ zTUX>Y)NU?E_fCY%^AhfEd*3DyflF!IJwyUsTih~K-6yi;*bOpUTl#B z=`C?pIAPTav@M67pOPl`-{A7-QWnuWTLwds$9ayFS2=T~k}Q-?Y!K?JkZZ`#tukT` zeH+Zy*SD*~edUZXjM|YRJ4Qz0kw^5P4&Qjkfxbi7b^N%?7%Vson<@yG-lh%olQxbC zOn$h7J$&=U7oYeWsV!c*Yh+v=UR-I2z(NRZmJ|cEG=7dmSW`71A$iVGs37R#zc)I^HK@XBeQHfS@F+k|x7% z;CJe)-Rq56bPAleM57PvEBTLgY}g;$w^;PY7Bp@Tl|Lah6F^9qi>0;I4-=nO!*H}z ze177K=Erm8OsZ-eJ@?krAA(2w#$(>fbsSr_^UHH3^)-LNohK2zVpR=ELM6cvJpf)q zOu3*Vt(JMirr=XX8OK;8>RX=dVXuyqL8MZ47o{r^^L2^Dk2t<(laL;?1ctL#3~c^n zqmcG~E%-k6M0L9Ov!A@-Jrsn^;8IT!r%I(66etUqdBHCIsfcRbRF3QrdBf=n8TAGr@NJm>a z7QdJ0>vp`+_xL23=}&YLBc^m^RUIc7wRb`UV^M`vz_{f>3G`P!k( z(>)^3ubcgTP$iA*(e?(*-F?&P{B_2k>mumGm=U{0<}*%TF4*AeY$ZN5Pjuc>-hFJ{ z+0txIPlIHg7YYz4S`)GN3XyL{cYw0mX{12tguql(s>ARqlnq@peL5@UK8SbjGsfBA()KKquaaf2b9VwO&*xJ=uTYM@&J4KW<{Xrp z5D9ExgT_d->87CITk3*&je?`Mvdi}MI}E1>d|g^d3KfLH>9=7ft$sVIU6DyuSk|O_ z))hI&v~|d=7t9laAWs8iA{vWGiFdm6g@jB9(YFK&chj{WZoWs0qXM$Ud43t zZ}0}`lj(VRBo_~a>=|Ff@?y-f6heHoL`^sO5vTm|s~nFf!ke!Rtzqvn{d)G+L(^{H zijV=>g#6!^k~1k6+3QWF8mNUDJ03DkssyRN8uA6%9(_!& zYM>A07{bST$Vi&w6Dm=k=EMv&gDv433YN9j^6j8!rypSU$#sFREsZ+^EQ}k5tZ|r8(6*DkVe~2bbQNL{fs){h;#v93*9_Xf}lIx4#Kq`Dp zR30$4P!g&P#*5B}|18c>9a~0BcyA8+Uy!V`MA>;{XPm?vpN*7mw~rM>0?eQz64QG5 zP(Q0Wix~x=)SCou;qx2Ljg}hxU6PE4yhshj(sx z;>mVfQgAWDHO293Z)HSbg;S3WR0ih^2Xs+Ci5F4>?!IYk`!5f{q@r%S<{0}D$gvkx zb9eVxyssSd$c$##;*=VT!hKESg8zZa2@xil!)NE?K|~d#Nf?`a68#7bj$PdAUu38YB!_D7yhho+|y!K~9*NiOQn0_-$>?suwUNP%(_rBGa{CBG-TGkRCh-^n4XOEWw0xL^lgHvKHZHp5&6sI*+i72A#~QekHooiVyWl z^@|(VZ%S=)7=@eH2J6gcu~xg*IqTh=XM>VB@Fo@(do@XF#8p+CUwt=ERG}J|-U-HN zm~!}WME$7wWzm^dTd^4^`BPzO>RX)B2V+MGH3Wfbm`AVFS+P_~EgHcpU&dV6c~7L= zDPD)dAYk2 zDhYoy8L=w6C|T1I@f(g%wX=JJMrSH>#fHs?m65`6c(}Nb*;EP>`X4=>>a#*O`U?_v z%0q;HOV)&C0Re8nTSBS?Nn{PaX#;AxTa6s!a65nTqWl{QOeVyRhpcW$0VXA~fGB)1 z!paQRqpsL;%LuJ6hl~uWTV)@Y`#)kVcO_%d;)`WhB&o9%NHO4>z;3YfI^!2JS2g z>e{#Hw1?P1h0Hm%E z5{?LIhZ3!B=)2}rUt4w;x=#j)8OV!^RL3n_L3k_kit~%S4{ni`pmm5<{9=NQTwBSf zo8e8CYW2}9zs?k2DjB8<%i;4vqc&R@eV39)EHDy=&i0h;&Lo#;Q|t{jb*e4%rc99j z8#cW%D2wjodEat5tGG;bSTi;uL$)_;N0gm`e!jP?KSj2N(KmBPjRdp?O9sMH$7G77 zhbEIbv|Hi{I!RBaVr@RtI<(k1W$3bh=j@QcOR#mDBma8G2%lpSJT0Q*GFE0n!f$aZ zqJgJW$GlnTC|vys=OVdVk0bAZFoxh&J{#o+=VDfxOjAP7KzLSd;3} z{OpY*D=d^zWT-F{b?=VieUH?!qilj3R{w$9nrV~QOP-RkOgJugCW8e{fnfN=;N1LE z(k9E`+PF7{5oxwM9<6|n+zAtNMQr0a$0nkqg*)D@{*_&!_oDZRatlTHY!qu}4$)kE(L)+ZE+d%w#yQV@I*DaLjBRzs2hiS9w~Ri3fJoYEc3 zJ2^GNvY1j*#$mL8Av1DPEP7p;^}PqLw<8+g=~kDigy;8Rw5HOJ6a*FhPPVC#+Z8j~sZKUENJv&USV|LN;lMYR zQ4~78B`hT;dfWR})W`ZvI7eW9%P49iucp%{SBXTF`%N9If4CKAkEe`a&d$i5Y^J_@ zs&N6_>UXrNzN%`v)S{6e?F_5d@{@-wS+oNug@oY|D0+g{<2P}GMvS};d)D`{(lKBC z!>@h!D^yR+Xy zQb#(9$-2dj>tPJKVoN3qPV6J?uuhlk#-4~XTOa_DbG2&y(0wPm;!P)#xy=nA zC2(qdFNwB@XFVA^Z$~?6?7x^!QhgoTcs(xso7MuOev&#FiuYWMxhRt{Fw6s^oqb=Q zTTpmnGDn+(_-l2n@;MSe3tnhZb;BX82EzN9^j;_^yOvvgVg0yx+k`UeMnako z1fcY}T}R`)g}kZ(lP+!N(O?&%a)F-0Ok@08tCm;Ne$WB7R*phsUzUgm-My^~D3w|r zNs_I^0=#5ZzrzEZ{~v2_0TpMjwv7S<4DJ+%p|}-haCeG36nA%bT3lM7xVzIrp=fa} z#T|-Ep}2d2!WsJR{@?rU-T&-&opn}Mnvgt6?p(Q&+{yDy?gkEhYPghxu{%EubIdkn zu&^oj_a8!LS4D9dG5`iQQeX~r{{HMI_vTT= ziEBdUHb_m?PG`q$J1sP+EY3Q)y*6<{uij|*Yue`n5&RL2#zHcZ#4#tzONR_`1K*K} zT=mqB^$ji#+Q|F4h%OeN`|MpiUIs`EFFDh2sJwH19r8$WEKmPx>p*5njF8a{?jGl8cS$=1s~@wVo}); zO}(hZDbEE7S_gAQ<|bWSI@%C(_q(JDeqGFWNekNPlvq_mN){xgl!%L?MmU_kmmfJs zq%lLaiJ3{5GFS)EKuUdu+SnD;pU)=PNswIaQ=^h$ZD>>=C!}3bGsL@<=5!Tz`2TX=ESPOOvg_^yC zmrgBr6k0c`B0%uwNGGy|+?N_l<9s!a$jw1U$Y>6p=Cpc}gibo7X=0#~Kg*mtjkA|} zF*R%>c0Q6|?n_>?5CY4W?R+}z5RrJwj2S4TOzhug!PdTg#T6&5eok9fb3Y`!Kusz` zuWy}+L1eXg3lWu`K~<}fF4*g^RC&F@z!@^Nsj3ayw0FLL$#yq4U+#4N9a)PnMq4D2 zNF@o!S;A5YZDXW#cMdFKSvn`dm4PVStm9m~)A_^q9XVrgdAVx3k2V#Zgjx1|ftg#5 zt?K(|Oj~_)z8e$fK09(H3N|#k2@B$^*e|5RqA6lM->cB2oIS}#+1_-Sk_@k$A*OYz zYG&i?)7~$K9hpIwfV8n{w4`x{nPRqijM#e#b~3^gDt|>|nQie<0WC{b76@}3E>L+@%+vJLW%GmCs>ipbJGVq@F~sM)}Hj?2rv2ggt(B>;YM!& z!xMQ(kS9AP8F;6H`0c0k!&_$8&}aNDBwh%xH$uU(Q7+c(6>=}$+>fBqE}+GcS}LmU zPC(wt8;oa2>~W5O&gs}u-PLL>LnGWcB9}_ghv;KPz7=URa;U4Fw>JxTkbajwdup03Cuf2?-jaC3Txu$fLh1He1F0x7aCozN zhjdz(5=miTc+@wCVZB}nOB#vP|6^I|P0W^WSiVBy4ME*qDiU_?4`u#uCz3 zoo%T*H(XEA@P6@b?(em*D4A0@{-&Tdiypz>Z}Db@3w3wnhu7;*=x;$o%yWKMvXq`C z@o0gW4yoNzC9%GVU^ShrsI(=lji@#eQufyu_)4PB-|2{jEt58dNw+WbYfhDpG?HW# znvqiR4Qs>3KA4U785Gmn&0I$NT1^M}*VnQGf}MC3RlEDVX&deD#QM)Kv_D%i)Np2h z#6R_U8V`qxPSAN70H0rI>s1fmZ=dzGPNnFZ%m9`It|PFfF;}<7ykX4NX40}|4hV59 zuNkAVwzM|1omrIsic6zkEo7M=qk`k*#Up!eCf%wKD1~ncz@EGD?xCt7uG5Z297&u@ zX&kb9YPO{OfjdSd`Rnr5D3vKc!(KhDy%g{KvS*h_RwE6y_z7O|_U@B|6zLK|iHW?7 zNQ(r7jh;1Hl~aN?8F#$-Z8Q+&>Krn{=ohtq#@>vj34xD(1(O+(k^$jb1Mt{390An6 z0;0tA*7un!rQD? zca?*gTgDTf4D%tpCs75wuU8}jWa&8cLWDX9N^dgG3K3}p5GIKULaE0W0f+g7j;Rt@ zZ${fNKqsi7DV<&XX&Cj!=V`lzg*`A8yLY9EE{65A7uQdAz*tDziC!7|)BuUSMt$Gy zLY8>RHmd%8GqYW>K3Huc!cHAu1?AoM(fe#EVh?4s&zcbs*g%EwpLFq72(z9#?pdN! zUp3;L@owliapc&udcPA3xDn*dHWTIRFPyW0ZN>D4nbsS}%Zc_2BgI^HYmh*(w36vl zB>A~&u!sh#GUV{mW2i!c77R?%~VFy@|x10g&EIlmxT3e$GKa(Rld_~PX70Y1(=9OjIZ)ron zdNbmV?H56B>Eoc*u@*A(l4^)^@k@i5mDqH2A5=h-dIn$4KW}|t>P-v&dYpPOd(fD2 ze&p`zS!$K?6Eal1S1U+!V3f54tVkO~^M?62u7G~sh~s9{7{?CZNL^$ZgCOEXqz+S} zm7fS-p+96r2JglN>~>sFLfjlK!oHvw}U#jAl)$B>W3;uyy{J4B{IFz`N{|Hj4z<{5cMLC3P3w zH|g#TMMCdW(KmIu@mGNE6I~&BffzuNS(poevjt9o!dHH1X-RkTqTG;%(kr2?5Lw%T zB~n$=ah(LV564dw7Uk;Uv_%&h+}%}*E#EfinF9q8hLvP%75U7Ghw?tklS_lnp1u)+ zd8c^R6(iJ0fn=`@k5^2IRLgYMWx?2pe7x5c4?|v4Kq7x?mqM)emRxP2#?F9(({ho0 z>HSt{HKNk$NBMd;)zg*j>UF6s744-XqDwYqTQbVlUM)8N`ka2#62gc>+spWP-@z_e z9D}Nw__BG~C!P~!Cy3$`K9O#b^JXrL0$U&)InP)Y2!wMvF)V(Zrrd0%Hcvi0Xi!!r zq0p0!U8`J@Ho7gx4*7wD2`}*AL;P+kJL2ZXS%PAmXBhX8(Ru9$qO5uSu;F=wC-MO@ z4f8TQBvyDqB4=VMu*|TMGMZd>h1xa1r*6RJ;ta*64D__)0HoX^xK8tM;Ss`3qaaxb#vrefyU&^_}4`nUwa1-ecg829Wi zo9A*wMkNp|+z~1%g`sAtygW<#{NZJ%WTM%;}>IqgAr*oCFyGat{4cXTc%L>khvU-Yva^}o2aJ~w$ktX= zIgo_c%=e3)nn-pp=T4>eJcRDoB z58k$$Xv-?8Yk{n02-s1bW8GYYxK8 zU_#o0*^+LzFcoP!VfQv$0`Z4S)C z2k-zlHBw07sgqIP)dS-BE)o4xMMu{r2=iMYuZWEk77XI3)72AAQaQNP4{wL@p~ojK z)0X1E}T1AJN;!%?S zx#0UA9^<z7%Th=JX9;Zpft86IK{_ z+UgV6+76$~?}z#o`cQaV#0@OfyqQK;X=X8R>Pa;-d6^IEzGqZ+o_^;jKF?~j?+zD1wwa-8Pl@rtjlZRKn~)p!e0=X56~37?J$6hDP#i}PdlRC&e)?13FJ-yKa}8`UPi_= z`#STf!e+)5lirq!(7t47S991M8Er^KuTozEz7JjeLU+yHsrY_r(MIES6FQO5=T@(1 zyv;X})~CXamDIL4+fS@n{mN5?;8d5d=F8&QF5~W_YhPgpzeS{=J!WjSOejASsTmip z{f>wdR0{pFiTb)TH%y|;SkX5ph)=<(jlN_ho7z;cx{|~6X!vDtCXw0OJyASboOkc* z#^ODkinfbSh_+}C%3mh3;SaccD;Myfo1{lV5JR!Lt8kX+dcK)cY5nEUd=oFeMY@>& zut1cW8PWXO2w|T)wpxldTer!3&m$-bL(g?~ivq^yvkhpk^{s**8E&iY0P_)mOAtbWuL>mbxTew+AOD`!cQ4GJZ zp^zHo60>=rooY@fd<)Okkh8-uhJfcV*!CS3q-RF1B8;ViS7cfC(ZrM-gSw?egz`%N zUA_QX-UpK6MNvW|L*AfXuhr$rPu~N1>`gtHr9)BDk@VexDUtVbFv;DQ0Q)_ggrN_$ z(&Q2SVsAX39@M3LH4=_hhn?|0`WbWuw;v?rHfjLkQ0Cvpl)x>@hct)~pV9#*=~Btd zLWa{al#4b?0Sx1cPupftrG$?n&rrK?UsftY_Ukj?NNlo-c*`|hNqP`m!|*}_#g8zT zu}B0W)|z3m6i%$O#d_m0LUPvR(ILcHF3U?lwDMk2tt+RacbiCBmDkB||dCJrJORNaDeoBQNihect+yqOU6DqwW> zJmeT>LDR2qFVk9Wy6MdVH|>e{h^XTZp31%J4IZTi3Yalt*y1krHb*o}_~~IDG7;5@ z&c!(b`K|Qt5dy{wR&GQttHWD~lum=IzQtO8vXrSNVu4#s#q{I0c&bv+vsjrR0HS1I z!}u1djWhHX9YNU@`xfEcUeHtmYUUg*sLQ|Pm%T>;8Xn**{V5{noLi4%- z%7jT;fL#Lg*9@Ss?&T($-dRfw>r`g6_h`U67DIbQvHdJ(!`pJQ@-ShHZ%ZHK%XFfh z_pC~TL$=(on(J&r^tIK}0AHO5Vl#m_2-5VimnJ;)hf$mJQZCI0)VZ2yX5xo>D|F^o zs0Frfd+tSllJFo$iDoI!H}j*a(>ke`8bU5uzRx;GS>?^&^8o^68{~+6JCW7ts%HE{ zf^o?A>~sCgqk`r6I=(jPQW3VCUao~HEz`|*U#<VZXZZKI-7 ze)HrCxgPqZXrTWE7i5#_#LaYpgn!qy@gC`erGSr~?CGfW(eM<0&3z1c&a9qCV4;3X4CmX~;_>mF1<9@Ap*XJoCwk~NrGkU2tXsErMKK3Rc(^WdtwM1C z1AM06V(WFK@7UtD-rSNE77^w#tauypUDqv8bDgxn@;X$Y;S?3dOYv1jS;tkA$-HEp zg(0HljRE=_p};sGx6qE!-7XUpPs;SftIi*~*UrhqXpx&~w14{^5Kh=cdAtwTiMT1y z(aFn0Qe3`UjRBgizXb;A*2qMdf2A`e`T|aUBBOT^K-0MdfKZa+1&m3*V2UxMzg>06 zlZY6n%_IHveJFp%_W9=KKGpinG6kF(8Agk)^2U4b_Y9iPTj z!D#%)1D%{nL_8QF#4R1Ay_!D!rQ(RhO}m=rfR!cvL=3lF1sDF;PmN3Z{pXW?hmS=D ztKI#hfD@b+-3$i-&nPwsq}zQVN%j2)|)h zU122KO$PMFK;T; zJeIhE250hMTkbPA)o7ok+Y}gP?gU#rOENi*__GV6jzVh}MxU)I@O6v7L{^;X?DK4P zzFFrnaGGSc_#z2H=W7A>9~;3&U748QJNNn6c=~d8dY@<)-O{VpM%wv&53XOV4Eq`^ zu7_S?^nMb_7B66XK=cNs#|67CHqvE|x<$-1Uycr$aNTFyCL;o@)N z9LX^gd@y*^MbwzFk)|MqQZLH zha4*f^W_C?3b~bOFNRDN7`W4M15PCuATtuH-vSitzpqt1ded8=U^61$Yz_3lXJagj z&jn2TIOf0rF|$Pl#wsoHz2G*4jyU}ReF`^qJrxVe<;by(IkUyUd3Uzjjw;$q^CjaE z1XP?%U%7-19Intaiar+Y6C?N=ASg)G39md_h90K?|#|cTyc$NidFbYQAa+88DW+8JDbW z@MgkFLJQ;SsI8dKn`8FWK|Bj&a6h5Ox&%qjXQONihKPXAGO&lR-9`Y77T6Tb zU*eT^Qhd?Z+*n+5CnCp;*w*+S37?7?OeY22K>{M#JK$iGM5>81w zifB-#lA9g$T{DbjD|RT7i@!F^1;#v<&?2ZEf$zTa64rhiy;JaD@-55?EGyo;XyPib zHh{hOKtU^aj)6F(kdL0bv#xwdwur~b(LR7`9`*ynEm91A>aB8F{4z<_8*{zm+42aFR)R`koj zC;*ce?dZ|L3#=j1;kphK_*7vXO1B+nR&db zkC@eYQ7qF5J0}Y_Pw@2TdgAQyPtZ^xka4MyERM`M`DAP~b1z?>W$IST*Qc<2+UQ}k zgsCCUs`usqLBfkn503_SBA9Lw@QDmJD3__w8i|)2o(atit2V!O)CIOquQiJGoq{5y zXGF@rb{S(AiI&hYce^GF1_<#E!De5QddP~1i212;SEeJ#wz{^}Vb%%FS${Y5VC%(u>Yes9Yic>9o9eRvg6BEVZR zi$vpgwiWoYsH7k$(AC}$#a2+Al3g77-3D{2JxIyG(p!1`0blStkmi^!24?abVRDDS z2Ycl=Dsj<`GrXjqHsz^~n^dU6gA?h`nu8@uD0CGJ?W$=$?vUCec5cpfln^MrtI7@$ zB4PJ)<0|aE3>0#tk~B$lbaPUwbzPNW$34#t4gGjHV@kS5e)Jw}(V}5tLRP4n4O)oC zp7XT~PYcaz#8@@Yp?7==UqlX?a|YZR<&LRX3rg~!r2{|QgUqBRhWWzg4Tustk3@z6 zXw}?ZsYBxzvn(SfYsdp zMFv|-Yp@HTys{v3Rc;;&pLaeHk~L)r4*wL3$|LehR^ARljN!V#DKz>Tz7llE(pjnH z8JSrmyi=UJsE{G##}`M>Rnvykit1~na?VIKux-FsMqSDkRG~cA(q40(=F1nuupt=m zZJjtMinX!&EkBMH%t_y5-^(7saf_($EcH|hJ^({f(uoiE0(b-#I_jbWu}Cj{k$IK9Ak>_tt+jnyDk1MaOpfz9SnIb z(cLEGkc!aRITd_KvFg*ZsLVPda4oxO;b}dzdsQT3$a1g%37)(9Ab=j~WU8AvKv8SOO_O-m~i;iu<_(%-()g5WL7rDhj`}h(^p}@);j|S1w=mip2>0E zV-zZd7*^YT>xFEu)0p&yRtSNgV}}@kF?!G!6=qm$O)Ncts;NT_!nq8FcSb|U+HuwU z99>+`t=@}cxU!e79AwYRONw-0_QJD$;|<{6g5IyXi1wM25UukvdjvLvS{@V22|Qf% zq;q8DIRxitKoZe3-72CPrL+x%`8%kZF6ZgfZMARQ5Kox#k5NEB#@_DO7nQU?paDSdP3+Z2tHiMVkUq2g zU>O9_UNMsvIRUFM{+Ra zhBU{fn!Ko)%_}7{ceSg6{ZjtuT41G+_QnsL;Pts5`0<)l(U(*Raa<~$N=<->y4TKI`=wFCfkx0`FdUZ%Y@5BWC|X&0V2PJeFD`NC zgT?VatRi00TXT+#{2FuONQBqrQNCBM$PA6!Fg_*0A_0D4PGPv8T7i>%J%&Yc`%!@K zd8w#?C$1e$PwG*qh=Uf%2V(l&(8{PQnk|X+Z5weR1J(vNA=%EG}He|`-^Fl4b*;M`My=`@{pDxDfc?&k&AJQNNR&%_>j_}*IS0W#a z1dG$`5Pw_^4V>QI&Z8rb0Y139$iz=a-rD2+=FQtw;+RYma77?S|E%W`#S-55UyXtht2~!W7=t%-CcNV zyOB3>9&o8QNKrlGB-2)(_)E>jAj&jOPhWrsX|4&mTi(XaM!@wp#wPF~$(ucoY6=wf z36Qo_OCqZ9ooH1^qm6bDr|^z%NE7PRGEHYcC$+4FNoMQ^x5gNX^4?bb@(1xVw-@js&@yDw05eK|oM+Vt~x)j&WWronT6mQnP@kEfpV9i$}1Y$t-WDcGt zj2Gl+I3l`(P14S~axrii>g77b4g0Lf8Zl_+y~LJib7BEJN6(QZvd)Q3@=i zp|}BRmCIsmmSyA+vv#uWpZmE5_Ad^H4fK^bB!~d=iu3JU7dQ|mruBY|vBk3U(U6=; za%!pP;vK3n?z8Mf(kN5_*q1oAvHrA{-qM#@8|hXL6QVj<>zz&-B!_VU4UP+I-##PW z4Sbj~d`D-Jp(@dvg>@=yHM7Iz!wUFPd-O#z89s7Q;I7{Hs|5FP)U+CU^Hps}WZCR< zXWw&#v77l+q!dLgzI=^Cx$iL6?jd25KjmR4y!-+VQg}ri99uOZPM>Z3FnWv-r3x%J zZOJ$aIOhUX5347XXg||M1xk>&A62}<;v)pcU!GOl#e|^gZQPB|+IJfc>N|c6o&IhY zuw#o>IJcW=k_yj3h`*4OfIjruX%r*u&GXz<`Zwpk6kmJ-MT|`6-B+GgDU51LMMmk< zk{{~lZ=i8xI&=c4KR9>~O<~MyRlzY-l83;J_D0BHZ%HV=VvXOaE9Bvhb{b2QiB=*t z+ET8qvh>NNVW;p>Rz+~2aXm7_C+!@7*)q%!{Y0MI|C*}Muyd1?KZc&&e2?f|$py;jTWb*@r$O*_^tlle5H_fi%c7bc2|{}A4zAZ; zj_5I7Nl}@Ve=3V6?V5>%hqfzkKPGCId3#XSP{d;Pkfb@C2zxG!MALsz(%jT9!D~b| z&qi3Zy+>}B-Xx8}(Jf>2Rn}faE~db7kbY1CKZ!TN8H^EiD=&*o05A$@-R&Qmjw2kRLuSOR5|=wXLn9Cwy(cM>fm&PZXt>~W z4ztdvtA9FlC8Gy6KLs7wJAk!ZzZV7wtJ2GtMHH@%z_B6SGp)a;p~dq#>A8P0A%I%VbzKi!qgEv=!Ub&muHxH{Quj4~c~3KqjpY!< zhl)#X6zLE|sh`gTov=yNbQLR}n;m-mB3gbxvzu&5$$+ zlRt;N()TWZ9VTQFVE9gzlJ;wRvHj}S+~_kff^t{(3RNx-V2-8c9;t!0hG`<;IW39% zUgG(+mogumg*Y8c#?R#-`nV_V-seCKzU#)MD6Rz6hLpMhE!p)z;XVRK2uF^WtAcBMrP1PtV&fk#F4z^vWHU=FjSd?IOuw zc@u&icni84vk0b5pLpx~18V)mQ21Vi7(4emlOp=C^0^q!)fb9ax7W>I+J#=85S*Me z3-3fe+#!7sxc_-_g(=VNG+{?xDZOkcS9vNG&HwxsLaWADJE%M;W-cBdC^DEEYe9TB&$^U;yUnhjftAX=9yYI(7tDlsrHS2E_h(m_1Pk{p7j?I z-E~MK-f>=nzM+I4JOazssDh5KeL|t)F(r0J81eeFl4pw#LAO&)1$%VNcaM3vW7`PT zU$Wh-#!|TT-=8s=*?#JSk@0^ON?}FOCT3w$v^JH*D0e4MAM!@OkJ=>zNU(l^!UV0W z5A7`X*mca#l|JQzM;^Bmg}*l8H{kr2N>C_FgwQ9`pU2Polz)s?Dmxvg9k;UeVAf(9 z=1Kb!Z=m`k?Cj&;w{MQnmKcM=x_lMEbRzwC;iw%?F5vN_CAi8G5kCpNMAi^YLN}yyIs04_?JCS|zYD1N*Oeb_vaJqFRM>#z=cKDfjS zICE#OcnfMmc)GE8sc)JP)QH|<+KvDJ~EnV{?XD5h;(LNTH*?#M5rM`p&m2&pO^n_GW{%EfkebxqG@kQP& zDl<88MofSvCWk!$sy-nVmVboq4vIfmu8qyA!|5;(vho)m#J$57l_l95E&mu8DsDQ7 zU4NPSrCbXsAOp)LgNv&<`U}#c;LpJggHq4?%NML$Z@+)Tn7rd^>h{q~Z!%-LE3Kb@ z)Bd1#orlppMe8#TQHdF@xq!>7SGg zDB56?!Sne4X$|X_rKv~jQxI-257%!yf#64*RgbZ{P#b^d4rY%*@NWa6e~W|swkP=a zxZhUb{ucMk8sNW`_sfRczs3DBj`i|K__LrvFjZD`pbdEs;n^D< zP$HO~;w&|(qh^(9jpH2Y6Blk*+l)?3EBZ(45pIYgP46~S>QvdvHW0aWHczj?QoWk! zowRZ&irAtMBsz-y&18`A3vk*Qv*nAJvm&oUgTgD}hW3Xi`}#Q)#n|%c|Ji?>zkBL$ z=x}rVcW~Hi*fiOct2p3rij$L3Hl>nL7{c^Kgd&m#5pV}#Yvfsm0-33xgCjYzj8ghZ zsJWdHa*|{fS^&pvI%(<-uO5+}H4JAL;ocu4dNgqM*oS|U2n_xQA_zF7N}z2) zQUWal0m9r`Y>v$gfn+8>3xOpechM{S@tk1tCd0lHf`>oI^k@q1-{6G&6F51Q8>0bW zv>^=y61EP*MV}eAXU-2h&ZX|4xPVO!1Y9*kRxqQ*{D<@8@5G6j=laRMTxtdMbgd!z zqE{22()$Y!H(nMOtru}uD2S9Eu1QO_i%2day~~$9?rcd(8F`W(98nsMOYjy&Rm6QF z3X_Wm5J<67Ci6PCV$z+BH!GCUp5pif*^^J8I4E;rCyz?PT}meRO; zN13cxGVu#V4l=>e;o`uoIu1-^xH1=<^Z7SRthWUUFu6+9W7s$@NI%~P%I3U^!dLJ9 z13bTHpue+?i}N4!Sqv|ZI3=36ui_e20QahZaDt5J;fmC$(u5Y~hUBDtlbxi@RGBV$ zf&b1@i|@E~TMfO-z`mS|yv*^mmRiB)j;cQ{-rj zd3Tz;+5fpf!zoQ8Rxkb9uvRUV%9w1H1 zx#+?-@rryZ&l>e46qD?cyxT)(n`(Awx-1ohFEj}%W37SJ%F^}Kv#gGGUFlx=K8y_z z5^76BpaT%%V9&#*3Zc}A?xuaeR1Cg|mRV@4`o@OLY@a`y`E+!;c5)+2Ht(g1sxmi( zgFS9C(fL_pdOz!X){p3_q>@snbmFJ4kt4J8ao>2lQuaOwA?82`7f09MP|lGZ8d0x9 z@6^a^X}o`(s_R&&j^O2%WEs=wPi^r$g;Yl|N7hVmX1#^JbxeOMM>kNfVDi%*vQ7W^uKFLotW z_}wQ363DqRzse;LomU`)eeVmdWO5ZEy5uoC)vUb4yAWDpMD*LZI8m4q>#+B%%_uBHOq&GNhog*7z z&=K09`&`RBJXoj};=g!bS&ySs8c@d&C*QLCpA ztWB@V;~8bmv4 zS-fOBBA2;NkX{ns$_7*A(}ZP?!}xSerr_6aKD#RG$$2425E@KJ6WC4AmqL=*Te05-a@aF9F-~*0>kMe<4m3zsMRl~ z6kNzUN5_C`wT}jSZHgi>g~=dvH`i)QJri|W`?mAXIiOIk$YZ!3Rk6>P?{?0;eXH*>7h~i!;Biu7`|n zldTM6rgZ4Lp&vuS@iV)1wat~ZHa))xEPUSoRGpovDZL3V^T~B~TTG`Z{=PB1;in&r zw!dEMA7J~FcFGC<@30A~@H)UlXXKwKFO6XLYJX-U2jSH`IEq`xS4c}kB*z=MXcv!h z#OEb8dQzhDo8XSZtyjt4of^Z=VO$_12U~~@IiU-}VTUp>C48zqAr|svzuINKE)KKv z5JS%oe;>KO=_ajNLTG-3Et3|S8@w?n(@bMcQVHOHf1qndrQf55tvC@i=8zoDxsdsy zL^P!iBkHGy=v1w%j98uvoSwJtv2{Qf#wUshYcH2yAAVOqR)+0~zGdGY=y zs$+&O<9}kv^#>i62lC$`?_h(fUGkksB7yX`>Kf$z_bXvoLf2-d?|E^mC)zOi zxpuB0A=saChrFdZrTGdfR1d?loowh?a*;sSg<#@W(gr_#spCY7Wzt}Da* zX+Bn1n1WRCl+1Xs?oL#6^>n(z4N2Sys6?=w9ug_%C_lx4yG<^@Y9>f@{njjALcu8e z08}nbu*~>06K{~)guKOm^NClUuZgImnU`oqA3^MvR4;iftZf71<*M_N{8X;ig6Ne1 zh8sCBsKP_oxs@zpF|KR@wqylk)|KDrZeDcVd90kj#IJ)xgzw&bH)D-l2iBh|Sk`g5 zr@4l85vRFp>DFbyV@37nqADtOx1hGlq-$y(@iP`Sn#-R*M)lS?U#>LzI}7{);Xf$c zU|zQWPDjaDxkrS>uY^Kl0z|J}vlWXhz>N=FkRL4jC1MVoY|Y(OrMB>K-A;Q4eGwf6 ziEHiPuPOZv$3qP$u52Yj`Kb|ZdMWerOIj5|j&XagtPZ`5EpbAk zwj4dK)vK=d>qtxnUQ(O~EvY_kOlidgtq@b|ykWC2tTmdH)gNH^gJS)Uw%yQu2Rd{; z*b@wWf@HK?G>$XGYS+h6_D8uxmj$H|n3{l1{t zqC7%^)23Mg>f}Vy&V&Eq$)Q8}4SMEE$N6nvJ&COvSoUs zFrJ-nn7W{TcyWEsbe&XPbFvGgo+C|aDLoH9Z%1TmZF?G~ZuAFq z{-B6+{)0U(I~SYUr2Ls;^tPQNn z0$_;cFp~aI`ybRyFy}v5t0k31aR8LiNdP4~Ab$HM`@qd-_WohMqkb}&Hbh?JWA`G) zc9+Mxo9}rt$EER67((!5hy6B0)IB^4ohnHK|0ekrK{m^^hTCe@N9&Kpe5UTG%p|g? zIibN+{3Z@B-YcNcuV~!l1b&YBU|xWe>MBM!)Sn>rZtqGhz@XGVNd-k5gC>&h9s5>A zdC&Si;%0rj;ivicg2ImWJj{P+<{uO}@IPoJJJ&~Lwt#Ltap){cP6YUavcvTUWe3du z50)Isx1t#0lt>a+nL$)YCgW$b9Qw1jj{JQOD2oa#yoo-~xD zo#j_yrZLhlibDnzSWOT5KKqizt$3@lfXMVXYn|=O5{BYuinAckG$LPq=<8DNmB-Q3 z)ABHq`3xk?We!Oq@0(vEdwnrevz^cCf3FRK$p0ry9Dj%DFXQ!6P&11lRxx8&^S?rp znlj>|N=y>wHpZInYQ_$(EMks!W{*>vi>sTsm9Yzm4TV+S_+McN=i~kvT7-nTtEr2% zlbfRph~trRsvuT1cM~^nCvy<%V@2#Jtcu3=&_DjK4I3mY}{-h77plc z2+RvTQ~`szIiZJ(zv@vic5|_Qe2DEC8{6ZO3;p+hLi!-qM+uc(98Fz89FHBRioz=C zVCraQ?O+Ld)K>p$^Y17WRtakh3v(B92UBxbXqk`Q>gr@{3Ki4b%hb-;9>mTIVs&jx?ujQ~qslvg@1>%H&prf&=VwRhX0#G`Fk1%w4smgv6A186IuqeGByx5luz7T5D+&z2Q(3i2$W5a zj^~$D+$iji!ayB2bTDv1_o%$QoY3dEK|IiNU2f>9`7a;JOm4_yUa0Wk$7ea93}a)5 z2Efn+Xax{Xl*iUQO8w|RO2G-O{Lyhi3+9H>k&_qtJk;lcQk#SQuQ+aKVUMRrkB$qP z7wSVH;(*owP2l3;{*~b|69+UiCsZ;HXy#vOoDe9BI6z!bQgcDWoKVSr6$UNj(T57~ zSHC?%2o>TNw9rbqpu+uv<+pT?{x1qWwvPjv6FPFZe`SHT={J2)e)R|x(Z}q+9CQHx z%K3=!{{$Wdd@O>E7b@9f%A-_JDStEJkpaIt4$AmP{}Ha=g8U^F*JG2oe&P4Xg#RV# zBd8qza?o5T+>fO|-J@`ir97r_{;L3J=RD^A{of-Sei7l34FBf9Y&@LMvY-|Jiy6QD z-^}>!|3$68nDNMp|K*@{|N0vYL3w1zZ}*EF&@^r^?=K=elHuPSw8No-Lp%798Gmyq zznSsd|BDcR5#urZclRjV@AO9k{C1DzfD#Dh(f>a$l77wMzb8HkD7k)3jL^CF*Siep z8Y2sWZlHgyHj2>U`&UQ_#QCU~LsOMOP(|xkNDTyCsGy<8nHf6Lf5n-Cpd|SfGJ_J} zuaG&keg6vmwE+Cua6*f+266qXB6|>Ykp3#l0otp7J?sc2!(XBQwOIT$k3(m9u76ee z*Wv=@?XPOxpg{a8>3?Q==oj+;afHwN54LQaQO%G0TM6IdPuSNy0auavc$fF~5iA6Y zf;*9Cc1C7d;Z(KjPpXTZ)Z|qaOFhwSinz)6w`qoda`gK=BRNW+WqNR2RNp==e_I*+^@4QKkbCEjn25s7PT5B;{(Y``cbiKs4uA5l ze(-*9amK_h`{alHfTEMI@~J|_S7aCMRLot4%hHlS$|voUgJjjrg``spjV}Rf`UgQ; zN)-b8Sk}v$K63a&n?vyOME3mU0d?A@UFyikPlhR{bRvcb_$nfPY%Gp>n0mcdX1H(T6`gNboSTwe%D3E?(ep4+vbQ!QbjRaHh{K@ZZvp;zy2<>EFV5jI zgCRVrkdlWbfFd$aeH5b-oA#vzwbwwl;!y!d%cbb#MXRsqa|c>8#s+06#_V#A-l)8r zXe-*Xqlduki`KS#7;y#l#y?>8IG}z{4F7ul2L4ARjP1mJ)XI-EKhgvQhzRzZxm>qD z*oq|RnUNV?mGKA0s)Vz5TYl>gp8Uw+{$0+ zYp?xTpG6+&aH^7d>y#Q=d~0W0nm0ZkYLR)YCGVncl5SCAMOswW26;Qhhn+m~j&B+ir6Rk@Y{JrE$+mNEE&v8)cSQsas)+=a`jsSt6+RygjD1O>dijp$pk~*3xS*;dVQ_T zoEq~>6L`Yuw9H+jdgI#p@E^(?dQUod+i{6u+a~MtaHy@NguW4i2r8*p+ml3yt*LCw zU%4q)velXG8vZ!w`Pg=a`xvfQ)9c%lo@1QfOv9IhDo3IvOa_A%ufNUo<15|$blOWv z?%qAE4`QmhnwcukQ}MEP5(vJ>tax4w#Lua#8~fM}HF%FDMgsHN5&1^OqQd#5zSJ(-fN`T^(WW;Q=Lw0geF zJ`Ih`041@2SqnvMpvB9VJA;-akFuwG{7d$T_2yHm{+5XCs@6B&$Gr|#!cP_Bv5|5izS6zh3XW zNMTy9F)0(e)FE$*lkUF}_#$JEy888n8TtIg3pz%Oq!3LOcTHjHT}9z$+_BsF0hdd6 z^}i5%aSa)r_pE-p+gA53mh*v(_?s!(X+^`yi({W9fJ0l>y`RxvQkV(*qJ6Y!aQ@Ss z4i6`RriVhc?4^MUg+ZH2wbW@795%g<#XU}(dQe>9(#X39I`W2un?l1-zxbsfsG+)N z8C%}rGxp>dO494?ykcS$7NmGIp>+(2lGTSdKMd`q+m{!_T$es8&{eQ#+uXD4eT0&z zZp?9RJf=2Ch*ML-k8-@hl@S~)DkS%^+MP-M!NIoWq_P$5J@GF7+=DF zAXkU()b*-T+0E3pOtCx; zOlD4CPaA(=vArxuIM2|Rbp~0_ATzhXisQFkBdDQL0esBxElQ~ZDUwsmsii*Y3wI9q zizA3%ykz+NAw?u4nR}09Pp{d!Y>Hp|W{(z6zNgs-DUVmiU9dN}(WYT)YjrP;>mMuS zS1`(+Rc4-bl}*;D<2cWH=USRkBS!m|EtrGs;*h8WRug+Veh^>}767 zK0O~ndAo;z%YE+Egf{WEM2D?pn97sxo#N@#i;aA}F%xMjIBEjZV;j9>!{WmEF~f|~ zTE+nm)-m5!lkcOG@IxB7r93a^Ww6@!j6VhF*JTECW1T1@gNDR3;zmaH(7!`ma z0-KQ;G72b1eqZV)Rs-hl7!9Pl2a;J1@Ljy@zLpr;)145tYiFq*#AE!{o%)hC)xC%& z8caeBEDZf_Tfe4LGoalC9k znn|5s%Rd-1 z6K{M$UzC=#LegX;#t-yQT#}&4t{4Ewf@t!cSK^tJXM5LE9b-ycH%YJ4lG=U= zJC#p+SY~dcLE0P|Ec@I{ECqK!ie;H^Rhi!N zvne^QDD?^r+qo`f%$4das5(69ex2^>)zt4WlhpnswEwHfl8)#DPKQ__;#4^Ynl^d* zwW^W(59|gEIOl{T)l5heJJP$>4d(k^-`%J*UV6^mHn`i-@RXL}^85;C&s2ZVv{wAP zanbF{q_M9qbrHcnuM~XQ6kR+XFzOb(7@pDg`KCR`^7R|~R@-GdBprstfPw)4;qTue zEwS-*=~Ew`YOkPW87vIk$mq-8b2}136C>xYl0h3s!%p=MXDZdK*ke`aH)a3i^+Y@> zyQe9}@RLqH!y_T5rEmJRY`97C+`-d5Eex!0yssY8ph-~Lw@0eV6R@CPfBIs1#3!C3 zbkR@1=@J905@8-@lTk-BA0AZEN2?<)^RTJ{Q;#wi$IViWg}F$t@ZJ6t%($D>lBGBe zlVz23wIUzuillLL$}>=4Ws@!ZmnP(KyskDr z;o8~$GRP7+OY5npT-YW`ux=eM8eml^#^%_*MD%4rA|N(J_-O=B09Uf;q}A7I205D5 zz}H_h9&pr>vmVMOidVm;KFVG^*#C0)9Xs;eZDu5+^RpHC!@v`j8=F*Ox>0ozrdyy74wrC_Pt8(8cf! zQ4A1uA^04wa}MWSklu6AP+cki;N1uD3$B+eGXDBv%&+%3kz;nLeUk{Jf8DMk=(2q>;#MDd_Lp z2*8(`K*6C{z}#S%Zpc(5EP-egRD#?XP$QKEjR5JZb#KmfBwbto#=fi zKqW?6RO8z*^wJ$llKX3Yz%J!4c9iHJ)Cb@f|Jq0=LhA$@=?tsi7uks4V>4h)pD{D9 z6@1w?-I_c;Bm5*^E`H{e-b1;0jjSfJ0(}#O0@Zf)RIxi+?zA6W2xb>wycJPfTDh`2 z`VEy(nH~>G9g)~7F#bPK$P)itAxvj|$v@YavT$dXoptpt?KFkl)ucvM18w>kf%u?1 ztDjt|GdIN?M{I0f8s;U3zfD*kCAV15yE#25Z(_jS^*+eve8Y;SC?&R#T50wJ?t*z< zZ-Plc)U-ozj!+mZb-G#(fVXp)aqhnmB4b@MRD;y{Nmgf~^pK z)-suLc}W^)Rk_{HFf7qa4qbYEv##G4XMJ|zYHtm8Ny@MohfUQ7cP(CL%1|cy-p%4q zZ|_ez$c*e&aj|YGPKCf++1Ks5-#zPn9rJx53Pa^yg&otEmzB;hc+Hx%&&yd@U2-k6 zkI=oet3fvLxZ?>v(YB+UEQed{^Y>RehVdMd94Wk&zhctgpJ!7c(3_quzO+#;K0Uh2 zU^wZ{fTZt{7-Ud^Kk)L%#j3etGMx(A&C28ZRQl$!#7MrDf$I9x(?iWHT!z8#&?nGH z6#!agEj*Ew0eX{f1$wgqD)#C8bgGN`BWL3d>kgQ?(MvJT0WniquX?QunYpDK)S?T# zT7vG@R@pweX?TM%VHLy%?p(;i6(gmMdiB* z7Lv|GVo3gxf`)r^{#vBGSZ`+U$_d>$n$+qiJ0-*Ygk^iUQJp%@4x~(q|z!;j7^@j7i`B z!dsYL)iP@h$tO)rl5CB0V)Sd*FRliVI6e64c2I`rbTC7D$m~B| zE`2Ns&BrQ`TQTn)bJ=s7&t(TXN?r}Sn0(ImWq4cLxlw|(#~*E9>oVDy%ojOX7RxL5 zOgufUrWSW<|00fUy}&xf_d6rNb#y>@iAjeTNrxdZa)B%Kw;BK@Bm~TVegTYFd3h|{ zLMgqwOq7bPqrDfIlJQthS%6guSX1VSU{|n(a-k1m*#{S6S2Mnk;)%-TB<2otgs(@6 zdMPWBR0xSx35ETM#h@-42TZJi8^MzNx@Tsko)#6=3r+3cX^cy~6!VFxZ-SIA)jO4$ zxyq(z*3gR4=B2i3*VCOkLekRRm-J^7BJj*ZVqIR8EjedJQyzIHj_6}=2<_c9FeRF5 zSAB7r!j^Xy)6zbpQ?$=H(uA}+%A>qUTVH`c7eC0oz9WIrO(%b=FBxvv!p#gd+<%Ya=l7q*OUm4jeqthyIUV6%xHwBrW3(O9qPDYA!`o{fGF zi{{AU^v(9B;QrVvWW*jP@rLo0mR5LQL3Tp_1*iT5-19bL7~CX}Us{o>i3~I;X+=^c zK;P3BdY-*iCfK?62!<8AUONl-1^n-gDb;E(8GI zQ8O!RyQKDIy968*l!=hrq_b^6gO z38-`x%R3I4Zf8xZG!kiI^W|9b;(U#4gZ4$u^$)vH%&XTT3-Byyw{Yq(WtvLB?6lXo1?n5qj>t^H%N9w4;A;8UpO*_oOgb)8UB_5)&(UQHd7Hb%S~QV zcsXm}L0m!i_|dStKSSaR7uC8Cw|lqMzq`*G4tX@Yp30MYt0w)aiJ(Hhy0a51nHk05 zlgC{}oz^T5YcnSzezDupi%$K`wV6nu$l+|A{@L`078j%LI=V?b!FZv%(Ixt#gS6do zBlAUW4GhM~6fn~ma`l6hXKl+Dm-_4ru$4R~Bx8!qvW0J=|IXW@2CySw?T)^2BRE@% zQObKnPnU4^=!(je=L8;opP!>9mOpE#cPP!Pv01`JIU#b}rHm_D92QI|>Mq&*+O$y| zUYnNp{H`Zl!9TEBIPM4PmZU$_GmqMS0T|U_O#px51!1VW86N}AN%H+ z6a|kZ`+}bN9mmR@LzH1h0VO02LSn^2|Kv4S(^R^P4!m#65__>sZo=PaZs$!m8TiEYO&_H!pVijf(<b4`?-KxD$kgNTCM9=pRq_oi;R^ETPei2;fS1f#aU1*I7FF zvi98zPhnY}&6mZuK7X~-OEwE0Q`y${z7i~%Qj+Sq9w+1TN=@wSAU$qARE&q~p>OAV zXaBDk2DCnX>)YL?#mA)XI1Y<48vu2N!onoh=?%22n)N zn`X-d>p_@!^*7`hFcaBK$@Q_7E$1}uNj-Vx%F{Q8N$A3UXJ%9HSLH*WQ*|Nez0auo@g?makn4XFEdK^^BqSj)I$+BlY zFI8{J7fzEKZm*9O4CQGK_4kVQ66I!*H`e$*Bbff=W31S?kubzWJ8-1o&uu;Lipwf( zH0rfRyhun_5x^B-Ugf|W_|Y2r?C5JKh4IF%U5g%GKZ>&DmNo*E;v&*I%wDRQbs-Kr z6y3%MrS|)qKF*$bj`wv}_i0gJ1pKb8VXQ`UEn^EPNw-W^roO0Xw#GuJV!3B1FzJ#Tkx^2D~26p*LJI7OsBn#9aOO^5Qzfs|-Hl zoez{MYeAjJqzH+5{zv;?axp#EF#!aBqZEym-w$`Ytmg~ihwqgkTNBbgXJqg#)Z43! zFJCuz@3_~%13P_3foi?`E7Y?2xjrNN3Oasl_=BFTU6Z7Gv+&gmSiw) z>46mU&GR)q@%8nvZ|dfUJ z6Jq6AN!}vprTHi5hOeF^K9HE%xIpiDeiK=1k=WCJ2BC!M0Q0a~haW1W$uA^;WiQ_B-^~nsz}>=_X9%Fk9*@jA{e!hK4BDUp*6sqRBq0>QXO0}&UB{Xc z5N7L_!stZQ#Yl0nIyIc?B3%Y*>;N3o$vR8Xhpa+KtkXZ)13_A$BqBXtN>y&`LA0?X z7u>U1ZHZVYk8Mx0BYQ3~EANlMq_M#9$Ot?&Spm#`hi^cVJZm`I`Sh@rkY{r7w;*Na zLK5y}vK#!j`ZkBG>Q}u)^~7i}$>h$Z^y#itqdgGQ?o4$Ezxu(_`pu&WF|00{VH433 ztLyh8J#;)cT~tRBw>9VYJq;3WDG*rWPL;XQI=<|8>NvwLpI*LBup7>(&aSP{^^tM< z6E=I)t;7dhE}4!yV=DXv7~f<_Q{F^KeZcqc9@5@=L$bECB~!{#fpzWmwV`#;z_{VUKJCJK;!zwvo5K?MM)SV&N}_bpUd z2(8w&m7Cd(nGugA!c5u<2HpVjFSs|6RTh~c1NLx#`#`xEF@Pe5hfd})?=hoGj@Ups zgm$l4PC06|PUtfRHjxHAn+XVC2?(ucoyXiL$FQ|LTorN_Bzc43;is3sWS5|I>v5NT zqWfn%AvZ3(a#A`Q0-uEYR5?WF$rnE6y>TsEAKsUB*}k&<)M8#VJ>0*qpYGJqd+oV> zA+fy1#=h_QJ#%G~x4G7I&Fu|T25=fOn{1?T=CnoOP+|t$^E=2Ii^Rx*LI2==7u0NW z>f*;=6U3zCXS~fiw8W?w+i)jcY^FqUqM1S06k1@+>(R$)V7^%(>v6a6W|mFiA;EUc zQVRVvQ=|5aB;PX>N#Qq?>8jJXcU|y=T2RtR-0(i{p8CY)r`dWj#~E;KG#ZddQIj2# zt)x9j(zif69T8kUP5xSMoL}~aJ`pp z8bKr3@mZEEHjdz(;F12fO}EpP+V|LFov*dLHq9s@Kqsb+OKWB}(XZwuA+{HkRvT$G zWM({e>Ab!??}1 z1K=U;3xIEv)%g+oyo50wPAZ2ptV0$n3!gj05&J%yUM9)^f`g?J7~fcHOqC7!oulCxKQZ$_g?bS*^9jr9yrM#?|rDIU9;C_>0jsy&u?{XwjM z0x6cPt-@siyn^0-9~r^+DAC;1Cmk+k=mWJu`32=#&4kvdD`+M&=a&SO1ENvd7tcj* zf5j?2G!;hHaY#%a2tYmlw&Nt!MZu;B{wxR9R_iZcW=#o zOH2ZYzfuovcyb-Xp+J-rcVHT`k4zUow~LUHf2g#c(q&1b3{|Fwn5K$*FUZ7c)T{!s zdcx=)#rLhCZ)h1+d#s>-B+_Lusl~UfE^l;^Fvk$=@*#h>&2JeVr;x3y@&Of-bZ>nB z&bsr4=PgvTE`9`@^8`uogCPW<-+$SEfc!}?q~N`Q)1ZB_8)*S}7^5RK;_7#ELfwxX zSc@?6Q~3MXrzrwsBV|?1)>15AvFfOoOR_LGG%AS_m1qcKsN1RH!!q(!UM8-#Sc^A! zi9I+gw2w~c&ZlPh!H{LhgMAQBtxPud3)sslzNzW_gWoP3B>Qf#;k+^31s zIrugJ=iSf#%s{TwD6i^HryzSa5<3k7kgUHI9=RA%65w~haC;KKu4o*vTQ=~p?dwqq z_6vo)RWt$GMhdllA2K`1)oS_N#!-hAZsoX(Z4_~I8by|2V&@c96)ocw>*)iT)AVai zQc`s|Yqeh0*>|-+oi=>ldoju->M(p2OXpDiy>Y@PwZO_a$k#JE&0>L(N_(bqbB2(Q z-!Jl4Q|;hflVigS3a>7iZW%EP8QjI46zTbLS~WoU1G1L>Y(@P5kpYPGZ(GWvrQ+Jf zh(8M=k#k^s*l)yF-5b5Z-eFw4oIHWq7UXS6xbjT{#)kVZIvb(}rGj5oJ{SxhQJn2P zA2tvZ@hFZb*!Lp$8%U>kXTZjT&-C>BWrKaS*xh{7UW8mb&w{L(mb$2B%*lJq$=~Q# zc`%jwdTZx*l(0-KMI1?Xx$$7|+>{^nyYeQxGWz9y9EJUEiqk}ePnD`zX?_*0qq>_Z z+Z99G$LHxsu={o@riitg(8z)x!-m~notw~;ucPo3QoL&HcrE&yzVc|Wv$6bL=9uh~ zv*J1VeS>VcukU5hyK$spQ?mKcqF9~Ue#1nh_B+0|&BGh) zBd&SRN=|K5+p3I?idJMjNiwi=PbY284;tydq<*R8GVBABC^=4vh2iBkd$LpuoLQ19 z+M3#FK2+ZiHrGFk-Tscs?RL`**|;Dv82<#60fPy_gWmW7*Gyr^Kkz&Vu9-n@1lLSV z5-qPRo+ZDNg=4;%jRlsK0V9O55x#;;Nh3u^Rvn}!-5aT-K9@yiSeTeW#)5e`!e`=ciCqO>){^ zq&2KH%T%Px^b4sdL2H<0W*RrXr5#(5Pjqn5ZygS~AYg^=dJ7#SHX5!x%)&i!KId0(IZIP)XUR}o;) z`1F^@|A{$?n3YfB1nJ(PYaweH5>p2P(5=6nisS%{0Rs^zm>1pJ!?1iEx*F_u^wBTV znbcAuSjYcyK&}L@19~Po0~1v}2oy|;mNSsl@zK&F?0Xqn);+*9m&5%jK#4PiNh2bK zI+n-UxxsH%=yXj@qIN-aRm6S%dpmWGMBdZF2;*ey|nXV)S#K-IxZepu?x9OQy-tue)m{h#Bg54ul5lJX~|% zuS=XE5q{%k9o&i2_kiT1R)}J8vmVp)jZ|o5gP<)&uU0|(oBTJ|`vNX>6cpk0?AKRU zzU`&EN3T!LoO`hq+SR#2oiw~&Wn*+Z$-ecTcTfh`3!WZ*Of8yuk&kzGb8qm}V1+$N zO}dLqrx#9fTl#a3Le#CDGdlW=V;^aiTq46YV*Nb*XB6>8eYrg~8Ma{OAfziZG;(v& z{;Y8+qFs&dTZ?<6)j8j%B88ux=Ug>1G_V|(r1_Z8%?(}k(cg0a@}egYPhcbOaQQht ziGX!fTsL<4jA%$+s-~WIgKYfiAxIC?7lNz~Djemn;$%1=$oWS|)RazB!sa)sKyJIzr0b@O4gPHY4}t=C`pA>^c@O?vco@ksx6<_rLS@@9Y5k07&7@Ky zRl2XtxRaHXQuVnxvn|1eR48wqn}jgqvb=t>d5p6jHH;)$AXt$pxHLU~7(<#QvO+#e z$d&A%fbX7rh;CFdWX+Ao>B@+jh{Ncbe%a=m1bUO}yzG>GWwcw{LgU-Nw^$S!@sg!#9jB;^L#%S$N&8RwARh(rK4k zYZ)zHkBJ=hJ(BE%NYCf-=qG#jOb?Q{wL>9j_40kw;(4uvJ;tXh8BLK z;>J_DKBbt7J*jDmAr0A{cix(>3eU8>8cK@h|HR`$BSPK7hqEjfiTaMCKWOV&#Z-Ej zC4QJ#J0nbar_cto@ogRsv-HlE8oOQ4<9C*gI_}Rpw+)R}>ED+KHhI*du{L)=Lofu; z%O;--HPiNgNsqg2UzpEhVI|4m#IK&7*-Ny|Vf0pq-1k6WsFhyA;qmIZNz3OuLyRpe z`edeb&$n{jzS`qIU02+>-YB=+mnP}HxPO41^G4GhSti8u#+bg2Ci->t%2(p1IGbCe-7<}=c7DEP+V*pW z*GW6+1To)pywAAbO!PRr#CA#`ZeE}^&h63Zr@kZFlSf4lng{kVpLp?pN75`1gZt|o z`$x!YO;G~i^jzBoaak;peBSDL=Z3yN9f};&qi-_~sY_r=-PDRNW_qhF>66Nv(Kt6b zcvWCIsgB3Z?@@g-UwK?>xJ_7Kn|;b7)HHEa*~zhDO<79%(PstP=c?u~B=0sj6V6*c z^iQs5S`KHlI7h`TJyaq`O5rrZo-g~}IidDV=e6ps zTCB@ozoTX==p_BxK_H~~@BaBaD*b0N0U0$-&5L?JkqN+GrN4Ltfb04v9)aVy^q*qd z!FmA^Cizz+5&Rjzo;nV5|K-;Ij%!B}X9S)OSO)%Q3IW`oCYk(cV*&3J6 z-3yf5?Od$^8-dL=4_ENl-V>DF0Q~`f=CsK4*Jp@Pp+w$7>)Y72I4Ji~)$$ zhf@{s$$@@hWH;)0I3y8!9Iuq2-4bEi1=fTM5$;jsgLc0K5>INeegyr&iSlI)X z1vh)Qe-a%C!6%NNi4G7${$tnqZ;|>*D1IV3_(cN1@csvm1EjL>e}m%y214Thj^;(Q z58j;rA8;IiRgM7I)E5A5AVdf@ObAyL&Kv*&s-Qr{!2lo_e!%0?;lIPc`7jWHgZKn2 z1Mv9u|KY&>@f!@B3qOuy(2r>j1Oadk5g;!l0uNq?htz{Wb1>FIAZS4l&N=`;fSf=@ z1#!UvWEijzfZ%Kqu*D<{V!(yrG30O-1NbNWcMvWw1l-=j;ClEyMS(sP0wN6m4*rQ~ z3=B{vfCj-U@*{L#2yg+zuN4FhKfVVz7d#hH23{j5h--%*g2JMRPf=hD3c&;JL9qPs zIdI+p2!bdA2GG**-vq&@2hvLGYDG09l2WtSl zk`TE3z@G$Xi9oQRAUG}HO@rU!_;SRUz%K^}uHea-{dKG30ucD)TL8ui1i=BkR-oqa z;e?C9kKy|932F-*?Z?B9xPt(g-9fXDWdX>+U-$nZAP8XscY^=_*InS10)z*M-%f5I z3SNRA2V4-2WdJS*|2ztW0*o_olO4>|a^PJ#E&=|TJzka3oM8J5W4%=!ph}vzs8Uf%EH}w9gR~Paf<9 ziijuU0@@wGpmBUILSe$`co0{@$#_m$2Q)AE=cIvf&30k|{H@3#Kz0yUA-+0R=x_yi z{N1rOhXDCO{C~{v1B$pnIRYA^<8P0ZHw3&ah?C$79Rg@Kj{iZ}0T6_n47B%{Fyh#4 z1#aRFwCYLok1YcbKYe^1+*&yH3V^FTK=Gsoe`{U{^rRm=!LKJZfotiLTHFA|lR6-b z5D3Ej{aHCffCT?-@SQl@!NC7^b%cl+VZ=v2N^NIm!9rWp*1Z)3;v+|QNVBc>Un`$0 zlPIc-?2qnXDQSa2I(;yoFta$^Sap!)HLn+>j=QPdOSWn(ux=H&t)w@h@a%c{$Y+Pb z&1O_c5-k!+R|uGUf2rl4L~+7k{3j8vg;?E4T+i^RWlL>9a~=%QAsOaJ88<_3Bh#Pi zeMZ#gw}Pn*xEN{R9_YyTH2%NZ$`GIpeyJ4{OcB5DyjT@yIpAwpE^Aj|yi=29q)Ruq ze#a19oz5?cXjrMYB&E2I$6kfz3bYL4UG$=PCO_9B?i8#!@2p^3ZxMIFW)kX<6s*;E zYFUF`B8&8yrwuOI&Uvus%NRV%XbiZfn)sM>M%rYw?*2&3+S0<3=R&ST&Gp4xYX;lv znGyFdzSQiU*}qg%Ctu8{XPTJ#G!w=(l{g#3m!(uA#-g(GQ9|(`wI6nyK;`S}AcDn4 z+U@Iww=*y=Z}`$@d6s(4~;4y>g@W2h#W4ix{PyEr9xMC7yjuWtP{Y!;lK^qgmIY$JpOo^&Jemnj6Un@(NtWws)8IKkOfzQ{G(Ol?%)o{`Tx6nx(-LYRz*= zbX2E2rSHPll>nOr53knNhkK-yrr=j! zeJn%gpSN6Zvu>#WA zL}lwWYIWVQ8?VUt($bai;wa+KrR2_cRJPZ5=8y;rSZ@6?Hh}B<_hTcc`S?1BxdG(& z)Q_lMvl`qJr^j|ZD)DNor9(yY|EAMn)k|=rL?vXPjETWVj=Oqwb3QXvX^*VO!Y+@& z^-7CLo-!?awT!XtiZ?(vrsVE{JDdg`~2u80)IH`jvbx)$z`q1dPxo&w^Jl zCX4eidQVQ1G99w4c$BJ@nip}U%Wlt+brD_UczWJfLA>lGHlEmfZ>FYGskB^YH9BQ0 zGmSoaIm%`^9Fc!-D_TCHleqBdXdk`efiW^b;a|x#ex{%ShVX9%TSDtcz|vPK6sLuS z_`DS+KtYqAvD!w|L&iEDpUcZxsKQIb`&j#(#(s^ww^QFDBW#m0p%^SyJ@mJax3h4 zlF+LbtxOH=3*$Aho@@LApC{UAFWm2dHmgfqFLdJ~9qy#K`0j&7pHIVxuS9S+Z_pxN zjT_~I0e$ZhduaFTPRVLK=T{DUCAa9YzO+eVbHq#G=!#tO+&SYan!!7F&NRAW*M@pd zL^!X*HmSQwGms;iHsk7=nc6MkncI}6&jT)yg?d&RSYSMIITN;s!CbR&bfs_N_RKEB zbbzxmGU0?P?SIW=e?-MA0pF(PKQ((mX3PkCXX%*Do+)gSzq_c{Zh!e4BkI{<=W)4% zIF$(&&)z}aD>Ix8YPZuuD16c?40_3~2dCVJHq8xL^a^S@rWkwXIa8d7#z= z=#Q%i@mbl%&n}tNhiAz&W5ygY1+t$_=Y>^iUrtT&Q#+jzJ&z89Bwy$DWV!2Jj5kJn zD0+2n@w$i03%_>F%-8f?w#_t0r^d8toHf<)cLLk4?93quvQ}MiSx;0k8TmUHm7${A0Dx z4~1%v3JM76@I!?3_=RB_fM^vh+&|^9F@GB^|G*##;3yH0wOCD~lCh4}XZ+&&=%xH} ztJ>f(^{T@#B`dN~hg*gJ8udTh1czH9`~tN)$5j#%);_5cg8Q|~k$}>eDZw7gg32I; zsd|w6bWY7Firx&`N8!g2zaHNxZIM+6iTMVn-}&uJ&ZGI05Mo(b03p0YSqX&jtckbA z-l_0s`01WWCRz12XG`s`NMcydZnXM~;(KtPR%{y2LVfzt;+;34%@d4~FbNt=Tk*jM zWdS$Xvtey&E!s@Z+oVtrO_Wi_cbm)=MYlg3p*`L2*XW8m_AnP(@ zw&Wk|f&LJxdW^Vk_=Z3TB4-XGDiJIhk1Yk=hAN!~&4AGW1Mh4)strwA>z1EBkpz~8 z2S4@McsTyLJ^Bn1?ig$0c%N_Ge)kw`5zjt7&9Q_V z7(^epxx!{5pPbxiwFqrnoRMkh%y6Z&{5+YpV}5amo;E$dDd;XAB{~bmo2%7AcPJ@PLs`P-!# z3h8Cn4ZJRW>Q?O1?)B`FcW|g;Yf74U0^71#qaG%cNSnTGsB}sDOPyPnUKwOxu!}ZA zWc$yiAu_WtiqC?yzXbVSyC@v zuxh`IRfAq>MpEu!yQx;G$a!AkNo?+pV_~-`uko^TQCh7|O08Z~c=hb^SGSuJ-wr5U z?-?y1kvatRoO7@n2|46XVN4`JT8n54Mf&;6K zLi}0vU{(3&g5E?z(1!h%BYp*y-h#>Nh_3Z&bllDcL=9aR#bxlEvk)tzE2cj-4GMoZ!sUNcRS* z``XZan3ho*4QQSz-IwX6DaI?!RT?E0tZe(-jV=(Gj)Q;JH%uWnqdUTKE#)kp^wgQr zK(EM-yP-MbJS9vO;p^hbms^9cjF5JTpA^EdLnq9Z|D3A3HI;H3N01c z9}Wn|IgGZEb>h!^H$QN5e-z$-V(`HO`B2JD7hrM_c!YYK>kK^_q((J1F0`>2KXI0) zz+L6laz{lSk7V*q;k(>QCM3R!WlU6Tm0a@51Xk^=m+xNtxYsD&)0g5ET>fni@*=RS zG%YdqmWzxPXIa&Zg&FLc5T1dGc#*0srKOZ_9cB)0hSm#?A`Ox{Orr0GH%vAn2=2eU z`_efvEQH1YOG6p0xW!3~^8`9@kgmYF~2!|F^nz^mE+Hv$CBE-$cJu>v&(d@}%z$8OFwkFW4zBLy>T=L4b?+ z7vli}ctn3YQOIeifNWK#W_|C{qB4bL^;{~7m^`?Kx6?3c6%^D<6l`|Gbm24vqhaB0 z8jm<3sd3cTp3<9B8D{LoLFFh^Q$rU1O#GK!ILqz@HrapTx5ieG^7S;d|JqNFv$b0F z>RN(s;;QZDSLxv9;e%|5;AL&4MRzUTN4U?v7zqu(@w3g^+|n^%lUBP?2(Jrk{h=HsINX)fA;a-)CY9a>QtFpf8ZKtN*#rwO8Qucj!!dH&Q zdV~nXq=XW?HS76Z$+j`ewi*< zFB>?pq&<>iq^w^S3_Z`ezvRY!-WdBC?mDFouGg0b_t~2tWj(T5a$d%LN5OP!C705k zITv%_;+dWyHxtMb%{^I3o4_RIN4+i=4EEm>Vw@0ec zV_&%{{;2V-)uPl``<76|^qj7FY-8hf{*G|VP17i?jnAGr+KW>Jcg<<2S6{~1+;h!& z@lDsaCgT=e?^SgY6QaVl{mu&7DK@0YjQ&rE$KMJ^0%ru zh0{sbFcPcC8POGL_03KVl4sz4uv{7wc$8Wb$ebe~RxWVM`X0tt_o+{K*1FbTFPYx? zq$;|5)RB@_Tath6d+26_KTa%M*rPeWimb)RY)!y7{@WG{YW@f$5h9W*dRt-}$e8`{h(>c*5-ZO-Jk&Nq zx(e^gdqwp(6gNM(YIfo@CtDLhNjn!ti->r=d3ZuEpw43vreB@^Ky&uGi;T9dg5W_2 z_qJdD+|@+h%D0+slKrzWhYW;bvP4X>A|5PZ=lXHdUJgH$BcWzxkGhlB{9>ciqJ4L> zRJv{On;hibdWjD7x%-nF`ju?4^%pJ`XG~8gu!M8=_Ic&waxJ}MoWF@u{M9MwPwI@QuNmDCV1NHTE{^V;NOVo-Q9r)&!I(zc6Ke9jjOgh)!bOsnN`M85Dy-K~j!Ad_*_+m82=B$$GNicn4?Ms-sP&;ZA?Ef<7(m=pBrd@#n0| zK~e0(Uei8GJH|_Ys671=*~5`oA^(L}MF4i3y#@ln z+yBg7lU!XwOoGRfj($10BeBie>MF`}bPH*n_Q;SbMo#nhQaQG9%u*pd=1e(f zW>PlKoU3tfNom9KTi82e%KkbfVCr?eU51-VOTxJqued+*-N}4{=_84*Cp5k>!`{A69C>vqpG2?BoV_Yy*&wJ~@3!xwZr|n> zHkb&~u}5N(z`!=#Z`&#urxa}k0Rh)XWX3rx@`THMS`R3vcNcd8%MA7XBbfqPRZMhL zwOhFJ6KZu@KK7Yt-F~xpF_m+J{fn}mvX5d#+#@5`(B`Zuwi~Kv`{<#bSXb_CNW|Jc zm3ZA!loUyl819k0C;d@X;kpj3LACK*#YOu`3}I=?y>?279Y~`|Em1zMdYC6*EjQ6mrR z?hm_`)o+ug8Ut!7W<)JZUWQL526xudr1Y_$>eMDm&!g5%=~8m5HkGq{SQgGp;glBt z8L!(Z#LYmOZ|p-Y{@D;`%d(O&^Xcw~bT%ogTUy*UuL8Y>&f$3thbCu*ys42;Q7Ng< zn7DIJUzXXjRZN&W?F}3fj;48I+75Jv#d*aU*?^X|2(kA6|*6vT5oa?*fZm%Z>H7n2>e zjYox5_RW_rIRJ*I(8Y^{ew3w%FZ2zgk+8xjK&h3I8>73d>C{WGeV6FWYJx)n>)I6g z=pD{1&apP!k__f)UaDo(SBJq)4n|lPnjX@39d zr*v7K*T=P`OAF3LkOX_>am(|N%VAewCsKBQFL5WjW~I!#ofyq!6Mg9(jSgPdhReQ~ z1%_PflV@Y@6`FR;@w0UU#p6n14F%)**6z^6OCc&`?hEp}u%kDRp4QN7M+Hhgj}VSv z*@>_rOs&2;_V7Rf+R@j^)-zdMH-mv4jy{6_Rz(Pl)B@~zF?_w=&aFil%fndT)|g|d=BP91)A7k-;I4a5 zgJPm&)aQ}X8<0!b$pJE_!pW!76fh zOzFmEw{g6xv-F=}%igsrp4Jx5q%&;Ouc)cAw%>nTa7KMF;oD4l(l}Ks|J4@g<4Mip zy9Xq`z70#*PBkK~$dnEeOXg1;JcySI#0W84YGc9}6(&c=RY8;x-tcfRzx1Z4F_t1e zNSh{Oa?C@m5JZTCY9G!V{ENK)2@~Cq1RMwH0mOSV!yuSCU~Li0$P;~oK!TbJvaieB zey(|fr&xS7@DZ=_8N4XXmr-6GLFc<~#%MG=Y{>The3*EV(OOBlA68Q7jrjEln(X9=S&)DIp zC=lqMtFydTNNRMdj9$3r+2dCIEF0Du?79l!XaWz8USl@5=QtwU@2%&m8aOEF*vIO^ zoC&*|V~WZN5=|N1FdghCa`Y7DM>Y#4n;%}q>{4UHdC__Q#+&kQlMEw!_ixP{VAtgf zA&U+B2O{c+d;1TN?k9oln7)PH%u*u)QT`OY8_@f3f*{)#eem57p8*1?!TyGUj#v9i7_n(`$bWX;NeB|sz*%P&( zBDuOrjU~?A)z3n^l3{YCmzeHPjW`g+UdgAH`}>F#(pJ>a}8fx?<})Hkhl4kn}y+pZfcK{2LMc zGss+CS;0_6?I)1A+BIhYo&Iy6IXs;o0(^xWX^!;%|3sS$0ume^K#>nJ7{O7U08<50 z(fxN=a{&E>SK|L=zB)LP2$6co%EFNm?#H#f=HvuG-2Z}^!#5iKf|vueHUNzYg3Npf zU^$$}=Vu5xeAZKf!w3OF9DoD?7!d9Zvv4)Hbg*_uVA|o>uap0eac}^>%je;2W$o%_ zdCe7|`F;VC!@+t8B>8bVJn+Q-j3fuQf+NXKVB~y0$Ir4xJQ)seLp&Pb?ck~C{*gWp z0#K&#;rttieLTQFY^uMd2sy4C>3D6{j14a0<0u;?b@dW#80g^19z^>s_`^j{Rz=weo5nmloxe(yKLtOl` zMmR>{044ShwE)sK9-nFnZhlgyV@(73kt_CCT^$1)z;mp?iHN%4!n3FTNc)FSQ*aU- z#B%{E94rbDMSu!NY?py6f1tvDqyPk{@Z%hSxIooFj*|ewgO?ybvH${9_>VMz073mD z4az;OSqM0 z-|FIL8%iDua&sP6RUSmpob;eP$j5~+wSK1R;<7xc7z}5gK!aosr6!;%bg_|t!Q$>zX75tB<9zUF)7Y0s* zpL;?>^ph9o?=OGapZfdDpWPTr5XK)Ud^LE?&T((5fgqU^Iah-MNjmu!#`puvuLkmq zBDx!wPYp!)oSZ8U;rS`ovYZEq{{z5Fged*{{i?Bmrn!4Z5c<3dZ07u$k4?$R+!hSnNm5?WgC{wBVS*r`uPlfmbk#*l(w{~T|(Nj zlr9urpiA;(aX~;O3L2^?vWWP+@5s5cGyASCAflT5TBGW* zZp*!X{?(=2+yFZ-z`U}YE(w@R1EzAxxPJxZmJAO{`n}D%#ZuSxxdf@>!|(FrY+5<4Y(dJ{j#pFdg5&sv@T z%c5LIr)pETV|;aDhm^idvbk#KeSbys-4*pWS0*`PD*kwR+#fEFQ$5~w@p8ONyFyCn z4t#oX-06$sPhGsj?(!};{)L(=_P_OoMzy)i5nq+q6*zRkm3HugOS}I%xc73z$Se=z$mD(z)p6nEkE}oXZcB-u-_i%kqVORFy28|S$ zJgYnF&=Sjietx!}JGhK>xdP1m0DWM&d11hGpvhP%QB{&*U$eee((P%<^hy~GQu^*T zm$pfArMglzUwsvaP`|V@OQ`)7>#gaq4 z*m~<6oky~boUhu}K`#hQadzihR%;dTdZFNMsCSlo^Xh6muyDXr*)Xey@o&O&>;OvF4+>ZuLx?piSO0Ufj1Ylthr_i3GrGnJw0ZuFfnOD!jw#Le6l}o)L7w>I4MR5#J(MU$>%H= zOT|4xs55*iyjTcb@_B{ObK-Np3eg$sn;zaPgtS6ze~hMgWVk`oyEE(#7ixMl`wN4c zl~o5Qd(^%6gpvP2`vfxV)Ua0ow7Lkspz17jDD=pqTM$pZe;Mqm`9^CMCT{beTs>qvu`999c)#e ziJB)mzw*uXJtaENix2pm*}gvCWYM`)u|<4Mbhasu_)Z9+*L<)0oGoITcvJ}KgxuhE zWz{Q#j}1Pe9GFz{ys~QZpg7o}9LVW!>Q`urO8Q-z-qFJH;DY^k@l!(@@rU9~n!_4# zeBul2+8QRPkJFnXh}V%hFVR`Y)%x@6TXgpF;-|vawTTM@TItP=Q6}Y z(`ToPS$VS)wyMP{!-TYE_-p1S!)OMQeSgZSP#F}J18pTegIkpY*8W>cHYuw%c=k|pqP|@>_>yuUEx5|F zL0PqGcfm5}tRuRu`khOnc8Ci%RE54CF>$gncNqUpM19ec#j+6^MD%N9Mg6&DO#ZAW zxL@F|F?_)9a?M9-Ho{T;X9#g?)=e8Fs&BLYE!%1sytBYs{?(}m4?bRa@#-tLES_3; zo<4bWO6$yjZWx@gi_3+3hZ|KJO(B1B#>)JoX4b(%voFT6aY)bR$8eu2O z-Ei79M)(A-m6HcA`=Xs^8t~zGs_Po6-3!%A%z7iGr!v)*4Go_9EQ6uu`v3HG-U=-o zJH2&fjivBX@QuklWsB4cQCy(5>{=Ee8Y#eCSDTL<1!?(NCJMgh3D@V17Pw1AWl*M6 zib~bqP`_o~@Qc5Gb0xYE{9kRWElK`96A_gMrKTg(ZEB2=SNY9aZN45C;S% z2c3msG)EI-WaJUS2$t4jL>w3L{49t-Q;tD?gl@nDA^QM0Zg5>QQ5169%F2NJi19UA zEXW;&$(hj;KQfF5-W|NX42a=!jEO@;9kR5H$&7rYa*QVDCDe}?bDHCji&vJG=E(Na zJVlN?;fUS^_S|BE}#r zEswSpGK{4nlrR`Zqo@mpfw99epw7YT$3pZ6FPDW-3f?cQi9+AVke6XH5&Z(1OW0V} z#1P}gnphN_!Sb^Z>nbb(C=er;1Cx#SJ!|F>HIZD7rVtwvhM}9CIL4sZ5hlkWJ|;N^ z28JkuiG=ttIWvkFVHiiuH?TtF7(s*axyrIMu|}~hBJRSrTjV zq^?1i5_0@7_KG)?`oc4WzQEK%ELO6*@W6Y7>nTsMgbu+>K(R1cS`I>}_!vQRiGJZA zfQR=LXG8>bSbipQETLCWJXBt;36bDom>CQ?ficAV;LKp02szlr;TV{ELJsUWDMzEQ zs=R(++KKkU0!PR(G_h`Ru!SSq%Tg#ZDldaI5_?e&IOmA^!BK(e7nUc+oa4xK2DTu? zngUdFWWR7c**4H!T-Uf^beH4J#QFi*5w-*{QqDpVddgYIy(4E~$?>p|_62rugzoY{ zeu4KNPm%Kt&Lrgd2vqF`JK5kZaaWgn6^Kvf9(NN2@xdY)X4TaJgIVM#3$evT5N{u1 bxPyR3eM5=2A#wubIWzF