mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2026-07-19 08:18:50 +00:00
eb9e966de9
FAstAPI is an async framework. Data may be imported and exported, load and save, set and get asynchronously. Prevent interleaving data operations to corrupt the data. In the previous design sync and async data access was intermixed leading to data corruption. The basic data classes DataSequence and DataContainer and the derived classes like Provider and Measurement now are async. Data access is protected by several async locks. To support the async design of the data classes the database interface became async. The energy management is also adapted to the new async design. Optimization is still off-loaded to another thread, but the prepration for the optimization and the post optimization actions now follow the async design. Adapter operations are now also protected by async locks. Tests were adapted to the async design and new tests were created. Besides this major fix several other improvements and fixes are included in this PR. * fix: key_to_dict/list/array only regard data records with key value set. Before the exclusion of no value data records was only done if the dropna flag was set. * fix: test for visual result pdf generation Due to updates in the library the generated charts text was a little bit different. Adapt the test to create the comaprison pdf in the test data durectory and update the reference pdf. * chore: Remove MutableMapping from DataSequence and DataContainer. Mutable Mapping does not fit to the now async design. * chore: Add NoDB database backend This backend implements the full database backend interface but performs no actual persistence. It is intended for configurations where database persistence is disabled (`provider=None`). * chore: Improve measurement data import testing with real world scenarios. Added two new endpoints to support testing. * chore: Add mermaid to supported documentation tools * chore: Add documentation about async design * chore: Add documentation about generic data handling Covers the basics of measurement and prediction time series data handling. * chore: Add empty lines around markdown lists. * chore: sync pre-commit config to updated package versions Signed-off-by: Bobby Noelte <b0661n0e17e@gmail.com>
702 lines
27 KiB
Python
702 lines
27 KiB
Python
"""Pytest tests for async DataSequence with persistence.
|
||
|
||
Tests the async DataSequence with database persistence.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import shutil
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
from typing import AsyncIterator, Optional, Type
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from pydantic import Field
|
||
|
||
from akkudoktoreos.core.coreabc import get_database
|
||
from akkudoktoreos.core.dataabc import DataProvider, DataRecord, DataSequence
|
||
from akkudoktoreos.core.database import Database, LMDBDatabase, SQLiteDatabase
|
||
from akkudoktoreos.core.databaseabc import (
|
||
DatabaseRecordProtocolLoadPhase,
|
||
DatabaseTimestamp,
|
||
)
|
||
from akkudoktoreos.utils.datetimeutil import (
|
||
DateTime,
|
||
Duration,
|
||
to_datetime,
|
||
to_duration,
|
||
)
|
||
|
||
# ==================== Test Fixtures ====================
|
||
|
||
@pytest.fixture
|
||
def temp_dir():
|
||
"""Create a temporary directory for test databases."""
|
||
temp_path = Path(tempfile.mkdtemp())
|
||
yield temp_path
|
||
shutil.rmtree(temp_path, ignore_errors=True)
|
||
|
||
|
||
@pytest.fixture(params=["LMDB", "SQLite"])
|
||
def database_provider(request) -> str:
|
||
"""Parametrize all database backend tests."""
|
||
return request.param
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def async_database_instance(
|
||
config_eos,
|
||
database_provider: str,
|
||
) -> AsyncIterator[Database]:
|
||
"""Open a database instance for testing and close it afterwards."""
|
||
config_eos.database.compression_level = 6
|
||
config_eos.database.provider = database_provider
|
||
|
||
db = get_database()
|
||
|
||
await db.open()
|
||
|
||
assert db.provider_id() == database_provider
|
||
assert db.is_open is True
|
||
|
||
yield db
|
||
|
||
await db.close()
|
||
|
||
config_eos.database.provider = None
|
||
|
||
|
||
# ==================== Helpers ====================
|
||
|
||
async def _clear_sequence_state(sequence) -> None:
|
||
"""Clear runtime DB state without re-instantiating the singleton.
|
||
|
||
Does _NOT_ initialize the DB state.
|
||
"""
|
||
await sequence.db_delete_records()
|
||
try:
|
||
sequence._db_metadata = None
|
||
await sequence.database().set_metadata(None, namespace=sequence.db_namespace())
|
||
except Exception:
|
||
# Database may not be available, just skip
|
||
pass
|
||
try:
|
||
del sequence._db_initialized
|
||
except Exception:
|
||
# May not be set
|
||
pass
|
||
|
||
|
||
async def _reset_sequence_state(sequence) -> None:
|
||
"""Reset runtime DB state without re-instantiating the singleton."""
|
||
try:
|
||
sequence.records = []
|
||
del sequence._db_initialized
|
||
except Exception:
|
||
# May not be set
|
||
pass
|
||
await sequence._db_ensure_initialized()
|
||
|
||
|
||
# Sample Data
|
||
|
||
class SampleDataRecord(DataRecord):
|
||
"""Minimal DataRecord for testing."""
|
||
temperature: float = Field(default=0.0)
|
||
humidity: float = Field(default=0.0)
|
||
pressure: float = Field(default=0.0)
|
||
|
||
|
||
class SampleDataSequence(DataSequence):
|
||
"""DataSequence subclass with database support."""
|
||
records: list[SampleDataRecord] = Field(default_factory=list)
|
||
|
||
@classmethod
|
||
def record_class(cls) -> Type[SampleDataRecord]:
|
||
return SampleDataRecord
|
||
|
||
def db_namespace(self) -> str:
|
||
return "SampleDataSequence"
|
||
|
||
|
||
class SampleDataProvider(DataProvider):
|
||
"""DataProvider subclass with database support."""
|
||
records: list[SampleDataRecord] = Field(default_factory=list)
|
||
|
||
@classmethod
|
||
def record_class(cls) -> Type[SampleDataRecord]:
|
||
return SampleDataRecord
|
||
|
||
def provider_id(self) -> str:
|
||
return "SampleDataProvider"
|
||
|
||
def enabled(self) -> bool:
|
||
return True
|
||
|
||
async def _update_data(self, force_update: Optional[bool] = False) -> None:
|
||
pass
|
||
|
||
def db_namespace(self) -> str:
|
||
return "SampleDataProvider"
|
||
|
||
|
||
# ==================== DatabaseRecordProtocolMixin Tests ====================
|
||
|
||
@pytest.mark.asyncio
|
||
class TestDataSequenceDatabaseProtocol:
|
||
"""Tests for DatabaseRecordProtocolMixin via SampleDataSequence."""
|
||
|
||
async def test_db_enabled_when_db_open(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
assert sequence.db_enabled is True
|
||
|
||
async def test_db_disabled_when_db_closed(self, config_eos):
|
||
config_eos.database.provider = None
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
assert sequence.db_enabled is False
|
||
|
||
async def test_insert_and_save_records(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
||
)
|
||
|
||
# All 10 are dirty/new, none persisted yet
|
||
assert len(sequence.records) == 10
|
||
assert len(sequence._db_new_timestamps) == 10
|
||
|
||
saved = await sequence.db_save_records()
|
||
assert saved == 10 # 10 inserts + 0 deletes
|
||
assert len(sequence._db_dirty_timestamps) == 0
|
||
assert len(sequence._db_new_timestamps) == 0
|
||
|
||
async def test_save_returns_insert_plus_delete_count(self, async_database_instance):
|
||
"""db_save_records() return value = saved_inserts + deleted_count."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(5):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
# Persist the 5 records
|
||
await sequence.db_save_records()
|
||
|
||
# Delete 2 of them
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
|
||
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
# Insert 3 new ones
|
||
for i in range(10, 13):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
|
||
result = await sequence.db_save_records()
|
||
# 3 inserts + 2 deletes = 5
|
||
assert result == 5
|
||
|
||
async def test_load_records_from_db(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
||
)
|
||
await sequence.db_save_records()
|
||
|
||
# Clear memory, then reload from DB
|
||
await _reset_sequence_state(sequence)
|
||
loaded = await sequence.db_load_records()
|
||
|
||
assert loaded == 10
|
||
assert len(sequence.records) == 10
|
||
for i, record in enumerate(sequence.records):
|
||
assert record.temperature == 20.0 + i
|
||
|
||
async def test_load_records_with_range(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Load [hours=3, hours=7) → 4 records (3, 4, 5, 6)
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
||
loaded = await sequence.db_load_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
assert loaded == 4
|
||
assert sequence.records[0].temperature == 23.0
|
||
assert sequence.records[-1].temperature == 26.0
|
||
|
||
async def test_iterate_records_triggers_lazy_load(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# db_iterate_records calls _db_ensure_loaded internally
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
assert len(records) == 3
|
||
assert all(base_time.add(hours=2) <= r.date_time < base_time.add(hours=5) for r in records)
|
||
|
||
async def test_delete_records(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(6):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
|
||
)
|
||
await sequence.db_save_records()
|
||
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
|
||
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
assert deleted == 3
|
||
|
||
# Persist the deletions
|
||
await sequence.db_save_records()
|
||
|
||
await _reset_sequence_state(sequence)
|
||
await sequence.db_load_records()
|
||
assert len(sequence.records) == 3
|
||
|
||
async def test_delete_tombstone_prevents_resurrection(self, async_database_instance):
|
||
"""Deleted records must not re-appear when db_load_records is called."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(3):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
|
||
# Delete middle record
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=1))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
assert deleted == 1
|
||
|
||
# Do NOT persist yet — tombstone lives only in memory
|
||
# Loading should not resurrect the tombstoned record
|
||
loaded = await sequence.db_load_records()
|
||
assert all(r.date_time != base_time.add(hours=1) for r in sequence.records)
|
||
|
||
async def test_insert_after_delete_clears_tombstone(self, async_database_instance):
|
||
"""Re-inserting a deleted datetime must clear its tombstone."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
dt = base_time.add(hours=5)
|
||
|
||
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=10.0))
|
||
await sequence.db_save_records()
|
||
|
||
db_start = DatabaseTimestamp.from_datetime(dt)
|
||
db_end = sequence._db_timestamp_after(db_start)
|
||
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
assert deleted == 1
|
||
|
||
await sequence.db_save_records()
|
||
|
||
# Re-insert the same datetime
|
||
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=99.0))
|
||
assert dt not in sequence._db_deleted_timestamps
|
||
await sequence.db_save_records()
|
||
|
||
await _reset_sequence_state(sequence)
|
||
await sequence.db_load_records()
|
||
assert any(r.date_time == dt and r.temperature == 99.0 for r in sequence.records)
|
||
|
||
async def test_db_count_records_memory_only(self):
|
||
"""When db is disabled, count reflects memory only."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Without a live DB, db_enabled is False
|
||
if sequence.db_enabled:
|
||
pytest.skip("DB is open; this test requires it to be closed")
|
||
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
for i in range(5):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)),
|
||
mark_dirty=False,
|
||
)
|
||
count = await sequence.db_count_records()
|
||
assert count == 5
|
||
|
||
async def test_db_count_records_combined(self, async_database_instance):
|
||
"""db_count_records = storage + new_unpersisted - pending_deletes."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
# Persist 10 records
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
|
||
# Add 3 new unpersisted records
|
||
for i in range(10, 13):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
|
||
# Delete 2 persisted records (not yet saved)
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
||
assert deleted == 2
|
||
|
||
# storage=10, new=3, pending_deletes=2 → expected=11
|
||
count = await sequence.db_count_records()
|
||
assert count == 11
|
||
|
||
async def test_db_timestamp_range_empty(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
min_dt, max_dt = await sequence.db_timestamp_range()
|
||
assert min_dt is None
|
||
assert max_dt is None
|
||
|
||
async def test_db_timestamp_range_with_records(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for hours in [0, 5, 10]:
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=hours), temperature=20.0)
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
min_dt, max_dt = await sequence.db_timestamp_range()
|
||
assert min_dt == DatabaseTimestamp.from_datetime(base_time)
|
||
assert max_dt == DatabaseTimestamp.from_datetime(base_time.add(hours=10))
|
||
|
||
async def test_db_mark_dirty_triggers_save(self, async_database_instance):
|
||
"""Marking a record dirty causes it to be re-saved."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
record = SampleDataRecord(date_time=base_time, temperature=20.0)
|
||
await sequence.db_insert_record(record)
|
||
await sequence.db_save_records()
|
||
|
||
# Mutate and mark dirty
|
||
record.temperature = 99.0
|
||
await sequence.db_mark_dirty_record(record)
|
||
await sequence.db_save_records()
|
||
|
||
# Reload and verify update was persisted
|
||
await _reset_sequence_state(sequence)
|
||
await sequence.db_load_records()
|
||
assert sequence.records[0].temperature == 99.0
|
||
|
||
async def test_db_vacuum_keep_hours(self, async_database_instance):
|
||
"""db_vacuum(keep_hours=N) retains only the last N hours of records."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
# 240 hourly records = 10 days
|
||
for i in range(240):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
keep_hours = 5 * 24 # keep last 5 days
|
||
deleted = await sequence.db_vacuum(keep_hours=keep_hours)
|
||
|
||
assert deleted == 240 - keep_hours
|
||
count = await sequence.db_count_records()
|
||
assert count == keep_hours
|
||
|
||
async def test_db_vacuum_keep_timestamp(self, async_database_instance):
|
||
"""db_vacuum(keep_timestamp=T) deletes everything before T (exclusive)."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Keep from hours=5 onward — delete [0, 5), i.e. 5 records
|
||
cutoff = base_time.add(hours=5)
|
||
db_cutoff = DatabaseTimestamp.from_datetime(cutoff)
|
||
deleted = await sequence.db_vacuum(keep_timestamp=db_cutoff)
|
||
|
||
assert deleted == 5
|
||
count = await sequence.db_count_records()
|
||
assert count == 5
|
||
|
||
# Verify the boundary record (hours=5) was NOT deleted
|
||
await _reset_sequence_state(sequence)
|
||
await sequence.db_load_records()
|
||
assert any(r.date_time == cutoff for r in sequence.records)
|
||
|
||
async def test_db_vacuum_no_argument(self, async_database_instance, config_eos):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
record = SampleDataRecord(date_time=base_time, temperature=20.0)
|
||
await sequence.db_insert_record(record)
|
||
await sequence.db_save_records()
|
||
|
||
config_eos.database.keep_duration_h = None
|
||
deleted = await sequence.db_vacuum()
|
||
assert deleted == 0
|
||
|
||
config_eos.database.keep_duration_h = 0
|
||
deleted = await sequence.db_vacuum()
|
||
assert deleted == 1
|
||
|
||
async def test_db_vacuum_keep_hours_zero_deletes_all(self, async_database_instance):
|
||
"""keep_hours=0 should delete all records."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(5):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
deleted = await sequence.db_vacuum(keep_hours=0)
|
||
assert deleted == 5
|
||
count = await sequence.db_count_records()
|
||
assert count == 0
|
||
|
||
async def test_db_get_stats(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
stats = await sequence.db_get_stats()
|
||
|
||
assert stats["enabled"] is True
|
||
assert "backend" in stats
|
||
assert "path" in stats
|
||
assert "memory_records" in stats
|
||
assert "total_records" in stats
|
||
assert "compression_enabled" in stats
|
||
assert "timestamp_range" in stats
|
||
assert stats["timestamp_range"]["min"] == "None"
|
||
assert stats["timestamp_range"]["max"] == "None"
|
||
|
||
async def test_db_get_stats_disabled(self, config_eos):
|
||
config_eos.database.provider = None
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
stats = await sequence.db_get_stats()
|
||
assert stats == {"enabled": False}
|
||
|
||
async def test_lazy_load_phase_none_to_initial(self, async_database_instance):
|
||
"""Phase transitions from NONE to INITIAL when a range is loaded via ensure_loaded."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE
|
||
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
|
||
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|
||
|
||
async def test_lazy_load_phase_initial_to_full(self, async_database_instance):
|
||
"""Phase transitions from INITIAL to FULL when iterate is called without range."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Load partial range → INITIAL
|
||
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|
||
|
||
# Iterate without range → escalates to FULL
|
||
records = [record async for record in sequence.db_iterate_records()]
|
||
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL
|
||
|
||
async def test_range_covered_skips_redundant_load(self, async_database_instance):
|
||
"""_db_range_covered prevents a second DB query for the same range."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(10):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
|
||
# Loaded range is now set
|
||
assert sequence._db_loaded_range is not None
|
||
assert sequence._db_range_covered(db_start, db_end) is True
|
||
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=20))
|
||
assert sequence._db_range_covered(db_start, db_end) is False
|
||
|
||
async def test_loaded_range_not_clobbered_by_expansion(self, async_database_instance):
|
||
"""Expanding left or right must not narrow the tracked loaded range."""
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
for i in range(24):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
||
)
|
||
await sequence.db_save_records()
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Initial window: hours 8–16
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
|
||
assert sequence._db_loaded_range is not None
|
||
initial_start, initial_end = sequence._db_loaded_range
|
||
assert initial_start is not None
|
||
assert initial_end is not None
|
||
|
||
# Expand left: load hours 4–8
|
||
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
|
||
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
|
||
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
||
|
||
assert sequence._db_loaded_range is not None
|
||
expanded_start, expanded_end = sequence._db_loaded_range
|
||
assert expanded_start is not None
|
||
assert expanded_end is not None
|
||
|
||
# Left boundary must have moved left; right must not have shrunk
|
||
assert expanded_start <= initial_start
|
||
assert expanded_end >= initial_end
|
||
|
||
async def test_duplicate_insert_raises(self, async_database_instance):
|
||
sequence = SampleDataSequence()
|
||
await _reset_sequence_state(sequence)
|
||
dt = to_datetime("2024-01-01T00:00:00Z")
|
||
|
||
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=1.0))
|
||
with pytest.raises(ValueError, match="Duplicate timestamp"):
|
||
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=2.0))
|
||
|
||
async def test_metadata_round_trip(self, async_database_instance):
|
||
"""Metadata can be saved and loaded back correctly."""
|
||
sequence = SampleDataSequence()
|
||
|
||
await _clear_sequence_state(sequence)
|
||
assert sequence._db_metadata is None
|
||
|
||
await _reset_sequence_state(sequence)
|
||
assert sequence._db_metadata is not None
|
||
created = sequence._db_metadata["created"]
|
||
assert sequence._db_metadata["version"] == 1
|
||
|
||
await _reset_sequence_state(sequence)
|
||
assert sequence._db_metadata is not None
|
||
assert sequence._db_metadata["created"] == created
|
||
assert sequence._db_metadata["version"] == 1
|
||
|
||
async def test_initial_load_window_respected(self, async_database_instance):
|
||
"""db_initial_time_window limits the initial load from DB."""
|
||
|
||
class WindowedSequence(SampleDataSequence):
|
||
def db_namespace(self) -> str:
|
||
return "WindowedSequence"
|
||
|
||
def db_initial_time_window(self) -> Optional[Duration]:
|
||
return to_duration("2 hours")
|
||
|
||
sequence = WindowedSequence()
|
||
await _reset_sequence_state(sequence)
|
||
base_time = to_datetime("2024-01-01T12:00:00Z")
|
||
|
||
# Store 24 hourly records centred on base_time
|
||
for i in range(24):
|
||
await sequence.db_insert_record(
|
||
SampleDataRecord(
|
||
date_time=base_time.subtract(hours=12).add(hours=i),
|
||
temperature=float(i),
|
||
)
|
||
)
|
||
await sequence.db_save_records()
|
||
|
||
await _reset_sequence_state(sequence)
|
||
|
||
# Trigger initial window load centred on base_time
|
||
sequence.config.database.initial_load_window_h = 2
|
||
db_center = DatabaseTimestamp.from_datetime(base_time)
|
||
await sequence._db_load_initial_window(center_timestamp=db_center)
|
||
|
||
# Only records within ±2h of base_time should be in memory
|
||
assert len(sequence.records) <= 5 # at most 4h window = 4–5 records
|
||
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|