mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2026-07-20 00:38:12 +00:00
702 lines
27 KiB
Python
702 lines
27 KiB
Python
|
|
"""Pytest tests for async DataSequence with persistence.
|
|||
|
|
|
|||
|
|
Tests the async DataSequence with database persistence.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import shutil
|
|||
|
|
import tempfile
|
|||
|
|
import time
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import AsyncIterator, Optional, Type
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
import pytest_asyncio
|
|||
|
|
from pydantic import Field
|
|||
|
|
|
|||
|
|
from akkudoktoreos.core.coreabc import get_database
|
|||
|
|
from akkudoktoreos.core.dataabc import DataProvider, DataRecord, DataSequence
|
|||
|
|
from akkudoktoreos.core.database import Database, LMDBDatabase, SQLiteDatabase
|
|||
|
|
from akkudoktoreos.core.databaseabc import (
|
|||
|
|
DatabaseRecordProtocolLoadPhase,
|
|||
|
|
DatabaseTimestamp,
|
|||
|
|
)
|
|||
|
|
from akkudoktoreos.utils.datetimeutil import (
|
|||
|
|
DateTime,
|
|||
|
|
Duration,
|
|||
|
|
to_datetime,
|
|||
|
|
to_duration,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# ==================== Test Fixtures ====================
|
|||
|
|
|
|||
|
|
@pytest.fixture
|
|||
|
|
def temp_dir():
|
|||
|
|
"""Create a temporary directory for test databases."""
|
|||
|
|
temp_path = Path(tempfile.mkdtemp())
|
|||
|
|
yield temp_path
|
|||
|
|
shutil.rmtree(temp_path, ignore_errors=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.fixture(params=["LMDB", "SQLite"])
|
|||
|
|
def database_provider(request) -> str:
|
|||
|
|
"""Parametrize all database backend tests."""
|
|||
|
|
return request.param
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest_asyncio.fixture
|
|||
|
|
async def async_database_instance(
|
|||
|
|
config_eos,
|
|||
|
|
database_provider: str,
|
|||
|
|
) -> AsyncIterator[Database]:
|
|||
|
|
"""Open a database instance for testing and close it afterwards."""
|
|||
|
|
config_eos.database.compression_level = 6
|
|||
|
|
config_eos.database.provider = database_provider
|
|||
|
|
|
|||
|
|
db = get_database()
|
|||
|
|
|
|||
|
|
await db.open()
|
|||
|
|
|
|||
|
|
assert db.provider_id() == database_provider
|
|||
|
|
assert db.is_open is True
|
|||
|
|
|
|||
|
|
yield db
|
|||
|
|
|
|||
|
|
await db.close()
|
|||
|
|
|
|||
|
|
config_eos.database.provider = None
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ==================== Helpers ====================
|
|||
|
|
|
|||
|
|
async def _clear_sequence_state(sequence) -> None:
|
|||
|
|
"""Clear runtime DB state without re-instantiating the singleton.
|
|||
|
|
|
|||
|
|
Does _NOT_ initialize the DB state.
|
|||
|
|
"""
|
|||
|
|
await sequence.db_delete_records()
|
|||
|
|
try:
|
|||
|
|
sequence._db_metadata = None
|
|||
|
|
await sequence.database().set_metadata(None, namespace=sequence.db_namespace())
|
|||
|
|
except Exception:
|
|||
|
|
# Database may not be available, just skip
|
|||
|
|
pass
|
|||
|
|
try:
|
|||
|
|
del sequence._db_initialized
|
|||
|
|
except Exception:
|
|||
|
|
# May not be set
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def _reset_sequence_state(sequence) -> None:
|
|||
|
|
"""Reset runtime DB state without re-instantiating the singleton."""
|
|||
|
|
try:
|
|||
|
|
sequence.records = []
|
|||
|
|
del sequence._db_initialized
|
|||
|
|
except Exception:
|
|||
|
|
# May not be set
|
|||
|
|
pass
|
|||
|
|
await sequence._db_ensure_initialized()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# Sample Data
|
|||
|
|
|
|||
|
|
class SampleDataRecord(DataRecord):
|
|||
|
|
"""Minimal DataRecord for testing."""
|
|||
|
|
temperature: float = Field(default=0.0)
|
|||
|
|
humidity: float = Field(default=0.0)
|
|||
|
|
pressure: float = Field(default=0.0)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class SampleDataSequence(DataSequence):
|
|||
|
|
"""DataSequence subclass with database support."""
|
|||
|
|
records: list[SampleDataRecord] = Field(default_factory=list)
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def record_class(cls) -> Type[SampleDataRecord]:
|
|||
|
|
return SampleDataRecord
|
|||
|
|
|
|||
|
|
def db_namespace(self) -> str:
|
|||
|
|
return "SampleDataSequence"
|
|||
|
|
|
|||
|
|
|
|||
|
|
class SampleDataProvider(DataProvider):
|
|||
|
|
"""DataProvider subclass with database support."""
|
|||
|
|
records: list[SampleDataRecord] = Field(default_factory=list)
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def record_class(cls) -> Type[SampleDataRecord]:
|
|||
|
|
return SampleDataRecord
|
|||
|
|
|
|||
|
|
def provider_id(self) -> str:
|
|||
|
|
return "SampleDataProvider"
|
|||
|
|
|
|||
|
|
def enabled(self) -> bool:
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
async def _update_data(self, force_update: Optional[bool] = False) -> None:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
def db_namespace(self) -> str:
|
|||
|
|
return "SampleDataProvider"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ==================== DatabaseRecordProtocolMixin Tests ====================
|
|||
|
|
|
|||
|
|
@pytest.mark.asyncio
|
|||
|
|
class TestDataSequenceDatabaseProtocol:
|
|||
|
|
"""Tests for DatabaseRecordProtocolMixin via SampleDataSequence."""
|
|||
|
|
|
|||
|
|
async def test_db_enabled_when_db_open(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
assert sequence.db_enabled is True
|
|||
|
|
|
|||
|
|
async def test_db_disabled_when_db_closed(self, config_eos):
|
|||
|
|
config_eos.database.provider = None
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
assert sequence.db_enabled is False
|
|||
|
|
|
|||
|
|
async def test_insert_and_save_records(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# All 10 are dirty/new, none persisted yet
|
|||
|
|
assert len(sequence.records) == 10
|
|||
|
|
assert len(sequence._db_new_timestamps) == 10
|
|||
|
|
|
|||
|
|
saved = await sequence.db_save_records()
|
|||
|
|
assert saved == 10 # 10 inserts + 0 deletes
|
|||
|
|
assert len(sequence._db_dirty_timestamps) == 0
|
|||
|
|
assert len(sequence._db_new_timestamps) == 0
|
|||
|
|
|
|||
|
|
async def test_save_returns_insert_plus_delete_count(self, async_database_instance):
|
|||
|
|
"""db_save_records() return value = saved_inserts + deleted_count."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(5):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
# Persist the 5 records
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Delete 2 of them
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
|
|||
|
|
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
# Insert 3 new ones
|
|||
|
|
for i in range(10, 13):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
result = await sequence.db_save_records()
|
|||
|
|
# 3 inserts + 2 deletes = 5
|
|||
|
|
assert result == 5
|
|||
|
|
|
|||
|
|
async def test_load_records_from_db(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Clear memory, then reload from DB
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
loaded = await sequence.db_load_records()
|
|||
|
|
|
|||
|
|
assert loaded == 10
|
|||
|
|
assert len(sequence.records) == 10
|
|||
|
|
for i, record in enumerate(sequence.records):
|
|||
|
|
assert record.temperature == 20.0 + i
|
|||
|
|
|
|||
|
|
async def test_load_records_with_range(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Load [hours=3, hours=7) → 4 records (3, 4, 5, 6)
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
|||
|
|
loaded = await sequence.db_load_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
assert loaded == 4
|
|||
|
|
assert sequence.records[0].temperature == 23.0
|
|||
|
|
assert sequence.records[-1].temperature == 26.0
|
|||
|
|
|
|||
|
|
async def test_iterate_records_triggers_lazy_load(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# db_iterate_records calls _db_ensure_loaded internally
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
assert len(records) == 3
|
|||
|
|
assert all(base_time.add(hours=2) <= r.date_time < base_time.add(hours=5) for r in records)
|
|||
|
|
|
|||
|
|
async def test_delete_records(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(6):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
|
|||
|
|
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
assert deleted == 3
|
|||
|
|
|
|||
|
|
# Persist the deletions
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
await sequence.db_load_records()
|
|||
|
|
assert len(sequence.records) == 3
|
|||
|
|
|
|||
|
|
async def test_delete_tombstone_prevents_resurrection(self, async_database_instance):
|
|||
|
|
"""Deleted records must not re-appear when db_load_records is called."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(3):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Delete middle record
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=1))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
assert deleted == 1
|
|||
|
|
|
|||
|
|
# Do NOT persist yet — tombstone lives only in memory
|
|||
|
|
# Loading should not resurrect the tombstoned record
|
|||
|
|
loaded = await sequence.db_load_records()
|
|||
|
|
assert all(r.date_time != base_time.add(hours=1) for r in sequence.records)
|
|||
|
|
|
|||
|
|
async def test_insert_after_delete_clears_tombstone(self, async_database_instance):
|
|||
|
|
"""Re-inserting a deleted datetime must clear its tombstone."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
dt = base_time.add(hours=5)
|
|||
|
|
|
|||
|
|
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=10.0))
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(dt)
|
|||
|
|
db_end = sequence._db_timestamp_after(db_start)
|
|||
|
|
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
assert deleted == 1
|
|||
|
|
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Re-insert the same datetime
|
|||
|
|
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=99.0))
|
|||
|
|
assert dt not in sequence._db_deleted_timestamps
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
await sequence.db_load_records()
|
|||
|
|
assert any(r.date_time == dt and r.temperature == 99.0 for r in sequence.records)
|
|||
|
|
|
|||
|
|
async def test_db_count_records_memory_only(self):
|
|||
|
|
"""When db is disabled, count reflects memory only."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Without a live DB, db_enabled is False
|
|||
|
|
if sequence.db_enabled:
|
|||
|
|
pytest.skip("DB is open; this test requires it to be closed")
|
|||
|
|
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
for i in range(5):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)),
|
|||
|
|
mark_dirty=False,
|
|||
|
|
)
|
|||
|
|
count = await sequence.db_count_records()
|
|||
|
|
assert count == 5
|
|||
|
|
|
|||
|
|
async def test_db_count_records_combined(self, async_database_instance):
|
|||
|
|
"""db_count_records = storage + new_unpersisted - pending_deletes."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
# Persist 10 records
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Add 3 new unpersisted records
|
|||
|
|
for i in range(10, 13):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# Delete 2 persisted records (not yet saved)
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
|
|||
|
|
assert deleted == 2
|
|||
|
|
|
|||
|
|
# storage=10, new=3, pending_deletes=2 → expected=11
|
|||
|
|
count = await sequence.db_count_records()
|
|||
|
|
assert count == 11
|
|||
|
|
|
|||
|
|
async def test_db_timestamp_range_empty(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
min_dt, max_dt = await sequence.db_timestamp_range()
|
|||
|
|
assert min_dt is None
|
|||
|
|
assert max_dt is None
|
|||
|
|
|
|||
|
|
async def test_db_timestamp_range_with_records(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for hours in [0, 5, 10]:
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=hours), temperature=20.0)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
min_dt, max_dt = await sequence.db_timestamp_range()
|
|||
|
|
assert min_dt == DatabaseTimestamp.from_datetime(base_time)
|
|||
|
|
assert max_dt == DatabaseTimestamp.from_datetime(base_time.add(hours=10))
|
|||
|
|
|
|||
|
|
async def test_db_mark_dirty_triggers_save(self, async_database_instance):
|
|||
|
|
"""Marking a record dirty causes it to be re-saved."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
record = SampleDataRecord(date_time=base_time, temperature=20.0)
|
|||
|
|
await sequence.db_insert_record(record)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Mutate and mark dirty
|
|||
|
|
record.temperature = 99.0
|
|||
|
|
await sequence.db_mark_dirty_record(record)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
# Reload and verify update was persisted
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
await sequence.db_load_records()
|
|||
|
|
assert sequence.records[0].temperature == 99.0
|
|||
|
|
|
|||
|
|
async def test_db_vacuum_keep_hours(self, async_database_instance):
|
|||
|
|
"""db_vacuum(keep_hours=N) retains only the last N hours of records."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
# 240 hourly records = 10 days
|
|||
|
|
for i in range(240):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
keep_hours = 5 * 24 # keep last 5 days
|
|||
|
|
deleted = await sequence.db_vacuum(keep_hours=keep_hours)
|
|||
|
|
|
|||
|
|
assert deleted == 240 - keep_hours
|
|||
|
|
count = await sequence.db_count_records()
|
|||
|
|
assert count == keep_hours
|
|||
|
|
|
|||
|
|
async def test_db_vacuum_keep_timestamp(self, async_database_instance):
|
|||
|
|
"""db_vacuum(keep_timestamp=T) deletes everything before T (exclusive)."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Keep from hours=5 onward — delete [0, 5), i.e. 5 records
|
|||
|
|
cutoff = base_time.add(hours=5)
|
|||
|
|
db_cutoff = DatabaseTimestamp.from_datetime(cutoff)
|
|||
|
|
deleted = await sequence.db_vacuum(keep_timestamp=db_cutoff)
|
|||
|
|
|
|||
|
|
assert deleted == 5
|
|||
|
|
count = await sequence.db_count_records()
|
|||
|
|
assert count == 5
|
|||
|
|
|
|||
|
|
# Verify the boundary record (hours=5) was NOT deleted
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
await sequence.db_load_records()
|
|||
|
|
assert any(r.date_time == cutoff for r in sequence.records)
|
|||
|
|
|
|||
|
|
async def test_db_vacuum_no_argument(self, async_database_instance, config_eos):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
record = SampleDataRecord(date_time=base_time, temperature=20.0)
|
|||
|
|
await sequence.db_insert_record(record)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
config_eos.database.keep_duration_h = None
|
|||
|
|
deleted = await sequence.db_vacuum()
|
|||
|
|
assert deleted == 0
|
|||
|
|
|
|||
|
|
config_eos.database.keep_duration_h = 0
|
|||
|
|
deleted = await sequence.db_vacuum()
|
|||
|
|
assert deleted == 1
|
|||
|
|
|
|||
|
|
async def test_db_vacuum_keep_hours_zero_deletes_all(self, async_database_instance):
|
|||
|
|
"""keep_hours=0 should delete all records."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(5):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
deleted = await sequence.db_vacuum(keep_hours=0)
|
|||
|
|
assert deleted == 5
|
|||
|
|
count = await sequence.db_count_records()
|
|||
|
|
assert count == 0
|
|||
|
|
|
|||
|
|
async def test_db_get_stats(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
stats = await sequence.db_get_stats()
|
|||
|
|
|
|||
|
|
assert stats["enabled"] is True
|
|||
|
|
assert "backend" in stats
|
|||
|
|
assert "path" in stats
|
|||
|
|
assert "memory_records" in stats
|
|||
|
|
assert "total_records" in stats
|
|||
|
|
assert "compression_enabled" in stats
|
|||
|
|
assert "timestamp_range" in stats
|
|||
|
|
assert stats["timestamp_range"]["min"] == "None"
|
|||
|
|
assert stats["timestamp_range"]["max"] == "None"
|
|||
|
|
|
|||
|
|
async def test_db_get_stats_disabled(self, config_eos):
|
|||
|
|
config_eos.database.provider = None
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
stats = await sequence.db_get_stats()
|
|||
|
|
assert stats == {"enabled": False}
|
|||
|
|
|
|||
|
|
async def test_lazy_load_phase_none_to_initial(self, async_database_instance):
|
|||
|
|
"""Phase transitions from NONE to INITIAL when a range is loaded via ensure_loaded."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE
|
|||
|
|
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
|
|||
|
|
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|
|||
|
|
|
|||
|
|
async def test_lazy_load_phase_initial_to_full(self, async_database_instance):
|
|||
|
|
"""Phase transitions from INITIAL to FULL when iterate is called without range."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Load partial range → INITIAL
|
|||
|
|
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|
|||
|
|
|
|||
|
|
# Iterate without range → escalates to FULL
|
|||
|
|
records = [record async for record in sequence.db_iterate_records()]
|
|||
|
|
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL
|
|||
|
|
|
|||
|
|
async def test_range_covered_skips_redundant_load(self, async_database_instance):
|
|||
|
|
"""_db_range_covered prevents a second DB query for the same range."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(10):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
|
|||
|
|
# Loaded range is now set
|
|||
|
|
assert sequence._db_loaded_range is not None
|
|||
|
|
assert sequence._db_range_covered(db_start, db_end) is True
|
|||
|
|
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=20))
|
|||
|
|
assert sequence._db_range_covered(db_start, db_end) is False
|
|||
|
|
|
|||
|
|
async def test_loaded_range_not_clobbered_by_expansion(self, async_database_instance):
|
|||
|
|
"""Expanding left or right must not narrow the tracked loaded range."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
for i in range(24):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Initial window: hours 8–16
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
|
|||
|
|
assert sequence._db_loaded_range is not None
|
|||
|
|
initial_start, initial_end = sequence._db_loaded_range
|
|||
|
|
assert initial_start is not None
|
|||
|
|
assert initial_end is not None
|
|||
|
|
|
|||
|
|
# Expand left: load hours 4–8
|
|||
|
|
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
|
|||
|
|
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
|
|||
|
|
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
|
|||
|
|
|
|||
|
|
assert sequence._db_loaded_range is not None
|
|||
|
|
expanded_start, expanded_end = sequence._db_loaded_range
|
|||
|
|
assert expanded_start is not None
|
|||
|
|
assert expanded_end is not None
|
|||
|
|
|
|||
|
|
# Left boundary must have moved left; right must not have shrunk
|
|||
|
|
assert expanded_start <= initial_start
|
|||
|
|
assert expanded_end >= initial_end
|
|||
|
|
|
|||
|
|
async def test_duplicate_insert_raises(self, async_database_instance):
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
dt = to_datetime("2024-01-01T00:00:00Z")
|
|||
|
|
|
|||
|
|
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=1.0))
|
|||
|
|
with pytest.raises(ValueError, match="Duplicate timestamp"):
|
|||
|
|
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=2.0))
|
|||
|
|
|
|||
|
|
async def test_metadata_round_trip(self, async_database_instance):
|
|||
|
|
"""Metadata can be saved and loaded back correctly."""
|
|||
|
|
sequence = SampleDataSequence()
|
|||
|
|
|
|||
|
|
await _clear_sequence_state(sequence)
|
|||
|
|
assert sequence._db_metadata is None
|
|||
|
|
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
assert sequence._db_metadata is not None
|
|||
|
|
created = sequence._db_metadata["created"]
|
|||
|
|
assert sequence._db_metadata["version"] == 1
|
|||
|
|
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
assert sequence._db_metadata is not None
|
|||
|
|
assert sequence._db_metadata["created"] == created
|
|||
|
|
assert sequence._db_metadata["version"] == 1
|
|||
|
|
|
|||
|
|
async def test_initial_load_window_respected(self, async_database_instance):
|
|||
|
|
"""db_initial_time_window limits the initial load from DB."""
|
|||
|
|
|
|||
|
|
class WindowedSequence(SampleDataSequence):
|
|||
|
|
def db_namespace(self) -> str:
|
|||
|
|
return "WindowedSequence"
|
|||
|
|
|
|||
|
|
def db_initial_time_window(self) -> Optional[Duration]:
|
|||
|
|
return to_duration("2 hours")
|
|||
|
|
|
|||
|
|
sequence = WindowedSequence()
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
base_time = to_datetime("2024-01-01T12:00:00Z")
|
|||
|
|
|
|||
|
|
# Store 24 hourly records centred on base_time
|
|||
|
|
for i in range(24):
|
|||
|
|
await sequence.db_insert_record(
|
|||
|
|
SampleDataRecord(
|
|||
|
|
date_time=base_time.subtract(hours=12).add(hours=i),
|
|||
|
|
temperature=float(i),
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
await sequence.db_save_records()
|
|||
|
|
|
|||
|
|
await _reset_sequence_state(sequence)
|
|||
|
|
|
|||
|
|
# Trigger initial window load centred on base_time
|
|||
|
|
sequence.config.database.initial_load_window_h = 2
|
|||
|
|
db_center = DatabaseTimestamp.from_datetime(base_time)
|
|||
|
|
await sequence._db_load_initial_window(center_timestamp=db_center)
|
|||
|
|
|
|||
|
|
# Only records within ±2h of base_time should be in memory
|
|||
|
|
assert len(sequence.records) <= 5 # at most 4h window = 4–5 records
|
|||
|
|
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
|