Files
EOS/tests/test_dataabcsequencedb.py
T

702 lines
27 KiB
Python
Raw Normal View History

2026-07-15 16:38:53 +02:00
"""Pytest tests for async DataSequence with persistence.
Tests the async DataSequence with database persistence.
"""
from __future__ import annotations
import asyncio
import shutil
import tempfile
import time
from pathlib import Path
from typing import AsyncIterator, Optional, Type
import pytest
import pytest_asyncio
from pydantic import Field
from akkudoktoreos.core.coreabc import get_database
from akkudoktoreos.core.dataabc import DataProvider, DataRecord, DataSequence
from akkudoktoreos.core.database import Database, LMDBDatabase, SQLiteDatabase
from akkudoktoreos.core.databaseabc import (
DatabaseRecordProtocolLoadPhase,
DatabaseTimestamp,
)
from akkudoktoreos.utils.datetimeutil import (
DateTime,
Duration,
to_datetime,
to_duration,
)
# ==================== Test Fixtures ====================
@pytest.fixture
def temp_dir():
"""Create a temporary directory for test databases."""
temp_path = Path(tempfile.mkdtemp())
yield temp_path
shutil.rmtree(temp_path, ignore_errors=True)
@pytest.fixture(params=["LMDB", "SQLite"])
def database_provider(request) -> str:
"""Parametrize all database backend tests."""
return request.param
@pytest_asyncio.fixture
async def async_database_instance(
config_eos,
database_provider: str,
) -> AsyncIterator[Database]:
"""Open a database instance for testing and close it afterwards."""
config_eos.database.compression_level = 6
config_eos.database.provider = database_provider
db = get_database()
await db.open()
assert db.provider_id() == database_provider
assert db.is_open is True
yield db
await db.close()
config_eos.database.provider = None
# ==================== Helpers ====================
async def _clear_sequence_state(sequence) -> None:
"""Clear runtime DB state without re-instantiating the singleton.
Does _NOT_ initialize the DB state.
"""
await sequence.db_delete_records()
try:
sequence._db_metadata = None
await sequence.database().set_metadata(None, namespace=sequence.db_namespace())
except Exception:
# Database may not be available, just skip
pass
try:
del sequence._db_initialized
except Exception:
# May not be set
pass
async def _reset_sequence_state(sequence) -> None:
"""Reset runtime DB state without re-instantiating the singleton."""
try:
sequence.records = []
del sequence._db_initialized
except Exception:
# May not be set
pass
await sequence._db_ensure_initialized()
# Sample Data
class SampleDataRecord(DataRecord):
"""Minimal DataRecord for testing."""
temperature: float = Field(default=0.0)
humidity: float = Field(default=0.0)
pressure: float = Field(default=0.0)
class SampleDataSequence(DataSequence):
"""DataSequence subclass with database support."""
records: list[SampleDataRecord] = Field(default_factory=list)
@classmethod
def record_class(cls) -> Type[SampleDataRecord]:
return SampleDataRecord
def db_namespace(self) -> str:
return "SampleDataSequence"
class SampleDataProvider(DataProvider):
"""DataProvider subclass with database support."""
records: list[SampleDataRecord] = Field(default_factory=list)
@classmethod
def record_class(cls) -> Type[SampleDataRecord]:
return SampleDataRecord
def provider_id(self) -> str:
return "SampleDataProvider"
def enabled(self) -> bool:
return True
async def _update_data(self, force_update: Optional[bool] = False) -> None:
pass
def db_namespace(self) -> str:
return "SampleDataProvider"
# ==================== DatabaseRecordProtocolMixin Tests ====================
@pytest.mark.asyncio
class TestDataSequenceDatabaseProtocol:
"""Tests for DatabaseRecordProtocolMixin via SampleDataSequence."""
async def test_db_enabled_when_db_open(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
assert sequence.db_enabled is True
async def test_db_disabled_when_db_closed(self, config_eos):
config_eos.database.provider = None
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
assert sequence.db_enabled is False
async def test_insert_and_save_records(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
)
# All 10 are dirty/new, none persisted yet
assert len(sequence.records) == 10
assert len(sequence._db_new_timestamps) == 10
saved = await sequence.db_save_records()
assert saved == 10 # 10 inserts + 0 deletes
assert len(sequence._db_dirty_timestamps) == 0
assert len(sequence._db_new_timestamps) == 0
async def test_save_returns_insert_plus_delete_count(self, async_database_instance):
"""db_save_records() return value = saved_inserts + deleted_count."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(5):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
# Persist the 5 records
await sequence.db_save_records()
# Delete 2 of them
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
# Insert 3 new ones
for i in range(10, 13):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
result = await sequence.db_save_records()
# 3 inserts + 2 deletes = 5
assert result == 5
async def test_load_records_from_db(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
)
await sequence.db_save_records()
# Clear memory, then reload from DB
await _reset_sequence_state(sequence)
loaded = await sequence.db_load_records()
assert loaded == 10
assert len(sequence.records) == 10
for i, record in enumerate(sequence.records):
assert record.temperature == 20.0 + i
async def test_load_records_with_range(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Load [hours=3, hours=7) → 4 records (3, 4, 5, 6)
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
loaded = await sequence.db_load_records(start_timestamp=db_start, end_timestamp=db_end)
assert loaded == 4
assert sequence.records[0].temperature == 23.0
assert sequence.records[-1].temperature == 26.0
async def test_iterate_records_triggers_lazy_load(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0 + i)
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# db_iterate_records calls _db_ensure_loaded internally
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
assert len(records) == 3
assert all(base_time.add(hours=2) <= r.date_time < base_time.add(hours=5) for r in records)
async def test_delete_records(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(6):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
)
await sequence.db_save_records()
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=5))
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
assert deleted == 3
# Persist the deletions
await sequence.db_save_records()
await _reset_sequence_state(sequence)
await sequence.db_load_records()
assert len(sequence.records) == 3
async def test_delete_tombstone_prevents_resurrection(self, async_database_instance):
"""Deleted records must not re-appear when db_load_records is called."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(3):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
# Delete middle record
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=1))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
assert deleted == 1
# Do NOT persist yet — tombstone lives only in memory
# Loading should not resurrect the tombstoned record
loaded = await sequence.db_load_records()
assert all(r.date_time != base_time.add(hours=1) for r in sequence.records)
async def test_insert_after_delete_clears_tombstone(self, async_database_instance):
"""Re-inserting a deleted datetime must clear its tombstone."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
dt = base_time.add(hours=5)
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=10.0))
await sequence.db_save_records()
db_start = DatabaseTimestamp.from_datetime(dt)
db_end = sequence._db_timestamp_after(db_start)
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
assert deleted == 1
await sequence.db_save_records()
# Re-insert the same datetime
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=99.0))
assert dt not in sequence._db_deleted_timestamps
await sequence.db_save_records()
await _reset_sequence_state(sequence)
await sequence.db_load_records()
assert any(r.date_time == dt and r.temperature == 99.0 for r in sequence.records)
async def test_db_count_records_memory_only(self):
"""When db is disabled, count reflects memory only."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
# Without a live DB, db_enabled is False
if sequence.db_enabled:
pytest.skip("DB is open; this test requires it to be closed")
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(5):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i)),
mark_dirty=False,
)
count = await sequence.db_count_records()
assert count == 5
async def test_db_count_records_combined(self, async_database_instance):
"""db_count_records = storage + new_unpersisted - pending_deletes."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
# Persist 10 records
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
# Add 3 new unpersisted records
for i in range(10, 13):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
# Delete 2 persisted records (not yet saved)
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
deleted = await sequence.db_delete_records(start_timestamp=db_start, end_timestamp=db_end)
assert deleted == 2
# storage=10, new=3, pending_deletes=2 → expected=11
count = await sequence.db_count_records()
assert count == 11
async def test_db_timestamp_range_empty(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
min_dt, max_dt = await sequence.db_timestamp_range()
assert min_dt is None
assert max_dt is None
async def test_db_timestamp_range_with_records(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for hours in [0, 5, 10]:
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=hours), temperature=20.0)
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
min_dt, max_dt = await sequence.db_timestamp_range()
assert min_dt == DatabaseTimestamp.from_datetime(base_time)
assert max_dt == DatabaseTimestamp.from_datetime(base_time.add(hours=10))
async def test_db_mark_dirty_triggers_save(self, async_database_instance):
"""Marking a record dirty causes it to be re-saved."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
record = SampleDataRecord(date_time=base_time, temperature=20.0)
await sequence.db_insert_record(record)
await sequence.db_save_records()
# Mutate and mark dirty
record.temperature = 99.0
await sequence.db_mark_dirty_record(record)
await sequence.db_save_records()
# Reload and verify update was persisted
await _reset_sequence_state(sequence)
await sequence.db_load_records()
assert sequence.records[0].temperature == 99.0
async def test_db_vacuum_keep_hours(self, async_database_instance):
"""db_vacuum(keep_hours=N) retains only the last N hours of records."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
# 240 hourly records = 10 days
for i in range(240):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=20.0)
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
keep_hours = 5 * 24 # keep last 5 days
deleted = await sequence.db_vacuum(keep_hours=keep_hours)
assert deleted == 240 - keep_hours
count = await sequence.db_count_records()
assert count == keep_hours
async def test_db_vacuum_keep_timestamp(self, async_database_instance):
"""db_vacuum(keep_timestamp=T) deletes everything before T (exclusive)."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Keep from hours=5 onward — delete [0, 5), i.e. 5 records
cutoff = base_time.add(hours=5)
db_cutoff = DatabaseTimestamp.from_datetime(cutoff)
deleted = await sequence.db_vacuum(keep_timestamp=db_cutoff)
assert deleted == 5
count = await sequence.db_count_records()
assert count == 5
# Verify the boundary record (hours=5) was NOT deleted
await _reset_sequence_state(sequence)
await sequence.db_load_records()
assert any(r.date_time == cutoff for r in sequence.records)
async def test_db_vacuum_no_argument(self, async_database_instance, config_eos):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
record = SampleDataRecord(date_time=base_time, temperature=20.0)
await sequence.db_insert_record(record)
await sequence.db_save_records()
config_eos.database.keep_duration_h = None
deleted = await sequence.db_vacuum()
assert deleted == 0
config_eos.database.keep_duration_h = 0
deleted = await sequence.db_vacuum()
assert deleted == 1
async def test_db_vacuum_keep_hours_zero_deletes_all(self, async_database_instance):
"""keep_hours=0 should delete all records."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(5):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
deleted = await sequence.db_vacuum(keep_hours=0)
assert deleted == 5
count = await sequence.db_count_records()
assert count == 0
async def test_db_get_stats(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
stats = await sequence.db_get_stats()
assert stats["enabled"] is True
assert "backend" in stats
assert "path" in stats
assert "memory_records" in stats
assert "total_records" in stats
assert "compression_enabled" in stats
assert "timestamp_range" in stats
assert stats["timestamp_range"]["min"] == "None"
assert stats["timestamp_range"]["max"] == "None"
async def test_db_get_stats_disabled(self, config_eos):
config_eos.database.provider = None
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
stats = await sequence.db_get_stats()
assert stats == {"enabled": False}
async def test_lazy_load_phase_none_to_initial(self, async_database_instance):
"""Phase transitions from NONE to INITIAL when a range is loaded via ensure_loaded."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.NONE
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
async def test_lazy_load_phase_initial_to_full(self, async_database_instance):
"""Phase transitions from INITIAL to FULL when iterate is called without range."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Load partial range → INITIAL
# Use db_iterate_records — it calls _db_ensure_loaded which owns phase transitions
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=3))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=7))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL
# Iterate without range → escalates to FULL
records = [record async for record in sequence.db_iterate_records()]
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.FULL
async def test_range_covered_skips_redundant_load(self, async_database_instance):
"""_db_range_covered prevents a second DB query for the same range."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(10):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=2))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
# Loaded range is now set
assert sequence._db_loaded_range is not None
assert sequence._db_range_covered(db_start, db_end) is True
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=0))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=20))
assert sequence._db_range_covered(db_start, db_end) is False
async def test_loaded_range_not_clobbered_by_expansion(self, async_database_instance):
"""Expanding left or right must not narrow the tracked loaded range."""
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T00:00:00Z")
for i in range(24):
await sequence.db_insert_record(
SampleDataRecord(date_time=base_time.add(hours=i), temperature=float(i))
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Initial window: hours 816
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=8))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
assert sequence._db_loaded_range is not None
initial_start, initial_end = sequence._db_loaded_range
assert initial_start is not None
assert initial_end is not None
# Expand left: load hours 48
db_start = DatabaseTimestamp.from_datetime(base_time.add(hours=4))
db_end = DatabaseTimestamp.from_datetime(base_time.add(hours=16))
records = [record async for record in sequence.db_iterate_records(start_timestamp=db_start, end_timestamp=db_end)]
assert sequence._db_loaded_range is not None
expanded_start, expanded_end = sequence._db_loaded_range
assert expanded_start is not None
assert expanded_end is not None
# Left boundary must have moved left; right must not have shrunk
assert expanded_start <= initial_start
assert expanded_end >= initial_end
async def test_duplicate_insert_raises(self, async_database_instance):
sequence = SampleDataSequence()
await _reset_sequence_state(sequence)
dt = to_datetime("2024-01-01T00:00:00Z")
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=1.0))
with pytest.raises(ValueError, match="Duplicate timestamp"):
await sequence.db_insert_record(SampleDataRecord(date_time=dt, temperature=2.0))
async def test_metadata_round_trip(self, async_database_instance):
"""Metadata can be saved and loaded back correctly."""
sequence = SampleDataSequence()
await _clear_sequence_state(sequence)
assert sequence._db_metadata is None
await _reset_sequence_state(sequence)
assert sequence._db_metadata is not None
created = sequence._db_metadata["created"]
assert sequence._db_metadata["version"] == 1
await _reset_sequence_state(sequence)
assert sequence._db_metadata is not None
assert sequence._db_metadata["created"] == created
assert sequence._db_metadata["version"] == 1
async def test_initial_load_window_respected(self, async_database_instance):
"""db_initial_time_window limits the initial load from DB."""
class WindowedSequence(SampleDataSequence):
def db_namespace(self) -> str:
return "WindowedSequence"
def db_initial_time_window(self) -> Optional[Duration]:
return to_duration("2 hours")
sequence = WindowedSequence()
await _reset_sequence_state(sequence)
base_time = to_datetime("2024-01-01T12:00:00Z")
# Store 24 hourly records centred on base_time
for i in range(24):
await sequence.db_insert_record(
SampleDataRecord(
date_time=base_time.subtract(hours=12).add(hours=i),
temperature=float(i),
)
)
await sequence.db_save_records()
await _reset_sequence_state(sequence)
# Trigger initial window load centred on base_time
sequence.config.database.initial_load_window_h = 2
db_center = DatabaseTimestamp.from_datetime(base_time)
await sequence._db_load_initial_window(center_timestamp=db_center)
# Only records within ±2h of base_time should be in memory
assert len(sequence.records) <= 5 # at most 4h window = 45 records
assert sequence._db_load_phase is DatabaseRecordProtocolLoadPhase.INITIAL