From 2ed04d557355b2f7a022a5a533860102bb70868f Mon Sep 17 00:00:00 2001 From: Bobby Noelte Date: Mon, 20 Jul 2026 12:59:08 +0200 Subject: [PATCH] fix: db compaction run (#1156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These fixes were taken from https://github.com/arneman/EOS/tree/refactor/economic-objective. 1. Fix compaction job registration bug in eos.py: - compact_eos_database was registered with save_eos_database function - Now correctly calls compact_eos_database() - Root cause: compaction/vacuum never ran, allowing records to grow unbounded 2. Optimize db_iterate_records() from O(n) to O(log n): - Previous: linear scan from index 0 for every key_to_array() call - Now: uses bisect_left on _db_sorted_timestamps to skip to start position - Critical for large datasets: 9400 measurement records → 94s/call overhead → 5 calls/optimization 3. Reduce compaction_interval_sec default from 604800s (7 days) to 3600s (1 hour): - With 7-day interval: 65000 records accumulate before first cleanup - Bridge pushes ~1 record/9s (grid_export_emr) - Hourly compaction + 2h data window → stable ~950 records Signed-off-by: Bobby Noelte --- .env | 2 +- config.yaml | 2 +- docs/_generated/configdatabase.md | 6 +++--- docs/_generated/configexample.md | 2 +- docs/_generated/openapi.md | 2 +- openapi.json | 10 +++++----- src/akkudoktoreos/core/database.py | 4 ++-- src/akkudoktoreos/core/databaseabc.py | 11 +++++++++-- src/akkudoktoreos/server/eos.py | 4 +++- tests/testdata/docs/_generated/configdatabase.md | 6 +++--- tests/testdata/docs/_generated/configexample.md | 2 +- 11 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.env b/.env index b814f002..e84f7b08 100644 --- a/.env +++ b/.env @@ -11,7 +11,7 @@ DOCKER_COMPOSE_DATA_DIR=${HOME}/.local/share/net.akkudoktor.eos # ----------------------------------------------------------------------------- # Image / build # ----------------------------------------------------------------------------- -VERSION=0.3.0.dev2607171544999797 +VERSION=0.3.0.dev2607200446185246 PYTHON_VERSION=3.13.9 # ----------------------------------------------------------------------------- diff --git a/config.yaml b/config.yaml index 1b97772e..6e082a01 100644 --- a/config.yaml +++ b/config.yaml @@ -6,7 +6,7 @@ # the root directory (no add-on folder as usual). name: "Akkudoktor-EOS" -version: "0.3.0.dev2607171544999797" +version: "0.3.0.dev2607200446185246" slug: "eos" description: "Akkudoktor-EOS add-on" url: "https://github.com/Akkudoktor-EOS/EOS" diff --git a/docs/_generated/configdatabase.md b/docs/_generated/configdatabase.md index 92c6ec49..3a46194d 100644 --- a/docs/_generated/configdatabase.md +++ b/docs/_generated/configdatabase.md @@ -16,7 +16,7 @@ Attributes: | autosave_interval_sec | `EOS_DATABASE__AUTOSAVE_INTERVAL_SEC` | `Optional[int]` | `rw` | `10` | Automatic saving interval [seconds]. Set to None to disable automatic saving. | | batch_size | `EOS_DATABASE__BATCH_SIZE` | `int` | `rw` | `100` | Number of records to process in batch operations. | -| compaction_interval_sec | `EOS_DATABASE__COMPACTION_INTERVAL_SEC` | `Optional[int]` | `rw` | `604800` | Interval in between automatic tiered compaction runs [seconds]. +| compaction_interval_sec | `EOS_DATABASE__COMPACTION_INTERVAL_SEC` | `Optional[int]` | `rw` | `3600` | Interval in between automatic tiered compaction runs [seconds]. Compaction downsamples old records to reduce storage while retaining coverage. Set to None to disable automatic compaction. | | compression_level | `EOS_DATABASE__COMPRESSION_LEVEL` | `int` | `rw` | `9` | Compression level for database record data. | | initial_load_window_h | `EOS_DATABASE__INITIAL_LOAD_WINDOW_H` | `Optional[int]` | `rw` | `None` | Specifies the default duration of the initial load window when loading records from the database, in hours. If set to None, the full available range is loaded. The window is centered around the current time by default, unless a different center time is specified. Different database namespaces may define their own default windows. | @@ -40,7 +40,7 @@ None indicates forever. Database namespaces may have diverging definitions. | "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100 } } @@ -60,7 +60,7 @@ None indicates forever. Database namespaces may have diverging definitions. | "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100, "providers": [ "LMDB", diff --git a/docs/_generated/configexample.md b/docs/_generated/configexample.md index aaad1b7a..31f6b665 100644 --- a/docs/_generated/configexample.md +++ b/docs/_generated/configexample.md @@ -32,7 +32,7 @@ "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100 }, "devices": { diff --git a/docs/_generated/openapi.md b/docs/_generated/openapi.md index de1cea46..1640cb52 100644 --- a/docs/_generated/openapi.md +++ b/docs/_generated/openapi.md @@ -1,6 +1,6 @@ # Akkudoktor-EOS -**Version**: `v0.3.0.dev2607181096091018` +**Version**: `v0.3.0.dev2607200446185246` **Description**: This project provides a comprehensive solution for simulating and optimizing an energy system based on renewable energy sources. With a focus on photovoltaic (PV) systems, battery storage (batteries), load management (consumer requirements), heat pumps, electric vehicles, and consideration of electricity price data, this system enables forecasting and optimization of energy flow and costs over a specified period. diff --git a/openapi.json b/openapi.json index c7f9db52..72215515 100644 --- a/openapi.json +++ b/openapi.json @@ -8,7 +8,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.html" }, - "version": "v0.3.0.dev2607181096091018" + "version": "v0.3.0.dev2607200446185246" }, "paths": { "/v1/admin/cache/clear": { @@ -2959,9 +2959,9 @@ ], "title": "Compaction Interval Sec", "description": "Interval in between automatic tiered compaction runs [seconds].\nCompaction downsamples old records to reduce storage while retaining coverage. Set to None to disable automatic compaction.", - "default": 604800, + "default": 3600, "examples": [ - 604800 + 3600 ] }, "batch_size": { @@ -3070,9 +3070,9 @@ ], "title": "Compaction Interval Sec", "description": "Interval in between automatic tiered compaction runs [seconds].\nCompaction downsamples old records to reduce storage while retaining coverage. Set to None to disable automatic compaction.", - "default": 604800, + "default": 3600, "examples": [ - 604800 + 3600 ] }, "batch_size": { diff --git a/src/akkudoktoreos/core/database.py b/src/akkudoktoreos/core/database.py index 69b74b6a..449f34af 100644 --- a/src/akkudoktoreos/core/database.py +++ b/src/akkudoktoreos/core/database.py @@ -109,7 +109,7 @@ class DatabaseCommonSettings(SettingsBaseModel): ) compaction_interval_sec: Optional[int] = Field( - default=7 * 24 * 3600, # weekly + default=3600, # hourly ge=0, json_schema_extra={ "description": ( @@ -117,7 +117,7 @@ class DatabaseCommonSettings(SettingsBaseModel): "Compaction downsamples old records to reduce storage while retaining " "coverage. Set to None to disable automatic compaction." ), - "examples": [604800], # 1 week + "examples": [3600], # 1 hour }, ) diff --git a/src/akkudoktoreos/core/databaseabc.py b/src/akkudoktoreos/core/databaseabc.py index ee5192f0..1052755b 100644 --- a/src/akkudoktoreos/core/databaseabc.py +++ b/src/akkudoktoreos/core/databaseabc.py @@ -1603,7 +1603,14 @@ class DatabaseRecordProtocolMixin( end_timestamp=end_timestamp, ) - for record in self.records: + # Use bisect to find the first record at or after start_timestamp. + # _db_sorted_timestamps and self.records are always kept in parallel + # (same insertion index), so bisect_left gives the correct slice start. + start_idx = 0 + if start_timestamp and not isinstance(start_timestamp, _DatabaseTimestampUnbound): + start_idx = bisect.bisect_left(self._db_sorted_timestamps, start_timestamp) + + for record in self.records[start_idx:]: record_date_time_timestamp = DatabaseTimestamp.from_datetime(record.date_time) if start_timestamp and record_date_time_timestamp < start_timestamp: @@ -1702,7 +1709,7 @@ class DatabaseRecordProtocolMixin( f"Vacuum requested for database '{self.db_namespace()}' but keep limit is infinite." ) return 0 - keep_hours = keep_duration.hours + keep_hours = int(keep_duration.total_seconds() // 3600) if keep_hours is not None: _, db_max = await self.db_timestamp_range() diff --git a/src/akkudoktoreos/server/eos.py b/src/akkudoktoreos/server/eos.py index 3ddc8375..0c97edd8 100755 --- a/src/akkudoktoreos/server/eos.py +++ b/src/akkudoktoreos/server/eos.py @@ -184,7 +184,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: "save_eos_database", save_eos_database, interval_attr="database/autosave_interval_sec" ) manager.register( - "compact_eos_database", save_eos_database, interval_attr="database/compaction_interval_sec" + "compact_eos_database", + compact_eos_database, + interval_attr="database/compaction_interval_sec", ) manager.register("manage_energy", ems_manage_energy, interval_attr="ems/interval") diff --git a/tests/testdata/docs/_generated/configdatabase.md b/tests/testdata/docs/_generated/configdatabase.md index 92c6ec49..3a46194d 100644 --- a/tests/testdata/docs/_generated/configdatabase.md +++ b/tests/testdata/docs/_generated/configdatabase.md @@ -16,7 +16,7 @@ Attributes: | autosave_interval_sec | `EOS_DATABASE__AUTOSAVE_INTERVAL_SEC` | `Optional[int]` | `rw` | `10` | Automatic saving interval [seconds]. Set to None to disable automatic saving. | | batch_size | `EOS_DATABASE__BATCH_SIZE` | `int` | `rw` | `100` | Number of records to process in batch operations. | -| compaction_interval_sec | `EOS_DATABASE__COMPACTION_INTERVAL_SEC` | `Optional[int]` | `rw` | `604800` | Interval in between automatic tiered compaction runs [seconds]. +| compaction_interval_sec | `EOS_DATABASE__COMPACTION_INTERVAL_SEC` | `Optional[int]` | `rw` | `3600` | Interval in between automatic tiered compaction runs [seconds]. Compaction downsamples old records to reduce storage while retaining coverage. Set to None to disable automatic compaction. | | compression_level | `EOS_DATABASE__COMPRESSION_LEVEL` | `int` | `rw` | `9` | Compression level for database record data. | | initial_load_window_h | `EOS_DATABASE__INITIAL_LOAD_WINDOW_H` | `Optional[int]` | `rw` | `None` | Specifies the default duration of the initial load window when loading records from the database, in hours. If set to None, the full available range is loaded. The window is centered around the current time by default, unless a different center time is specified. Different database namespaces may define their own default windows. | @@ -40,7 +40,7 @@ None indicates forever. Database namespaces may have diverging definitions. | "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100 } } @@ -60,7 +60,7 @@ None indicates forever. Database namespaces may have diverging definitions. | "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100, "providers": [ "LMDB", diff --git a/tests/testdata/docs/_generated/configexample.md b/tests/testdata/docs/_generated/configexample.md index aaad1b7a..31f6b665 100644 --- a/tests/testdata/docs/_generated/configexample.md +++ b/tests/testdata/docs/_generated/configexample.md @@ -32,7 +32,7 @@ "initial_load_window_h": 48, "keep_duration_h": 48, "autosave_interval_sec": 5, - "compaction_interval_sec": 604800, + "compaction_interval_sec": 3600, "batch_size": 100 }, "devices": {