mirror of
https://github.com/Akkudoktor-EOS/EOS.git
synced 2026-10-09 07:56:40 +00:00
Andreas
a2f4ef6f54
The optimizer treated the end of `optimization.horizon_hours` as the end of the world: energy left in the battery there was worth a single configured price per kWh, so it either dumped the battery into the last hours or hoarded it, depending on that one number. The horizon is now two spans. `horizon_hours` still receives every control command. The new `optimization.tail_horizon_hours` (default 48 h) is a pure lookahead that never produces a command. In AUTO terminal-value mode a deterministic dynamic program solves that tail backwards on a 101-point SoC grid using the production battery and inverter models - SoC bounds, power caps, conversion losses, configured charge and export rates, direct-marketing permission and LCOS on delivered DC energy - and the existing AUTO proxy supplies the continuation value at the tail end. Genetic fitness reads the resulting curve. `tail_horizon_hours: 0` restores the plain proxy at the control end, FIXED is unchanged. The forecast budget is reported, never enforced by refusal: a tail that does not fit is shortened to what the forecast covers and reported as `effective_tail_hours`, and a control horizon that does not fit is warned about at configuration time and rejected by the optimizer at run time, which knows which series ran out. `prediction.hours` defaults to 72 so the new defaults fit out of the box; existing shorter configurations keep starting. Control arrays and warm-start genomes now begin at the run timestamp rather than midnight, flagged by `controls_start_at_now` so the adapters still read older solutions. `forecast_interval_seconds` declares the resolution of shortened native quarter-hour inputs. Required forecasts are no longer silently replaced by demo providers. A missing PV, price, load, feed-in or weather forecast used to rewrite the configured provider and retry, so a run could quietly optimize against invented data. Missing values now stay missing, and provider values are held only within their own source interval instead of being extended indefinitely. Also fixes a config update that could leave EOS half-updated: the merged candidate is validated before the singleton is reinitialized. Four provider tests that hard-coded the old 48 h prediction default are rewritten to derive their expectations from the configured horizon.
579 lines
17 KiB
Python
579 lines
17 KiB
Python
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, patch
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pvlib
|
|
import pytest
|
|
from bs4 import BeautifulSoup
|
|
|
|
from akkudoktoreos.core.cache import CacheFileStore
|
|
from akkudoktoreos.core.coreabc import get_ems
|
|
from akkudoktoreos.prediction.weatherclearoutside import WeatherClearOutside
|
|
from akkudoktoreos.utils.datetimeutil import compare_datetimes, to_datetime
|
|
|
|
DIR_TESTDATA = Path(__file__).absolute().parent.joinpath("testdata")
|
|
|
|
FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_HTML = DIR_TESTDATA.joinpath("weatherforecast_clearout_1.html")
|
|
FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_DATA = DIR_TESTDATA.joinpath("weatherforecast_clearout_1.json")
|
|
|
|
|
|
@pytest.fixture
|
|
def provider(config_eos):
|
|
"""Fixture to create a WeatherProvider instance."""
|
|
settings = {
|
|
"weather": {
|
|
"provider": "ClearOutside",
|
|
},
|
|
"general": {
|
|
"latitude": 50.0,
|
|
"longitude": 10.0,
|
|
},
|
|
}
|
|
config_eos.merge_settings_from_dict(settings)
|
|
return WeatherClearOutside()
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_clearout_1_html():
|
|
"""Fixture that returns sample forecast data report."""
|
|
with FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_HTML.open(
|
|
"r", encoding="utf-8", newline=None
|
|
) as f_res:
|
|
input_data = f_res.read()
|
|
return input_data
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_clearout_1_data():
|
|
"""Fixture that returns sample forecast data."""
|
|
with FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_DATA.open("r", encoding="utf-8", newline=None) as f_in:
|
|
json_str = f_in.read()
|
|
data = WeatherClearOutside.from_json(json_str)
|
|
return data
|
|
|
|
|
|
@pytest.fixture
|
|
def cache_store():
|
|
"""A pytest fixture that creates a new CacheFileStore instance for testing."""
|
|
return CacheFileStore()
|
|
|
|
|
|
# ------------------------------------------------
|
|
# General WeatherProvider
|
|
# ------------------------------------------------
|
|
|
|
|
|
def test_singleton_instance(provider):
|
|
"""Test that WeatherForecast behaves as a singleton."""
|
|
another_instance = WeatherClearOutside()
|
|
assert provider is another_instance
|
|
|
|
|
|
def test_invalid_provider(provider, config_eos):
|
|
"""Test requesting an unsupported provider."""
|
|
settings = {
|
|
"weather": {
|
|
"provider": "<invalid>",
|
|
}
|
|
}
|
|
with pytest.raises(ValueError, match="not a valid weather provider"):
|
|
config_eos.merge_settings_from_dict(settings)
|
|
|
|
|
|
def test_invalid_coordinates(provider, config_eos):
|
|
"""Test invalid coordinates raise ValueError."""
|
|
settings = {
|
|
"weather": {
|
|
"provider": "ClearOutside",
|
|
},
|
|
"general": {
|
|
"latitude": 1000.0,
|
|
"longitude": 1000.0,
|
|
},
|
|
}
|
|
with pytest.raises(
|
|
ValueError, # match="Latitude '1000' and/ or longitude `1000` out of valid range."
|
|
):
|
|
config_eos.merge_settings_from_dict(settings)
|
|
|
|
|
|
# ------------------------------------------------
|
|
# Irradiance caclulation
|
|
# ------------------------------------------------
|
|
|
|
|
|
def test_irridiance_estimate_from_cloud_cover(provider):
|
|
"""Test cloud cover to irradiance estimation."""
|
|
cloud_cover_data = pd.Series(
|
|
data=[20, 50, 80], index=pd.date_range("2023-10-22", periods=3, freq="h")
|
|
)
|
|
|
|
ghi, dni, dhi = provider.estimate_irradiance_from_cloud_cover(50.0, 10.0, cloud_cover_data)
|
|
|
|
assert ghi == [0, 0, 0]
|
|
assert dhi == [0, 0, 0]
|
|
assert dni == [0, 0, 0]
|
|
|
|
|
|
# ------------------------------------------------
|
|
# ClearOutside
|
|
# ------------------------------------------------
|
|
|
|
|
|
@patch("requests.get")
|
|
def test_request_forecast(mock_get, provider, sample_clearout_1_html, config_eos):
|
|
"""Test fetching forecast from ClearOutside."""
|
|
# Mock response object
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.content = sample_clearout_1_html
|
|
mock_get.return_value = mock_response
|
|
|
|
# Preset, as this is usually done by update()
|
|
config_eos.update()
|
|
|
|
# Test function
|
|
response = provider._request_forecast()
|
|
|
|
assert response.status_code == 200
|
|
assert response.content == sample_clearout_1_html
|
|
|
|
|
|
@patch("requests.get")
|
|
def test_update_data(mock_get, provider, sample_clearout_1_html, sample_clearout_1_data):
|
|
# Mock response object
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.content = sample_clearout_1_html
|
|
mock_get.return_value = mock_response
|
|
|
|
expected_start = to_datetime("2024-10-26 00:00:00", in_timezone="Europe/Berlin")
|
|
expected_end = to_datetime("2024-10-29 00:00:00", in_timezone="Europe/Berlin")
|
|
expected_keep = to_datetime("2024-10-24 00:00:00", in_timezone="Europe/Berlin")
|
|
|
|
# Call the method
|
|
ems_eos = get_ems()
|
|
ems_eos.set_start_datetime(expected_start)
|
|
provider.update_data()
|
|
|
|
# Check for correct prediction time window
|
|
assert provider.config.prediction.hours == 72
|
|
assert provider.config.prediction.historic_hours == 48
|
|
assert compare_datetimes(provider.ems_start_datetime, expected_start).equal
|
|
assert compare_datetimes(provider.end_datetime, expected_end).equal
|
|
assert compare_datetimes(provider.keep_datetime, expected_keep).equal
|
|
|
|
# Verify the data
|
|
assert len(provider) == 165 # 6 days, 24 hours per day - 7th day 21 hours
|
|
|
|
# Check that specific values match the expected output
|
|
# for i, record in enumerate(weather_data.records):
|
|
# # Compare datetime and specific values
|
|
# assert record.datetime == sample_clearout_1_data.records[i].datetime
|
|
# assert record.data['total_clouds'] == sample_clearout_1_data.records[i].data['total_clouds']
|
|
# # Check additional weather attributes as necessary
|
|
|
|
|
|
@pytest.mark.skip(reason="Test fixture to be improved")
|
|
@patch("requests.get")
|
|
def test_cache_forecast(mock_get, provider, sample_clearout_1_html, cache_store):
|
|
"""Test that ClearOutside forecast data is cached with TTL.
|
|
|
|
This can not be tested with mock_get. Mock objects are not pickable and therefor can not be
|
|
cached to a file. Keep it for documentation.
|
|
"""
|
|
# Mock response object
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.content = sample_clearout_1_html
|
|
mock_get.return_value = mock_response
|
|
|
|
cache_store.clear(clear_all=True)
|
|
|
|
provider.update_data()
|
|
mock_get.assert_called_once()
|
|
forecast_data_first = provider.to_json()
|
|
|
|
provider.update_data()
|
|
forecast_data_second = provider.to_json()
|
|
# Verify that cache returns the same object without calling the method again
|
|
assert forecast_data_first == forecast_data_second
|
|
# A mock object is not pickable and therefor can not be chached to file
|
|
assert mock_get.call_count == 2
|
|
|
|
|
|
# ------------------------------------------------
|
|
# Development ClearOutside
|
|
# ------------------------------------------------
|
|
|
|
|
|
@pytest.mark.skip(reason="For development only")
|
|
@patch("requests.get")
|
|
def test_development_forecast_data(mock_get, provider, sample_clearout_1_html):
|
|
# Mock response object
|
|
mock_response = Mock()
|
|
mock_response.status_code = 200
|
|
mock_response.content = sample_clearout_1_html
|
|
mock_get.return_value = mock_response
|
|
|
|
# Fill the instance
|
|
provider.update_data(force_enable=True)
|
|
|
|
with FILE_TESTDATA_WEATHERCLEAROUTSIDE_1_DATA.open(
|
|
"w", encoding="utf-8", newline="\n"
|
|
) as f_out:
|
|
f_out.write(provider.to_json())
|
|
|
|
|
|
@pytest.mark.skip(reason="For development only")
|
|
def test_clearoutsides_development_scraper(provider, sample_clearout_1_html):
|
|
"""Test scraping from ClearOutside."""
|
|
soup = BeautifulSoup(sample_clearout_1_html, "html.parser")
|
|
|
|
# Sample was created for the loacation
|
|
lat = 50.0
|
|
lon = 10.0
|
|
|
|
# Find generation data
|
|
p_generated = soup.find("h2", string=lambda text: text and text.startswith("Generated:"))
|
|
assert p_generated is not None
|
|
|
|
# Extract forecast start and end dates
|
|
forecast_pattern = r"Forecast: (\d{2}/\d{2}/\d{2}) to (\d{2}/\d{2}/\d{2})"
|
|
forecast_match = re.search(forecast_pattern, p_generated.get_text())
|
|
if forecast_match:
|
|
forecast_start_date = forecast_match.group(1)
|
|
forecast_end_date = forecast_match.group(2)
|
|
else:
|
|
assert False
|
|
assert forecast_start_date == "26/10/24"
|
|
assert forecast_end_date == "01/11/24"
|
|
|
|
# Extract timezone offset
|
|
timezone_pattern = r"Timezone: UTC([+-]\d+)\.(\d+)"
|
|
timezone_match = re.search(timezone_pattern, p_generated.get_text())
|
|
if timezone_match:
|
|
hours = int(timezone_match.group(1))
|
|
assert hours == 2
|
|
# Convert the decimal part to minutes (e.g., .50 -> 30 minutes)
|
|
minutes = int(timezone_match.group(2)) * 6 # Multiply by 6 to convert to minutes
|
|
assert minutes == 0
|
|
|
|
# Create the timezone object using timedelta for the offset
|
|
forecast_timezone = timezone(timedelta(hours=hours, minutes=minutes))
|
|
else:
|
|
assert False
|
|
|
|
forecast_start_datetime = to_datetime(
|
|
forecast_start_date, in_timezone=forecast_timezone, to_naiv=False, to_maxtime=False
|
|
)
|
|
assert forecast_start_datetime == datetime(2024, 10, 26, 0, 0)
|
|
|
|
# Find all paragraphs with id 'day_<x>'. There should be seven.
|
|
p_days = soup.find_all(id=re.compile(r"day_[0-9]"))
|
|
assert len(p_days) == 7
|
|
p_day = p_days[0]
|
|
|
|
# Within day_x paragraph find the details labels
|
|
p_detail_labels = p_day.find_all(class_="fc_detail_label")
|
|
detail_names = [p.get_text() for p in p_detail_labels]
|
|
|
|
assert detail_names == [
|
|
"Total Clouds (% Sky Obscured)",
|
|
"Low Clouds (% Sky Obscured)",
|
|
"Medium Clouds (% Sky Obscured)",
|
|
"High Clouds (% Sky Obscured)",
|
|
"ISS Passover",
|
|
"Visibility (miles)",
|
|
"Fog (%)",
|
|
"Precipitation Type",
|
|
"Precipitation Probability (%)",
|
|
"Precipitation Amount (mm)",
|
|
"Wind Speed/Direction (mph)",
|
|
"Chance of Frost",
|
|
"Temperature (°C)",
|
|
"Feels Like (°C)",
|
|
"Dew Point (°C)",
|
|
"Relative Humidity (%)",
|
|
"Pressure (mb)",
|
|
"Ozone (du)",
|
|
]
|
|
|
|
# Find all the paragraphs that are associated to the details.
|
|
# Beware there is one ul paragraph before that is not associated to a detail
|
|
p_detail_tables = p_day.find_all("ul")
|
|
assert len(p_detail_tables) == len(detail_names) + 1
|
|
p_detail_tables.pop(0)
|
|
|
|
# Create clearout data
|
|
clearout_data = {}
|
|
# Add data values
|
|
for i, detail_name in enumerate(detail_names):
|
|
p_detail_values = p_detail_tables[i].find_all("li")
|
|
detail_data = []
|
|
for p_detail_value in p_detail_values:
|
|
if (
|
|
detail_name in ("Precipitation Type", "Chance of Frost")
|
|
and hasattr(p_detail_value, "title")
|
|
and p_detail_value.title
|
|
):
|
|
value_str = p_detail_value.title.string
|
|
else:
|
|
value_str = p_detail_value.get_text()
|
|
try:
|
|
value = float(value_str)
|
|
except ValueError:
|
|
value = value_str
|
|
detail_data.append(value)
|
|
assert len(detail_data) == 24
|
|
clearout_data[detail_name] = detail_data
|
|
|
|
assert clearout_data["Temperature (°C)"] == [
|
|
14.0,
|
|
14.0,
|
|
13.0,
|
|
12.0,
|
|
11.0,
|
|
11.0,
|
|
10.0,
|
|
10.0,
|
|
9.0,
|
|
9.0,
|
|
9.0,
|
|
9.0,
|
|
9.0,
|
|
10.0,
|
|
9.0,
|
|
9.0,
|
|
10.0,
|
|
11.0,
|
|
13.0,
|
|
14.0,
|
|
15.0,
|
|
16.0,
|
|
16.0,
|
|
16.0,
|
|
]
|
|
assert clearout_data["Relative Humidity (%)"] == [
|
|
59.0,
|
|
68.0,
|
|
75.0,
|
|
81.0,
|
|
84.0,
|
|
85.0,
|
|
85.0,
|
|
91.0,
|
|
91.0,
|
|
93.0,
|
|
93.0,
|
|
93.0,
|
|
93.0,
|
|
93.0,
|
|
95.0,
|
|
95.0,
|
|
93.0,
|
|
87.0,
|
|
81.0,
|
|
76.0,
|
|
70.0,
|
|
66.0,
|
|
66.0,
|
|
69.0,
|
|
]
|
|
assert clearout_data["Wind Speed/Direction (mph)"] == [
|
|
7.0,
|
|
6.0,
|
|
4.0,
|
|
4.0,
|
|
4.0,
|
|
4.0,
|
|
4.0,
|
|
4.0,
|
|
3.0,
|
|
3.0,
|
|
3.0,
|
|
2.0,
|
|
1.0,
|
|
1.0,
|
|
1.0,
|
|
2.0,
|
|
2.0,
|
|
2.0,
|
|
4.0,
|
|
5.0,
|
|
6.0,
|
|
6.0,
|
|
5.0,
|
|
5.0,
|
|
]
|
|
|
|
# Add datetimes of the scrapped data
|
|
clearout_data["DateTime"] = [forecast_start_datetime + timedelta(hours=i) for i in range(24)]
|
|
detail_names.append("DateTime")
|
|
|
|
assert len(clearout_data["DateTime"]) == 24
|
|
assert clearout_data["DateTime"][0] == to_datetime(
|
|
"2024-10-26 00:00:00", in_timezone=forecast_timezone
|
|
)
|
|
assert clearout_data["DateTime"][23] == to_datetime(
|
|
"2024-10-26 23:00:00", in_timezone=forecast_timezone
|
|
)
|
|
|
|
# Converting the cloud cover into Global Horizontal Irradiance (GHI) with a PVLib method
|
|
offset = 35 # The default
|
|
offset_fraction = offset / 100.0 # Adjust percentage to scaling factor
|
|
cloud_cover = pd.Series(clearout_data["Total Clouds (% Sky Obscured)"])
|
|
|
|
# Convert datetime list to a pandas DatetimeIndex
|
|
cloud_cover_times = pd.DatetimeIndex(clearout_data["DateTime"])
|
|
|
|
# Create a location object
|
|
location = pvlib.location.Location(latitude=lat, longitude=lon)
|
|
|
|
# Get solar position and clear-sky GHI using the Ineichen model
|
|
solpos = location.get_solarposition(cloud_cover_times)
|
|
clear_sky = location.get_clearsky(cloud_cover_times, model="ineichen")
|
|
|
|
# Convert cloud cover percentage to a scaling factor
|
|
cloud_cover_fraction = np.array(cloud_cover) / 100.0
|
|
|
|
# Calculate adjusted GHI with proportional offset adjustment
|
|
adjusted_ghi = clear_sky["ghi"] * (
|
|
offset_fraction + (1 - offset_fraction) * (1 - cloud_cover_fraction)
|
|
)
|
|
adjusted_ghi.fillna(0.0, inplace=True)
|
|
|
|
# Apply DISC model to estimate Direct Normal Irradiance (DNI) from adjusted GHI
|
|
disc_output = pvlib.irradiance.disc(adjusted_ghi, solpos["zenith"], cloud_cover_times)
|
|
adjusted_dni = disc_output["dni"]
|
|
adjusted_dni.fillna(0.0, inplace=True)
|
|
|
|
# Calculate Diffuse Horizontal Irradiance (DHI) as DHI = GHI - DNI * cos(zenith)
|
|
zenith_rad = np.radians(solpos["zenith"])
|
|
adjusted_dhi = adjusted_ghi - adjusted_dni * np.cos(zenith_rad)
|
|
adjusted_dhi.fillna(0.0, inplace=True)
|
|
|
|
# Add GHI, DNI, DHI to clearout data
|
|
clearout_data["Global Horizontal Irradiance (W/m2)"] = adjusted_ghi.to_list()
|
|
detail_names.append("Global Horizontal Irradiance (W/m2)")
|
|
clearout_data["Direct Normal Irradiance (W/m2)"] = adjusted_dni.to_list()
|
|
detail_names.append("Direct Normal Irradiance (W/m2)")
|
|
clearout_data["Diffuse Horizontal Irradiance (W/m2)"] = adjusted_dhi.to_list()
|
|
detail_names.append("Diffuse Horizontal Irradiance (W/m2)")
|
|
|
|
assert clearout_data["Global Horizontal Irradiance (W/m2)"] == [
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
24.291000436601216,
|
|
85.88494154645998,
|
|
136.09269403109946,
|
|
139.26925350542064,
|
|
146.7174434892616,
|
|
149.0167479382964,
|
|
138.97458866666065,
|
|
103.47132353697396,
|
|
46.81279774519421,
|
|
0.12972168074047014,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
]
|
|
assert clearout_data["Direct Normal Irradiance (W/m2)"] == [
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
10.19687368654253,
|
|
0.0,
|
|
0.0,
|
|
2.9434862632289804,
|
|
9.621272744657047,
|
|
9.384995789935898,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
]
|
|
assert clearout_data["Diffuse Horizontal Irradiance (W/m2)"] == [
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
24.291000436601216,
|
|
85.88494154645998,
|
|
132.32210426501337,
|
|
139.26925350542064,
|
|
146.7174434892616,
|
|
147.721968406295,
|
|
135.32240392326145,
|
|
100.82522311704261,
|
|
46.81279774519421,
|
|
0.12972168074047014,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
]
|
|
|
|
# Preciptable Water (PWAT) with a PVLib method
|
|
clearout_data["Preciptable Water (cm)"] = pvlib.atmosphere.gueymard94_pw(
|
|
pd.Series(data=clearout_data["Temperature (°C)"]),
|
|
pd.Series(data=clearout_data["Relative Humidity (%)"]),
|
|
).to_list()
|
|
detail_names.append("Preciptable Water (cm)")
|
|
|
|
assert clearout_data["Preciptable Water (cm)"] == [
|
|
1.5345406562673334,
|
|
1.7686231292572652,
|
|
1.8354895631381385,
|
|
1.8651290310892348,
|
|
1.8197998755611786,
|
|
1.8414641597940502,
|
|
1.7325709431177607,
|
|
1.8548700685143087,
|
|
1.7453005409540279,
|
|
1.783658794601369,
|
|
1.783658794601369,
|
|
1.783658794601369,
|
|
1.783658794601369,
|
|
1.8956364436464912,
|
|
1.8220170482487101,
|
|
1.8220170482487101,
|
|
1.8956364436464912,
|
|
1.8847927282597918,
|
|
1.9823287281891897,
|
|
1.9766964385816497,
|
|
1.9346943880237457,
|
|
1.9381315133101413,
|
|
1.9381315133101413,
|
|
2.026228400278784,
|
|
]
|