From b620250536a9e88e8a6ea79759d5765ebe919f1a Mon Sep 17 00:00:00 2001 From: Dominique Lasserre Date: Mon, 7 Oct 2024 20:56:10 +0200 Subject: [PATCH] Streamline Dockerfile, remove unused deps * Dockerfile: Use non-root user, buildx cache, setup for readonly container, remove unused apt deps. For now don't install pip package and keep development flask server as this will be replaced in the future (fastapi). Then a proper webserver (e.g. nginx) should be used and the pip package can be created and deployed just to the run-stage (with the webserver). * docker-compose: Set to readonly (anonymous volumes declared in Dockerfile should maintain all writable data). Mount config.py for easier development. Should be replaced by environment support for all config file variables. * Remove unused runtime dependencies: mariadb, joblib, pytest, pytest-cov. * Move pytest-cov to dev dependencies. * Add output_dir to config.py. * Fix visualization_results.pdf endpoint. * Update docs. --- .github/workflows/pytest.yml | 2 -- .gitignore | 3 +++ CONTRIBUTING.md | 4 +-- Dockerfile | 36 ++++++++++++++++--------- Makefile | 24 ++++++++++------- docker-compose.yaml | 16 +++-------- requirements-dev.txt | 4 +-- requirements.txt | 4 --- src/akkudoktoreos/config.py | 2 ++ src/akkudoktoreos/visualize.py | 7 ++++- src/akkudoktoreosserver/flask_server.py | 11 +++++--- tests/test_class_optimize.py | 3 ++- 12 files changed, 65 insertions(+), 51 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 55de900..183a82a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,9 +20,7 @@ jobs: - name: Install dependencies run: | - sudo apt install -y libmariadb3 libmariadb-dev python -m pip install --upgrade pip - pip install -r requirements.txt pip install -r requirements-dev.txt - name: Run Pytest diff --git a/.gitignore b/.gitignore index 6d705e5..2b43d60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +cache/ +output/ + # Default ignore folders and files for VS Code, Python .vscode/* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4fbc3d..229d33b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ To make collaboration easier, we require pull requests to pass code style and un Our code style checks use [`pre-commit`](https://pre-commit.com). ```bash -pip install -r requirements.txt +pip install -r requirements-dev.txt ``` To run formatting automatically before every commit: @@ -36,7 +36,7 @@ pre-commit install Or run them manually: ```bash -pre-commit --all +pre-commit run --all-files ``` ### Tests diff --git a/Dockerfile b/Dockerfile index dbbb1d2..87ebca6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,23 +3,33 @@ FROM python:${PYTHON_VERSION}-slim LABEL source="https://github.com/Akkudoktor-EOS/EOS" -EXPOSE 5000 +ENV VIRTUAL_ENV="/opt/venv" +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +ENV MPLCONFIGDIR="/tmp/mplconfigdir" +ENV EOS_DIR="/opt/eos" +ENV EOS_CACHE_DIR="${EOS_DIR}/cache" +ENV EOS_OUTPUT_DIR="${EOS_DIR}/output" -WORKDIR /opt/eos +WORKDIR ${EOS_DIR} -COPY . . +RUN adduser --system --group --no-create-home eos \ + && mkdir -p "${MPLCONFIGDIR}" \ + && chown eos "${MPLCONFIGDIR}" \ + && mkdir -p "${EOS_CACHE_DIR}" \ + && chown eos "${EOS_CACHE_DIR}" \ + && mkdir -p "${EOS_OUTPUT_DIR}" \ + && chown eos "${EOS_OUTPUT_DIR}" -ARG APT_OPTS="--yes --auto-remove --no-install-recommends --no-install-suggests" +COPY requirements.txt . -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get update \ - && apt-get install ${APT_OPTS} gcc libhdf5-dev libmariadb-dev pkg-config mariadb-common libmariadb3 \ - && rm -rf /var/lib/apt/lists/* \ - && pip install --no-cache-dir -r requirements.txt \ - && pip install --no-cache-dir build \ - && pip install --no-cache-dir -e . \ - && apt remove ${APT_OPTS} gcc libhdf5-dev libmariadb-dev pkg-config +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install -r requirements.txt +COPY src . + +USER eos ENTRYPOINT [] -CMD ["python", "-m", "akkudoktoreos.flask_server"] +CMD ["python", "-m", "akkudoktoreosserver.flask_server"] + +VOLUME ["${MPLCONFIGDIR}", "${EOS_CACHE_DIR}", "${EOS_OUTPUT_DIR}"] diff --git a/Makefile b/Makefile index f3d04c8..a50c2de 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,16 @@ all: help # Target to display help information help: @echo "Available targets:" - @echo " venv - Set up a Python 3 virtual environment." - @echo " pip - Install dependencies from requirements.txt." - @echo " pip-dev - Install dependencies from requirements-dev.txt." - @echo " install - Install EOS in editable form (development mode) into virtual environment." - @echo " docker-run - Run entire setup on docker - @echo " docs - Generate HTML documentation (in build/docs/html/)." - @echo " run - Run flask_server in the virtual environment (needs install before)." - @echo " dist - Create distribution (in dist/)." - @echo " clean - Remove generated documentation, distribution and virtual environment." + @echo " venv - Set up a Python 3 virtual environment." + @echo " pip - Install dependencies from requirements.txt." + @echo " pip-dev - Install dependencies from requirements-dev.txt." + @echo " install - Install EOS in editable form (development mode) into virtual environment." + @echo " docker-run - Run entire setup on docker" + @echo " docker-build - Rebuild docker image" + @echo " docs - Generate HTML documentation (in build/docs/html/)." + @echo " run - Run flask_server in the virtual environment (needs install before)." + @echo " dist - Create distribution (in dist/)." + @echo " clean - Remove generated documentation, distribution and virtual environment." # Target to set up a Python 3 virtual environment venv: @@ -71,4 +72,7 @@ test: # Run entire setup on docker docker-run: - @docker compose up + @docker compose up --remove-orphans + +docker-build: + @docker compose build diff --git a/docker-compose.yaml b/docker-compose.yaml index 34650d4..cdc5ad6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,28 +5,18 @@ networks: services: eos: image: 'akkudoktor/eos:${EOS_VERSION}' + read_only: true build: context: . dockerfile: 'Dockerfile' args: PYTHON_VERSION: '${PYTHON_VERSION}' - depends_on: - - 'mariadb' init: true environment: FLASK_RUN_PORT: '${EOS_PORT}' networks: - 'eos' + volumes: + - ./src/akkudoktoreos/config.py:/opt/eos/akkudoktoreos/config.py:ro ports: - '${EOS_PORT}:${EOS_PORT}' - mariadb: - image: 'mariadb:${MARIADB_VERSION}-jammy' - environment: - MARIADB_ROOT_PASSWORD: '${MARIADB_ROOT_PASSWORD}' - MARIADB_DATABASE: '${MARIADB_DATABASE}' - MARIADB_USER: '${MARIADB_USER}' - MARIADB_PASSWORD: '${MARIADB_PASSWORD}' - networks: - - 'eos' - volumes: - - ./data/mariadb:/var/lib/mysql diff --git a/requirements-dev.txt b/requirements-dev.txt index 038a491..52cc46c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ -build==1.2.2.post1 +-r requirements.txt myst-parser==4.0.0 sphinx==8.0.2 sphinx_rtd_theme==3.0.1 pytest==8.3.3 +pytest-cov==5.0.0 pytest-xprocess==1.0.2 -requests==2.32.3 pre-commit diff --git a/requirements.txt b/requirements.txt index dfa8727..fd82718 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,7 @@ numpy==2.1.2 -mariadb==1.1.10 matplotlib==3.9.2 flask==3.0.3 scikit-learn==1.5.2 deap==1.4.1 -joblib==1.4.2 requests==2.32.3 -pytest==8.3.3 -pytest-cov==5.0.0 pandas==2.2.3 diff --git a/src/akkudoktoreos/config.py b/src/akkudoktoreos/config.py index 109c3b0..9a1ee37 100644 --- a/src/akkudoktoreos/config.py +++ b/src/akkudoktoreos/config.py @@ -1,5 +1,7 @@ from datetime import datetime, timedelta +output_dir = "output" + prediction_hours = 48 optimization_hours = 24 strafe = 10 diff --git a/src/akkudoktoreos/visualize.py b/src/akkudoktoreos/visualize.py index 9046438..219e348 100644 --- a/src/akkudoktoreos/visualize.py +++ b/src/akkudoktoreos/visualize.py @@ -1,4 +1,5 @@ import datetime +import os # Set the backend for matplotlib to Agg import matplotlib @@ -7,6 +8,7 @@ import numpy as np from matplotlib.backends.backend_pdf import PdfPages from akkudoktoreos.class_sommerzeit import ist_dst_wechsel +from akkudoktoreos.config import output_dir matplotlib.use("Agg") @@ -28,7 +30,10 @@ def visualisiere_ergebnisse( ##################### # 24-hour visualization ##################### - with PdfPages(filename) as pdf: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + output_file = os.path.join(output_dir, filename) + with PdfPages(output_file) as pdf: # Load and PV generation plt.figure(figsize=(14, 14)) plt.subplot(3, 3, 1) diff --git a/src/akkudoktoreosserver/flask_server.py b/src/akkudoktoreosserver/flask_server.py index 743da59..0a06d4c 100755 --- a/src/akkudoktoreosserver/flask_server.py +++ b/src/akkudoktoreosserver/flask_server.py @@ -18,7 +18,12 @@ from akkudoktoreos.class_load_corrector import LoadPredictionAdjuster from akkudoktoreos.class_optimize import optimization_problem from akkudoktoreos.class_pv_forecast import PVForecast from akkudoktoreos.class_strompreis import HourlyElectricityPriceForecast -from akkudoktoreos.config import get_start_enddate, optimization_hours, prediction_hours +from akkudoktoreos.config import ( + get_start_enddate, + optimization_hours, + output_dir, + prediction_hours, +) app = Flask(__name__) @@ -262,11 +267,11 @@ def flask_optimize(): return jsonify(result) -@app.route("/visualisierungsergebnisse.pdf") +@app.route("/visualization_results.pdf") def get_pdf(): # Endpoint to serve the generated PDF with visualization results return send_from_directory( - "", "visualisierungsergebnisse.pdf" + os.path.abspath(output_dir), "visualization_results.pdf" ) # Adjust the directory if needed diff --git a/tests/test_class_optimize.py b/tests/test_class_optimize.py index 647ad8a..057ed50 100644 --- a/tests/test_class_optimize.py +++ b/tests/test_class_optimize.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest from akkudoktoreos.class_optimize import optimization_problem +from akkudoktoreos.config import output_dir DIR_TESTDATA = Path(__file__).parent / "testdata" @@ -35,5 +36,5 @@ def test_optimize(fn_in, fn_out): assert set(ergebnis) == set(expected_output_data) # The function creates a visualization result PDF as a side-effect. - fp_viz = Path(".") / "visualization_results.pdf" + fp_viz = Path(output_dir) / "visualization_results.pdf" assert fp_viz.exists()