From 87ac1278171f04257236d514640b277185a0b145 Mon Sep 17 00:00:00 2001 From: Dominique Lasserre Date: Sun, 26 Jan 2025 21:13:23 +0100 Subject: [PATCH] Workflow: Docker: Add more archs: armv6/v7, i386 * Build amd64 on any PR. --- .env | 2 + .github/workflows/docker-build.yml | 89 ++++++++++++++++++++++++---- Dockerfile | 88 +++++++++++++++++++++++++-- docker-compose.yaml | 6 ++ src/akkudoktoreos/config/config.py | 7 ++- src/akkudoktoreos/core/decorators.py | 18 ++++-- 6 files changed, 183 insertions(+), 27 deletions(-) diff --git a/.env b/.env index c58592f..beb7e52 100644 --- a/.env +++ b/.env @@ -3,3 +3,5 @@ EOS_SERVER__PORT=8503 EOS_SERVER__EOSDASH_PORT=8504 PYTHON_VERSION=3.12.6 +BASE_IMAGE=python +IMAGE_SUFFIX=-slim diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a60da76..5de04e1 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,13 +7,11 @@ on: push: branches: - 'main' - - 'feature/config-overhaul' tags: - 'v*' pull_request: branches: - - 'main' - - 'feature/config-overhaul' + - '**' env: DOCKERHUB_REPO: akkudoktor/eos @@ -40,7 +38,9 @@ jobs: run: | if ${{ github.event_name == 'pull_request' }}; then echo 'matrix=[ - {"platform": "linux/arm64"} + {"platform": {"name": "linux/amd64"}}, + {"platform": {"name": "linux/arm64"}}, + {"platform": {"name": "linux/386"}}, ]' | tr -d '[:space:]' >> $GITHUB_OUTPUT else echo 'matrix=[]' >> $GITHUB_OUTPUT @@ -58,13 +58,69 @@ jobs: fail-fast: false matrix: platform: - - linux/amd64 - - linux/arm64 + - name: linux/amd64 + base: python + python: 3.12 # pendulum not yet on pypi for 3.13 + rustup_install: "" + apt_packages: "" + apt_build_packages: "" + pip_extra_url: "" + - name: linux/arm64 + base: python + python: 3.12 # pendulum not yet on pypi for 3.13 + rustup_install: "" + apt_packages: "" + apt_build_packages: "" + pip_extra_url: "" + - name: linux/arm/v6 + base: python + python: 3.11 # highest version on piwheels + rustup_install: true + # numpy: libopenblas0 + # h5py: libhdf5-hl-310 + #apt_packages: "libopenblas0 libhdf5-hl-310" + apt_packages: "" #TODO verify + # pendulum: git (apply patch) + # matplotlib (countourpy): g++ + # fastapi (MarkupSafe): gcc + # rustup installer: curl + apt_build_packages: "curl git g++" + pip_extra_url: "https://www.piwheels.org/simple" # armv6/v7 packages + - name: linux/arm/v7 + base: python + python: 3.11 # highest version on piwheels + rustup_install: true + # numpy: libopenblas0 + # h5py: libhdf5-hl-310 + #apt_packages: "libopenblas0 libhdf5-hl-310" + apt_packages: "" #TODO verify + # pendulum: git (apply patch) + # matplotlib (countourpy): g++ + # fastapi (MarkupSafe): gcc + # rustup installer: curl + apt_build_packages: "curl git g++" + pip_extra_url: "https://www.piwheels.org/simple" # armv6/v7 packages + - name: linux/386 + # Get 32bit distributor fix for pendulum, not yet officially released. + # Needs Debian testing instead of python:xyz which is based on Debian stable. + base: debian + python: trixie + rustup_install: "" + # numpy: libopenblas0 + # h5py: libhdf5-hl-310 + apt_packages: "python3-pendulum python3-pip libopenblas0 libhdf5-hl-310" + # numpy: g++, libc-dev + # skikit: pkgconf python3-dev, libopenblas-dev + # uvloop: make + # h5py: libhdf5-dev + # many others g++/gcc + apt_build_packages: "g++ pkgconf libc-dev python3-dev make libopenblas-dev libhdf5-dev" + pip_extra_url: "" exclude: ${{ fromJSON(needs.platform-excludes.outputs.excludes) }} steps: - name: Prepare run: | - platform=${{ matrix.platform }} + platform=${{ matrix.platform.name }} echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Docker meta @@ -98,7 +154,8 @@ jobs: - name: Login to GHCR uses: docker/login-action@v3 # skip for pull requests - if: ${{ github.event_name != 'pull_request' }} + #TODO: uncomment again + #if: ${{ github.event_name != 'pull_request' }} with: registry: ghcr.io username: ${{ github.actor }} @@ -106,8 +163,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # skip for pull requests - if: ${{ github.event_name != 'pull_request' }} + #if: ${{ github.event_name != 'pull_request' }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -116,10 +172,19 @@ jobs: id: build uses: docker/build-push-action@v6 with: - platforms: ${{ matrix.platform }} + platforms: ${{ matrix.platform.name }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - outputs: type=image,"name=${{ env.DOCKERHUB_REPO }},${{ env.GHCR_REPO }}",push-by-digest=true,name-canonical=true,"push=${{ github.event_name != 'pull_request' }}","annotation-index.org.opencontainers.image.description=${{ env.EOS_REPO_DESCRIPTION }}" + #TODO: uncomment again + #outputs: type=image,"name=${{ env.DOCKERHUB_REPO }},${{ env.GHCR_REPO }}",push-by-digest=true,name-canonical=true,"push=${{ github.event_name != 'pull_request' }}","annotation-index.org.opencontainers.image.description=${{ env.EOS_REPO_DESCRIPTION }}" + outputs: type=image,"name=${{ env.DOCKERHUB_REPO }},${{ env.GHCR_REPO }}",push-by-digest=true,name-canonical=true,push=true,"annotation-index.org.opencontainers.image.description=${{ env.EOS_REPO_DESCRIPTION }}" + build-args: | + BASE_IMAGE=${{ matrix.platform.base }} + PYTHON_VERSION=${{ matrix.platform.python }} + PIP_EXTRA_INDEX_URL=${{ matrix.platform.pip_extra_url }} + APT_PACKAGES=${{ matrix.platform.apt_packages }} + APT_BUILD_PACKAGES=${{ matrix.platform.apt_build_packages }} + RUSTUP_INSTALL=${{ matrix.platform.rustup_install }} - name: Generate artifact attestation DockerHub uses: actions/attest-build-provenance@v2 diff --git a/Dockerfile b/Dockerfile index a00e419..2971781 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ -ARG PYTHON_VERSION=3.12.7 -FROM python:${PYTHON_VERSION}-slim +ARG PYTHON_VERSION=3.12.8 +ARG BASE_IMAGE=python +ARG IMAGE_SUFFIX=-slim +FROM ${BASE_IMAGE}:${PYTHON_VERSION}${IMAGE_SUFFIX} AS base LABEL source="https://github.com/Akkudoktor-EOS/EOS" @@ -11,7 +13,8 @@ ENV EOS_CONFIG_DIR="${EOS_DIR}/config" WORKDIR ${EOS_DIR} -RUN adduser --system --group --no-create-home eos \ +# Use useradd over adduser to support both debian:x-slim and python:x-slim base images +RUN useradd --system --no-create-home --shell /usr/sbin/nologin eos \ && mkdir -p "${MPLCONFIGDIR}" \ && chown eos "${MPLCONFIGDIR}" \ && mkdir -p "${EOS_CACHE_DIR}" \ @@ -21,13 +24,85 @@ RUN adduser --system --group --no-create-home eos \ && mkdir -p "${EOS_CONFIG_DIR}" \ && chown eos "${EOS_CONFIG_DIR}" +ARG APT_PACKAGES +ENV APT_PACKAGES="${APT_PACKAGES}" +RUN --mount=type=cache,sharing=locked,target=/var/lib/apt/lists \ + --mount=type=cache,sharing=locked,target=/var/cache/apt \ + rm /etc/apt/apt.conf.d/docker-clean; \ + if [ -n "${APT_PACKAGES}" ]; then \ + apt-get update \ + && apt-get install -y --no-install-recommends ${APT_PACKAGES}; \ + fi + +FROM base AS build +ARG APT_BUILD_PACKAGES +ENV APT_BUILD_PACKAGES="${APT_BUILD_PACKAGES}" +RUN --mount=type=cache,sharing=locked,target=/var/lib/apt/lists \ + --mount=type=cache,sharing=locked,target=/var/cache/apt \ + rm /etc/apt/apt.conf.d/docker-clean; \ + if [ -n "${APT_BUILD_PACKAGES}" ]; then \ + apt-get update \ + && apt-get install -y --no-install-recommends ${APT_BUILD_PACKAGES}; \ + fi + +ARG RUSTUP_INSTALL +ENV RUSTUP_INSTALL="${RUSTUP_INSTALL}" +ENV RUSTUP_HOME=/opt/rust +ENV CARGO_HOME=/opt/rust +ENV PATH="$RUSTUP_HOME/bin:$PATH" +ARG PIP_EXTRA_INDEX_URL +ENV PIP_EXTRA_INDEX_URL="${PIP_EXTRA_INDEX_URL}" +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=tmpfs,target=/root/.cargo \ + dpkgArch=$(dpkg --print-architecture) \ + && if [ -n "${RUSTUP_INSTALL}" ]; then \ + case "$dpkgArch" in \ + # armv6 + armel) \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --target arm-unknown-linux-gnueabi --no-modify-path \ + ;; \ + *) \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --no-modify-path \ + ;; \ + esac \ + && rustc --version \ + && cargo --version; \ + fi \ + # Install 32bit fix for pendulum, can be removed after next pendulum release (> 3.0.0) + && case "$dpkgArch" in \ + # armv7/armv6 + armhf|armel) \ + git clone https://github.com/python-pendulum/pendulum.git \ + && git -C pendulum checkout -b 3.0.0 3.0.0 \ + # Apply 32bit patch + && git -C pendulum -c user.name=ci -c user.email=ci@github.com cherry-pick b84b97625cdea00f8ab150b8b35aa5ccaaf36948 \ + && cd pendulum \ + # Use pip3 over pip to support both debian:x and python:x base images + && pip3 install maturin \ + && maturin build --release --out dist \ + && pip3 install dist/*.whl --break-system-packages \ + && cd - \ + ;; \ + esac + + COPY requirements.txt . +# Use tmpfs for cargo due to qemu (multiarch) limitations RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements.txt + --mount=type=tmpfs,target=/root/.cargo \ + # Use pip3 over pip to support both debian:x and python:x base images + pip3 install -r requirements.txt --break-system-packages + +FROM base AS final +# Copy all python dependencies previously installed or built to the final stage. +COPY --from=build /usr/local/ /usr/local/ +COPY --from=build /opt/eos/requirements.txt . COPY pyproject.toml . -RUN mkdir -p src && pip install -e . +RUN --mount=type=cache,target=/root/.cache/pip \ + # Use pip3 over pip to support both debian:x and python:x base images + mkdir -p src && pip3 install -e . --break-system-packages COPY src src @@ -37,6 +112,7 @@ ENTRYPOINT [] EXPOSE 8503 EXPOSE 8504 -CMD ["python", "src/akkudoktoreos/server/eos.py", "--host", "0.0.0.0"] +# Use python3 over python to support both debian:x and python:x base images +CMD ["python3", "src/akkudoktoreos/server/eos.py", "--host", "0.0.0.0"] VOLUME ["${MPLCONFIGDIR}", "${EOS_CACHE_DIR}", "${EOS_OUTPUT_DIR}", "${EOS_CONFIG_DIR}"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 7f88bb8..30d448e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,12 @@ services: dockerfile: "Dockerfile" args: PYTHON_VERSION: "${PYTHON_VERSION}" + BASE_IMAGE: "${BASE_IMAGE}" + IMAGE_SUFFIX: "${IMAGE_SUFFIX}" + APT_PACKAGES: "${APT_PACKAGES:-}" + APT_BUILD_PACKAGES: "${APT_BUILD_PACKAGES:-}" + PIP_EXTRA_INDEX_URL: "${PIP_EXTRA_INDEX_URL:-}" + RUSTUP_INSTALL: "${RUSTUP_INSTALL:-}" env_file: - .env environment: diff --git a/src/akkudoktoreos/config/config.py b/src/akkudoktoreos/config/config.py index 8e658aa..fd1be9b 100644 --- a/src/akkudoktoreos/config/config.py +++ b/src/akkudoktoreos/config/config.py @@ -167,7 +167,10 @@ class SettingsEOS(BaseSettings): utils: Optional[UtilsCommonSettings] = None model_config = SettingsConfigDict( - env_nested_delimiter="__", nested_model_default_partial_update=True, env_prefix="EOS_" + env_nested_delimiter="__", + nested_model_default_partial_update=True, + env_prefix="EOS_", + ignored_types=(classproperty,), ) @@ -307,13 +310,11 @@ class ConfigEOS(SingletonMixin, SettingsEOSDefaults): default_settings, ) - @classmethod @classproperty def config_default_file_path(cls) -> Path: """Compute the default config file path.""" return cls.package_root_path.joinpath("data/default.config.json") - @classmethod @classproperty def package_root_path(cls) -> Path: """Compute the package root path.""" diff --git a/src/akkudoktoreos/core/decorators.py b/src/akkudoktoreos/core/decorators.py index 7811a9c..d51207a 100644 --- a/src/akkudoktoreos/core/decorators.py +++ b/src/akkudoktoreos/core/decorators.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import Any, Optional from akkudoktoreos.core.logging import get_logger @@ -5,18 +6,20 @@ from akkudoktoreos.core.logging import get_logger logger = get_logger(__name__) -class classproperty(property): +class classproperty: """A decorator to define a read-only property at the class level. - This class extends the built-in `property` to allow a method to be accessed - as a property on the class itself, rather than an instance. This is useful - when you want a property-like syntax for methods that depend on the class - rather than any instance of the class. + This class replaces the built-in `property` which is no longer available in + combination with @classmethod since Python 3.13 to allow a method to be + accessed as a property on the class itself, rather than an instance. This + is useful when you want a property-like syntax for methods that depend on + the class rather than any instance of the class. Example: class MyClass: _value = 42 + @classmethod @classproperty def value(cls): return cls._value @@ -28,13 +31,16 @@ class classproperty(property): decorated method on the class. Parameters: - fget (Callable[[type], Any]): A method that takes the class as an + fget (Callable[[Any], Any]): A method that takes the class as an argument and returns a value. Raises: AssertionError: If `fget` is not defined when `__get__` is called. """ + def __init__(self, fget: Callable[[Any], Any]) -> None: + self.fget = fget + def __get__(self, _: Any, owner_cls: Optional[type[Any]] = None) -> Any: if owner_cls is None: return self