From faab1824047167f70e2ee4b8386b1a2e32a3b801 Mon Sep 17 00:00:00 2001 From: "Jens L." Date: Tue, 21 Jan 2025 15:36:25 +0100 Subject: [PATCH] lifecycle: build binary dependencies which link against SSL directly (#12724) * lifecycle: install binary dependencies in dockerfile directly Signed-off-by: Jens Langhammer * install ua-parser-builtins manually as its only distributed as binary Signed-off-by: Jens Langhammer * build duo_client from scratch, sigh Signed-off-by: Jens Langhammer * deps for kadmin Signed-off-by: Jens Langhammer * ok fine Signed-off-by: Jens Langhammer * run on arm runner? Signed-off-by: Jens Langhammer * fix yaml format Signed-off-by: Jens Langhammer * rewrite release pipeline to use re-usable workflows Signed-off-by: Jens Langhammer * fix typo Signed-off-by: Jens Langhammer * re-usable multi-arch build? Signed-off-by: Jens Langhammer * also add suffix for amd64 Signed-off-by: Jens Langhammer * parameterise image name Signed-off-by: Jens Langhammer * re-use workflow for CI images...? Signed-off-by: Jens Langhammer * fix missing checkout Signed-off-by: Jens Langhammer * inherit secrets Signed-off-by: Jens Langhammer * temp build directly Signed-off-by: Jens Langhammer * get cache-to from python script Signed-off-by: Jens Langhammer * better name? Signed-off-by: Jens Langhammer * matrix for merging images? Signed-off-by: Jens Langhammer * re-add build dep Signed-off-by: Jens Langhammer * use multi-image tag Signed-off-by: Jens Langhammer * include arch in buildcache Signed-off-by: Jens Langhammer --------- Signed-off-by: Jens Langhammer --- .../comment-pr-instructions/action.yml | 20 ---- .../actions/docker-push-variables/action.yml | 6 ++ .../docker-push-variables/push_vars.py | 14 ++- .github/actions/docker-push-variables/test.sh | 11 ++ .../_reusable-docker-build-single.yaml | 94 ++++++++++++++++ .github/workflows/_reusable-docker-build.yaml | 102 ++++++++++++++++++ .github/workflows/ci-main.yml | 64 ++--------- .github/workflows/ci-outpost.yml | 2 +- .github/workflows/release-publish.yml | 61 ++--------- Dockerfile | 29 +++-- website/docs/install-config/beta.mdx | 4 - 11 files changed, 261 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/_reusable-docker-build-single.yaml create mode 100644 .github/workflows/_reusable-docker-build.yaml diff --git a/.github/actions/comment-pr-instructions/action.yml b/.github/actions/comment-pr-instructions/action.yml index 47f7cc212e..c113deeebb 100644 --- a/.github/actions/comment-pr-instructions/action.yml +++ b/.github/actions/comment-pr-instructions/action.yml @@ -35,14 +35,6 @@ runs: AUTHENTIK_OUTPOSTS__CONTAINER_IMAGE_BASE=ghcr.io/goauthentik/dev-%(type)s:gh-%(build_hash)s ``` - For arm64, use these values: - - ```shell - AUTHENTIK_IMAGE=ghcr.io/goauthentik/dev-server - AUTHENTIK_TAG=${{ inputs.tag }}-arm64 - AUTHENTIK_OUTPOSTS__CONTAINER_IMAGE_BASE=ghcr.io/goauthentik/dev-%(type)s:gh-%(build_hash)s - ``` - Afterwards, run the upgrade commands from the latest release notes.
@@ -60,18 +52,6 @@ runs: tag: ${{ inputs.tag }} ``` - For arm64, use these values: - - ```yaml - authentik: - outposts: - container_image_base: ghcr.io/goauthentik/dev-%(type)s:gh-%(build_hash)s - global: - image: - repository: ghcr.io/goauthentik/dev-server - tag: ${{ inputs.tag }}-arm64 - ``` - Afterwards, run the upgrade commands from the latest release notes.
edit-mode: replace diff --git a/.github/actions/docker-push-variables/action.yml b/.github/actions/docker-push-variables/action.yml index e95361a5d8..04fdb8d6b2 100644 --- a/.github/actions/docker-push-variables/action.yml +++ b/.github/actions/docker-push-variables/action.yml @@ -29,9 +29,15 @@ outputs: imageTags: description: "Docker image tags" value: ${{ steps.ev.outputs.imageTags }} + imageTagsJSON: + description: "Docker image tags, as a JSON array" + value: ${{ steps.ev.outputs.imageTagsJSON }} attestImageNames: description: "Docker image names used for attestation" value: ${{ steps.ev.outputs.attestImageNames }} + cacheTo: + description: "cache-to value for the docker build step" + value: ${{ steps.ev.outputs.cacheTo }} imageMainTag: description: "Docker image main tag" value: ${{ steps.ev.outputs.imageMainTag }} diff --git a/.github/actions/docker-push-variables/push_vars.py b/.github/actions/docker-push-variables/push_vars.py index 7599633401..feeae09b8c 100644 --- a/.github/actions/docker-push-variables/push_vars.py +++ b/.github/actions/docker-push-variables/push_vars.py @@ -2,6 +2,7 @@ import configparser import os +from json import dumps from time import time parser = configparser.ConfigParser() @@ -48,7 +49,7 @@ if is_release: ] else: suffix = "" - if image_arch and image_arch != "amd64": + if image_arch: suffix = f"-{image_arch}" for name in image_names: image_tags += [ @@ -70,12 +71,23 @@ def get_attest_image_names(image_with_tags: list[str]): return ",".join(set(image_tags)) +# Generate `cache-to` param +cache_to = "" +if should_push: + _cache_tag = "buildcache" + if image_arch: + _cache_tag += f"-{image_arch}" + cache_to = f"type=registry,ref={get_attest_image_names(image_tags)}:{_cache_tag},mode=max" + + with open(os.environ["GITHUB_OUTPUT"], "a+", encoding="utf-8") as _output: print(f"shouldPush={str(should_push).lower()}", file=_output) print(f"sha={sha}", file=_output) print(f"version={version}", file=_output) print(f"prerelease={prerelease}", file=_output) print(f"imageTags={','.join(image_tags)}", file=_output) + print(f"imageTagsJSON={dumps(image_tags)}", file=_output) print(f"attestImageNames={get_attest_image_names(image_tags)}", file=_output) print(f"imageMainTag={image_main_tag}", file=_output) print(f"imageMainName={image_tags[0]}", file=_output) + print(f"cacheTo={cache_to}", file=_output) diff --git a/.github/actions/docker-push-variables/test.sh b/.github/actions/docker-push-variables/test.sh index bfb6cca8af..aa2d1579db 100755 --- a/.github/actions/docker-push-variables/test.sh +++ b/.github/actions/docker-push-variables/test.sh @@ -1,7 +1,18 @@ #!/bin/bash -x SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# Non-pushing PR GITHUB_OUTPUT=/dev/stdout \ GITHUB_REF=ref \ GITHUB_SHA=sha \ IMAGE_NAME=ghcr.io/goauthentik/server,beryju/authentik \ + GITHUB_REPOSITORY=goauthentik/authentik \ + python $SCRIPT_DIR/push_vars.py + +# Pushing PR/main +GITHUB_OUTPUT=/dev/stdout \ + GITHUB_REF=ref \ + GITHUB_SHA=sha \ + IMAGE_NAME=ghcr.io/goauthentik/server,beryju/authentik \ + GITHUB_REPOSITORY=goauthentik/authentik \ + DOCKER_USERNAME=foo \ python $SCRIPT_DIR/push_vars.py diff --git a/.github/workflows/_reusable-docker-build-single.yaml b/.github/workflows/_reusable-docker-build-single.yaml new file mode 100644 index 0000000000..cca6a9a213 --- /dev/null +++ b/.github/workflows/_reusable-docker-build-single.yaml @@ -0,0 +1,94 @@ +# Re-usable workflow for a single-architecture build +name: Single-arch Container build + +on: + workflow_call: + inputs: + image_name: + required: true + type: string + image_arch: + required: true + type: string + runs-on: + required: true + type: string + registry_dockerhub: + default: false + type: boolean + registry_ghcr: + default: false + type: boolean + release: + default: false + type: boolean + outputs: + image-digest: + value: ${{ jobs.build.outputs.image-digest }} + +jobs: + build: + name: Build ${{ inputs.image_arch }} + runs-on: ${{ inputs.runs-on }} + outputs: + image-digest: ${{ steps.push.outputs.digest }} + permissions: + # Needed to upload container images to ghcr.io + packages: write + # Needed for attestation + id-token: write + attestations: write + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3.3.0 + - uses: docker/setup-buildx-action@v3 + - name: prepare variables + uses: ./.github/actions/docker-push-variables + id: ev + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + with: + image-name: ${{ inputs.image_name }} + image-arch: ${{ inputs.image_arch }} + - name: Login to Docker Hub + if: ${{ inputs.registry_dockerhub }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to GitHub Container Registry + if: ${{ inputs.registry_ghcr }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: make empty clients + if: ${{ inputs.release }} + run: | + mkdir -p ./gen-ts-api + mkdir -p ./gen-go-api + - name: generate ts client + if: ${{ !inputs.release }} + run: make gen-client-ts + - name: Build Docker Image + uses: docker/build-push-action@v6 + id: push + with: + context: . + push: true + secrets: | + GEOIPUPDATE_ACCOUNT_ID=${{ secrets.GEOIPUPDATE_ACCOUNT_ID }} + GEOIPUPDATE_LICENSE_KEY=${{ secrets.GEOIPUPDATE_LICENSE_KEY }} + build-args: | + VERSION=${{ github.ref }} + tags: ${{ steps.ev.outputs.imageTags }} + platforms: linux/${{ inputs.image_arch }} + cache-from: type=registry,ref=${{ steps.ev.outputs.attestImageNames }}:buildcache-${{ inputs.image_arch }} + cache-to: ${{ steps.ev.outputs.cacheTo }} + - uses: actions/attest-build-provenance@v2 + id: attest + with: + subject-name: ${{ steps.ev.outputs.attestImageNames }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/_reusable-docker-build.yaml b/.github/workflows/_reusable-docker-build.yaml new file mode 100644 index 0000000000..833c9d443d --- /dev/null +++ b/.github/workflows/_reusable-docker-build.yaml @@ -0,0 +1,102 @@ +# Re-usable workflow for a multi-architecture build +name: Multi-arch container build + +on: + workflow_call: + inputs: + image_name: + required: true + type: string + registry_dockerhub: + default: false + type: boolean + registry_ghcr: + default: true + type: boolean + release: + default: false + type: boolean + outputs: {} + +jobs: + build-server-amd64: + uses: ./.github/workflows/_reusable-docker-build-single.yaml + secrets: inherit + with: + image_name: ${{ inputs.image_name }} + image_arch: amd64 + runs-on: ubuntu-latest + registry_dockerhub: ${{ inputs.registry_dockerhub }} + registry_ghcr: ${{ inputs.registry_ghcr }} + release: ${{ inputs.release }} + build-server-arm64: + uses: ./.github/workflows/_reusable-docker-build-single.yaml + secrets: inherit + with: + image_name: ${{ inputs.image_name }} + image_arch: arm64 + runs-on: ubuntu-22.04-arm + registry_dockerhub: ${{ inputs.registry_dockerhub }} + registry_ghcr: ${{ inputs.registry_ghcr }} + release: ${{ inputs.release }} + get-tags: + runs-on: ubuntu-latest + needs: + - build-server-amd64 + - build-server-arm64 + outputs: + tags: ${{ steps.ev.outputs.imageTagsJSON }} + steps: + - uses: actions/checkout@v4 + - name: prepare variables + uses: ./.github/actions/docker-push-variables + id: ev + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + with: + image-name: ${{ inputs.image_name }} + merge-server: + runs-on: ubuntu-latest + needs: + - get-tags + - build-server-amd64 + - build-server-arm64 + strategy: + fail-fast: false + matrix: + tag: ${{ fromJson(needs.get-tags.outputs.tags) }} + steps: + - uses: actions/checkout@v4 + - name: prepare variables + uses: ./.github/actions/docker-push-variables + id: ev + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + with: + image-name: ${{ inputs.image_name }} + - name: Login to Docker Hub + if: ${{ inputs.registry_dockerhub }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to GitHub Container Registry + if: ${{ inputs.registry_ghcr }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: int128/docker-manifest-create-action@v2 + id: build + with: + tags: ${{ matrix.tag }} + sources: | + ${{ steps.ev.outputs.attestImageNames }}@${{ needs.build-server-amd64.outputs.image-digest }} + ${{ steps.ev.outputs.attestImageNames }}@${{ needs.build-server-arm64.outputs.image-digest }} + - uses: actions/attest-build-provenance@v2 + id: attest + with: + subject-name: ${{ steps.ev.outputs.attestImageNames }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 3aca6533a7..d88da9dd6f 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -223,68 +223,18 @@ jobs: with: jobs: ${{ toJSON(needs) }} build: - strategy: - fail-fast: false - matrix: - arch: - - amd64 - - arm64 - needs: ci-core-mark - runs-on: ubuntu-latest permissions: - # Needed to upload contianer images to ghcr.io + # Needed to upload container images to ghcr.io packages: write # Needed for attestation id-token: write attestations: write - timeout-minutes: 120 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.3.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: prepare variables - uses: ./.github/actions/docker-push-variables - id: ev - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - with: - image-name: ghcr.io/goauthentik/dev-server - image-arch: ${{ matrix.arch }} - - name: Login to Container Registry - if: ${{ steps.ev.outputs.shouldPush == 'true' }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: generate ts client - run: make gen-client-ts - - name: Build Docker Image - uses: docker/build-push-action@v6 - id: push - with: - context: . - secrets: | - GEOIPUPDATE_ACCOUNT_ID=${{ secrets.GEOIPUPDATE_ACCOUNT_ID }} - GEOIPUPDATE_LICENSE_KEY=${{ secrets.GEOIPUPDATE_LICENSE_KEY }} - tags: ${{ steps.ev.outputs.imageTags }} - push: ${{ steps.ev.outputs.shouldPush == 'true' }} - build-args: | - GIT_BUILD_HASH=${{ steps.ev.outputs.sha }} - cache-from: type=registry,ref=ghcr.io/goauthentik/dev-server:buildcache - cache-to: ${{ steps.ev.outputs.shouldPush == 'true' && 'type=registry,ref=ghcr.io/goauthentik/dev-server:buildcache,mode=max' || '' }} - platforms: linux/${{ matrix.arch }} - - uses: actions/attest-build-provenance@v2 - id: attest - if: ${{ steps.ev.outputs.shouldPush == 'true' }} - with: - subject-name: ${{ steps.ev.outputs.attestImageNames }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + needs: ci-core-mark + uses: ./.github/workflows/_reusable-docker-build.yaml + secrets: inherit + with: + image_name: ghcr.io/goauthentik/dev-server + release: false pr-comment: needs: - build diff --git a/.github/workflows/ci-outpost.yml b/.github/workflows/ci-outpost.yml index ef68a6af2c..49be7efae0 100644 --- a/.github/workflows/ci-outpost.yml +++ b/.github/workflows/ci-outpost.yml @@ -72,7 +72,7 @@ jobs: - rac runs-on: ubuntu-latest permissions: - # Needed to upload contianer images to ghcr.io + # Needed to upload container images to ghcr.io packages: write # Needed for attestation id-token: write diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index f4f3bd0f83..3a828e7fc1 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -7,64 +7,15 @@ on: jobs: build-server: - runs-on: ubuntu-latest - permissions: - # Needed to upload contianer images to ghcr.io - packages: write - # Needed for attestation - id-token: write - attestations: write - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.3.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: prepare variables - uses: ./.github/actions/docker-push-variables - id: ev - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - with: - image-name: ghcr.io/goauthentik/server,beryju/authentik - - name: Docker Login Registry - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: make empty clients - run: | - mkdir -p ./gen-ts-api - mkdir -p ./gen-go-api - - name: Build Docker Image - uses: docker/build-push-action@v6 - id: push - with: - context: . - push: true - secrets: | - GEOIPUPDATE_ACCOUNT_ID=${{ secrets.GEOIPUPDATE_ACCOUNT_ID }} - GEOIPUPDATE_LICENSE_KEY=${{ secrets.GEOIPUPDATE_LICENSE_KEY }} - build-args: | - VERSION=${{ github.ref }} - tags: ${{ steps.ev.outputs.imageTags }} - platforms: linux/amd64,linux/arm64 - - uses: actions/attest-build-provenance@v2 - id: attest - with: - subject-name: ${{ steps.ev.outputs.attestImageNames }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true + uses: ./.github/workflows/_reusable-docker-build.yaml + secrets: inherit + with: + image_name: ghcr.io/goauthentik/server,beryju/authentik + release: true build-outpost: runs-on: ubuntu-latest permissions: - # Needed to upload contianer images to ghcr.io + # Needed to upload container images to ghcr.io packages: write # Needed for attestation id-token: write diff --git a/Dockerfile b/Dockerfile index ea9f491a12..f71c93aba7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ RUN --mount=type=secret,id=GEOIPUPDATE_ACCOUNT_ID \ /bin/sh -c "/usr/bin/entry.sh || echo 'Failed to get GeoIP database, disabling'; exit 0" # Stage 5: Python dependencies -FROM ghcr.io/goauthentik/fips-python:3.12.7-slim-bookworm-fips-full AS python-deps +FROM ghcr.io/goauthentik/fips-python:3.12.7-slim-bookworm-fips AS python-deps ARG TARGETARCH ARG TARGETVARIANT @@ -116,15 +116,29 @@ RUN --mount=type=bind,target=./pyproject.toml,src=./pyproject.toml \ --mount=type=bind,target=./poetry.lock,src=./poetry.lock \ --mount=type=cache,target=/root/.cache/pip \ --mount=type=cache,target=/root/.cache/pypoetry \ + pip install --no-cache cffi && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential libffi-dev \ + # Required for cryptography + curl pkg-config \ + # Required for lxml + libxslt-dev zlib1g-dev \ + # Required for xmlsec + libltdl-dev \ + # Required for kadmin + sccache clang && \ + curl https://sh.rustup.rs -sSf | sh -s -- -y && \ + . "$HOME/.cargo/env" && \ python -m venv /ak-root/venv/ && \ bash -c "source ${VENV_PATH}/bin/activate && \ pip3 install --upgrade pip && \ pip3 install poetry && \ - poetry install --only=main --no-ansi --no-interaction --no-root && \ - pip install --force-reinstall /wheels/*" + poetry config --local installer.no-binary cryptography,xmlsec,lxml,python-kadmin-rs && \ + poetry install --only=main --no-ansi --no-interaction --no-root" # Stage 6: Run -FROM ghcr.io/goauthentik/fips-python:3.12.7-slim-bookworm-fips-full AS final-image +FROM ghcr.io/goauthentik/fips-python:3.12.7-slim-bookworm-fips AS final-image ARG VERSION ARG GIT_BUILD_HASH @@ -141,7 +155,7 @@ WORKDIR / # We cannot cache this layer otherwise we'll end up with a bigger image RUN apt-get update && \ # Required for runtime - apt-get install -y --no-install-recommends libpq5 libmaxminddb0 ca-certificates libkrb5-3 libkadm5clnt-mit12 libkdb5-10 && \ + apt-get install -y --no-install-recommends libpq5 libmaxminddb0 ca-certificates libkrb5-3 libkadm5clnt-mit12 libkdb5-10 libltdl7 libxslt1.1 && \ # Required for bootstrap & healtcheck apt-get install -y --no-install-recommends runit && \ apt-get clean && \ @@ -176,9 +190,8 @@ ENV TMPDIR=/dev/shm/ \ PYTHONUNBUFFERED=1 \ PATH="/ak-root/venv/bin:/lifecycle:$PATH" \ VENV_PATH="/ak-root/venv" \ - POETRY_VIRTUALENVS_CREATE=false - -ENV GOFIPS=1 + POETRY_VIRTUALENVS_CREATE=false \ + GOFIPS=1 HEALTHCHECK --interval=30s --timeout=30s --start-period=60s --retries=3 CMD [ "ak", "healthcheck" ] diff --git a/website/docs/install-config/beta.mdx b/website/docs/install-config/beta.mdx index 6a548934d7..85b49922c7 100644 --- a/website/docs/install-config/beta.mdx +++ b/website/docs/install-config/beta.mdx @@ -27,8 +27,6 @@ AUTHENTIK_TAG=gh-next AUTHENTIK_OUTPOSTS__CONTAINER_IMAGE_BASE=ghcr.io/goauthentik/dev-%(type)s:gh-%(build_hash)s ``` -The Beta image is amd64 only. For arm64 platforms, append `-arm64` to the tag name (no spaces). - Next, run the upgrade commands below. @@ -47,8 +45,6 @@ image: pullPolicy: Always ``` -The Beta image is amd64 only. For arm64 platforms, append `-arm64` to the tag name (no spaces). - Next, run the upgrade commands below.