Speed up multi-arch Docker builds

2026-06-07 07:55:16 +02:00 · 2026-05-28 06:56:56 +00:00 · 2026-05-28 06:56:56 +00:00 · 2a3d7b00f1
commit 2a3d7b00f1
parent 93edef35e8
4 changed files with 201 additions and 64 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,4 +1,17 @@
 api/.env
+.git
+.github
+.claude
+**/.claude
+**/.next
+**/__pycache__
+**/*.pyc
+**/node_modules
+.mypy_cache
+.pytest_cache
+.ruff_cache
+.venv
 evals/
 api/mcp_server/ts_validator/node_modules/
 sdk/
+venv
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@ -3,24 +3,83 @@ name: Build and Push Docker Images
 on:
  release:
    types: [published]
+  push:
+    branches:
+      - docker-build-speedup
+      - docker-build-speedup-*
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Tag to publish for a manual test run. Defaults to test-<short-sha>."
+        required: false
+        type: string
+      push_latest:
+        description: "Also update :latest. Leave false for test runs."
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+  packages: write

-# Ensure only one workflow run per branch at a time; cancel any in-progress runs on new push
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

-jobs:
-  build:
-    runs-on: ubuntu-latest
-    env:
-      COMMIT_SHA: ${{ github.sha }}
+env:
+  REGISTRY_GHCR: ghcr.io

+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      short_sha: ${{ steps.tags.outputs.short_sha }}
+      version: ${{ steps.tags.outputs.version }}
+      push_latest: ${{ steps.tags.outputs.push_latest }}
+    steps:
+      - name: Compute tags
+        id: tags
+        run: |
+          SHORT_SHA="${GITHUB_SHA::8}"
+
+          if [ "${{ github.event_name }}" = "release" ]; then
+            VERSION="${{ github.event.release.tag_name }}"
+            VERSION="${VERSION#dograh-}"
+            VERSION="${VERSION#v}"
+            PUSH_LATEST="true"
+          else
+            VERSION="${{ inputs.image_tag }}"
+            if [ -z "$VERSION" ]; then
+              VERSION="test-${SHORT_SHA}"
+            fi
+            PUSH_LATEST="${{ inputs.push_latest }}"
+          fi
+
+          echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "push_latest=${PUSH_LATEST}" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: prepare
    strategy:
+      fail-fast: false
      matrix:
        service:
-          - "dograh-api|api/Dockerfile|."
-          - "dograh-ui|ui/Dockerfile|."
-
+          - name: dograh-api
+            dockerfile: api/Dockerfile
+            context: .
+          - name: dograh-ui
+            dockerfile: ui/Dockerfile
+            context: .
+        platform:
+          - name: linux/amd64
+            runner: ubuntu-24.04
+            short: amd64
+          - name: linux/arm64
+            runner: ubuntu-24.04-arm
+            short: arm64
+    runs-on: ${{ matrix.platform.runner }}
    steps:
      - name: Free Disk Space
        uses: jlumbroso/free-disk-space@main
@ -38,90 +97,153 @@ jobs:
        with:
          submodules: true

-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4

      - name: Log in to DockerHub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Log in to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
        with:
-          registry: ghcr.io
+          registry: ${{ env.REGISTRY_GHCR }}
          username: ${{ secrets.GHCR_USERNAME }}
          password: ${{ secrets.GHCR_TOKEN }}

-      - name: Set build variables
-        id: build-vars
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          context: ${{ matrix.service.context }}
+          file: ${{ matrix.service.dockerfile }}
+          platforms: ${{ matrix.platform.name }}
+          outputs: 'type=image,"name=${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.service.name }},${{ env.REGISTRY_GHCR }}/${{ secrets.GHCR_USERNAME }}/${{ matrix.service.name }}",push-by-digest=true,name-canonical=true,push=true'
+          cache-from: type=gha,scope=${{ matrix.service.name }}-${{ matrix.platform.short }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service.name }}-${{ matrix.platform.short }}
+
+      - name: Export digest
        run: |
-          SERVICE="${{ matrix.service }}"
-          IMAGE_NAME=$(echo "$SERVICE" | cut -d '|' -f1)
-          SHORT_SHA=${COMMIT_SHA::8}
+          mkdir -p "/tmp/digests/${{ matrix.service.name }}"
+          echo "${{ steps.build.outputs.digest }}" | sed 's/^sha256://' > "/tmp/digests/${{ matrix.service.name }}/${{ matrix.platform.short }}"

-          # Get version from release tag (removes 'dograh-' and 'v' prefixes if present)
-          VERSION="${{ github.event.release.tag_name }}"
-          VERSION="${VERSION#dograh-}"
-          VERSION="${VERSION#v}"
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digest-${{ matrix.service.name }}-${{ matrix.platform.short }}
+          path: /tmp/digests/${{ matrix.service.name }}/${{ matrix.platform.short }}
+          retention-days: 1

-          echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
-          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+  merge:
+    needs:
+      - prepare
+      - build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download API digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digest-dograh-api-*
+          merge-multiple: true
+          path: /tmp/digests/dograh-api

-      - name: Build and Push ${{ matrix.service }}
-        id: docker-build
+      - name: Download UI digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digest-dograh-ui-*
+          merge-multiple: true
+          path: /tmp/digests/dograh-ui
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY_GHCR }}
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Create manifest lists
+        env:
+          DH_NAMESPACE: ${{ secrets.DOCKERHUB_USERNAME }}
+          GH_NAMESPACE: ${{ env.REGISTRY_GHCR }}/${{ secrets.GHCR_USERNAME }}
+          VERSION: ${{ needs.prepare.outputs.version }}
+          SHORT_SHA: ${{ needs.prepare.outputs.short_sha }}
+          PUSH_LATEST: ${{ needs.prepare.outputs.push_latest }}
        run: |
-          SERVICE="${{ matrix.service }}"
-          IMAGE_NAME=$(echo "$SERVICE" | cut -d '|' -f1)
-          DOCKERFILE=$(echo "$SERVICE" | cut -d '|' -f2)
-          CONTEXT=$(echo "$SERVICE" | cut -d '|' -f3)
-          SHORT_SHA=${COMMIT_SHA::8}
-          VERSION="${{ steps.build-vars.outputs.version }}"
+          inspect_digests() {
+            service="$1"
+            digest_dir="/tmp/digests/$service"
+            dh_image="$DH_NAMESPACE/$service"
+            gh_image="$GH_NAMESPACE/$service"

-          echo "Building and pushing image: $IMAGE_NAME"
-          echo "Dockerfile: $DOCKERFILE"
-          echo "Context: $CONTEXT"
-          echo "Version: $VERSION"
+            for digest_file in "$digest_dir"/*; do
+              digest="$(cat "$digest_file")"
+              docker buildx imagetools inspect "$dh_image@sha256:$digest" >/dev/null
+              docker buildx imagetools inspect "$gh_image@sha256:$digest" >/dev/null
+            done
+          }

-          echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT
-          echo "dockerhub_tag=${{ secrets.DOCKERHUB_USERNAME }}/${IMAGE_NAME}:${SHORT_SHA}" >> $GITHUB_OUTPUT
-          echo "ghcr_tag=ghcr.io/${{ secrets.GHCR_USERNAME }}/${IMAGE_NAME}:${SHORT_SHA}" >> $GITHUB_OUTPUT
-          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+          create_manifests() {
+            service="$1"
+            digest_dir="/tmp/digests/$service"
+            dh_image="$DH_NAMESPACE/$service"
+            gh_image="$GH_NAMESPACE/$service"

-          docker buildx build \
-            -f "$DOCKERFILE" \
-            --platform linux/amd64,linux/arm64 \
-            --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:$VERSION \
-            --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:$SHORT_SHA \
-            --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:latest \
-            --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:$VERSION \
-            --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:$SHORT_SHA \
-            --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:latest \
-            --push "$CONTEXT"
+            dh_refs=$(printf "${dh_image}@sha256:%s " $(cat "$digest_dir"/*))
+            gh_refs=$(printf "${gh_image}@sha256:%s " $(cat "$digest_dir"/*))

-      - name: Send Slack notification - Success
-        if: success()
+            dh_tags=(-t "$dh_image:$VERSION" -t "$dh_image:$SHORT_SHA")
+            gh_tags=(-t "$gh_image:$VERSION" -t "$gh_image:$SHORT_SHA")
+
+            if [ "$PUSH_LATEST" = "true" ]; then
+              dh_tags+=(-t "$dh_image:latest")
+              gh_tags+=(-t "$gh_image:latest")
+            fi
+
+            docker buildx imagetools create "${dh_tags[@]}" $dh_refs
+            docker buildx imagetools create "${gh_tags[@]}" $gh_refs
+          }
+
+          inspect_digests dograh-api
+          inspect_digests dograh-ui
+          create_manifests dograh-api
+          create_manifests dograh-ui
+
+  notify:
+    needs:
+      - prepare
+      - merge
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Slack success
+        if: needs.merge.result == 'success'
        uses: slackapi/slack-github-action@v1.26.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          payload: |
            {
-              "text": "✅ Docker Build Successful - ${{ steps.build-vars.outputs.image_name }} (${{ steps.build-vars.outputs.version }}) on ${{ github.ref_name }} by ${{ github.actor }}"
+              "text": "✅ Docker images built for ${{ needs.prepare.outputs.version }} on ${{ github.ref_name }} by ${{ github.actor }}"
            }

-      - name: Send Slack notification - Failure
-        if: failure()
+      - name: Slack failure
+        if: needs.merge.result != 'success'
        uses: slackapi/slack-github-action@v1.26.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          payload: |
            {
-              "text": "❌ Docker Build Failed - ${{ steps.build-vars.outputs.image_name }} (${{ steps.build-vars.outputs.version }}) on ${{ github.ref_name }} by ${{ github.actor }} - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>"
+              "text": "❌ Docker build failed for ${{ needs.prepare.outputs.version }} on ${{ github.ref_name }} by ${{ github.actor }} - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>"
            }
--- a/api/Dockerfile
+++ b/api/Dockerfile
@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 # Multi-stage Dockerfile
 # Stage 1: Builder - Install Python dependencies into a venv via uv
 # (mirrors .devcontainer/Dockerfile's venv-builder stage).
@ -58,9 +59,10 @@ RUN npm ci --omit=dev && npm cache clean --force
 # Stage 3: Static ffmpeg binary (avoids apt ffmpeg pulling mesa/libllvm for
 # hardware acceleration we don't use server-side).
 FROM debian:trixie-slim AS ffmpeg-static
+ARG TARGETARCH
 RUN apt-get update && apt-get install -y --no-install-recommends \
        curl ca-certificates xz-utils \
-    && curl -fsSL -o /tmp/ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz \
+    && curl -fsSL -o /tmp/ffmpeg.tar.xz "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-${TARGETARCH}-static.tar.xz" \
    && mkdir -p /tmp/ffmpeg \
    && tar -xJf /tmp/ffmpeg.tar.xz -C /tmp/ffmpeg --strip-components=1 \
    && mv /tmp/ffmpeg/ffmpeg /tmp/ffmpeg/ffprobe /usr/local/bin/ \
--- a/ui/Dockerfile
+++ b/ui/Dockerfile
@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 # Multi-stage build
 # Stage 1: Dependencies
 FROM node:20-alpine AS deps
@ -11,8 +12,7 @@ RUN apk add --no-cache python3 make g++ libc6-compat
 COPY ui/package*.json ./

 # Clean install with proper handling of native modules
-RUN rm -rf node_modules && \
-    npm ci || npm install
+RUN --mount=type=cache,target=/root/.npm npm ci

 # Stage 2: Builder
 FROM node:20-alpine AS builder
@ -71,4 +71,4 @@ USER nextjs
 EXPOSE 3010

 # Start the production server using the standalone Node.js server
-CMD sh -c "echo '🚀 Application ready at http://localhost:3010' && PORT=3010 node server.js"
+CMD sh -c "echo '🚀 Application ready at http://localhost:3010' && PORT=3010 node server.js"