diff --git a/.dockerignore b/.dockerignore index 38086c3..ebcee79 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,17 @@ api/.env +.git +.github +.claude +**/.claude +**/.next +**/__pycache__ +**/*.pyc +**/node_modules +.mypy_cache +.pytest_cache +.ruff_cache +.venv evals/ api/mcp_server/ts_validator/node_modules/ sdk/ +venv diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3747b77..1bea8a4 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -3,24 +3,83 @@ name: Build and Push Docker Images on: release: types: [published] + push: + branches: + - docker-build-speedup + - docker-build-speedup-* + workflow_dispatch: + inputs: + image_tag: + description: "Tag to publish for a manual test run. Defaults to test-." + required: false + type: string + push_latest: + description: "Also update :latest. Leave false for test runs." + required: false + default: false + type: boolean + +permissions: + contents: read + packages: write -# Ensure only one workflow run per branch at a time; cancel any in-progress runs on new push concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -jobs: - build: - runs-on: ubuntu-latest - env: - COMMIT_SHA: ${{ github.sha }} +env: + REGISTRY_GHCR: ghcr.io +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + short_sha: ${{ steps.tags.outputs.short_sha }} + version: ${{ steps.tags.outputs.version }} + push_latest: ${{ steps.tags.outputs.push_latest }} + steps: + - name: Compute tags + id: tags + run: | + SHORT_SHA="${GITHUB_SHA::8}" + + if [ "${{ github.event_name }}" = "release" ]; then + VERSION="${{ github.event.release.tag_name }}" + VERSION="${VERSION#dograh-}" + VERSION="${VERSION#v}" + PUSH_LATEST="true" + else + VERSION="${{ inputs.image_tag }}" + if [ -z "$VERSION" ]; then + VERSION="test-${SHORT_SHA}" + fi + PUSH_LATEST="${{ inputs.push_latest }}" + fi + + echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "push_latest=${PUSH_LATEST}" >> "$GITHUB_OUTPUT" + + build: + needs: prepare strategy: + fail-fast: false matrix: service: - - "dograh-api|api/Dockerfile|." - - "dograh-ui|ui/Dockerfile|." - + - name: dograh-api + dockerfile: api/Dockerfile + context: . + - name: dograh-ui + dockerfile: ui/Dockerfile + context: . + platform: + - name: linux/amd64 + runner: ubuntu-24.04 + short: amd64 + - name: linux/arm64 + runner: ubuntu-24.04-arm + short: arm64 + runs-on: ${{ matrix.platform.runner }} steps: - name: Free Disk Space uses: jlumbroso/free-disk-space@main @@ -38,90 +97,153 @@ jobs: with: submodules: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Log in to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: - registry: ghcr.io + registry: ${{ env.REGISTRY_GHCR }} username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - - name: Set build variables - id: build-vars + - name: Build and push by digest + id: build + uses: docker/build-push-action@v7 + with: + context: ${{ matrix.service.context }} + file: ${{ matrix.service.dockerfile }} + platforms: ${{ matrix.platform.name }} + outputs: 'type=image,"name=${{ secrets.DOCKERHUB_USERNAME }}/${{ matrix.service.name }},${{ env.REGISTRY_GHCR }}/${{ secrets.GHCR_USERNAME }}/${{ matrix.service.name }}",push-by-digest=true,name-canonical=true,push=true' + cache-from: type=gha,scope=${{ matrix.service.name }}-${{ matrix.platform.short }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }}-${{ matrix.platform.short }} + + - name: Export digest run: | - SERVICE="${{ matrix.service }}" - IMAGE_NAME=$(echo "$SERVICE" | cut -d '|' -f1) - SHORT_SHA=${COMMIT_SHA::8} + mkdir -p "/tmp/digests/${{ matrix.service.name }}" + echo "${{ steps.build.outputs.digest }}" | sed 's/^sha256://' > "/tmp/digests/${{ matrix.service.name }}/${{ matrix.platform.short }}" - # Get version from release tag (removes 'dograh-' and 'v' prefixes if present) - VERSION="${{ github.event.release.tag_name }}" - VERSION="${VERSION#dograh-}" - VERSION="${VERSION#v}" + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.service.name }}-${{ matrix.platform.short }} + path: /tmp/digests/${{ matrix.service.name }}/${{ matrix.platform.short }} + retention-days: 1 - echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT - echo "version=${VERSION}" >> $GITHUB_OUTPUT + merge: + needs: + - prepare + - build + runs-on: ubuntu-latest + steps: + - name: Download API digests + uses: actions/download-artifact@v4 + with: + pattern: digest-dograh-api-* + merge-multiple: true + path: /tmp/digests/dograh-api - - name: Build and Push ${{ matrix.service }} - id: docker-build + - name: Download UI digests + uses: actions/download-artifact@v4 + with: + pattern: digest-dograh-ui-* + merge-multiple: true + path: /tmp/digests/dograh-ui + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Log in to GHCR + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY_GHCR }} + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Create manifest lists + env: + DH_NAMESPACE: ${{ secrets.DOCKERHUB_USERNAME }} + GH_NAMESPACE: ${{ env.REGISTRY_GHCR }}/${{ secrets.GHCR_USERNAME }} + VERSION: ${{ needs.prepare.outputs.version }} + SHORT_SHA: ${{ needs.prepare.outputs.short_sha }} + PUSH_LATEST: ${{ needs.prepare.outputs.push_latest }} run: | - SERVICE="${{ matrix.service }}" - IMAGE_NAME=$(echo "$SERVICE" | cut -d '|' -f1) - DOCKERFILE=$(echo "$SERVICE" | cut -d '|' -f2) - CONTEXT=$(echo "$SERVICE" | cut -d '|' -f3) - SHORT_SHA=${COMMIT_SHA::8} - VERSION="${{ steps.build-vars.outputs.version }}" + inspect_digests() { + service="$1" + digest_dir="/tmp/digests/$service" + dh_image="$DH_NAMESPACE/$service" + gh_image="$GH_NAMESPACE/$service" - echo "Building and pushing image: $IMAGE_NAME" - echo "Dockerfile: $DOCKERFILE" - echo "Context: $CONTEXT" - echo "Version: $VERSION" + for digest_file in "$digest_dir"/*; do + digest="$(cat "$digest_file")" + docker buildx imagetools inspect "$dh_image@sha256:$digest" >/dev/null + docker buildx imagetools inspect "$gh_image@sha256:$digest" >/dev/null + done + } - echo "image_name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - echo "dockerhub_tag=${{ secrets.DOCKERHUB_USERNAME }}/${IMAGE_NAME}:${SHORT_SHA}" >> $GITHUB_OUTPUT - echo "ghcr_tag=ghcr.io/${{ secrets.GHCR_USERNAME }}/${IMAGE_NAME}:${SHORT_SHA}" >> $GITHUB_OUTPUT - echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + create_manifests() { + service="$1" + digest_dir="/tmp/digests/$service" + dh_image="$DH_NAMESPACE/$service" + gh_image="$GH_NAMESPACE/$service" - docker buildx build \ - -f "$DOCKERFILE" \ - --platform linux/amd64,linux/arm64 \ - --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:$VERSION \ - --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:$SHORT_SHA \ - --tag ${{ secrets.DOCKERHUB_USERNAME }}/$IMAGE_NAME:latest \ - --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:$VERSION \ - --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:$SHORT_SHA \ - --tag ghcr.io/${{ secrets.GHCR_USERNAME }}/$IMAGE_NAME:latest \ - --push "$CONTEXT" + dh_refs=$(printf "${dh_image}@sha256:%s " $(cat "$digest_dir"/*)) + gh_refs=$(printf "${gh_image}@sha256:%s " $(cat "$digest_dir"/*)) - - name: Send Slack notification - Success - if: success() + dh_tags=(-t "$dh_image:$VERSION" -t "$dh_image:$SHORT_SHA") + gh_tags=(-t "$gh_image:$VERSION" -t "$gh_image:$SHORT_SHA") + + if [ "$PUSH_LATEST" = "true" ]; then + dh_tags+=(-t "$dh_image:latest") + gh_tags+=(-t "$gh_image:latest") + fi + + docker buildx imagetools create "${dh_tags[@]}" $dh_refs + docker buildx imagetools create "${gh_tags[@]}" $gh_refs + } + + inspect_digests dograh-api + inspect_digests dograh-ui + create_manifests dograh-api + create_manifests dograh-ui + + notify: + needs: + - prepare + - merge + if: always() + runs-on: ubuntu-latest + steps: + - name: Slack success + if: needs.merge.result == 'success' uses: slackapi/slack-github-action@v1.26.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} with: payload: | { - "text": "✅ Docker Build Successful - ${{ steps.build-vars.outputs.image_name }} (${{ steps.build-vars.outputs.version }}) on ${{ github.ref_name }} by ${{ github.actor }}" + "text": "✅ Docker images built for ${{ needs.prepare.outputs.version }} on ${{ github.ref_name }} by ${{ github.actor }}" } - - name: Send Slack notification - Failure - if: failure() + - name: Slack failure + if: needs.merge.result != 'success' uses: slackapi/slack-github-action@v1.26.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} with: payload: | { - "text": "❌ Docker Build Failed - ${{ steps.build-vars.outputs.image_name }} (${{ steps.build-vars.outputs.version }}) on ${{ github.ref_name }} by ${{ github.actor }} - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>" + "text": "❌ Docker build failed for ${{ needs.prepare.outputs.version }} on ${{ github.ref_name }} by ${{ github.actor }} - <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Logs>" } diff --git a/api/Dockerfile b/api/Dockerfile index 85c5f56..a56dd9e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Multi-stage Dockerfile # Stage 1: Builder - Install Python dependencies into a venv via uv # (mirrors .devcontainer/Dockerfile's venv-builder stage). @@ -58,9 +59,10 @@ RUN npm ci --omit=dev && npm cache clean --force # Stage 3: Static ffmpeg binary (avoids apt ffmpeg pulling mesa/libllvm for # hardware acceleration we don't use server-side). FROM debian:trixie-slim AS ffmpeg-static +ARG TARGETARCH RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates xz-utils \ - && curl -fsSL -o /tmp/ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz \ + && curl -fsSL -o /tmp/ffmpeg.tar.xz "https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-${TARGETARCH}-static.tar.xz" \ && mkdir -p /tmp/ffmpeg \ && tar -xJf /tmp/ffmpeg.tar.xz -C /tmp/ffmpeg --strip-components=1 \ && mv /tmp/ffmpeg/ffmpeg /tmp/ffmpeg/ffprobe /usr/local/bin/ \ diff --git a/ui/Dockerfile b/ui/Dockerfile index 314cea7..80dac3f 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Multi-stage build # Stage 1: Dependencies FROM node:20-alpine AS deps @@ -11,8 +12,7 @@ RUN apk add --no-cache python3 make g++ libc6-compat COPY ui/package*.json ./ # Clean install with proper handling of native modules -RUN rm -rf node_modules && \ - npm ci || npm install +RUN --mount=type=cache,target=/root/.npm npm ci # Stage 2: Builder FROM node:20-alpine AS builder @@ -71,4 +71,4 @@ USER nextjs EXPOSE 3010 # Start the production server using the standalone Node.js server -CMD sh -c "echo '🚀 Application ready at http://localhost:3010' && PORT=3010 node server.js" \ No newline at end of file +CMD sh -c "echo '🚀 Application ready at http://localhost:3010' && PORT=3010 node server.js"