mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-08 20:25:19 +02:00
6ebee03931
- Added support for version tagging with 'v*' and 'beta-v*' patterns in the Docker build workflow. - Updated the tag_release job to identify release tags and calculate the next Docker version accordingly. - Improved error handling for version validation and adjusted conditions for build and release processes.
398 lines
15 KiB
YAML
398 lines
15 KiB
YAML
name: Build and Push Docker Images
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- dev
|
|
tags:
|
|
- 'v*'
|
|
- 'beta-v*'
|
|
paths:
|
|
- 'surfsense_backend/**'
|
|
- 'surfsense_web/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
branch:
|
|
description: 'Branch to build from (leave empty for default branch)'
|
|
required: false
|
|
default: ''
|
|
|
|
concurrency:
|
|
group: docker-build
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: write
|
|
packages: write
|
|
|
|
jobs:
|
|
tag_release:
|
|
runs-on: ubuntu-latest
|
|
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/beta-v')
|
|
outputs:
|
|
new_tag: ${{ steps.tag_version.outputs.next_version }}
|
|
commit_sha: ${{ steps.tag_version.outputs.commit_sha }}
|
|
is_release_tag: ${{ steps.tag_version.outputs.is_release_tag }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ github.event.inputs.branch }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
# Compute-only: tag is pushed by finalize_release after everything succeeds.
|
|
- name: Read app version and calculate next Docker build version
|
|
id: tag_version
|
|
run: |
|
|
if [[ "$GITHUB_REF" == refs/tags/beta-v* ]]; then
|
|
VERSION="${GITHUB_REF#refs/tags/beta-v}"
|
|
NEXT_VERSION="beta-${VERSION}"
|
|
IS_RELEASE_TAG="true"
|
|
|
|
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
|
echo "::error::Version '$VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Docker beta release version from git tag: $NEXT_VERSION"
|
|
elif [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
|
NEXT_VERSION="${GITHUB_REF#refs/tags/v}"
|
|
IS_RELEASE_TAG="true"
|
|
|
|
if ! echo "$NEXT_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
|
echo "::error::Version '$NEXT_VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Docker release version from git tag: $NEXT_VERSION"
|
|
else
|
|
APP_VERSION=$(tr -d '[:space:]' < VERSION)
|
|
echo "App version from VERSION file: $APP_VERSION"
|
|
|
|
if [ -z "$APP_VERSION" ]; then
|
|
echo "Error: Could not read version from VERSION file"
|
|
exit 1
|
|
fi
|
|
|
|
git fetch --tags
|
|
|
|
LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
|
|
|
|
if [ -z "$LATEST_BUILD_TAG" ]; then
|
|
echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
|
|
NEXT_VERSION="${APP_VERSION}.1"
|
|
else
|
|
echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
|
|
BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
|
|
NEXT_BUILD=$((BUILD_NUMBER + 1))
|
|
NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
|
|
fi
|
|
|
|
IS_RELEASE_TAG="false"
|
|
echo "Calculated next Docker version: $NEXT_VERSION"
|
|
fi
|
|
|
|
echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
|
|
echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
|
echo "is_release_tag=$IS_RELEASE_TAG" >> $GITHUB_OUTPUT
|
|
|
|
build:
|
|
needs: tag_release
|
|
if: always() && (needs.tag_release.result == 'success' || needs.tag_release.result == 'skipped')
|
|
runs-on: ${{ matrix.os }}
|
|
permissions:
|
|
packages: write
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
platform: [linux/amd64, linux/arm64]
|
|
image: [backend, web]
|
|
variant: [cpu, cuda, cuda126]
|
|
exclude:
|
|
- image: web
|
|
variant: cuda
|
|
- image: web
|
|
variant: cuda126
|
|
include:
|
|
- platform: linux/amd64
|
|
suffix: amd64
|
|
os: ubuntu-latest
|
|
- platform: linux/arm64
|
|
suffix: arm64
|
|
os: ubuntu-24.04-arm
|
|
- image: backend
|
|
name: surfsense-backend
|
|
context: ./surfsense_backend
|
|
file: ./surfsense_backend/Dockerfile
|
|
target: production
|
|
- image: web
|
|
name: surfsense-web
|
|
context: ./surfsense_web
|
|
file: ./surfsense_web/Dockerfile
|
|
target: runner
|
|
- variant: cpu
|
|
tag_suffix: ""
|
|
use_cuda: "false"
|
|
cuda_extra: cpu
|
|
- variant: cuda
|
|
tag_suffix: "-cuda"
|
|
use_cuda: "true"
|
|
cuda_extra: cu128
|
|
- variant: cuda126
|
|
tag_suffix: "-cuda126"
|
|
use_cuda: "true"
|
|
cuda_extra: cu126
|
|
env:
|
|
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
|
|
- name: Set lowercase image name
|
|
id: image
|
|
run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Docker meta
|
|
id: meta
|
|
uses: docker/metadata-action@v6
|
|
with:
|
|
images: ${{ steps.image.outputs.name }}
|
|
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.repository_owner }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v4
|
|
|
|
- name: Free up disk space
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf /usr/local/share/boost
|
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
|
|
docker system prune -af
|
|
|
|
- name: Build and push by digest ${{ matrix.name }} (${{ matrix.variant }}, ${{ matrix.suffix }})
|
|
id: build
|
|
uses: docker/build-push-action@v7
|
|
with:
|
|
context: ${{ matrix.context }}
|
|
file: ${{ matrix.file }}
|
|
target: ${{ matrix.target }}
|
|
labels: ${{ steps.meta.outputs.labels }}
|
|
tags: ${{ steps.image.outputs.name }}
|
|
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
|
platforms: ${{ matrix.platform }}
|
|
cache-from: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }}
|
|
cache-to: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }},mode=max,image-manifest=true,oci-mediatypes=true
|
|
provenance: false
|
|
build-args: |
|
|
${{ matrix.image == 'backend' && format('USE_CUDA={0}', matrix.use_cuda) || '' }}
|
|
${{ matrix.image == 'backend' && format('CUDA_EXTRA={0}', matrix.cuda_extra) || '' }}
|
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
|
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
|
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
|
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_ZERO_CACHE_URL=__NEXT_PUBLIC_ZERO_CACHE_URL__' || '' }}
|
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__' || '' }}
|
|
|
|
- name: Export digest
|
|
run: |
|
|
mkdir -p /tmp/digests
|
|
digest="${{ steps.build.outputs.digest }}"
|
|
touch "/tmp/digests/${digest#sha256:}"
|
|
|
|
- name: Upload digest
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: digests-${{ matrix.image }}-${{ matrix.variant }}-${{ matrix.suffix }}
|
|
path: /tmp/digests/*
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
# Release gate: require both arches for every variant, else block publishing.
|
|
# Release-only; skipped on dev so the tolerant create_manifest path is kept.
|
|
verify_digests:
|
|
runs-on: ubuntu-latest
|
|
needs: [tag_release, build]
|
|
if: ${{ always() && needs.tag_release.result == 'success' && needs.tag_release.outputs.new_tag != '' }}
|
|
steps:
|
|
- name: Download all digests
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: digests-*
|
|
path: /tmp/digests
|
|
merge-multiple: false
|
|
|
|
- name: Require both arches for every required variant
|
|
run: |
|
|
fail=0
|
|
check() {
|
|
c=$(find /tmp/digests -type f -path "*/digests-$1-*/*" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$c" -lt 2 ]; then
|
|
echo "::error::$1 has $c/2 arch digests — blocking release"
|
|
fail=1
|
|
else
|
|
echo "OK: $1 ($c/2)"
|
|
fi
|
|
}
|
|
check backend-cpu
|
|
check backend-cuda
|
|
check backend-cuda126
|
|
check web-cpu
|
|
[ "$fail" -eq 0 ] || exit 1
|
|
|
|
create_manifest:
|
|
runs-on: ubuntu-latest
|
|
needs: [tag_release, build, verify_digests]
|
|
if: ${{ !cancelled() && needs.verify_digests.result != 'failure' }}
|
|
permissions:
|
|
packages: write
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: surfsense-backend
|
|
image: backend
|
|
variant: cpu
|
|
tag_suffix: ""
|
|
- name: surfsense-backend
|
|
image: backend
|
|
variant: cuda
|
|
tag_suffix: "-cuda"
|
|
- name: surfsense-backend
|
|
image: backend
|
|
variant: cuda126
|
|
tag_suffix: "-cuda126"
|
|
- name: surfsense-web
|
|
image: web
|
|
variant: cpu
|
|
tag_suffix: ""
|
|
env:
|
|
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
|
|
|
steps:
|
|
- name: Set lowercase image name
|
|
id: image
|
|
run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Download digests
|
|
id: download
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
pattern: digests-${{ matrix.image }}-${{ matrix.variant }}-*
|
|
path: /tmp/digests
|
|
merge-multiple: true
|
|
continue-on-error: true
|
|
|
|
- name: Check digests
|
|
id: check
|
|
run: |
|
|
count=$(find /tmp/digests -type f 2>/dev/null | wc -l | tr -d ' ')
|
|
echo "digest_count=$count" >> $GITHUB_OUTPUT
|
|
if [ "$count" -lt 2 ]; then
|
|
echo "::warning::${{ matrix.variant }}: $count/2 digests, skipping merge"
|
|
echo "skip=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "skip=false" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Set up Docker Buildx
|
|
if: steps.check.outputs.skip != 'true'
|
|
uses: docker/setup-buildx-action@v4
|
|
|
|
- name: Login to GitHub Container Registry
|
|
if: steps.check.outputs.skip != 'true'
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.repository_owner }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Compute app version
|
|
if: steps.check.outputs.skip != 'true'
|
|
id: appver
|
|
run: |
|
|
VERSION_TAG="${{ needs.tag_release.outputs.new_tag }}"
|
|
if [ -n "$VERSION_TAG" ]; then
|
|
APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev)
|
|
else
|
|
APP_VERSION=""
|
|
fi
|
|
echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT
|
|
|
|
- name: Docker meta
|
|
if: steps.check.outputs.skip != 'true'
|
|
id: meta
|
|
uses: docker/metadata-action@v6
|
|
with:
|
|
images: ${{ steps.image.outputs.name }}
|
|
tags: |
|
|
type=raw,value=${{ needs.tag_release.outputs.new_tag }},enable=${{ needs.tag_release.outputs.new_tag != '' }}
|
|
type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.tag_release.outputs.new_tag != '' && needs.tag_release.outputs.is_release_tag != 'true' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }}
|
|
type=ref,event=branch
|
|
type=sha,prefix=git-
|
|
flavor: |
|
|
latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/v') }}
|
|
${{ matrix.tag_suffix != '' && format('suffix={0},onlatest=true', matrix.tag_suffix) || '' }}
|
|
|
|
- name: Create manifest list and push
|
|
if: steps.check.outputs.skip != 'true'
|
|
working-directory: /tmp/digests
|
|
run: |
|
|
docker buildx imagetools create \
|
|
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
|
$(printf '${{ steps.image.outputs.name }}@sha256:%s ' *)
|
|
- name: Inspect image
|
|
if: steps.check.outputs.skip != 'true'
|
|
run: |
|
|
docker buildx imagetools inspect ${{ steps.image.outputs.name }}:${{ steps.meta.outputs.version }}
|
|
|
|
- name: Summary
|
|
if: steps.check.outputs.skip != 'true'
|
|
run: |
|
|
echo "Multi-arch manifest created for ${{ matrix.name }}!"
|
|
echo "Tags: $(jq -cr '.tags | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")"
|
|
|
|
# Push the git tag only after build, gate, and manifest publish all succeed.
|
|
finalize_release:
|
|
runs-on: ubuntu-latest
|
|
needs: [tag_release, create_manifest]
|
|
if: ${{ success() && needs.tag_release.outputs.new_tag != '' && needs.tag_release.outputs.is_release_tag != 'true' }}
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ github.event.inputs.branch }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Create and push git tag
|
|
run: |
|
|
git config --global user.name 'github-actions[bot]'
|
|
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
|
|
|
NEXT_TAG="${{ needs.tag_release.outputs.new_tag }}"
|
|
COMMIT_SHA="${{ needs.tag_release.outputs.commit_sha }}"
|
|
echo "Tagging commit $COMMIT_SHA with $NEXT_TAG"
|
|
|
|
git tag -a "$NEXT_TAG" "$COMMIT_SHA" -m "Docker build $NEXT_TAG"
|
|
echo "Pushing tag $NEXT_TAG to origin"
|
|
git push origin "$NEXT_TAG"
|
|
|
|
- name: Verify tag push
|
|
run: |
|
|
echo "Checking if tag ${{ needs.tag_release.outputs.new_tag }} exists remotely..."
|
|
sleep 5
|
|
git ls-remote --tags origin | grep "refs/tags/${{ needs.tag_release.outputs.new_tag }}" || (echo "Tag push verification failed!" && exit 1)
|
|
echo "Tag successfully pushed."
|