mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-26 21:39:43 +02:00
Merge 901c72cdcc into 26a504f137
This commit is contained in:
commit
dba5cf6132
22 changed files with 2331 additions and 745 deletions
245
.github/workflows/docker-build.yml
vendored
245
.github/workflows/docker-build.yml
vendored
|
|
@ -5,6 +5,9 @@ on:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
- dev
|
- dev
|
||||||
|
tags:
|
||||||
|
- 'v*'
|
||||||
|
- 'beta-v*'
|
||||||
paths:
|
paths:
|
||||||
- 'surfsense_backend/**'
|
- 'surfsense_backend/**'
|
||||||
- 'surfsense_web/**'
|
- 'surfsense_web/**'
|
||||||
|
|
@ -24,11 +27,13 @@ permissions:
|
||||||
packages: write
|
packages: write
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
tag_release:
|
compute_version:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch'
|
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/beta-v')
|
||||||
outputs:
|
outputs:
|
||||||
new_tag: ${{ steps.tag_version.outputs.next_version }}
|
new_tag: ${{ steps.tag_version.outputs.next_version }}
|
||||||
|
commit_sha: ${{ steps.tag_version.outputs.commit_sha }}
|
||||||
|
is_release_tag: ${{ steps.tag_version.outputs.is_release_tag }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v6
|
||||||
|
|
@ -37,57 +42,65 @@ jobs:
|
||||||
ref: ${{ github.event.inputs.branch }}
|
ref: ${{ github.event.inputs.branch }}
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
# Compute-only: tag is pushed by finalize_release after everything succeeds.
|
||||||
- name: Read app version and calculate next Docker build version
|
- name: Read app version and calculate next Docker build version
|
||||||
id: tag_version
|
id: tag_version
|
||||||
run: |
|
run: |
|
||||||
APP_VERSION=$(tr -d '[:space:]' < VERSION)
|
if [[ "$GITHUB_REF" == refs/tags/beta-v* ]]; then
|
||||||
echo "App version from VERSION file: $APP_VERSION"
|
VERSION="${GITHUB_REF#refs/tags/beta-v}"
|
||||||
|
NEXT_VERSION="beta-${VERSION}"
|
||||||
|
IS_RELEASE_TAG="true"
|
||||||
|
|
||||||
if [ -z "$APP_VERSION" ]; then
|
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||||
echo "Error: Could not read version from VERSION file"
|
echo "::error::Version '$VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
git fetch --tags
|
echo "Docker beta release version from git tag: $NEXT_VERSION"
|
||||||
|
elif [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||||
|
NEXT_VERSION="${GITHUB_REF#refs/tags/v}"
|
||||||
|
IS_RELEASE_TAG="true"
|
||||||
|
|
||||||
LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
|
if ! echo "$NEXT_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||||
|
echo "::error::Version '$NEXT_VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -z "$LATEST_BUILD_TAG" ]; then
|
echo "Docker release version from git tag: $NEXT_VERSION"
|
||||||
echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
|
|
||||||
NEXT_VERSION="${APP_VERSION}.1"
|
|
||||||
else
|
else
|
||||||
echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
|
APP_VERSION=$(tr -d '[:space:]' < VERSION)
|
||||||
BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
|
echo "App version from VERSION file: $APP_VERSION"
|
||||||
NEXT_BUILD=$((BUILD_NUMBER + 1))
|
|
||||||
NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
|
if [ -z "$APP_VERSION" ]; then
|
||||||
|
echo "Error: Could not read version from VERSION file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git fetch --tags
|
||||||
|
|
||||||
|
LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
|
||||||
|
|
||||||
|
if [ -z "$LATEST_BUILD_TAG" ]; then
|
||||||
|
echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
|
||||||
|
NEXT_VERSION="${APP_VERSION}.1"
|
||||||
|
else
|
||||||
|
echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
|
||||||
|
BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
|
||||||
|
NEXT_BUILD=$((BUILD_NUMBER + 1))
|
||||||
|
NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
IS_RELEASE_TAG="false"
|
||||||
|
echo "Calculated next Docker version: $NEXT_VERSION"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Calculated next Docker version: $NEXT_VERSION"
|
|
||||||
echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
|
echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
|
||||||
|
echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||||
- name: Create and Push Tag
|
echo "is_release_tag=$IS_RELEASE_TAG" >> $GITHUB_OUTPUT
|
||||||
run: |
|
|
||||||
git config --global user.name 'github-actions[bot]'
|
|
||||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
|
||||||
|
|
||||||
NEXT_TAG="${{ steps.tag_version.outputs.next_version }}"
|
|
||||||
COMMIT_SHA=$(git rev-parse HEAD)
|
|
||||||
echo "Tagging commit $COMMIT_SHA with $NEXT_TAG"
|
|
||||||
|
|
||||||
git tag -a "$NEXT_TAG" -m "Docker build $NEXT_TAG"
|
|
||||||
echo "Pushing tag $NEXT_TAG to origin"
|
|
||||||
git push origin "$NEXT_TAG"
|
|
||||||
|
|
||||||
- name: Verify Tag Push
|
|
||||||
run: |
|
|
||||||
echo "Checking if tag ${{ steps.tag_version.outputs.next_version }} exists remotely..."
|
|
||||||
sleep 5
|
|
||||||
git ls-remote --tags origin | grep "refs/tags/${{ steps.tag_version.outputs.next_version }}" || (echo "Tag push verification failed!" && exit 1)
|
|
||||||
echo "Tag successfully pushed."
|
|
||||||
|
|
||||||
build:
|
build:
|
||||||
needs: tag_release
|
needs: compute_version
|
||||||
if: always() && (needs.tag_release.result == 'success' || needs.tag_release.result == 'skipped')
|
if: always() && (needs.compute_version.result == 'success' || needs.compute_version.result == 'skipped')
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
|
|
@ -97,6 +110,12 @@ jobs:
|
||||||
matrix:
|
matrix:
|
||||||
platform: [linux/amd64, linux/arm64]
|
platform: [linux/amd64, linux/arm64]
|
||||||
image: [backend, web]
|
image: [backend, web]
|
||||||
|
variant: [cpu, cuda, cuda126]
|
||||||
|
exclude:
|
||||||
|
- image: web
|
||||||
|
variant: cuda
|
||||||
|
- image: web
|
||||||
|
variant: cuda126
|
||||||
include:
|
include:
|
||||||
- platform: linux/amd64
|
- platform: linux/amd64
|
||||||
suffix: amd64
|
suffix: amd64
|
||||||
|
|
@ -114,6 +133,18 @@ jobs:
|
||||||
context: ./surfsense_web
|
context: ./surfsense_web
|
||||||
file: ./surfsense_web/Dockerfile
|
file: ./surfsense_web/Dockerfile
|
||||||
target: runner
|
target: runner
|
||||||
|
- variant: cpu
|
||||||
|
tag_suffix: ""
|
||||||
|
use_cuda: "false"
|
||||||
|
cuda_extra: cpu
|
||||||
|
- variant: cuda
|
||||||
|
tag_suffix: "-cuda"
|
||||||
|
use_cuda: "true"
|
||||||
|
cuda_extra: cu128
|
||||||
|
- variant: cuda126
|
||||||
|
tag_suffix: "-cuda126"
|
||||||
|
use_cuda: "true"
|
||||||
|
cuda_extra: cu126
|
||||||
env:
|
env:
|
||||||
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
||||||
|
|
||||||
|
|
@ -149,7 +180,7 @@ jobs:
|
||||||
sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
|
||||||
docker system prune -af
|
docker system prune -af
|
||||||
|
|
||||||
- name: Build and push by digest ${{ matrix.name }} (${{ matrix.suffix }})
|
- name: Build and push by digest ${{ matrix.name }} (${{ matrix.variant }}, ${{ matrix.suffix }})
|
||||||
id: build
|
id: build
|
||||||
uses: docker/build-push-action@v7
|
uses: docker/build-push-action@v7
|
||||||
with:
|
with:
|
||||||
|
|
@ -160,10 +191,14 @@ jobs:
|
||||||
tags: ${{ steps.image.outputs.name }}
|
tags: ${{ steps.image.outputs.name }}
|
||||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||||
platforms: ${{ matrix.platform }}
|
platforms: ${{ matrix.platform }}
|
||||||
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.suffix }}
|
cache-from: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }}
|
||||||
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.suffix }}
|
cache-to: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }},mode=max,image-manifest=true,oci-mediatypes=true
|
||||||
|
secrets: |
|
||||||
|
HF_TOKEN=${{ secrets.HF_TOKEN }}
|
||||||
provenance: false
|
provenance: false
|
||||||
build-args: |
|
build-args: |
|
||||||
|
${{ matrix.image == 'backend' && format('USE_CUDA={0}', matrix.use_cuda) || '' }}
|
||||||
|
${{ matrix.image == 'backend' && format('CUDA_EXTRA={0}', matrix.cuda_extra) || '' }}
|
||||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
|
||||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
|
||||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
|
${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
|
||||||
|
|
@ -179,15 +214,47 @@ jobs:
|
||||||
- name: Upload digest
|
- name: Upload digest
|
||||||
uses: actions/upload-artifact@v7
|
uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: digests-${{ matrix.image }}-${{ matrix.suffix }}
|
name: digests-${{ matrix.image }}-${{ matrix.variant }}-${{ matrix.suffix }}
|
||||||
path: /tmp/digests/*
|
path: /tmp/digests/*
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
|
|
||||||
|
# Release gate: require both arches for every variant, else block publishing.
|
||||||
|
# Release-only; skipped on dev so the tolerant create_manifest path is kept.
|
||||||
|
verify_digests:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [compute_version, build]
|
||||||
|
if: ${{ always() && needs.compute_version.result == 'success' && needs.compute_version.outputs.new_tag != '' }}
|
||||||
|
steps:
|
||||||
|
- name: Download all digests
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
pattern: digests-*
|
||||||
|
path: /tmp/digests
|
||||||
|
merge-multiple: false
|
||||||
|
|
||||||
|
- name: Require both arches for every required variant
|
||||||
|
run: |
|
||||||
|
fail=0
|
||||||
|
check() {
|
||||||
|
c=$(find /tmp/digests -type f -path "*/digests-$1-*/*" 2>/dev/null | wc -l | tr -d ' ')
|
||||||
|
if [ "$c" -lt 2 ]; then
|
||||||
|
echo "::error::$1 has $c/2 arch digests — blocking release"
|
||||||
|
fail=1
|
||||||
|
else
|
||||||
|
echo "OK: $1 ($c/2)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
check backend-cpu
|
||||||
|
check backend-cuda
|
||||||
|
check backend-cuda126
|
||||||
|
check web-cpu
|
||||||
|
[ "$fail" -eq 0 ] || exit 1
|
||||||
|
|
||||||
create_manifest:
|
create_manifest:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [tag_release, build]
|
needs: [compute_version, build, verify_digests]
|
||||||
if: always() && needs.build.result == 'success'
|
if: ${{ !cancelled() && needs.verify_digests.result != 'failure' }}
|
||||||
permissions:
|
permissions:
|
||||||
packages: write
|
packages: write
|
||||||
contents: read
|
contents: read
|
||||||
|
|
@ -197,8 +264,20 @@ jobs:
|
||||||
include:
|
include:
|
||||||
- name: surfsense-backend
|
- name: surfsense-backend
|
||||||
image: backend
|
image: backend
|
||||||
|
variant: cpu
|
||||||
|
tag_suffix: ""
|
||||||
|
- name: surfsense-backend
|
||||||
|
image: backend
|
||||||
|
variant: cuda
|
||||||
|
tag_suffix: "-cuda"
|
||||||
|
- name: surfsense-backend
|
||||||
|
image: backend
|
||||||
|
variant: cuda126
|
||||||
|
tag_suffix: "-cuda126"
|
||||||
- name: surfsense-web
|
- name: surfsense-web
|
||||||
image: web
|
image: web
|
||||||
|
variant: cpu
|
||||||
|
tag_suffix: ""
|
||||||
env:
|
env:
|
||||||
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
||||||
|
|
||||||
|
|
@ -207,22 +286,33 @@ jobs:
|
||||||
id: image
|
id: image
|
||||||
run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
|
run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
- name: Download amd64 digest
|
- name: Download digests
|
||||||
|
id: download
|
||||||
uses: actions/download-artifact@v8
|
uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
name: digests-${{ matrix.image }}-amd64
|
pattern: digests-${{ matrix.image }}-${{ matrix.variant }}-*
|
||||||
path: /tmp/digests
|
path: /tmp/digests
|
||||||
|
merge-multiple: true
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
- name: Download arm64 digest
|
- name: Check digests
|
||||||
uses: actions/download-artifact@v8
|
id: check
|
||||||
with:
|
run: |
|
||||||
name: digests-${{ matrix.image }}-arm64
|
count=$(find /tmp/digests -type f 2>/dev/null | wc -l | tr -d ' ')
|
||||||
path: /tmp/digests
|
echo "digest_count=$count" >> $GITHUB_OUTPUT
|
||||||
|
if [ "$count" -lt 2 ]; then
|
||||||
|
echo "::warning::${{ matrix.variant }}: $count/2 digests, skipping merge"
|
||||||
|
echo "skip=true" >> $GITHUB_OUTPUT
|
||||||
|
else
|
||||||
|
echo "skip=false" >> $GITHUB_OUTPUT
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
uses: docker/setup-buildx-action@v4
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Login to GitHub Container Registry
|
- name: Login to GitHub Container Registry
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
uses: docker/login-action@v4
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
|
|
@ -230,9 +320,10 @@ jobs:
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Compute app version
|
- name: Compute app version
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
id: appver
|
id: appver
|
||||||
run: |
|
run: |
|
||||||
VERSION_TAG="${{ needs.tag_release.outputs.new_tag }}"
|
VERSION_TAG="${{ needs.compute_version.outputs.new_tag }}"
|
||||||
if [ -n "$VERSION_TAG" ]; then
|
if [ -n "$VERSION_TAG" ]; then
|
||||||
APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev)
|
APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev)
|
||||||
else
|
else
|
||||||
|
|
@ -241,29 +332,69 @@ jobs:
|
||||||
echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT
|
echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
- name: Docker meta
|
- name: Docker meta
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v6
|
uses: docker/metadata-action@v6
|
||||||
with:
|
with:
|
||||||
images: ${{ steps.image.outputs.name }}
|
images: ${{ steps.image.outputs.name }}
|
||||||
tags: |
|
tags: |
|
||||||
type=raw,value=${{ needs.tag_release.outputs.new_tag }},enable=${{ needs.tag_release.outputs.new_tag != '' }}
|
type=raw,value=${{ needs.compute_version.outputs.new_tag }},enable=${{ needs.compute_version.outputs.new_tag != '' }}
|
||||||
type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.tag_release.outputs.new_tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }}
|
type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }}
|
||||||
type=ref,event=branch
|
type=ref,event=branch
|
||||||
type=sha,prefix=git-
|
type=sha,prefix=git-
|
||||||
flavor: |
|
flavor: |
|
||||||
latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch }}
|
latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/v') }}
|
||||||
|
${{ matrix.tag_suffix != '' && format('suffix={0},onlatest=true', matrix.tag_suffix) || '' }}
|
||||||
|
|
||||||
- name: Create manifest list and push
|
- name: Create manifest list and push
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
working-directory: /tmp/digests
|
working-directory: /tmp/digests
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools create \
|
docker buildx imagetools create \
|
||||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||||
$(printf '${{ steps.image.outputs.name }}@sha256:%s ' *)
|
$(printf '${{ steps.image.outputs.name }}@sha256:%s ' *)
|
||||||
- name: Inspect image
|
- name: Inspect image
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
run: |
|
run: |
|
||||||
docker buildx imagetools inspect ${{ steps.image.outputs.name }}:${{ steps.meta.outputs.version }}
|
docker buildx imagetools inspect ${{ steps.image.outputs.name }}:${{ steps.meta.outputs.version }}
|
||||||
|
|
||||||
- name: Summary
|
- name: Summary
|
||||||
|
if: steps.check.outputs.skip != 'true'
|
||||||
run: |
|
run: |
|
||||||
echo "Multi-arch manifest created for ${{ matrix.name }}!"
|
echo "Multi-arch manifest created for ${{ matrix.name }}!"
|
||||||
echo "Tags: $(jq -cr '.tags | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")"
|
echo "Tags: $(jq -cr '.tags | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")"
|
||||||
|
|
||||||
|
# Push the git tag only after build, gate, and manifest publish all succeed.
|
||||||
|
finalize_release:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [compute_version, create_manifest]
|
||||||
|
if: ${{ success() && needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' }}
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v6
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
ref: ${{ github.event.inputs.branch }}
|
||||||
|
token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Create and push git tag
|
||||||
|
run: |
|
||||||
|
git config --global user.name 'github-actions[bot]'
|
||||||
|
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||||
|
|
||||||
|
NEXT_TAG="${{ needs.compute_version.outputs.new_tag }}"
|
||||||
|
COMMIT_SHA="${{ needs.compute_version.outputs.commit_sha }}"
|
||||||
|
echo "Tagging commit $COMMIT_SHA with $NEXT_TAG"
|
||||||
|
|
||||||
|
git tag -a "$NEXT_TAG" "$COMMIT_SHA" -m "Docker build $NEXT_TAG"
|
||||||
|
echo "Pushing tag $NEXT_TAG to origin"
|
||||||
|
git push origin "$NEXT_TAG"
|
||||||
|
|
||||||
|
- name: Verify tag push
|
||||||
|
run: |
|
||||||
|
echo "Checking if tag ${{ needs.compute_version.outputs.new_tag }} exists remotely..."
|
||||||
|
sleep 5
|
||||||
|
git ls-remote --tags origin | grep "refs/tags/${{ needs.compute_version.outputs.new_tag }}" || (echo "Tag push verification failed!" && exit 1)
|
||||||
|
echo "Tag successfully pushed."
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,16 @@
|
||||||
# SurfSense version (use "latest" or a specific version like "0.0.14")
|
# SurfSense version (use "latest" or a specific version like "0.0.14")
|
||||||
SURFSENSE_VERSION=latest
|
SURFSENSE_VERSION=latest
|
||||||
|
|
||||||
|
# Image variant: empty = CPU (default), "cuda" = CUDA 12.8, "cuda126" = CUDA 12.6.
|
||||||
|
# GPU acceleration also requires the NVIDIA Container Toolkit on the host and
|
||||||
|
# the GPU overlay in COMPOSE_FILE. Linux/macOS use ":"; Windows uses ";".
|
||||||
|
# Example Linux/macOS: COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||||
|
# Example Windows: COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml
|
||||||
|
# Use "cuda126" for older NVIDIA driver stacks; use "cuda" for newer drivers.
|
||||||
|
SURFSENSE_VARIANT=
|
||||||
|
# COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||||
|
# SURFSENSE_GPU_COUNT=1
|
||||||
|
|
||||||
# Deployment environment: dev or production
|
# Deployment environment: dev or production
|
||||||
SURFSENSE_ENV=production
|
SURFSENSE_ENV=production
|
||||||
|
|
||||||
|
|
@ -92,6 +102,10 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
|
||||||
# Only change this if you manage publications manually.
|
# Only change this if you manage publications manually.
|
||||||
# ZERO_APP_PUBLICATIONS=zero_publication
|
# ZERO_APP_PUBLICATIONS=zero_publication
|
||||||
|
|
||||||
|
# Keep Zero's documented halt safety net enabled. If replication halts, Zero
|
||||||
|
# can wipe and re-sync its local SQLite replica without touching Postgres.
|
||||||
|
# ZERO_AUTO_RESET=true
|
||||||
|
|
||||||
# Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number
|
# Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number
|
||||||
# of CPU cores, which can exceed the connection pool limits on high-core machines.
|
# of CPU cores, which can exceed the connection pool limits on high-core machines.
|
||||||
# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR
|
# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR
|
||||||
|
|
|
||||||
|
|
@ -114,6 +114,7 @@ services:
|
||||||
- ZERO_REPLICA_FILE=/data/zero.db
|
- ZERO_REPLICA_FILE=/data/zero.db
|
||||||
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||||
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||||
|
- ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true}
|
||||||
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
||||||
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||||
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
||||||
|
|
@ -122,11 +123,13 @@ services:
|
||||||
volumes:
|
volumes:
|
||||||
- zero_cache_data:/data
|
- zero_cache_data:/data
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 300s
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
start_period: 600s
|
||||||
|
|
||||||
# OPTIONAL — Azurite emulates Azure Blob Storage for testing the Azure
|
# OPTIONAL — Azurite emulates Azure Blob Storage for testing the Azure
|
||||||
# original-file backend. The default filesystem backend needs none of this.
|
# original-file backend. The default filesystem backend needs none of this.
|
||||||
|
|
|
||||||
|
|
@ -46,8 +46,6 @@ services:
|
||||||
- PYTHONPATH=/app
|
- PYTHONPATH=/app
|
||||||
- SERVICE_ROLE=migrate
|
- SERVICE_ROLE=migrate
|
||||||
- MIGRATION_TIMEOUT=${MIGRATION_TIMEOUT:-900}
|
- MIGRATION_TIMEOUT=${MIGRATION_TIMEOUT:-900}
|
||||||
volumes:
|
|
||||||
- zero_init:/zero-init
|
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
@ -217,21 +215,6 @@ services:
|
||||||
celery_worker:
|
celery_worker:
|
||||||
condition: service_started
|
condition: service_started
|
||||||
|
|
||||||
# flower:
|
|
||||||
# build: *backend-build
|
|
||||||
# ports:
|
|
||||||
# - "${FLOWER_PORT:-5555}:5555"
|
|
||||||
# env_file:
|
|
||||||
# - ../surfsense_backend/.env
|
|
||||||
# environment:
|
|
||||||
# - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
|
|
||||||
# - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
|
|
||||||
# - PYTHONPATH=/app
|
|
||||||
# command: celery -A app.celery_app flower --port=5555
|
|
||||||
# depends_on:
|
|
||||||
# - redis
|
|
||||||
# - celery_worker
|
|
||||||
|
|
||||||
zero-cache:
|
zero-cache:
|
||||||
image: rocicorp/zero:1.4.0
|
image: rocicorp/zero:1.4.0
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -250,6 +233,7 @@ services:
|
||||||
- ZERO_REPLICA_FILE=/data/zero.db
|
- ZERO_REPLICA_FILE=/data/zero.db
|
||||||
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||||
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||||
|
- ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true}
|
||||||
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
||||||
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||||
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
||||||
|
|
@ -257,18 +241,14 @@ services:
|
||||||
- ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
- ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
||||||
volumes:
|
volumes:
|
||||||
- zero_cache_data:/data
|
- zero_cache_data:/data
|
||||||
- zero_init:/zero-init
|
|
||||||
# Wrapper: see docker/docker-compose.yml `zero-cache` for rationale.
|
|
||||||
entrypoint: ["sh", "-c"]
|
|
||||||
# Pass the script as a single list element so Compose does not tokenize it.
|
|
||||||
command:
|
|
||||||
- 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache'
|
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 300s
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
start_period: 600s
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
build:
|
build:
|
||||||
|
|
@ -300,7 +280,5 @@ volumes:
|
||||||
name: surfsense-dev-shared-temp
|
name: surfsense-dev-shared-temp
|
||||||
zero_cache_data:
|
zero_cache_data:
|
||||||
name: surfsense-dev-zero-cache
|
name: surfsense-dev-zero-cache
|
||||||
zero_init:
|
|
||||||
name: surfsense-dev-zero-init
|
|
||||||
whatsapp_sessions:
|
whatsapp_sessions:
|
||||||
name: surfsense-dev-whatsapp-sessions
|
name: surfsense-dev-whatsapp-sessions
|
||||||
|
|
|
||||||
30
docker/docker-compose.gpu.yml
Normal file
30
docker/docker-compose.gpu.yml
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
services:
|
||||||
|
backend:
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||||
|
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||||
|
capabilities:
|
||||||
|
- gpu
|
||||||
|
|
||||||
|
celery_worker:
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||||
|
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||||
|
capabilities:
|
||||||
|
- gpu
|
||||||
|
|
||||||
|
celery_beat:
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
reservations:
|
||||||
|
devices:
|
||||||
|
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||||
|
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||||
|
capabilities:
|
||||||
|
- gpu
|
||||||
|
|
@ -29,12 +29,11 @@ services:
|
||||||
|
|
||||||
# Short-lived schema runner. Executes `alembic upgrade head` and verifies
|
# Short-lived schema runner. Executes `alembic upgrade head` and verifies
|
||||||
# that the `zero_publication` Postgres logical-replication publication
|
# that the `zero_publication` Postgres logical-replication publication
|
||||||
# exists, then exits 0. Downstream services (backend, celery_*, zero-cache)
|
# matches the canonical shape, then exits 0. Downstream services gate on this
|
||||||
# gate on this with `condition: service_completed_successfully` so a failed
|
# with `condition: service_completed_successfully` so a failed migration halts
|
||||||
# migration halts the whole stack instead of silently producing a half-built
|
# the whole stack instead of booting zero-cache against a drifted publication.
|
||||||
# system that crash-loops zero-cache on missing publications.
|
|
||||||
migrations:
|
migrations:
|
||||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
environment:
|
||||||
|
|
@ -42,8 +41,6 @@ services:
|
||||||
PYTHONPATH: /app
|
PYTHONPATH: /app
|
||||||
SERVICE_ROLE: migrate
|
SERVICE_ROLE: migrate
|
||||||
MIGRATION_TIMEOUT: ${MIGRATION_TIMEOUT:-900}
|
MIGRATION_TIMEOUT: ${MIGRATION_TIMEOUT:-900}
|
||||||
volumes:
|
|
||||||
- zero_init:/zero-init
|
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
@ -61,28 +58,28 @@ services:
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
otel-collector:
|
# otel-collector:
|
||||||
image: otel/opentelemetry-collector-contrib:0.152.1
|
# image: otel/opentelemetry-collector-contrib:0.152.1
|
||||||
profiles:
|
# profiles:
|
||||||
- observability
|
# - observability
|
||||||
command: ["--config=/etc/otelcol/config.yaml"]
|
# command: ["--config=/etc/otelcol/config.yaml"]
|
||||||
volumes:
|
# volumes:
|
||||||
- ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro
|
# - ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro
|
||||||
environment:
|
# environment:
|
||||||
GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-}
|
# GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-}
|
||||||
GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-}
|
# GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-}
|
||||||
GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-}
|
# GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-}
|
||||||
ports:
|
# ports:
|
||||||
- "${OTEL_GRPC_PORT:-4317}:4317"
|
# - "${OTEL_GRPC_PORT:-4317}:4317"
|
||||||
- "${OTEL_HTTP_PORT:-4318}:4318"
|
# - "${OTEL_HTTP_PORT:-4318}:4318"
|
||||||
- "${OTEL_HEALTH_PORT:-13133}:13133"
|
# - "${OTEL_HEALTH_PORT:-13133}:13133"
|
||||||
mem_limit: 2g
|
# mem_limit: 2g
|
||||||
restart: unless-stopped
|
# restart: unless-stopped
|
||||||
healthcheck:
|
# healthcheck:
|
||||||
test: ["CMD", "/otelcol-contrib", "--version"]
|
# test: ["CMD", "/otelcol-contrib", "--version"]
|
||||||
interval: 30s
|
# interval: 30s
|
||||||
timeout: 5s
|
# timeout: 5s
|
||||||
retries: 3
|
# retries: 3
|
||||||
|
|
||||||
searxng:
|
searxng:
|
||||||
image: searxng/searxng:2026.3.13-3c1f68c59
|
image: searxng/searxng:2026.3.13-3c1f68c59
|
||||||
|
|
@ -98,7 +95,7 @@ services:
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
backend:
|
backend:
|
||||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||||
ports:
|
ports:
|
||||||
- "${BACKEND_PORT:-8929}:8000"
|
- "${BACKEND_PORT:-8929}:8000"
|
||||||
volumes:
|
volumes:
|
||||||
|
|
@ -144,28 +141,28 @@ services:
|
||||||
retries: 30
|
retries: 30
|
||||||
start_period: 200s
|
start_period: 200s
|
||||||
|
|
||||||
whatsapp-bridge:
|
# whatsapp-bridge:
|
||||||
build: ../surfsense_backend/scripts/whatsapp-bridge
|
# build: ../surfsense_backend/scripts/whatsapp-bridge
|
||||||
profiles:
|
# profiles:
|
||||||
- whatsapp
|
# - whatsapp
|
||||||
expose:
|
# expose:
|
||||||
- "9929"
|
# - "9929"
|
||||||
volumes:
|
# volumes:
|
||||||
- whatsapp_sessions:/data/sessions
|
# - whatsapp_sessions:/data/sessions
|
||||||
environment:
|
# environment:
|
||||||
PORT: 9929
|
# PORT: 9929
|
||||||
WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat}
|
# WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat}
|
||||||
WHATSAPP_SESSION_DIR: /data/sessions
|
# WHATSAPP_SESSION_DIR: /data/sessions
|
||||||
mem_limit: 512m
|
# mem_limit: 512m
|
||||||
restart: unless-stopped
|
# restart: unless-stopped
|
||||||
healthcheck:
|
# healthcheck:
|
||||||
test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"]
|
# test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"]
|
||||||
interval: 30s
|
# interval: 30s
|
||||||
timeout: 5s
|
# timeout: 5s
|
||||||
retries: 5
|
# retries: 5
|
||||||
|
|
||||||
celery_worker:
|
celery_worker:
|
||||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||||
volumes:
|
volumes:
|
||||||
- shared_temp:/shared_tmp
|
- shared_temp:/shared_tmp
|
||||||
env_file:
|
env_file:
|
||||||
|
|
@ -195,7 +192,7 @@ services:
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
celery_beat:
|
celery_beat:
|
||||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
environment:
|
||||||
|
|
@ -218,22 +215,6 @@ services:
|
||||||
- "com.centurylinklabs.watchtower.enable=true"
|
- "com.centurylinklabs.watchtower.enable=true"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
# flower:
|
|
||||||
# image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
|
||||||
# ports:
|
|
||||||
# - "${FLOWER_PORT:-5555}:5555"
|
|
||||||
# env_file:
|
|
||||||
# - .env
|
|
||||||
# environment:
|
|
||||||
# CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
|
|
||||||
# CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
|
|
||||||
# PYTHONPATH: /app
|
|
||||||
# command: celery -A app.celery_app flower --port=5555
|
|
||||||
# depends_on:
|
|
||||||
# - redis
|
|
||||||
# - celery_worker
|
|
||||||
# restart: unless-stopped
|
|
||||||
|
|
||||||
zero-cache:
|
zero-cache:
|
||||||
image: rocicorp/zero:1.4.0
|
image: rocicorp/zero:1.4.0
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -247,6 +228,7 @@ services:
|
||||||
ZERO_REPLICA_FILE: /data/zero.db
|
ZERO_REPLICA_FILE: /data/zero.db
|
||||||
ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||||
ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication}
|
ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||||
|
ZERO_AUTO_RESET: ${ZERO_AUTO_RESET:-true}
|
||||||
ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4}
|
ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4}
|
||||||
ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20}
|
ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||||
ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30}
|
ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30}
|
||||||
|
|
@ -254,16 +236,8 @@ services:
|
||||||
ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
||||||
volumes:
|
volumes:
|
||||||
- zero_cache_data:/data
|
- zero_cache_data:/data
|
||||||
- zero_init:/zero-init
|
|
||||||
# Wrapper: if the migrations service flagged a publication change via
|
|
||||||
# /zero-init/needs_reset, wipe the SQLite replica before starting so
|
|
||||||
# zero-cache does a clean initial sync. Recovers from the half-built
|
|
||||||
# replica state (`_zero.tableMetadata` missing) caused by earlier crashes.
|
|
||||||
entrypoint: ["sh", "-c"]
|
|
||||||
# Pass the script as a single list element so Compose does not tokenize it.
|
|
||||||
command:
|
|
||||||
- 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache'
|
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 300s
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
@ -274,6 +248,7 @@ services:
|
||||||
interval: 10s
|
interval: 10s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
start_period: 600s
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
image: ghcr.io/modsetter/surfsense-web:${SURFSENSE_VERSION:-latest}
|
image: ghcr.io/modsetter/surfsense-web:${SURFSENSE_VERSION:-latest}
|
||||||
|
|
@ -305,7 +280,5 @@ volumes:
|
||||||
name: surfsense-shared-temp
|
name: surfsense-shared-temp
|
||||||
zero_cache_data:
|
zero_cache_data:
|
||||||
name: surfsense-zero-cache
|
name: surfsense-zero-cache
|
||||||
zero_init:
|
|
||||||
name: surfsense-zero-init
|
|
||||||
whatsapp_sessions:
|
whatsapp_sessions:
|
||||||
name: surfsense-whatsapp-sessions
|
name: surfsense-whatsapp-sessions
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@
|
||||||
# To pass flags, save and run locally:
|
# To pass flags, save and run locally:
|
||||||
# .\install.ps1 -NoWatchtower
|
# .\install.ps1 -NoWatchtower
|
||||||
# .\install.ps1 -WatchtowerInterval 3600
|
# .\install.ps1 -WatchtowerInterval 3600
|
||||||
|
# .\install.ps1 -Variant cuda
|
||||||
|
# .\install.ps1 -Variant cuda -GpuCount all
|
||||||
#
|
#
|
||||||
# Handles two cases automatically:
|
# Handles two cases automatically:
|
||||||
# 1. Fresh install — no prior SurfSense data detected
|
# 1. Fresh install — no prior SurfSense data detected
|
||||||
|
|
@ -17,7 +19,11 @@
|
||||||
|
|
||||||
param(
|
param(
|
||||||
[switch]$NoWatchtower,
|
[switch]$NoWatchtower,
|
||||||
[int]$WatchtowerInterval = 86400
|
[int]$WatchtowerInterval = 86400,
|
||||||
|
[ValidateSet("cpu", "cuda", "cuda126")]
|
||||||
|
[string]$Variant,
|
||||||
|
[string]$GpuCount,
|
||||||
|
[switch]$Quiet
|
||||||
)
|
)
|
||||||
|
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
@ -34,6 +40,11 @@ $MigrationMode = $false
|
||||||
$SetupWatchtower = -not $NoWatchtower
|
$SetupWatchtower = -not $NoWatchtower
|
||||||
$WatchtowerContainer = "watchtower"
|
$WatchtowerContainer = "watchtower"
|
||||||
|
|
||||||
|
if ($GpuCount -and $GpuCount -notmatch '^([0-9]+|all)$') {
|
||||||
|
Write-Host "[SurfSense] ERROR: Invalid -GpuCount '$GpuCount'. Use a number or 'all'." -ForegroundColor Red
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
# ── Output helpers ──────────────────────────────────────────────────────────
|
# ── Output helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
function Write-Info { param([string]$Msg) Write-Host "[SurfSense] " -ForegroundColor Cyan -NoNewline; Write-Host $Msg }
|
function Write-Info { param([string]$Msg) Write-Host "[SurfSense] " -ForegroundColor Cyan -NoNewline; Write-Host $Msg }
|
||||||
|
|
@ -42,6 +53,27 @@ function Write-Warn { param([string]$Msg) Write-Host "[SurfSense] " -Foregrou
|
||||||
function Write-Step { param([string]$Msg) Write-Host "`n-- $Msg" -ForegroundColor Cyan }
|
function Write-Step { param([string]$Msg) Write-Host "`n-- $Msg" -ForegroundColor Cyan }
|
||||||
function Write-Err { param([string]$Msg) Write-Host "[SurfSense] ERROR: $Msg" -ForegroundColor Red; exit 1 }
|
function Write-Err { param([string]$Msg) Write-Host "[SurfSense] ERROR: $Msg" -ForegroundColor Red; exit 1 }
|
||||||
|
|
||||||
|
function Show-Banner {
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host @"
|
||||||
|
|
||||||
|
|
||||||
|
███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗
|
||||||
|
██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝
|
||||||
|
███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗
|
||||||
|
╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝
|
||||||
|
███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗
|
||||||
|
╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝
|
||||||
|
|
||||||
|
|
||||||
|
"@ -ForegroundColor White
|
||||||
|
Write-Host " OSS Alternative to NotebookLM for Teams" -ForegroundColor Yellow
|
||||||
|
Write-Host ("=" * 62) -ForegroundColor Cyan
|
||||||
|
Write-Info "This installer will create $InstallDir\ and start SurfSense with Docker Compose."
|
||||||
|
}
|
||||||
|
|
||||||
|
Show-Banner
|
||||||
|
|
||||||
function Invoke-NativeSafe {
|
function Invoke-NativeSafe {
|
||||||
param([scriptblock]$Command)
|
param([scriptblock]$Command)
|
||||||
$previousErrorActionPreference = $ErrorActionPreference
|
$previousErrorActionPreference = $ErrorActionPreference
|
||||||
|
|
@ -53,6 +85,28 @@ function Invoke-NativeSafe {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function Resolve-WatchtowerPreference {
|
||||||
|
if ($NoWatchtower -or $Quiet -or -not [Environment]::UserInteractive) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Automatic updates" -ForegroundColor Cyan
|
||||||
|
$choice = Read-Host "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]"
|
||||||
|
|
||||||
|
switch ($choice) {
|
||||||
|
"" { $script:SetupWatchtower = $true }
|
||||||
|
{ $_ -match '^(?i)y(es)?$' } { $script:SetupWatchtower = $true }
|
||||||
|
{ $_ -match '^(?i)n(o)?$' } { $script:SetupWatchtower = $false }
|
||||||
|
default {
|
||||||
|
Write-Warn "Unrecognized choice '$choice'; enabling Watchtower by default. Use -NoWatchtower to skip it."
|
||||||
|
$script:SetupWatchtower = $true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Resolve-WatchtowerPreference
|
||||||
|
|
||||||
# ── Pre-flight checks ──────────────────────────────────────────────────────
|
# ── Pre-flight checks ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
Write-Step "Checking prerequisites"
|
Write-Step "Checking prerequisites"
|
||||||
|
|
@ -97,143 +151,11 @@ function Wait-ForPostgres {
|
||||||
Write-Ok "PostgreSQL is ready."
|
Write-Ok "PostgreSQL is ready."
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Stack health helpers ────────────────────────────────────────────────────
|
# ── Stack startup helper ────────────────────────────────────────────────────
|
||||||
|
|
||||||
function Get-ComposeServices {
|
|
||||||
Push-Location $InstallDir
|
|
||||||
try {
|
|
||||||
$raw = Invoke-NativeSafe { docker compose ps -a --format json 2>$null }
|
|
||||||
} finally {
|
|
||||||
Pop-Location
|
|
||||||
}
|
|
||||||
if ([string]::IsNullOrWhiteSpace($raw)) { return @() }
|
|
||||||
|
|
||||||
# Compose v2.21+ emits a JSON array; older versions emit one object per line.
|
|
||||||
try {
|
|
||||||
$parsed = $raw | ConvertFrom-Json
|
|
||||||
if ($parsed -is [System.Collections.IEnumerable] -and -not ($parsed -is [string])) {
|
|
||||||
return @($parsed)
|
|
||||||
}
|
|
||||||
return @($parsed)
|
|
||||||
} catch {
|
|
||||||
$services = @()
|
|
||||||
foreach ($line in ($raw -split "`r?`n")) {
|
|
||||||
$line = $line.Trim()
|
|
||||||
if (-not $line) { continue }
|
|
||||||
try { $services += ($line | ConvertFrom-Json) } catch { }
|
|
||||||
}
|
|
||||||
return $services
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function Wait-StackHealthy {
|
|
||||||
param([int]$TimeoutSec = 300)
|
|
||||||
|
|
||||||
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
||||||
$lastReport = ""
|
|
||||||
|
|
||||||
while ((Get-Date) -lt $deadline) {
|
|
||||||
$services = Get-ComposeServices
|
|
||||||
if (-not $services -or $services.Count -eq 0) {
|
|
||||||
Start-Sleep -Seconds 3
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
$bad = @()
|
|
||||||
$waiting = @()
|
|
||||||
$good = @()
|
|
||||||
|
|
||||||
foreach ($svc in $services) {
|
|
||||||
$name = $svc.Service
|
|
||||||
$state = $svc.State
|
|
||||||
$health = if ($svc.PSObject.Properties.Name -contains 'Health') { $svc.Health } else { '' }
|
|
||||||
$exit = if ($svc.PSObject.Properties.Name -contains 'ExitCode') { $svc.ExitCode } else { $null }
|
|
||||||
|
|
||||||
if ($name -eq 'migrations') {
|
|
||||||
if ($state -eq 'exited' -and $exit -eq 0) { $good += $name }
|
|
||||||
elseif ($state -eq 'exited') { $bad += "${name} (exit=${exit})" }
|
|
||||||
else { $waiting += "${name} (${state})" }
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($state -eq 'running') {
|
|
||||||
if ([string]::IsNullOrEmpty($health) -or $health -eq 'healthy') {
|
|
||||||
$good += $name
|
|
||||||
} elseif ($health -eq 'starting') {
|
|
||||||
$waiting += "${name} (starting)"
|
|
||||||
} elseif ($health -eq 'unhealthy') {
|
|
||||||
$bad += "${name} (unhealthy)"
|
|
||||||
} else {
|
|
||||||
$waiting += "${name} (${health})"
|
|
||||||
}
|
|
||||||
} elseif ($state -eq 'restarting') {
|
|
||||||
$bad += "${name} (restarting)"
|
|
||||||
} elseif ($state -eq 'exited') {
|
|
||||||
$bad += "${name} (exited, code=${exit})"
|
|
||||||
} else {
|
|
||||||
$waiting += "${name} (${state})"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($bad.Count -gt 0) {
|
|
||||||
return @{ Ok = $false; Reason = 'failure'; Bad = $bad; Waiting = $waiting; Good = $good }
|
|
||||||
}
|
|
||||||
if ($waiting.Count -eq 0) {
|
|
||||||
return @{ Ok = $true; Reason = 'all_healthy'; Good = $good }
|
|
||||||
}
|
|
||||||
|
|
||||||
$report = "Waiting on: " + ($waiting -join ', ')
|
|
||||||
if ($report -ne $lastReport) {
|
|
||||||
Write-Info $report
|
|
||||||
$lastReport = $report
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
}
|
|
||||||
|
|
||||||
return @{ Ok = $false; Reason = 'timeout'; Bad = $bad; Waiting = $waiting; Good = $good }
|
|
||||||
}
|
|
||||||
|
|
||||||
function Test-StaleZeroCacheVolume {
|
|
||||||
$raw = Invoke-NativeSafe { docker volume ls --format '{{.Name}}' 2>$null }
|
|
||||||
if ([string]::IsNullOrWhiteSpace($raw)) { return $false }
|
|
||||||
$names = $raw -split "`r?`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ }
|
|
||||||
$hasZeroCache = $names -contains 'surfsense-zero-cache'
|
|
||||||
$hasZeroInit = $names -contains 'surfsense-zero-init'
|
|
||||||
# Pre-fix installs created surfsense-zero-cache but never surfsense-zero-init.
|
|
||||||
# Such a volume may hold a half-initialized SQLite replica from an earlier
|
|
||||||
# crash-loop. Wiping it forces zero-cache to do a fresh initial sync.
|
|
||||||
return ($hasZeroCache -and -not $hasZeroInit)
|
|
||||||
}
|
|
||||||
|
|
||||||
function Invoke-StaleZeroCacheCleanup {
|
|
||||||
if (-not (Test-StaleZeroCacheVolume)) { return }
|
|
||||||
|
|
||||||
Write-Warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that"
|
|
||||||
Write-Warn "predates the migrations-service fix. It may contain a half-initialized"
|
|
||||||
Write-Warn "SQLite replica that would block zero-cache from starting."
|
|
||||||
Write-Warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel."
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
|
|
||||||
Push-Location $InstallDir
|
|
||||||
Invoke-NativeSafe { docker compose down --remove-orphans 2>$null } | Out-Null
|
|
||||||
Pop-Location
|
|
||||||
Invoke-NativeSafe { docker volume rm surfsense-zero-cache 2>$null } | Out-Null
|
|
||||||
Write-Ok "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start."
|
|
||||||
}
|
|
||||||
|
|
||||||
function Write-Err-NoExit {
|
|
||||||
param([string]$Message)
|
|
||||||
Write-Host "[ERROR] $Message" -ForegroundColor Red
|
|
||||||
}
|
|
||||||
|
|
||||||
function Invoke-StackFailureReport {
|
function Invoke-StackFailureReport {
|
||||||
param([hashtable]$Result)
|
|
||||||
|
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
Write-Err-NoExit "Stack did not reach a healthy state."
|
Write-Host "[ERROR] Stack did not reach a healthy state." -ForegroundColor Red
|
||||||
if ($Result.Bad.Count -gt 0) { Write-Host (" Failed: " + ($Result.Bad -join ', ')) }
|
|
||||||
if ($Result.Waiting.Count -gt 0) { Write-Host (" Stuck: " + ($Result.Waiting -join ', ')) }
|
|
||||||
|
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
Write-Info "Recent logs from migrations / zero-cache / backend:"
|
Write-Info "Recent logs from migrations / zero-cache / backend:"
|
||||||
Push-Location $InstallDir
|
Push-Location $InstallDir
|
||||||
|
|
@ -247,11 +169,151 @@ function Invoke-StackFailureReport {
|
||||||
Write-Host "Recovery hints:" -ForegroundColor Yellow
|
Write-Host "Recovery hints:" -ForegroundColor Yellow
|
||||||
Write-Host " 1. Inspect migrations: cd $InstallDir; docker compose logs migrations"
|
Write-Host " 1. Inspect migrations: cd $InstallDir; docker compose logs migrations"
|
||||||
Write-Host " 2. Verify publication: cd $InstallDir; docker compose exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
Write-Host " 2. Verify publication: cd $InstallDir; docker compose exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
||||||
Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d"
|
Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d --wait"
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function Invoke-ComposeUpWait {
|
||||||
|
Push-Location $InstallDir
|
||||||
|
try {
|
||||||
|
Invoke-NativeSafe { docker compose up -d --wait }
|
||||||
|
} finally {
|
||||||
|
Pop-Location
|
||||||
|
}
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
Invoke-StackFailureReport
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Variant and .env helpers ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function Set-EnvValue {
|
||||||
|
param([string]$Path, [string]$Key, [string]$Value)
|
||||||
|
$lines = @()
|
||||||
|
if (Test-Path $Path) {
|
||||||
|
$lines = @(Get-Content $Path)
|
||||||
|
}
|
||||||
|
$updated = $false
|
||||||
|
$newLines = foreach ($line in $lines) {
|
||||||
|
if ($line -match "^$([regex]::Escape($Key))=") {
|
||||||
|
$updated = $true
|
||||||
|
"$Key=$Value"
|
||||||
|
} else {
|
||||||
|
$line
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (-not $updated) {
|
||||||
|
$newLines += "$Key=$Value"
|
||||||
|
}
|
||||||
|
Set-Content -Path $Path -Value $newLines
|
||||||
|
}
|
||||||
|
|
||||||
|
function Remove-EnvValue {
|
||||||
|
param([string]$Path, [string]$Key)
|
||||||
|
if (-not (Test-Path $Path)) { return }
|
||||||
|
$newLines = Get-Content $Path | Where-Object { $_ -notmatch "^$([regex]::Escape($Key))=" }
|
||||||
|
Set-Content -Path $Path -Value $newLines
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-NvidiaGpu {
|
||||||
|
if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) { return $false }
|
||||||
|
Invoke-NativeSafe { nvidia-smi *>$null } | Out-Null
|
||||||
|
return ($LASTEXITCODE -eq 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-NvidiaRuntime {
|
||||||
|
$info = Invoke-NativeSafe { docker info 2>$null }
|
||||||
|
if ($info -match 'nvidia') { return $true }
|
||||||
|
if (Get-Command nvidia-ctk -ErrorAction SilentlyContinue) { return $true }
|
||||||
|
if (Get-Command nvidia-container-runtime -ErrorAction SilentlyContinue) { return $true }
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-RecommendedVariant {
|
||||||
|
$driver = (Invoke-NativeSafe { nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>$null } | Select-Object -First 1)
|
||||||
|
$major = 0
|
||||||
|
if ($driver -match '^(\d+)') {
|
||||||
|
$major = [int]$Matches[1]
|
||||||
|
}
|
||||||
|
if ($major -gt 0 -and $major -lt 570) {
|
||||||
|
return "cuda126"
|
||||||
|
}
|
||||||
|
return "cuda"
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-Variant {
|
||||||
|
$hasGpu = Test-NvidiaGpu
|
||||||
|
$hasRuntime = $false
|
||||||
|
$recommended = "cpu"
|
||||||
|
|
||||||
|
if ($hasGpu) {
|
||||||
|
$recommended = Get-RecommendedVariant
|
||||||
|
$hasRuntime = Test-NvidiaRuntime
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($Variant) {
|
||||||
|
if ($Variant -eq "cpu") { return "cpu" }
|
||||||
|
if (-not $hasGpu) {
|
||||||
|
Write-Warn "No NVIDIA GPU detected; falling back to CPU variant."
|
||||||
|
return "cpu"
|
||||||
|
}
|
||||||
|
if (-not $hasRuntime) {
|
||||||
|
Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant."
|
||||||
|
Write-Warn "Install the toolkit before enabling SurfSense GPU acceleration."
|
||||||
|
return "cpu"
|
||||||
|
}
|
||||||
|
return $Variant
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($hasGpu -and -not $hasRuntime) {
|
||||||
|
Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant."
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($hasGpu -and $hasRuntime -and -not $Quiet -and [Environment]::UserInteractive) {
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "SurfSense detected an NVIDIA GPU." -ForegroundColor Cyan
|
||||||
|
$choice = Read-Host "Use GPU acceleration? [Y/n]"
|
||||||
|
switch ($choice) {
|
||||||
|
"" { return $recommended }
|
||||||
|
{ $_ -match '^(?i)y(es)?$' } { return $recommended }
|
||||||
|
{ $_ -match '^(?i)n(o)?$' } { return "cpu" }
|
||||||
|
default {
|
||||||
|
Write-Warn "Unrecognized choice '$choice'; using CPU variant."
|
||||||
|
return "cpu"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "cpu"
|
||||||
|
}
|
||||||
|
|
||||||
|
function Set-VariantEnv {
|
||||||
|
param([string]$Path, [string]$SelectedVariant, [bool]$AllowExistingUpdate)
|
||||||
|
|
||||||
|
if ((Test-Path $Path) -and -not $AllowExistingUpdate) {
|
||||||
|
Write-Warn ".env already exists - keeping your existing configuration."
|
||||||
|
Write-Info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in $Path, then run docker compose up -d --wait."
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($SelectedVariant -eq "cpu") {
|
||||||
|
Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value ""
|
||||||
|
Remove-EnvValue -Path $Path -Key "COMPOSE_FILE"
|
||||||
|
Remove-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT"
|
||||||
|
} else {
|
||||||
|
Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value $SelectedVariant
|
||||||
|
Set-EnvValue -Path $Path -Key "COMPOSE_FILE" -Value "docker-compose.yml;docker-compose.gpu.yml"
|
||||||
|
if ($GpuCount) {
|
||||||
|
Set-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT" -Value $GpuCount
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Remove-EnvValue -Path $Path -Key "COMPOSE_PROFILES"
|
||||||
|
}
|
||||||
|
|
||||||
|
$SelectedVariant = Resolve-Variant
|
||||||
|
|
||||||
# ── Download files ──────────────────────────────────────────────────────────
|
# ── Download files ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
Write-Step "Downloading SurfSense files"
|
Write-Step "Downloading SurfSense files"
|
||||||
|
|
@ -262,6 +324,7 @@ New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null
|
||||||
|
|
||||||
$Files = @(
|
$Files = @(
|
||||||
@{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" }
|
@{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" }
|
||||||
|
@{ Src = "docker/docker-compose.gpu.yml"; Dest = "docker-compose.gpu.yml" }
|
||||||
@{ Src = "docker/.env.example"; Dest = ".env.example" }
|
@{ Src = "docker/.env.example"; Dest = ".env.example" }
|
||||||
@{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" }
|
@{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" }
|
||||||
@{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" }
|
@{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" }
|
||||||
|
|
@ -339,15 +402,19 @@ if (-not (Test-Path $envPath)) {
|
||||||
$content = $content -replace 'SECRET_KEY=replace_me_with_a_random_string', "SECRET_KEY=$SecretKey"
|
$content = $content -replace 'SECRET_KEY=replace_me_with_a_random_string', "SECRET_KEY=$SecretKey"
|
||||||
Set-Content -Path $envPath -Value $content -NoNewline
|
Set-Content -Path $envPath -Value $content -NoNewline
|
||||||
|
|
||||||
|
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false
|
||||||
Write-Info "Created $envPath"
|
Write-Info "Created $envPath"
|
||||||
} else {
|
} else {
|
||||||
Write-Warn ".env already exists - keeping your existing configuration."
|
if ($PSBoundParameters.ContainsKey('Variant')) {
|
||||||
|
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $true
|
||||||
|
Write-Info "Updated SurfSense image variant in existing $envPath"
|
||||||
|
} else {
|
||||||
|
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Start containers ────────────────────────────────────────────────────────
|
# ── Start containers ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
Invoke-StaleZeroCacheCleanup
|
|
||||||
|
|
||||||
if ($MigrationMode) {
|
if ($MigrationMode) {
|
||||||
$envContent = Get-Content $envPath
|
$envContent = Get-Content $envPath
|
||||||
$DbUser = ($envContent | Select-String '^DB_USER=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
$DbUser = ($envContent | Select-String '^DB_USER=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||||
|
|
@ -405,31 +472,15 @@ if ($MigrationMode) {
|
||||||
}
|
}
|
||||||
|
|
||||||
Write-Step "Starting all SurfSense services"
|
Write-Step "Starting all SurfSense services"
|
||||||
Push-Location $InstallDir
|
Invoke-ComposeUpWait
|
||||||
Invoke-NativeSafe { docker compose up -d }
|
Write-Ok "All services started and healthy."
|
||||||
Pop-Location
|
|
||||||
Write-Ok "All containers started; waiting for stack to become healthy..."
|
|
||||||
|
|
||||||
$waitResult = Wait-StackHealthy -TimeoutSec 300
|
|
||||||
if (-not $waitResult.Ok) {
|
|
||||||
Invoke-StackFailureReport -Result $waitResult
|
|
||||||
}
|
|
||||||
Write-Ok "All services healthy."
|
|
||||||
|
|
||||||
Remove-Item $KeyFile -ErrorAction SilentlyContinue
|
Remove-Item $KeyFile -ErrorAction SilentlyContinue
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
Write-Step "Starting SurfSense"
|
Write-Step "Starting SurfSense"
|
||||||
Push-Location $InstallDir
|
Invoke-ComposeUpWait
|
||||||
Invoke-NativeSafe { docker compose up -d }
|
Write-Ok "All services started and healthy."
|
||||||
Pop-Location
|
|
||||||
Write-Ok "All containers started; waiting for stack to become healthy..."
|
|
||||||
|
|
||||||
$waitResult = Wait-StackHealthy -TimeoutSec 300
|
|
||||||
if (-not $waitResult.Ok) {
|
|
||||||
Invoke-StackFailureReport -Result $waitResult
|
|
||||||
}
|
|
||||||
Write-Ok "All services healthy."
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Watchtower (auto-update) ────────────────────────────────────────────────
|
# ── Watchtower (auto-update) ────────────────────────────────────────────────
|
||||||
|
|
@ -461,7 +512,7 @@ if ($SetupWatchtower) {
|
||||||
if ($LASTEXITCODE -eq 0) {
|
if ($LASTEXITCODE -eq 0) {
|
||||||
Write-Ok "Watchtower started - labeled SurfSense containers will auto-update."
|
Write-Ok "Watchtower started - labeled SurfSense containers will auto-update."
|
||||||
} else {
|
} else {
|
||||||
Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d"
|
Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d --wait"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -471,39 +522,26 @@ if ($SetupWatchtower) {
|
||||||
# ── Done ────────────────────────────────────────────────────────────────────
|
# ── Done ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
Write-Host @"
|
|
||||||
|
|
||||||
|
|
||||||
.d8888b. .d888 .d8888b.
|
|
||||||
d88P Y88b d88P" d88P Y88b
|
|
||||||
Y88b. 888 Y88b.
|
|
||||||
"Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b.
|
|
||||||
"Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b
|
|
||||||
"888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888
|
|
||||||
Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b.
|
|
||||||
"Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888
|
|
||||||
|
|
||||||
|
|
||||||
"@ -ForegroundColor White
|
|
||||||
|
|
||||||
$versionDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VERSION=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
$versionDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VERSION=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||||
if (-not $versionDisplay) { $versionDisplay = "latest" }
|
if (-not $versionDisplay) { $versionDisplay = "latest" }
|
||||||
Write-Host " OSS Alternative to NotebookLM for Teams [$versionDisplay]" -ForegroundColor Yellow
|
$variantDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VARIANT=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||||
Write-Host ("=" * 62) -ForegroundColor Cyan
|
if (-not $variantDisplay) { $variantDisplay = "cpu" }
|
||||||
Write-Host ""
|
$wtHours = [math]::Floor($WatchtowerInterval / 3600)
|
||||||
|
Write-Step "SurfSense is now installed [$versionDisplay]"
|
||||||
|
|
||||||
Write-Info " Frontend: http://localhost:3929"
|
Write-Info " Frontend: http://localhost:3929"
|
||||||
Write-Info " Backend: http://localhost:8929"
|
Write-Info " Backend: http://localhost:8929"
|
||||||
Write-Info " API Docs: http://localhost:8929/docs"
|
Write-Info " API Docs: http://localhost:8929/docs"
|
||||||
Write-Info ""
|
Write-Info ""
|
||||||
Write-Info " Config: $InstallDir\.env"
|
Write-Info " Config: $InstallDir\.env"
|
||||||
|
Write-Info " Variant: $variantDisplay"
|
||||||
Write-Info " Logs: cd $InstallDir; docker compose logs -f"
|
Write-Info " Logs: cd $InstallDir; docker compose logs -f"
|
||||||
Write-Info " Stop: cd $InstallDir; docker compose down"
|
Write-Info " Stop: cd $InstallDir; docker compose down"
|
||||||
Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d"
|
Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d --wait"
|
||||||
Write-Info ""
|
Write-Info ""
|
||||||
|
|
||||||
if ($SetupWatchtower) {
|
if ($SetupWatchtower) {
|
||||||
Write-Info " Watchtower: auto-updates every ${wtHours}h (stop: docker rm -f $WatchtowerContainer)"
|
Write-Info " Watchtower: auto-updates every ${wtHours}h (disable: docker rm -f $WatchtowerContainer)"
|
||||||
} else {
|
} else {
|
||||||
Write-Warn " Watchtower skipped. For auto-updates, re-run without -NoWatchtower."
|
Write-Warn " Watchtower skipped. For auto-updates, re-run without -NoWatchtower."
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,11 @@
|
||||||
# Flags:
|
# Flags:
|
||||||
# --no-watchtower Skip automatic Watchtower setup
|
# --no-watchtower Skip automatic Watchtower setup
|
||||||
# --watchtower-interval=SECS Check interval in seconds (default: 86400 = 24h)
|
# --watchtower-interval=SECS Check interval in seconds (default: 86400 = 24h)
|
||||||
|
# --variant=cpu|cuda|cuda126 Select backend image variant
|
||||||
|
# --gpu Alias for --variant=cuda
|
||||||
|
# --cpu Alias for --variant=cpu
|
||||||
|
# --gpu-count=N|all Number of GPUs to reserve when GPU is enabled
|
||||||
|
# --quiet Skip interactive prompts
|
||||||
#
|
#
|
||||||
# Handles two cases automatically:
|
# Handles two cases automatically:
|
||||||
# 1. Fresh install — no prior SurfSense data detected
|
# 1. Fresh install — no prior SurfSense data detected
|
||||||
|
|
@ -35,12 +40,22 @@ MIGRATION_MODE=false
|
||||||
SETUP_WATCHTOWER=true
|
SETUP_WATCHTOWER=true
|
||||||
WATCHTOWER_INTERVAL=86400
|
WATCHTOWER_INTERVAL=86400
|
||||||
WATCHTOWER_CONTAINER="watchtower"
|
WATCHTOWER_CONTAINER="watchtower"
|
||||||
|
WATCHTOWER_EXPLICIT=false
|
||||||
|
REQUESTED_VARIANT=""
|
||||||
|
VARIANT_EXPLICIT=false
|
||||||
|
GPU_COUNT=""
|
||||||
|
QUIET=false
|
||||||
|
|
||||||
# ── Parse flags ─────────────────────────────────────────────────────────────
|
# ── Parse flags ─────────────────────────────────────────────────────────────
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
case "$arg" in
|
case "$arg" in
|
||||||
--no-watchtower) SETUP_WATCHTOWER=false ;;
|
--no-watchtower) SETUP_WATCHTOWER=false; WATCHTOWER_EXPLICIT=true ;;
|
||||||
--watchtower-interval=*) WATCHTOWER_INTERVAL="${arg#*=}" ;;
|
--watchtower-interval=*) WATCHTOWER_INTERVAL="${arg#*=}" ;;
|
||||||
|
--variant=*) REQUESTED_VARIANT="${arg#*=}"; VARIANT_EXPLICIT=true ;;
|
||||||
|
--gpu) REQUESTED_VARIANT="cuda"; VARIANT_EXPLICIT=true ;;
|
||||||
|
--cpu) REQUESTED_VARIANT="cpu"; VARIANT_EXPLICIT=true ;;
|
||||||
|
--gpu-count=*) GPU_COUNT="${arg#*=}" ;;
|
||||||
|
--quiet) QUIET=true ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
@ -57,6 +72,57 @@ warn() { printf "${YELLOW}[SurfSense]${NC} %s\n" "$1"; }
|
||||||
error() { printf "${RED}[SurfSense]${NC} ERROR: %s\n" "$1" >&2; exit 1; }
|
error() { printf "${RED}[SurfSense]${NC} ERROR: %s\n" "$1" >&2; exit 1; }
|
||||||
step() { printf "\n${BOLD}${CYAN}── %s${NC}\n" "$1"; }
|
step() { printf "\n${BOLD}${CYAN}── %s${NC}\n" "$1"; }
|
||||||
|
|
||||||
|
show_banner() {
|
||||||
|
echo ""
|
||||||
|
printf '\033[1;37m'
|
||||||
|
cat << 'EOF'
|
||||||
|
|
||||||
|
|
||||||
|
███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗
|
||||||
|
██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝
|
||||||
|
███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗
|
||||||
|
╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝
|
||||||
|
███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗
|
||||||
|
╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝
|
||||||
|
|
||||||
|
|
||||||
|
EOF
|
||||||
|
printf "${YELLOW} OSS Alternative to NotebookLM for Teams${NC}\n"
|
||||||
|
printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n"
|
||||||
|
info "This installer will create ${INSTALL_DIR}/ and start SurfSense with Docker Compose."
|
||||||
|
}
|
||||||
|
|
||||||
|
show_banner
|
||||||
|
|
||||||
|
case "${REQUESTED_VARIANT}" in
|
||||||
|
""|cpu|cuda|cuda126) ;;
|
||||||
|
*) error "Invalid --variant='${REQUESTED_VARIANT}'. Use cpu, cuda, or cuda126." ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [[ -n "${GPU_COUNT}" && ! "${GPU_COUNT}" =~ ^([0-9]+|all)$ ]]; then
|
||||||
|
error "Invalid --gpu-count='${GPU_COUNT}'. Use a number or 'all'."
|
||||||
|
fi
|
||||||
|
|
||||||
|
resolve_watchtower_preference() {
|
||||||
|
if $WATCHTOWER_EXPLICIT || $QUIET || [[ ! -r /dev/tty || ! -w /dev/tty ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local choice
|
||||||
|
echo "" > /dev/tty
|
||||||
|
printf "${BOLD}${CYAN}Automatic updates${NC}\n" > /dev/tty
|
||||||
|
printf "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]: " > /dev/tty
|
||||||
|
read -r choice < /dev/tty || choice=""
|
||||||
|
|
||||||
|
case "$choice" in
|
||||||
|
""|[Yy]|[Yy][Ee][Ss]) SETUP_WATCHTOWER=true ;;
|
||||||
|
[Nn]|[Nn][Oo]) SETUP_WATCHTOWER=false ;;
|
||||||
|
*) warn "Unrecognized choice '${choice}', enabling Watchtower by default. Use --no-watchtower to skip it." >&2; SETUP_WATCHTOWER=true ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_watchtower_preference
|
||||||
|
|
||||||
# ── Pre-flight checks ────────────────────────────────────────────────────────
|
# ── Pre-flight checks ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
step "Checking prerequisites"
|
step "Checking prerequisites"
|
||||||
|
|
@ -97,126 +163,11 @@ wait_for_pg() {
|
||||||
success "PostgreSQL is ready."
|
success "PostgreSQL is ready."
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Stack health helpers ─────────────────────────────────────────────────────
|
# ── Stack startup helper ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
# Enumerate compose services for project `surfsense` as `service|state|health|exitcode`
|
|
||||||
# lines. Uses `docker inspect` so we don't depend on `jq`, `python3`, or the
|
|
||||||
# exact ordering of fields in `docker compose ps --format json` output.
|
|
||||||
get_compose_services() {
|
|
||||||
local containers
|
|
||||||
containers=$(docker ps -a --filter "label=com.docker.compose.project=surfsense" --format '{{.Names}}' 2>/dev/null) || true
|
|
||||||
[[ -z "$containers" ]] && return 0
|
|
||||||
|
|
||||||
while IFS= read -r container; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
local svc state health code
|
|
||||||
svc=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$container" 2>/dev/null || echo "")
|
|
||||||
state=$(docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null || echo "unknown")
|
|
||||||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$container" 2>/dev/null || echo "")
|
|
||||||
code=$(docker inspect -f '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "")
|
|
||||||
[[ -z "$svc" ]] && continue
|
|
||||||
printf '%s|%s|%s|%s\n' "$svc" "$state" "$health" "$code"
|
|
||||||
done <<< "$containers"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Globals populated by wait_stack_healthy / consumed by stack_failure_report.
|
|
||||||
STACK_BAD=()
|
|
||||||
STACK_WAITING=()
|
|
||||||
STACK_GOOD=()
|
|
||||||
STACK_TIMEOUT=false
|
|
||||||
|
|
||||||
wait_stack_healthy() {
|
|
||||||
local timeout_sec=${1:-300}
|
|
||||||
local deadline=$(($(date +%s) + timeout_sec))
|
|
||||||
local last_report=""
|
|
||||||
local bad=()
|
|
||||||
local waiting=()
|
|
||||||
local good=()
|
|
||||||
|
|
||||||
while [[ $(date +%s) -lt $deadline ]]; do
|
|
||||||
local lines
|
|
||||||
lines=$(get_compose_services)
|
|
||||||
if [[ -z "$lines" ]]; then
|
|
||||||
sleep 3
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
bad=()
|
|
||||||
waiting=()
|
|
||||||
good=()
|
|
||||||
|
|
||||||
while IFS='|' read -r name state health code; do
|
|
||||||
[[ -z "$name" ]] && continue
|
|
||||||
if [[ "$name" == "migrations" ]]; then
|
|
||||||
if [[ "$state" == "exited" && "$code" == "0" ]]; then
|
|
||||||
good+=("$name")
|
|
||||||
elif [[ "$state" == "exited" ]]; then
|
|
||||||
bad+=("${name} (exit=${code})")
|
|
||||||
else
|
|
||||||
waiting+=("${name} (${state})")
|
|
||||||
fi
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$state" == "running" ]]; then
|
|
||||||
if [[ -z "$health" || "$health" == "healthy" ]]; then
|
|
||||||
good+=("$name")
|
|
||||||
elif [[ "$health" == "starting" ]]; then
|
|
||||||
waiting+=("${name} (starting)")
|
|
||||||
elif [[ "$health" == "unhealthy" ]]; then
|
|
||||||
bad+=("${name} (unhealthy)")
|
|
||||||
else
|
|
||||||
waiting+=("${name} (${health})")
|
|
||||||
fi
|
|
||||||
elif [[ "$state" == "restarting" ]]; then
|
|
||||||
bad+=("${name} (restarting)")
|
|
||||||
elif [[ "$state" == "exited" ]]; then
|
|
||||||
bad+=("${name} (exited, code=${code})")
|
|
||||||
else
|
|
||||||
waiting+=("${name} (${state})")
|
|
||||||
fi
|
|
||||||
done <<< "$lines"
|
|
||||||
|
|
||||||
if (( ${#bad[@]} > 0 )); then
|
|
||||||
STACK_BAD=("${bad[@]}")
|
|
||||||
STACK_WAITING=("${waiting[@]}")
|
|
||||||
STACK_GOOD=("${good[@]}")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
if (( ${#waiting[@]} == 0 )); then
|
|
||||||
STACK_GOOD=("${good[@]}")
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
local report="Waiting on: ${waiting[*]}"
|
|
||||||
if [[ "$report" != "$last_report" ]]; then
|
|
||||||
info "$report"
|
|
||||||
last_report="$report"
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
# bad/waiting/good are declared at function scope so referencing them is
|
|
||||||
# safe even if the polling loop never executed its body.
|
|
||||||
STACK_BAD=()
|
|
||||||
[[ ${#bad[@]} -gt 0 ]] && STACK_BAD=("${bad[@]}")
|
|
||||||
STACK_WAITING=()
|
|
||||||
[[ ${#waiting[@]} -gt 0 ]] && STACK_WAITING=("${waiting[@]}")
|
|
||||||
STACK_GOOD=()
|
|
||||||
[[ ${#good[@]} -gt 0 ]] && STACK_GOOD=("${good[@]}")
|
|
||||||
STACK_TIMEOUT=true
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
stack_failure_report() {
|
stack_failure_report() {
|
||||||
echo ""
|
echo ""
|
||||||
echo -e "\033[31m[ERROR]\033[0m Stack did not reach a healthy state."
|
echo -e "\033[31m[ERROR]\033[0m Stack did not reach a healthy state."
|
||||||
if (( ${#STACK_BAD[@]} > 0 )) && [[ -n "${STACK_BAD[0]}" ]]; then
|
|
||||||
echo " Failed: ${STACK_BAD[*]}"
|
|
||||||
fi
|
|
||||||
if (( ${#STACK_WAITING[@]} > 0 )) && [[ -n "${STACK_WAITING[0]}" ]]; then
|
|
||||||
echo " Stuck: ${STACK_WAITING[*]}"
|
|
||||||
fi
|
|
||||||
echo ""
|
echo ""
|
||||||
info "Recent logs from migrations / zero-cache / backend:"
|
info "Recent logs from migrations / zero-cache / backend:"
|
||||||
(cd "${INSTALL_DIR}" && ${DC} logs --tail=60 migrations zero-cache backend 2>&1) || true
|
(cd "${INSTALL_DIR}" && ${DC} logs --tail=60 migrations zero-cache backend 2>&1) || true
|
||||||
|
|
@ -224,36 +175,158 @@ stack_failure_report() {
|
||||||
echo "Recovery hints:"
|
echo "Recovery hints:"
|
||||||
echo " 1. Inspect migrations: cd ${INSTALL_DIR} && ${DC} logs migrations"
|
echo " 1. Inspect migrations: cd ${INSTALL_DIR} && ${DC} logs migrations"
|
||||||
echo " 2. Verify publication: cd ${INSTALL_DIR} && ${DC} exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
echo " 2. Verify publication: cd ${INSTALL_DIR} && ${DC} exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
||||||
echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d"
|
echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d --wait"
|
||||||
echo ""
|
echo ""
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# True if `surfsense-zero-cache` exists but `surfsense-zero-init` does not.
|
compose_up_wait() {
|
||||||
# That signals an install that predates the migrations-service fix; the old
|
local service="${1:-}"
|
||||||
# replica may be half-initialized and would block zero-cache on next start.
|
if [[ -n "$service" ]]; then
|
||||||
test_stale_zero_cache_volume() {
|
(cd "${INSTALL_DIR}" && ${DC} up -d --wait "$service") < /dev/null
|
||||||
local has_zc has_zi
|
else
|
||||||
has_zc=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-cache' || true)
|
(cd "${INSTALL_DIR}" && ${DC} up -d --wait) < /dev/null
|
||||||
has_zi=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-init' || true)
|
fi
|
||||||
[[ -n "$has_zc" && -z "$has_zi" ]]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
invoke_stale_zero_cache_cleanup() {
|
# ── Variant and .env helpers ─────────────────────────────────────────────────
|
||||||
if ! test_stale_zero_cache_volume; then
|
|
||||||
|
set_env_value() {
|
||||||
|
local file="$1"
|
||||||
|
local key="$2"
|
||||||
|
local value="$3"
|
||||||
|
local tmp
|
||||||
|
tmp=$(mktemp)
|
||||||
|
|
||||||
|
if grep -q "^${key}=" "$file" 2>/dev/null; then
|
||||||
|
awk -v key="$key" -v value="$value" 'BEGIN { prefix = key "=" } $0 ~ "^" prefix { print prefix value; next } { print }' "$file" > "$tmp"
|
||||||
|
else
|
||||||
|
cp "$file" "$tmp"
|
||||||
|
printf '\n%s=%s\n' "$key" "$value" >> "$tmp"
|
||||||
|
fi
|
||||||
|
mv "$tmp" "$file"
|
||||||
|
}
|
||||||
|
|
||||||
|
remove_env_value() {
|
||||||
|
local file="$1"
|
||||||
|
local key="$2"
|
||||||
|
local tmp
|
||||||
|
tmp=$(mktemp)
|
||||||
|
awk -v key="$key" 'BEGIN { prefix = key "=" } $0 !~ "^" prefix { print }' "$file" > "$tmp"
|
||||||
|
mv "$tmp" "$file"
|
||||||
|
}
|
||||||
|
|
||||||
|
version_major() {
|
||||||
|
printf '%s' "$1" | cut -d. -f1
|
||||||
|
}
|
||||||
|
|
||||||
|
recommend_cuda_variant() {
|
||||||
|
local driver_version driver_major
|
||||||
|
driver_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n 1 | tr -d '[:space:]' || true)
|
||||||
|
driver_major=$(version_major "$driver_version")
|
||||||
|
|
||||||
|
# CUDA 12.8 generally requires an R570+ driver. Use CUDA 12.6 as the
|
||||||
|
# compatibility fallback for older 12.x driver stacks and GPUs.
|
||||||
|
if [[ "$driver_major" =~ ^[0-9]+$ && "$driver_major" -lt 570 ]]; then
|
||||||
|
printf 'cuda126'
|
||||||
|
else
|
||||||
|
printf 'cuda'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
gpu_runtime_available() {
|
||||||
|
docker info 2>/dev/null | grep -qi 'nvidia' \
|
||||||
|
|| command -v nvidia-ctk >/dev/null 2>&1 \
|
||||||
|
|| command -v nvidia-container-runtime >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
host_has_nvidia_gpu() {
|
||||||
|
command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_variant() {
|
||||||
|
local detected_variant="cpu"
|
||||||
|
local has_gpu=false
|
||||||
|
local has_runtime=false
|
||||||
|
|
||||||
|
if host_has_nvidia_gpu; then
|
||||||
|
has_gpu=true
|
||||||
|
detected_variant=$(recommend_cuda_variant)
|
||||||
|
if gpu_runtime_available; then
|
||||||
|
has_runtime=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if $VARIANT_EXPLICIT; then
|
||||||
|
if [[ "$REQUESTED_VARIANT" == "cpu" ]]; then
|
||||||
|
printf 'cpu'
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if ! $has_gpu; then
|
||||||
|
warn "No NVIDIA GPU detected; falling back to CPU variant." >&2
|
||||||
|
printf 'cpu'
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if ! $has_runtime; then
|
||||||
|
warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant." >&2
|
||||||
|
warn "Install the toolkit before enabling SurfSense GPU acceleration." >&2
|
||||||
|
printf 'cpu'
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
printf '%s' "$REQUESTED_VARIANT"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that"
|
|
||||||
warn "predates the migrations-service fix. It may contain a half-initialized"
|
|
||||||
warn "SQLite replica that would block zero-cache from starting."
|
|
||||||
warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel."
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
(cd "${INSTALL_DIR}" && ${DC} down --remove-orphans 2>/dev/null) || true
|
if $has_gpu && ! $has_runtime; then
|
||||||
docker volume rm surfsense-zero-cache 2>/dev/null || true
|
warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant." >&2
|
||||||
success "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start."
|
fi
|
||||||
|
|
||||||
|
if $has_gpu && $has_runtime && ! $QUIET && [[ -r /dev/tty && -w /dev/tty ]]; then
|
||||||
|
local choice
|
||||||
|
echo "" > /dev/tty
|
||||||
|
printf "${BOLD}${CYAN}SurfSense detected an NVIDIA GPU.${NC}\n" > /dev/tty
|
||||||
|
printf "Use GPU acceleration? [Y/n]: " > /dev/tty
|
||||||
|
read -r choice < /dev/tty || choice=""
|
||||||
|
case "$choice" in
|
||||||
|
"") printf '%s' "$detected_variant" ;;
|
||||||
|
[Yy]|[Yy][Ee][Ss]) printf '%s' "$detected_variant" ;;
|
||||||
|
[Nn]|[Nn][Oo]) printf 'cpu' ;;
|
||||||
|
*) warn "Unrecognized choice '${choice}', using CPU variant." >&2; printf 'cpu' ;;
|
||||||
|
esac
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'cpu'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
apply_variant_env() {
|
||||||
|
local env_file="$1"
|
||||||
|
local variant="$2"
|
||||||
|
local allow_existing_update="$3"
|
||||||
|
|
||||||
|
if [[ -f "$env_file" && "$allow_existing_update" != "true" ]]; then
|
||||||
|
warn ".env already exists — keeping your existing configuration."
|
||||||
|
info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in ${env_file}, then run ${DC} up -d --wait."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$variant" == "cpu" ]]; then
|
||||||
|
set_env_value "$env_file" "SURFSENSE_VARIANT" ""
|
||||||
|
remove_env_value "$env_file" "COMPOSE_FILE"
|
||||||
|
remove_env_value "$env_file" "SURFSENSE_GPU_COUNT"
|
||||||
|
else
|
||||||
|
set_env_value "$env_file" "SURFSENSE_VARIANT" "$variant"
|
||||||
|
set_env_value "$env_file" "COMPOSE_FILE" "docker-compose.yml:docker-compose.gpu.yml"
|
||||||
|
if [[ -n "$GPU_COUNT" ]]; then
|
||||||
|
set_env_value "$env_file" "SURFSENSE_GPU_COUNT" "$GPU_COUNT"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
remove_env_value "$env_file" "COMPOSE_PROFILES"
|
||||||
|
}
|
||||||
|
|
||||||
|
SELECTED_VARIANT=$(resolve_variant)
|
||||||
|
|
||||||
# ── Download files ───────────────────────────────────────────────────────────
|
# ── Download files ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
step "Downloading SurfSense files"
|
step "Downloading SurfSense files"
|
||||||
|
|
@ -263,6 +336,7 @@ mkdir -p "${INSTALL_DIR}/searxng"
|
||||||
|
|
||||||
FILES=(
|
FILES=(
|
||||||
"docker/docker-compose.yml:docker-compose.yml"
|
"docker/docker-compose.yml:docker-compose.yml"
|
||||||
|
"docker/docker-compose.gpu.yml:docker-compose.gpu.yml"
|
||||||
"docker/.env.example:.env.example"
|
"docker/.env.example:.env.example"
|
||||||
"docker/postgresql.conf:postgresql.conf"
|
"docker/postgresql.conf:postgresql.conf"
|
||||||
"docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
|
"docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
|
||||||
|
|
@ -336,15 +410,19 @@ if [ ! -f "${INSTALL_DIR}/.env" ]; then
|
||||||
else
|
else
|
||||||
sed -i "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env"
|
sed -i "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env"
|
||||||
fi
|
fi
|
||||||
|
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false"
|
||||||
info "Created ${INSTALL_DIR}/.env"
|
info "Created ${INSTALL_DIR}/.env"
|
||||||
else
|
else
|
||||||
warn ".env already exists — keeping your existing configuration."
|
if $VARIANT_EXPLICIT; then
|
||||||
|
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "true"
|
||||||
|
info "Updated SurfSense image variant in existing ${INSTALL_DIR}/.env"
|
||||||
|
else
|
||||||
|
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Start containers ─────────────────────────────────────────────────────────
|
# ── Start containers ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
invoke_stale_zero_cache_cleanup
|
|
||||||
|
|
||||||
if $MIGRATION_MODE; then
|
if $MIGRATION_MODE; then
|
||||||
# Read DB credentials from .env (fall back to defaults from docker-compose.yml)
|
# Read DB credentials from .env (fall back to defaults from docker-compose.yml)
|
||||||
DB_USER=$(grep '^DB_USER=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
DB_USER=$(grep '^DB_USER=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||||
|
|
@ -401,26 +479,20 @@ if $MIGRATION_MODE; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
step "Starting all SurfSense services"
|
step "Starting all SurfSense services"
|
||||||
(cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
|
if ! compose_up_wait; then
|
||||||
success "All containers started; waiting for stack to become healthy..."
|
|
||||||
|
|
||||||
if ! wait_stack_healthy 300; then
|
|
||||||
stack_failure_report
|
stack_failure_report
|
||||||
fi
|
fi
|
||||||
success "All services healthy."
|
success "All services started and healthy."
|
||||||
|
|
||||||
# Key file is no longer needed — SECRET_KEY is now in .env
|
# Key file is no longer needed — SECRET_KEY is now in .env
|
||||||
rm -f "${KEY_FILE}"
|
rm -f "${KEY_FILE}"
|
||||||
|
|
||||||
else
|
else
|
||||||
step "Starting SurfSense"
|
step "Starting SurfSense"
|
||||||
(cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
|
if ! compose_up_wait; then
|
||||||
success "All containers started; waiting for stack to become healthy..."
|
|
||||||
|
|
||||||
if ! wait_stack_healthy 300; then
|
|
||||||
stack_failure_report
|
stack_failure_report
|
||||||
fi
|
fi
|
||||||
success "All services healthy."
|
success "All services started and healthy."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Watchtower (auto-update) ─────────────────────────────────────────────────
|
# ── Watchtower (auto-update) ─────────────────────────────────────────────────
|
||||||
|
|
@ -445,7 +517,7 @@ if $SETUP_WATCHTOWER; then
|
||||||
--label-enable \
|
--label-enable \
|
||||||
--interval "${WATCHTOWER_INTERVAL}" >/dev/null 2>&1 < /dev/null \
|
--interval "${WATCHTOWER_INTERVAL}" >/dev/null 2>&1 < /dev/null \
|
||||||
&& success "Watchtower started — labeled SurfSense containers will auto-update." \
|
&& success "Watchtower started — labeled SurfSense containers will auto-update." \
|
||||||
|| warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d"
|
|| warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d --wait"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
info "Skipping Watchtower setup (--no-watchtower flag)."
|
info "Skipping Watchtower setup (--no-watchtower flag)."
|
||||||
|
|
@ -454,38 +526,25 @@ fi
|
||||||
# ── Done ─────────────────────────────────────────────────────────────────────
|
# ── Done ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
printf '\033[1;37m'
|
|
||||||
cat << 'EOF'
|
|
||||||
|
|
||||||
|
|
||||||
.d8888b. .d888 .d8888b.
|
|
||||||
d88P Y88b d88P" d88P Y88b
|
|
||||||
Y88b. 888 Y88b.
|
|
||||||
"Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b.
|
|
||||||
"Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b
|
|
||||||
"888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888
|
|
||||||
Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b.
|
|
||||||
"Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888
|
|
||||||
|
|
||||||
|
|
||||||
EOF
|
|
||||||
_version_display=$(grep '^SURFSENSE_VERSION=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
_version_display=$(grep '^SURFSENSE_VERSION=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||||
_version_display="${_version_display:-latest}"
|
_version_display="${_version_display:-latest}"
|
||||||
printf " OSS Alternative to NotebookLM for Teams ${YELLOW}[%s]${NC}\n" "${_version_display}"
|
_variant_display=$(grep '^SURFSENSE_VARIANT=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||||
printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n\n"
|
_variant_display="${_variant_display:-cpu}"
|
||||||
|
step "SurfSense is now installed [${_version_display}]"
|
||||||
|
|
||||||
info " Frontend: http://localhost:3929"
|
info " Frontend: http://localhost:3929"
|
||||||
info " Backend: http://localhost:8929"
|
info " Backend: http://localhost:8929"
|
||||||
info " API Docs: http://localhost:8929/docs"
|
info " API Docs: http://localhost:8929/docs"
|
||||||
info ""
|
info ""
|
||||||
info " Config: ${INSTALL_DIR}/.env"
|
info " Config: ${INSTALL_DIR}/.env"
|
||||||
|
info " Variant: ${_variant_display}"
|
||||||
info " Logs: cd ${INSTALL_DIR} && ${DC} logs -f"
|
info " Logs: cd ${INSTALL_DIR} && ${DC} logs -f"
|
||||||
info " Stop: cd ${INSTALL_DIR} && ${DC} down"
|
info " Stop: cd ${INSTALL_DIR} && ${DC} down"
|
||||||
info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d"
|
info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d --wait"
|
||||||
info ""
|
info ""
|
||||||
|
|
||||||
if $SETUP_WATCHTOWER; then
|
if $SETUP_WATCHTOWER; then
|
||||||
info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (stop: docker rm -f ${WATCHTOWER_CONTAINER})"
|
info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (disable: docker rm -f ${WATCHTOWER_CONTAINER})"
|
||||||
else
|
else
|
||||||
warn " Watchtower skipped. For auto-updates, re-run without --no-watchtower."
|
warn " Watchtower skipped. For auto-updates, re-run without --no-watchtower."
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
# syntax=docker.io/docker/dockerfile:1
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# SurfSense Backend — Multi-stage Dockerfile
|
# SurfSense Backend — Multi-stage Dockerfile
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
@ -61,15 +62,25 @@ COPY pyproject.toml uv.lock ./
|
||||||
# Exporting the lock to requirements.txt and feeding it to `uv pip install`
|
# Exporting the lock to requirements.txt and feeding it to `uv pip install`
|
||||||
# pins every transitive package to the exact version captured in uv.lock.
|
# pins every transitive package to the exact version captured in uv.lock.
|
||||||
#
|
#
|
||||||
# Note on torch/CUDA: we do NOT install torch from a separate cu* index here.
|
# Note on torch/CUDA: the export must always select either the cpu or CUDA
|
||||||
# PyPI's torch wheels for Linux x86_64 already ship CUDA-enabled and pull
|
# extra declared in pyproject.toml. A no-extra export would resolve torch from
|
||||||
# nvidia-cudnn-cu13, nvidia-nccl-cu13, triton, etc. as install deps (all
|
# PyPI on Linux, which currently pulls CUDA-enabled wheels and nvidia-* deps.
|
||||||
# captured in uv.lock). If a specific CUDA version is needed, wire it through
|
# Keep CUDA version selection in [tool.uv.sources] so uv.lock remains the
|
||||||
# [tool.uv.sources] in pyproject.toml so the lock stays the source of truth.
|
# source of truth. The install step also needs the matching PyTorch index,
|
||||||
|
# because requirements.txt preserves the +cpu/+cu wheel pins but not uv's
|
||||||
|
# package source metadata.
|
||||||
|
ARG USE_CUDA=false
|
||||||
|
ARG CUDA_EXTRA=cu128
|
||||||
RUN pip install --no-cache-dir uv && \
|
RUN pip install --no-cache-dir uv && \
|
||||||
|
if [ "$USE_CUDA" = "true" ]; then EXTRA="$CUDA_EXTRA"; else EXTRA="cpu"; fi && \
|
||||||
|
TORCH_INDEX="https://download.pytorch.org/whl/${EXTRA}" && \
|
||||||
uv export --frozen --no-dev --no-hashes --no-emit-project \
|
uv export --frozen --no-dev --no-hashes --no-emit-project \
|
||||||
|
--extra "$EXTRA" \
|
||||||
--format requirements-txt -o /tmp/requirements.txt && \
|
--format requirements-txt -o /tmp/requirements.txt && \
|
||||||
uv pip install --system --no-cache-dir -r /tmp/requirements.txt && \
|
uv pip install --system --no-cache-dir \
|
||||||
|
--index "$TORCH_INDEX" \
|
||||||
|
--index-strategy unsafe-best-match \
|
||||||
|
-r /tmp/requirements.txt && \
|
||||||
rm /tmp/requirements.txt
|
rm /tmp/requirements.txt
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -94,7 +105,9 @@ RUN printf '%s\n' \
|
||||||
| python || true
|
| python || true
|
||||||
|
|
||||||
ARG EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
|
ARG EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
|
||||||
RUN python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')"
|
RUN --mount=type=secret,id=HF_TOKEN \
|
||||||
|
HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
|
||||||
|
python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')"
|
||||||
|
|
||||||
# Install Playwright browsers (the playwright python package itself is in deps)
|
# Install Playwright browsers (the playwright python package itself is in deps)
|
||||||
RUN playwright install chromium --with-deps
|
RUN playwright install chromium --with-deps
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import os
|
||||||
import sys
|
import sys
|
||||||
from logging.config import fileConfig
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
from sqlalchemy import pool
|
from sqlalchemy import pool
|
||||||
from sqlalchemy.engine import Connection
|
from sqlalchemy.engine import Connection
|
||||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||||
|
|
@ -36,6 +37,9 @@ if config.config_file_name is not None:
|
||||||
# target_metadata = mymodel.Base.metadata
|
# target_metadata = mymodel.Base.metadata
|
||||||
target_metadata = Base.metadata
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
MIGRATION_ADVISORY_LOCK_NAMESPACE = "surfsense"
|
||||||
|
MIGRATION_ADVISORY_LOCK_NAME = "alembic_migrations"
|
||||||
|
|
||||||
# other values from the config, defined by the needs of env.py,
|
# other values from the config, defined by the needs of env.py,
|
||||||
# can be acquired:
|
# can be acquired:
|
||||||
# my_important_option = config.get_main_option("my_important_option")
|
# my_important_option = config.get_main_option("my_important_option")
|
||||||
|
|
@ -73,8 +77,22 @@ def do_run_migrations(connection: Connection) -> None:
|
||||||
transaction_per_migration=True,
|
transaction_per_migration=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
with context.begin_transaction():
|
lock_params = {
|
||||||
context.run_migrations()
|
"namespace": MIGRATION_ADVISORY_LOCK_NAMESPACE,
|
||||||
|
"name": MIGRATION_ADVISORY_LOCK_NAME,
|
||||||
|
}
|
||||||
|
connection.execute(
|
||||||
|
sa.text("SELECT pg_advisory_lock(hashtext(:namespace), hashtext(:name))"),
|
||||||
|
lock_params,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
finally:
|
||||||
|
connection.execute(
|
||||||
|
sa.text("SELECT pg_advisory_unlock(hashtext(:namespace), hashtext(:name))"),
|
||||||
|
lock_params,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def run_async_migrations() -> None:
|
async def run_async_migrations() -> None:
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,6 @@ depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
PUBLICATION_NAME = "zero_publication"
|
PUBLICATION_NAME = "zero_publication"
|
||||||
|
|
||||||
# Must stay in sync with the column lists in migrations 117 / 139 / 140.
|
|
||||||
DOCUMENT_COLS = [
|
DOCUMENT_COLS = [
|
||||||
"id",
|
"id",
|
||||||
"title",
|
"title",
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
"""reconcile zero_publication from canonical definition
|
||||||
|
|
||||||
|
Revision ID: 155
|
||||||
|
Revises: 154
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from app.zero_publication import apply_publication
|
||||||
|
|
||||||
|
revision: str = "155"
|
||||||
|
down_revision: str | None = "154"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
apply_publication(op.get_bind())
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""No-op. Historical publication shapes are immutable."""
|
||||||
229
surfsense_backend/app/zero_publication.py
Normal file
229
surfsense_backend/app/zero_publication.py
Normal file
|
|
@ -0,0 +1,229 @@
|
||||||
|
"""Canonical Zero publication definition for SurfSense.
|
||||||
|
|
||||||
|
This module is the single source of truth for ``zero_publication``. Future
|
||||||
|
publication changes should update ``ZERO_PUBLICATION`` and call
|
||||||
|
``apply_publication()`` from a migration instead of hand-copying table lists.
|
||||||
|
|
||||||
|
SurfSense runs Zero on Postgres with Zero's event triggers installed, so the
|
||||||
|
official Zero path is a plain ``ALTER PUBLICATION ... SET TABLE``. If a future
|
||||||
|
deployment cannot use event triggers, use Zero's documented
|
||||||
|
``zero_0.update_schemas()`` hook as the fallback instead of COMMENT bookends.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.engine import Connection
|
||||||
|
from sqlalchemy.ext.asyncio import create_async_engine
|
||||||
|
|
||||||
|
PUBLICATION_NAME = "zero_publication"
|
||||||
|
|
||||||
|
DOCUMENT_COLS = [
|
||||||
|
"id",
|
||||||
|
"title",
|
||||||
|
"document_type",
|
||||||
|
"search_space_id",
|
||||||
|
"folder_id",
|
||||||
|
"created_by_id",
|
||||||
|
"status",
|
||||||
|
"created_at",
|
||||||
|
"updated_at",
|
||||||
|
]
|
||||||
|
|
||||||
|
USER_COLS = [
|
||||||
|
"id",
|
||||||
|
"pages_limit",
|
||||||
|
"pages_used",
|
||||||
|
"premium_credit_micros_limit",
|
||||||
|
"premium_credit_micros_used",
|
||||||
|
]
|
||||||
|
|
||||||
|
AUTOMATION_RUN_COLS = [
|
||||||
|
"id",
|
||||||
|
"automation_id",
|
||||||
|
"trigger_id",
|
||||||
|
"status",
|
||||||
|
"step_results",
|
||||||
|
"started_at",
|
||||||
|
"finished_at",
|
||||||
|
"created_at",
|
||||||
|
]
|
||||||
|
|
||||||
|
ZERO_PUBLICATION: Mapping[str, Sequence[str] | None] = {
|
||||||
|
"notifications": None,
|
||||||
|
"documents": DOCUMENT_COLS,
|
||||||
|
"folders": None,
|
||||||
|
"search_source_connectors": None,
|
||||||
|
"new_chat_messages": None,
|
||||||
|
"chat_comments": None,
|
||||||
|
"chat_session_state": None,
|
||||||
|
"user": USER_COLS,
|
||||||
|
"automation_runs": AUTOMATION_RUN_COLS,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _quote_identifier(identifier: str) -> str:
|
||||||
|
return '"' + identifier.replace('"', '""') + '"'
|
||||||
|
|
||||||
|
|
||||||
|
def _column_exists(conn: Connection, table: str, column: str) -> bool:
|
||||||
|
return (
|
||||||
|
conn.execute(
|
||||||
|
text(
|
||||||
|
"SELECT 1 FROM information_schema.columns "
|
||||||
|
"WHERE table_schema = current_schema() "
|
||||||
|
"AND table_name = :table AND column_name = :column"
|
||||||
|
),
|
||||||
|
{"table": table, "column": column},
|
||||||
|
).fetchone()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _expected_columns(conn: Connection, table: str) -> list[str] | None:
|
||||||
|
columns = ZERO_PUBLICATION[table]
|
||||||
|
if columns is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
expected = list(columns)
|
||||||
|
if table in {"documents", "user"} and _column_exists(conn, table, "_0_version"):
|
||||||
|
expected.append("_0_version")
|
||||||
|
return expected
|
||||||
|
|
||||||
|
|
||||||
|
def _format_table_entry(conn: Connection, table: str) -> str:
|
||||||
|
columns = _expected_columns(conn, table)
|
||||||
|
table_sql = _quote_identifier(table)
|
||||||
|
if columns is None:
|
||||||
|
return table_sql
|
||||||
|
|
||||||
|
column_sql = ", ".join(_quote_identifier(column) for column in columns)
|
||||||
|
return f"{table_sql} ({column_sql})"
|
||||||
|
|
||||||
|
|
||||||
|
def build_set_table_sql(conn: Connection) -> str:
|
||||||
|
"""Build the canonical plain SET TABLE statement for Zero's event triggers."""
|
||||||
|
|
||||||
|
table_list = ", ".join(_format_table_entry(conn, table) for table in ZERO_PUBLICATION)
|
||||||
|
return f"ALTER PUBLICATION {_quote_identifier(PUBLICATION_NAME)} SET TABLE {table_list}"
|
||||||
|
|
||||||
|
|
||||||
|
def apply_publication(conn: Connection) -> None:
|
||||||
|
"""Reconcile ``zero_publication`` to the canonical shape."""
|
||||||
|
|
||||||
|
exists = conn.execute(
|
||||||
|
text("SELECT 1 FROM pg_publication WHERE pubname = :name"),
|
||||||
|
{"name": PUBLICATION_NAME},
|
||||||
|
).fetchone()
|
||||||
|
if not exists:
|
||||||
|
return
|
||||||
|
|
||||||
|
conn.execute(text(build_set_table_sql(conn)))
|
||||||
|
|
||||||
|
|
||||||
|
def _actual_publication_shape(conn: Connection) -> dict[str, list[str] | None]:
|
||||||
|
rows = conn.execute(
|
||||||
|
text(
|
||||||
|
"SELECT pt.tablename, pr.prattrs IS NULL AS all_columns, pt.attnames "
|
||||||
|
"FROM pg_publication_tables pt "
|
||||||
|
"JOIN pg_publication p ON p.pubname = pt.pubname "
|
||||||
|
"JOIN pg_class c ON c.relname = pt.tablename "
|
||||||
|
"JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = pt.schemaname "
|
||||||
|
"JOIN pg_publication_rel pr ON pr.prpubid = p.oid AND pr.prrelid = c.oid "
|
||||||
|
"WHERE pt.pubname = :name AND pt.schemaname = current_schema() "
|
||||||
|
"ORDER BY pt.tablename"
|
||||||
|
),
|
||||||
|
{"name": PUBLICATION_NAME},
|
||||||
|
).mappings()
|
||||||
|
|
||||||
|
return {
|
||||||
|
str(row["tablename"]): None
|
||||||
|
if row["all_columns"]
|
||||||
|
else list(row["attnames"] or [])
|
||||||
|
for row in rows
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def expected_publication_shape(conn: Connection) -> dict[str, list[str] | None]:
|
||||||
|
return {table: _expected_columns(conn, table) for table in ZERO_PUBLICATION}
|
||||||
|
|
||||||
|
|
||||||
|
def verify_publication(conn: Connection) -> list[str]:
|
||||||
|
"""Return human-readable mismatches between Postgres and the canonical shape."""
|
||||||
|
|
||||||
|
publication_exists = conn.execute(
|
||||||
|
text("SELECT 1 FROM pg_publication WHERE pubname = :name"),
|
||||||
|
{"name": PUBLICATION_NAME},
|
||||||
|
).fetchone()
|
||||||
|
if not publication_exists:
|
||||||
|
return [f"Publication {PUBLICATION_NAME!r} does not exist"]
|
||||||
|
|
||||||
|
actual = _actual_publication_shape(conn)
|
||||||
|
expected = expected_publication_shape(conn)
|
||||||
|
mismatches: list[str] = []
|
||||||
|
|
||||||
|
for table, expected_columns in expected.items():
|
||||||
|
if table not in actual:
|
||||||
|
mismatches.append(f"{table}: missing from publication")
|
||||||
|
continue
|
||||||
|
|
||||||
|
actual_columns = actual[table]
|
||||||
|
actual_key = sorted(actual_columns) if actual_columns is not None else None
|
||||||
|
expected_key = sorted(expected_columns) if expected_columns is not None else None
|
||||||
|
if actual_key != expected_key:
|
||||||
|
mismatches.append(
|
||||||
|
f"{table}: expected columns {expected_columns or 'ALL'}, "
|
||||||
|
f"got {actual_columns or 'ALL'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for table in sorted(set(actual) - set(expected)):
|
||||||
|
mismatches.append(f"{table}: unexpected table in publication")
|
||||||
|
|
||||||
|
return mismatches
|
||||||
|
|
||||||
|
|
||||||
|
async def _verify_cli() -> int:
|
||||||
|
database_url = os.getenv("DATABASE_URL")
|
||||||
|
if not database_url:
|
||||||
|
print("DATABASE_URL is required to verify zero_publication.", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
engine = create_async_engine(database_url)
|
||||||
|
async with engine.connect() as async_conn:
|
||||||
|
def run_verify(sync_conn: Connection) -> list[str]:
|
||||||
|
return verify_publication(sync_conn)
|
||||||
|
|
||||||
|
mismatches = await async_conn.run_sync(run_verify)
|
||||||
|
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
if mismatches:
|
||||||
|
print("zero_publication shape mismatch:", file=sys.stderr)
|
||||||
|
for mismatch in mismatches:
|
||||||
|
print(f" - {mismatch}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("zero_publication shape verified.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Manage SurfSense's Zero publication")
|
||||||
|
parser.add_argument("--verify", action="store_true", help="verify zero_publication shape")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.verify:
|
||||||
|
return asyncio.run(_verify_cli())
|
||||||
|
|
||||||
|
parser.print_help()
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -92,6 +92,11 @@ dependencies = [
|
||||||
"croniter>=2.0.0",
|
"croniter>=2.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
cpu = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||||
|
cu126 = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||||
|
cu128 = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"ruff>=0.12.5",
|
"ruff>=0.12.5",
|
||||||
|
|
@ -101,6 +106,36 @@ dev = [
|
||||||
"httpx>=0.28.1",
|
"httpx>=0.28.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[tool.uv]
|
||||||
|
conflicts = [[{ extra = "cpu" }, { extra = "cu126" }, { extra = "cu128" }]]
|
||||||
|
|
||||||
|
[tool.uv.sources]
|
||||||
|
torch = [
|
||||||
|
{ index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" },
|
||||||
|
{ index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" },
|
||||||
|
{ index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" },
|
||||||
|
]
|
||||||
|
torchvision = [
|
||||||
|
{ index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" },
|
||||||
|
{ index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" },
|
||||||
|
{ index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[[tool.uv.index]]
|
||||||
|
name = "pytorch-cpu"
|
||||||
|
url = "https://download.pytorch.org/whl/cpu"
|
||||||
|
explicit = true
|
||||||
|
|
||||||
|
[[tool.uv.index]]
|
||||||
|
name = "pytorch-cu126"
|
||||||
|
url = "https://download.pytorch.org/whl/cu126"
|
||||||
|
explicit = true
|
||||||
|
|
||||||
|
[[tool.uv.index]]
|
||||||
|
name = "pytorch-cu128"
|
||||||
|
url = "https://download.pytorch.org/whl/cu128"
|
||||||
|
explicit = true
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
# Exclude a variety of commonly ignored directories.
|
# Exclude a variety of commonly ignored directories.
|
||||||
exclude = [
|
exclude = [
|
||||||
|
|
|
||||||
|
|
@ -49,10 +49,10 @@ trap cleanup SIGTERM SIGINT
|
||||||
# ── Database migrations (only for migrate / all) ─────────────
|
# ── Database migrations (only for migrate / all) ─────────────
|
||||||
# Fail-fast contract:
|
# Fail-fast contract:
|
||||||
# - alembic upgrade head must succeed within ${MIGRATION_TIMEOUT:-900}s
|
# - alembic upgrade head must succeed within ${MIGRATION_TIMEOUT:-900}s
|
||||||
# - zero_publication must exist in pg_publication afterwards
|
# - zero_publication must match the canonical app.zero_publication shape
|
||||||
# Either failure exits non-zero so the dedicated `migrations` compose
|
# Either failure exits non-zero so the dedicated `migrations` compose
|
||||||
# service exits non-zero, halting the rest of the stack instead of
|
# service exits non-zero, halting the rest of the stack instead of
|
||||||
# silently producing a half-built system that crash-loops zero-cache.
|
# silently producing a drifted Zero publication.
|
||||||
run_migrations() {
|
run_migrations() {
|
||||||
echo "Running database migrations..."
|
echo "Running database migrations..."
|
||||||
for i in {1..30}; do
|
for i in {1..30}; do
|
||||||
|
|
@ -73,58 +73,13 @@ run_migrations() {
|
||||||
fi
|
fi
|
||||||
echo "Migrations completed successfully."
|
echo "Migrations completed successfully."
|
||||||
|
|
||||||
echo "Verifying zero_publication exists in Postgres..."
|
echo "Verifying zero_publication matches the canonical shape..."
|
||||||
local pub_oid
|
if ! python -m app.zero_publication --verify; then
|
||||||
pub_oid=$(python <<'PY' 2>/dev/null || true
|
echo "ERROR: zero_publication does not match the canonical shape." >&2
|
||||||
import asyncio
|
|
||||||
import sys
|
|
||||||
from sqlalchemy import text
|
|
||||||
from app.db import engine
|
|
||||||
|
|
||||||
|
|
||||||
async def get_oid():
|
|
||||||
async with engine.connect() as conn:
|
|
||||||
result = await conn.execute(
|
|
||||||
text("SELECT oid FROM pg_publication WHERE pubname = 'zero_publication'")
|
|
||||||
)
|
|
||||||
row = result.first()
|
|
||||||
if row is None:
|
|
||||||
sys.exit(1)
|
|
||||||
print(int(row[0]))
|
|
||||||
|
|
||||||
|
|
||||||
asyncio.run(get_oid())
|
|
||||||
PY
|
|
||||||
)
|
|
||||||
if [ -z "${pub_oid}" ]; then
|
|
||||||
echo "ERROR: zero_publication is missing from Postgres after running alembic." >&2
|
|
||||||
echo "This usually means migration 116 (or a later publication migration) did not run." >&2
|
|
||||||
echo "Inspect alembic state with:" >&2
|
echo "Inspect alembic state with:" >&2
|
||||||
echo " docker compose exec db psql -U \"\$DB_USER\" -d \"\$DB_NAME\" -c 'SELECT * FROM alembic_version;'" >&2
|
echo " docker compose exec db psql -U \"\$DB_USER\" -d \"\$DB_NAME\" -c 'SELECT * FROM alembic_version;'" >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "zero_publication verified (oid=${pub_oid})."
|
|
||||||
|
|
||||||
# Stale-replica safety net: if /zero-init is mounted (i.e. we are the
|
|
||||||
# dedicated `migrations` compose service), drop a marker file when the
|
|
||||||
# publication oid changed (or on first run) so the wrapped zero-cache
|
|
||||||
# entrypoint can wipe /data/zero.db before starting. This recovers from
|
|
||||||
# the case where a previous zero-cache crashed mid-init and left a
|
|
||||||
# half-built SQLite replica without a `_zero.tableMetadata` table.
|
|
||||||
if [ -d /zero-init ]; then
|
|
||||||
local stored_oid=""
|
|
||||||
[ -f /zero-init/last_pub_oid ] && stored_oid=$(cat /zero-init/last_pub_oid 2>/dev/null || true)
|
|
||||||
if [ -z "${stored_oid}" ] || [ "${stored_oid}" != "${pub_oid}" ]; then
|
|
||||||
echo "Publication oid changed (stored=${stored_oid:-<none>}, current=${pub_oid}); writing /zero-init/needs_reset."
|
|
||||||
: > /zero-init/needs_reset
|
|
||||||
chmod 666 /zero-init/needs_reset 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
echo "${pub_oid}" > /zero-init/last_pub_oid
|
|
||||||
chmod 666 /zero-init/last_pub_oid 2>/dev/null || true
|
|
||||||
# World-writable dir so the (possibly non-root) zero-cache container
|
|
||||||
# can `rm -f /zero-init/needs_reset` after acting on the marker.
|
|
||||||
chmod 777 /zero-init 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Service starters ─────────────────────────────────────────
|
# ── Service starters ─────────────────────────────────────────
|
||||||
|
|
|
||||||
1279
surfsense_backend/uv.lock
generated
1279
surfsense_backend/uv.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,12 +1,16 @@
|
||||||
import { mustGetQuery } from "@rocicorp/zero";
|
import { mustGetQuery } from "@rocicorp/zero";
|
||||||
import { handleQueryRequest } from "@rocicorp/zero/server";
|
import { handleQueryRequest } from "@rocicorp/zero/server";
|
||||||
import { NextResponse } from "next/server";
|
import { NextResponse } from "next/server";
|
||||||
import { BACKEND_URL } from "@/lib/env-config";
|
|
||||||
import type { Context } from "@/types/zero";
|
import type { Context } from "@/types/zero";
|
||||||
import { queries } from "@/zero/queries";
|
import { queries } from "@/zero/queries";
|
||||||
import { schema } from "@/zero/schema";
|
import { schema } from "@/zero/schema";
|
||||||
|
|
||||||
const backendURL = BACKEND_URL;
|
function getBackendBaseUrl() {
|
||||||
|
const base = process.env.FASTAPI_BACKEND_INTERNAL_URL || "http://localhost:8000";
|
||||||
|
return base.endsWith("/") ? base.slice(0, -1) : base;
|
||||||
|
}
|
||||||
|
|
||||||
|
const backendURL = getBackendBaseUrl();
|
||||||
|
|
||||||
async function authenticateRequest(
|
async function authenticateRequest(
|
||||||
request: Request
|
request: Request
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com
|
||||||
import { useFolderSync } from "@/hooks/use-folder-sync";
|
import { useFolderSync } from "@/hooks/use-folder-sync";
|
||||||
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
||||||
import { useElectronAPI } from "@/hooks/use-platform";
|
import { useElectronAPI } from "@/hooks/use-platform";
|
||||||
|
import { isLlmOnboardingComplete } from "@/lib/onboarding";
|
||||||
|
|
||||||
export function DashboardClientLayout({
|
export function DashboardClientLayout({
|
||||||
children,
|
children,
|
||||||
|
|
@ -47,9 +48,8 @@ export function DashboardClientLayout({
|
||||||
const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
|
const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
|
||||||
|
|
||||||
const isOnboardingComplete = useCallback(() => {
|
const isOnboardingComplete = useCallback(() => {
|
||||||
// Check that the Agent LLM ID is set, including 0 for Auto mode.
|
return isLlmOnboardingComplete(preferences.agent_llm_id, globalConfigs.length > 0);
|
||||||
return preferences.agent_llm_id !== null && preferences.agent_llm_id !== undefined;
|
}, [preferences.agent_llm_id, globalConfigs.length]);
|
||||||
}, [preferences.agent_llm_id]);
|
|
||||||
|
|
||||||
const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom);
|
const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom);
|
||||||
const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false);
|
const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false);
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import { Button } from "@/components/ui/button";
|
||||||
import { Spinner } from "@/components/ui/spinner";
|
import { Spinner } from "@/components/ui/spinner";
|
||||||
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
||||||
import { getBearerToken, redirectToLogin } from "@/lib/auth-utils";
|
import { getBearerToken, redirectToLogin } from "@/lib/auth-utils";
|
||||||
|
import { isLlmOnboardingComplete } from "@/lib/onboarding";
|
||||||
|
|
||||||
export default function OnboardPage() {
|
export default function OnboardPage() {
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
|
|
@ -52,15 +53,16 @@ export default function OnboardPage() {
|
||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Check if onboarding is already complete (including 0 for Auto mode)
|
const isOnboardingComplete = isLlmOnboardingComplete(
|
||||||
const isOnboardingComplete =
|
preferences.agent_llm_id,
|
||||||
preferences.agent_llm_id !== null && preferences.agent_llm_id !== undefined;
|
globalConfigs.length > 0
|
||||||
|
);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!preferencesLoading && isOnboardingComplete) {
|
if (!preferencesLoading && globalConfigsLoaded && isOnboardingComplete) {
|
||||||
router.push(`/dashboard/${searchSpaceId}/new-chat`);
|
router.push(`/dashboard/${searchSpaceId}/new-chat`);
|
||||||
}
|
}
|
||||||
}, [preferencesLoading, isOnboardingComplete, router, searchSpaceId]);
|
}, [preferencesLoading, globalConfigsLoaded, isOnboardingComplete, router, searchSpaceId]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const autoConfigureWithGlobal = async () => {
|
const autoConfigureWithGlobal = async () => {
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y
|
||||||
| Variable | Description | Default |
|
| Variable | Description | Default |
|
||||||
|----------|-------------|---------|
|
|----------|-------------|---------|
|
||||||
| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` |
|
| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` |
|
||||||
|
| `SURFSENSE_VARIANT` | Backend image variant. Leave empty for CPU, set `cuda` for CUDA 12.8, or `cuda126` for CUDA 12.6. | *(empty)* |
|
||||||
| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
|
| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
|
||||||
| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
|
| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
|
||||||
| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
|
| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
|
||||||
|
|
@ -42,6 +43,62 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y
|
||||||
| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
|
| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
|
||||||
| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
|
| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
|
||||||
|
|
||||||
|
### Image Variants
|
||||||
|
|
||||||
|
SurfSense publishes CPU and CUDA backend image variants. The frontend image is not variant-specific.
|
||||||
|
|
||||||
|
| Backend tag | Use case | `SURFSENSE_VARIANT` |
|
||||||
|
|-------------|----------|---------------------|
|
||||||
|
| `:latest` | CPU-only default | *(empty)* |
|
||||||
|
| `:latest-cuda` | NVIDIA CUDA 12.8 backend image | `cuda` |
|
||||||
|
| `:latest-cuda126` | NVIDIA CUDA 12.6 backend image for older driver stacks | `cuda126` |
|
||||||
|
|
||||||
|
All backend variants are published for `linux/amd64` and `linux/arm64`. CUDA on `linux/arm64` is best-effort.
|
||||||
|
|
||||||
|
<Callout type="info">
|
||||||
|
GPU acceleration needs two settings: `SURFSENSE_VARIANT` selects the CUDA image, and `COMPOSE_FILE` enables the GPU device overlay. The host must have the NVIDIA Container Toolkit installed.
|
||||||
|
</Callout>
|
||||||
|
|
||||||
|
### NVIDIA GPU Acceleration
|
||||||
|
|
||||||
|
For most NVIDIA systems, add these values to `.env` to use the CUDA 12.8 image:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
SURFSENSE_VARIANT=cuda
|
||||||
|
COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||||
|
SURFSENSE_GPU_COUNT=1
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks or older GPUs that need the CUDA 12.6 fallback image.
|
||||||
|
|
||||||
|
On Windows, use `;` instead of `:` in `COMPOSE_FILE` inside `.env`:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
To switch variants later, edit `SURFSENSE_VARIANT` and `COMPOSE_FILE` in `.env`, then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose pull
|
||||||
|
docker compose up -d --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
### Automatic Updates
|
||||||
|
|
||||||
|
Manual Docker Compose installs do not start Watchtower automatically. To enable external automatic updates, run Watchtower separately:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -d --name watchtower \
|
||||||
|
--restart unless-stopped \
|
||||||
|
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||||
|
nickfedor/watchtower \
|
||||||
|
--label-enable \
|
||||||
|
--interval 86400
|
||||||
|
```
|
||||||
|
|
||||||
|
SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services.
|
||||||
|
|
||||||
### Ports
|
### Ports
|
||||||
|
|
||||||
| Variable | Description | Default |
|
| Variable | Description | Default |
|
||||||
|
|
@ -270,11 +327,13 @@ Symptom (in `docker compose logs zero-cache`):
|
||||||
Error: Unknown or invalid publications. Specified: [zero_publication]. Found: []
|
Error: Unknown or invalid publications. Specified: [zero_publication]. Found: []
|
||||||
```
|
```
|
||||||
|
|
||||||
This means `zero-cache` started before `zero_publication` was created. With
|
This means `zero-cache` started before `zero_publication` was created or the
|
||||||
the current compose files this should be impossible. The `migrations`
|
publication does not match SurfSense's canonical Zero shape. With the current
|
||||||
service blocks `zero-cache` from starting. If you see it, your stack
|
compose files this should be impossible: the `migrations` service blocks
|
||||||
predates the fix or you brought up `zero-cache` manually with `docker
|
`zero-cache` from starting and verifies the publication before exiting
|
||||||
compose up zero-cache` before the migrations service ran.
|
successfully. If you see it, your stack predates the fix or you brought up
|
||||||
|
`zero-cache` manually with `docker compose up zero-cache` before the migrations
|
||||||
|
service ran.
|
||||||
|
|
||||||
Recovery:
|
Recovery:
|
||||||
|
|
||||||
|
|
@ -284,18 +343,13 @@ docker volume rm surfsense-zero-cache # wipe half-built SQLite replica
|
||||||
docker compose up -d # migrations runs first, then zero-cache
|
docker compose up -d # migrations runs first, then zero-cache
|
||||||
```
|
```
|
||||||
|
|
||||||
The install script (`install.ps1` / `install.sh`) detects this case
|
|
||||||
automatically: if it finds a `surfsense-zero-cache` volume from a previous
|
|
||||||
install with no matching `surfsense-zero-init` volume, it removes the stale
|
|
||||||
volume before bringing the stack up.
|
|
||||||
|
|
||||||
### Zero-cache crashes with `_zero.tableMetadata` errors
|
### Zero-cache crashes with `_zero.tableMetadata` errors
|
||||||
|
|
||||||
This indicates a half-initialized SQLite replica left behind by a previous
|
This indicates a half-initialized SQLite replica left behind by a previous
|
||||||
crash. The `migrations` service writes a marker file on a shared volume
|
crash. Zero's own event triggers and `ZERO_AUTO_RESET` handle schema and
|
||||||
(`surfsense-zero-init`) when the publication oid changes; zero-cache wipes
|
replication halts automatically. If the local SQLite replica is wedged, run the
|
||||||
its replica and re-syncs on next start. If the marker mechanism somehow did
|
recovery one-liner above to wipe `surfsense-zero-cache`; zero-cache will
|
||||||
not trigger, run the recovery one-liner above.
|
re-sync from Postgres on the next start.
|
||||||
|
|
||||||
### Ensuring `wal_level = logical`
|
### Ensuring `wal_level = logical`
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ title: One-Line Install Script
|
||||||
description: One-command installation of SurfSense using Docker
|
description: One-command installation of SurfSense using Docker
|
||||||
---
|
---
|
||||||
|
|
||||||
Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates.
|
Downloads the compose files, generates a `SECRET_KEY`, starts all services with `docker compose up -d --wait`, and starts [Watchtower](https://github.com/nicholas-fedor/watchtower) as an external updater for automatic daily updates.
|
||||||
|
|
||||||
**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running.
|
**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running.
|
||||||
|
|
||||||
|
|
@ -19,9 +19,38 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr
|
||||||
irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex
|
irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex
|
||||||
```
|
```
|
||||||
|
|
||||||
This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`.
|
This creates a `./surfsense/` directory with `docker-compose.yml`, `docker-compose.gpu.yml`, and `.env`, then runs `docker compose up -d --wait`.
|
||||||
|
|
||||||
To skip Watchtower (e.g. in production where you manage updates yourself):
|
If an NVIDIA GPU and NVIDIA Container Toolkit are detected, the installer asks whether to use GPU acceleration and chooses the compatible backend image automatically. Non-interactive installs default to CPU unless you pass an explicit flag.
|
||||||
|
|
||||||
|
Interactive installs also ask whether to enable automatic daily updates with Watchtower, noting that updates may download several GB in the background.
|
||||||
|
|
||||||
|
### GPU options
|
||||||
|
|
||||||
|
Linux/macOS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# CUDA 12.8
|
||||||
|
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda
|
||||||
|
|
||||||
|
# CUDA 12.6 fallback for older driver stacks
|
||||||
|
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda126
|
||||||
|
|
||||||
|
# Reserve all available GPUs
|
||||||
|
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --gpu --gpu-count=all
|
||||||
|
```
|
||||||
|
|
||||||
|
PowerShell:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Save the script locally first when passing PowerShell parameters.
|
||||||
|
.\install.ps1 -Variant cuda
|
||||||
|
.\install.ps1 -Variant cuda126 -GpuCount all
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer writes the same `.env` settings you would configure manually: `SURFSENSE_VARIANT` selects the backend image and `COMPOSE_FILE` enables the GPU overlay.
|
||||||
|
|
||||||
|
To skip Watchtower (e.g. in production where you manage updates yourself, or to avoid large background image downloads):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower
|
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower
|
||||||
|
|
@ -29,6 +58,16 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr
|
||||||
|
|
||||||
To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`.
|
To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`.
|
||||||
|
|
||||||
|
Manual updates use the same compose state stored in `.env`, so GPU overlays and variants are preserved:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd surfsense
|
||||||
|
docker compose pull
|
||||||
|
docker compose up -d --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
If Watchtower is enabled, it preserves the running image variant tag automatically. Because SurfSense images are large, use `--no-watchtower` when you prefer to manage update timing yourself.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Access SurfSense
|
## Access SurfSense
|
||||||
|
|
|
||||||
8
surfsense_web/lib/onboarding.ts
Normal file
8
surfsense_web/lib/onboarding.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
export function isLlmOnboardingComplete(
|
||||||
|
agentLlmId: number | null | undefined,
|
||||||
|
hasGlobalConfigs: boolean
|
||||||
|
): boolean {
|
||||||
|
if (agentLlmId === null || agentLlmId === undefined) return false;
|
||||||
|
if (agentLlmId === 0) return hasGlobalConfigs;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue