mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-06-08 20:25:19 +02:00
Merge 901c72cdcc into 26a504f137
This commit is contained in:
commit
dba5cf6132
22 changed files with 2331 additions and 745 deletions
245
.github/workflows/docker-build.yml
vendored
245
.github/workflows/docker-build.yml
vendored
|
|
@ -5,6 +5,9 @@ on:
|
|||
branches:
|
||||
- main
|
||||
- dev
|
||||
tags:
|
||||
- 'v*'
|
||||
- 'beta-v*'
|
||||
paths:
|
||||
- 'surfsense_backend/**'
|
||||
- 'surfsense_web/**'
|
||||
|
|
@ -24,11 +27,13 @@ permissions:
|
|||
packages: write
|
||||
|
||||
jobs:
|
||||
tag_release:
|
||||
compute_version:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch'
|
||||
if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/beta-v')
|
||||
outputs:
|
||||
new_tag: ${{ steps.tag_version.outputs.next_version }}
|
||||
commit_sha: ${{ steps.tag_version.outputs.commit_sha }}
|
||||
is_release_tag: ${{ steps.tag_version.outputs.is_release_tag }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
|
@ -37,57 +42,65 @@ jobs:
|
|||
ref: ${{ github.event.inputs.branch }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Compute-only: tag is pushed by finalize_release after everything succeeds.
|
||||
- name: Read app version and calculate next Docker build version
|
||||
id: tag_version
|
||||
run: |
|
||||
APP_VERSION=$(tr -d '[:space:]' < VERSION)
|
||||
echo "App version from VERSION file: $APP_VERSION"
|
||||
if [[ "$GITHUB_REF" == refs/tags/beta-v* ]]; then
|
||||
VERSION="${GITHUB_REF#refs/tags/beta-v}"
|
||||
NEXT_VERSION="beta-${VERSION}"
|
||||
IS_RELEASE_TAG="true"
|
||||
|
||||
if [ -z "$APP_VERSION" ]; then
|
||||
echo "Error: Could not read version from VERSION file"
|
||||
exit 1
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||
echo "::error::Version '$VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git fetch --tags
|
||||
echo "Docker beta release version from git tag: $NEXT_VERSION"
|
||||
elif [[ "$GITHUB_REF" == refs/tags/v* ]]; then
|
||||
NEXT_VERSION="${GITHUB_REF#refs/tags/v}"
|
||||
IS_RELEASE_TAG="true"
|
||||
|
||||
LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
|
||||
if ! echo "$NEXT_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then
|
||||
echo "::error::Version '$NEXT_VERSION' is not valid semver (expected X.Y.Z). Fix your tag name."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$LATEST_BUILD_TAG" ]; then
|
||||
echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
|
||||
NEXT_VERSION="${APP_VERSION}.1"
|
||||
echo "Docker release version from git tag: $NEXT_VERSION"
|
||||
else
|
||||
echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
|
||||
BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
|
||||
NEXT_BUILD=$((BUILD_NUMBER + 1))
|
||||
NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
|
||||
APP_VERSION=$(tr -d '[:space:]' < VERSION)
|
||||
echo "App version from VERSION file: $APP_VERSION"
|
||||
|
||||
if [ -z "$APP_VERSION" ]; then
|
||||
echo "Error: Could not read version from VERSION file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git fetch --tags
|
||||
|
||||
LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1)
|
||||
|
||||
if [ -z "$LATEST_BUILD_TAG" ]; then
|
||||
echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1"
|
||||
NEXT_VERSION="${APP_VERSION}.1"
|
||||
else
|
||||
echo "Latest Docker build tag found: $LATEST_BUILD_TAG"
|
||||
BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev)
|
||||
NEXT_BUILD=$((BUILD_NUMBER + 1))
|
||||
NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}"
|
||||
fi
|
||||
|
||||
IS_RELEASE_TAG="false"
|
||||
echo "Calculated next Docker version: $NEXT_VERSION"
|
||||
fi
|
||||
|
||||
echo "Calculated next Docker version: $NEXT_VERSION"
|
||||
echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create and Push Tag
|
||||
run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||
|
||||
NEXT_TAG="${{ steps.tag_version.outputs.next_version }}"
|
||||
COMMIT_SHA=$(git rev-parse HEAD)
|
||||
echo "Tagging commit $COMMIT_SHA with $NEXT_TAG"
|
||||
|
||||
git tag -a "$NEXT_TAG" -m "Docker build $NEXT_TAG"
|
||||
echo "Pushing tag $NEXT_TAG to origin"
|
||||
git push origin "$NEXT_TAG"
|
||||
|
||||
- name: Verify Tag Push
|
||||
run: |
|
||||
echo "Checking if tag ${{ steps.tag_version.outputs.next_version }} exists remotely..."
|
||||
sleep 5
|
||||
git ls-remote --tags origin | grep "refs/tags/${{ steps.tag_version.outputs.next_version }}" || (echo "Tag push verification failed!" && exit 1)
|
||||
echo "Tag successfully pushed."
|
||||
echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||
echo "is_release_tag=$IS_RELEASE_TAG" >> $GITHUB_OUTPUT
|
||||
|
||||
build:
|
||||
needs: tag_release
|
||||
if: always() && (needs.tag_release.result == 'success' || needs.tag_release.result == 'skipped')
|
||||
needs: compute_version
|
||||
if: always() && (needs.compute_version.result == 'success' || needs.compute_version.result == 'skipped')
|
||||
runs-on: ${{ matrix.os }}
|
||||
permissions:
|
||||
packages: write
|
||||
|
|
@ -97,6 +110,12 @@ jobs:
|
|||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
image: [backend, web]
|
||||
variant: [cpu, cuda, cuda126]
|
||||
exclude:
|
||||
- image: web
|
||||
variant: cuda
|
||||
- image: web
|
||||
variant: cuda126
|
||||
include:
|
||||
- platform: linux/amd64
|
||||
suffix: amd64
|
||||
|
|
@ -114,6 +133,18 @@ jobs:
|
|||
context: ./surfsense_web
|
||||
file: ./surfsense_web/Dockerfile
|
||||
target: runner
|
||||
- variant: cpu
|
||||
tag_suffix: ""
|
||||
use_cuda: "false"
|
||||
cuda_extra: cpu
|
||||
- variant: cuda
|
||||
tag_suffix: "-cuda"
|
||||
use_cuda: "true"
|
||||
cuda_extra: cu128
|
||||
- variant: cuda126
|
||||
tag_suffix: "-cuda126"
|
||||
use_cuda: "true"
|
||||
cuda_extra: cu126
|
||||
env:
|
||||
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
||||
|
||||
|
|
@ -149,7 +180,7 @@ jobs:
|
|||
sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
|
||||
docker system prune -af
|
||||
|
||||
- name: Build and push by digest ${{ matrix.name }} (${{ matrix.suffix }})
|
||||
- name: Build and push by digest ${{ matrix.name }} (${{ matrix.variant }}, ${{ matrix.suffix }})
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
|
|
@ -160,10 +191,14 @@ jobs:
|
|||
tags: ${{ steps.image.outputs.name }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.suffix }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.suffix }}
|
||||
cache-from: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }}
|
||||
cache-to: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }},mode=max,image-manifest=true,oci-mediatypes=true
|
||||
secrets: |
|
||||
HF_TOKEN=${{ secrets.HF_TOKEN }}
|
||||
provenance: false
|
||||
build-args: |
|
||||
${{ matrix.image == 'backend' && format('USE_CUDA={0}', matrix.use_cuda) || '' }}
|
||||
${{ matrix.image == 'backend' && format('CUDA_EXTRA={0}', matrix.cuda_extra) || '' }}
|
||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }}
|
||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }}
|
||||
${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }}
|
||||
|
|
@ -179,15 +214,47 @@ jobs:
|
|||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digests-${{ matrix.image }}-${{ matrix.suffix }}
|
||||
name: digests-${{ matrix.image }}-${{ matrix.variant }}-${{ matrix.suffix }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Release gate: require both arches for every variant, else block publishing.
|
||||
# Release-only; skipped on dev so the tolerant create_manifest path is kept.
|
||||
verify_digests:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [compute_version, build]
|
||||
if: ${{ always() && needs.compute_version.result == 'success' && needs.compute_version.outputs.new_tag != '' }}
|
||||
steps:
|
||||
- name: Download all digests
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: digests-*
|
||||
path: /tmp/digests
|
||||
merge-multiple: false
|
||||
|
||||
- name: Require both arches for every required variant
|
||||
run: |
|
||||
fail=0
|
||||
check() {
|
||||
c=$(find /tmp/digests -type f -path "*/digests-$1-*/*" 2>/dev/null | wc -l | tr -d ' ')
|
||||
if [ "$c" -lt 2 ]; then
|
||||
echo "::error::$1 has $c/2 arch digests — blocking release"
|
||||
fail=1
|
||||
else
|
||||
echo "OK: $1 ($c/2)"
|
||||
fi
|
||||
}
|
||||
check backend-cpu
|
||||
check backend-cuda
|
||||
check backend-cuda126
|
||||
check web-cpu
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
|
||||
create_manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [tag_release, build]
|
||||
if: always() && needs.build.result == 'success'
|
||||
needs: [compute_version, build, verify_digests]
|
||||
if: ${{ !cancelled() && needs.verify_digests.result != 'failure' }}
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
|
|
@ -197,8 +264,20 @@ jobs:
|
|||
include:
|
||||
- name: surfsense-backend
|
||||
image: backend
|
||||
variant: cpu
|
||||
tag_suffix: ""
|
||||
- name: surfsense-backend
|
||||
image: backend
|
||||
variant: cuda
|
||||
tag_suffix: "-cuda"
|
||||
- name: surfsense-backend
|
||||
image: backend
|
||||
variant: cuda126
|
||||
tag_suffix: "-cuda126"
|
||||
- name: surfsense-web
|
||||
image: web
|
||||
variant: cpu
|
||||
tag_suffix: ""
|
||||
env:
|
||||
REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }}
|
||||
|
||||
|
|
@ -207,22 +286,33 @@ jobs:
|
|||
id: image
|
||||
run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download amd64 digest
|
||||
- name: Download digests
|
||||
id: download
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: digests-${{ matrix.image }}-amd64
|
||||
pattern: digests-${{ matrix.image }}-${{ matrix.variant }}-*
|
||||
path: /tmp/digests
|
||||
merge-multiple: true
|
||||
continue-on-error: true
|
||||
|
||||
- name: Download arm64 digest
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: digests-${{ matrix.image }}-arm64
|
||||
path: /tmp/digests
|
||||
- name: Check digests
|
||||
id: check
|
||||
run: |
|
||||
count=$(find /tmp/digests -type f 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "digest_count=$count" >> $GITHUB_OUTPUT
|
||||
if [ "$count" -lt 2 ]; then
|
||||
echo "::warning::${{ matrix.variant }}: $count/2 digests, skipping merge"
|
||||
echo "skip=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "skip=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
|
|
@ -230,9 +320,10 @@ jobs:
|
|||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute app version
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
id: appver
|
||||
run: |
|
||||
VERSION_TAG="${{ needs.tag_release.outputs.new_tag }}"
|
||||
VERSION_TAG="${{ needs.compute_version.outputs.new_tag }}"
|
||||
if [ -n "$VERSION_TAG" ]; then
|
||||
APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev)
|
||||
else
|
||||
|
|
@ -241,29 +332,69 @@ jobs:
|
|||
echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Docker meta
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ steps.image.outputs.name }}
|
||||
tags: |
|
||||
type=raw,value=${{ needs.tag_release.outputs.new_tag }},enable=${{ needs.tag_release.outputs.new_tag != '' }}
|
||||
type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.tag_release.outputs.new_tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }}
|
||||
type=raw,value=${{ needs.compute_version.outputs.new_tag }},enable=${{ needs.compute_version.outputs.new_tag != '' }}
|
||||
type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }}
|
||||
type=ref,event=branch
|
||||
type=sha,prefix=git-
|
||||
flavor: |
|
||||
latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch }}
|
||||
latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/v') }}
|
||||
${{ matrix.tag_suffix != '' && format('suffix={0},onlatest=true', matrix.tag_suffix) || '' }}
|
||||
|
||||
- name: Create manifest list and push
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ steps.image.outputs.name }}@sha256:%s ' *)
|
||||
- name: Inspect image
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ steps.image.outputs.name }}:${{ steps.meta.outputs.version }}
|
||||
|
||||
- name: Summary
|
||||
if: steps.check.outputs.skip != 'true'
|
||||
run: |
|
||||
echo "Multi-arch manifest created for ${{ matrix.name }}!"
|
||||
echo "Tags: $(jq -cr '.tags | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")"
|
||||
|
||||
# Push the git tag only after build, gate, and manifest publish all succeed.
|
||||
finalize_release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [compute_version, create_manifest]
|
||||
if: ${{ success() && needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' }}
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.branch }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Create and push git tag
|
||||
run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||
|
||||
NEXT_TAG="${{ needs.compute_version.outputs.new_tag }}"
|
||||
COMMIT_SHA="${{ needs.compute_version.outputs.commit_sha }}"
|
||||
echo "Tagging commit $COMMIT_SHA with $NEXT_TAG"
|
||||
|
||||
git tag -a "$NEXT_TAG" "$COMMIT_SHA" -m "Docker build $NEXT_TAG"
|
||||
echo "Pushing tag $NEXT_TAG to origin"
|
||||
git push origin "$NEXT_TAG"
|
||||
|
||||
- name: Verify tag push
|
||||
run: |
|
||||
echo "Checking if tag ${{ needs.compute_version.outputs.new_tag }} exists remotely..."
|
||||
sleep 5
|
||||
git ls-remote --tags origin | grep "refs/tags/${{ needs.compute_version.outputs.new_tag }}" || (echo "Tag push verification failed!" && exit 1)
|
||||
echo "Tag successfully pushed."
|
||||
|
|
|
|||
|
|
@ -7,6 +7,16 @@
|
|||
# SurfSense version (use "latest" or a specific version like "0.0.14")
|
||||
SURFSENSE_VERSION=latest
|
||||
|
||||
# Image variant: empty = CPU (default), "cuda" = CUDA 12.8, "cuda126" = CUDA 12.6.
|
||||
# GPU acceleration also requires the NVIDIA Container Toolkit on the host and
|
||||
# the GPU overlay in COMPOSE_FILE. Linux/macOS use ":"; Windows uses ";".
|
||||
# Example Linux/macOS: COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||
# Example Windows: COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml
|
||||
# Use "cuda126" for older NVIDIA driver stacks; use "cuda" for newer drivers.
|
||||
SURFSENSE_VARIANT=
|
||||
# COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||
# SURFSENSE_GPU_COUNT=1
|
||||
|
||||
# Deployment environment: dev or production
|
||||
SURFSENSE_ENV=production
|
||||
|
||||
|
|
@ -92,6 +102,10 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
|
|||
# Only change this if you manage publications manually.
|
||||
# ZERO_APP_PUBLICATIONS=zero_publication
|
||||
|
||||
# Keep Zero's documented halt safety net enabled. If replication halts, Zero
|
||||
# can wipe and re-sync its local SQLite replica without touching Postgres.
|
||||
# ZERO_AUTO_RESET=true
|
||||
|
||||
# Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number
|
||||
# of CPU cores, which can exceed the connection pool limits on high-core machines.
|
||||
# Each sync worker needs at least 1 connection from both the UPSTREAM and CVR
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ services:
|
|||
- ZERO_REPLICA_FILE=/data/zero.db
|
||||
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||
- ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true}
|
||||
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
||||
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
||||
|
|
@ -122,11 +123,13 @@ services:
|
|||
volumes:
|
||||
- zero_cache_data:/data
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 300s
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 600s
|
||||
|
||||
# OPTIONAL — Azurite emulates Azure Blob Storage for testing the Azure
|
||||
# original-file backend. The default filesystem backend needs none of this.
|
||||
|
|
|
|||
|
|
@ -46,8 +46,6 @@ services:
|
|||
- PYTHONPATH=/app
|
||||
- SERVICE_ROLE=migrate
|
||||
- MIGRATION_TIMEOUT=${MIGRATION_TIMEOUT:-900}
|
||||
volumes:
|
||||
- zero_init:/zero-init
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -217,21 +215,6 @@ services:
|
|||
celery_worker:
|
||||
condition: service_started
|
||||
|
||||
# flower:
|
||||
# build: *backend-build
|
||||
# ports:
|
||||
# - "${FLOWER_PORT:-5555}:5555"
|
||||
# env_file:
|
||||
# - ../surfsense_backend/.env
|
||||
# environment:
|
||||
# - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0}
|
||||
# - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0}
|
||||
# - PYTHONPATH=/app
|
||||
# command: celery -A app.celery_app flower --port=5555
|
||||
# depends_on:
|
||||
# - redis
|
||||
# - celery_worker
|
||||
|
||||
zero-cache:
|
||||
image: rocicorp/zero:1.4.0
|
||||
ports:
|
||||
|
|
@ -250,6 +233,7 @@ services:
|
|||
- ZERO_REPLICA_FILE=/data/zero.db
|
||||
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||
- ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true}
|
||||
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
||||
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
||||
|
|
@ -257,18 +241,14 @@ services:
|
|||
- ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
||||
volumes:
|
||||
- zero_cache_data:/data
|
||||
- zero_init:/zero-init
|
||||
# Wrapper: see docker/docker-compose.yml `zero-cache` for rationale.
|
||||
entrypoint: ["sh", "-c"]
|
||||
# Pass the script as a single list element so Compose does not tokenize it.
|
||||
command:
|
||||
- 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache'
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 300s
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 600s
|
||||
|
||||
frontend:
|
||||
build:
|
||||
|
|
@ -300,7 +280,5 @@ volumes:
|
|||
name: surfsense-dev-shared-temp
|
||||
zero_cache_data:
|
||||
name: surfsense-dev-zero-cache
|
||||
zero_init:
|
||||
name: surfsense-dev-zero-init
|
||||
whatsapp_sessions:
|
||||
name: surfsense-dev-whatsapp-sessions
|
||||
|
|
|
|||
30
docker/docker-compose.gpu.yml
Normal file
30
docker/docker-compose.gpu.yml
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
services:
|
||||
backend:
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||
capabilities:
|
||||
- gpu
|
||||
|
||||
celery_worker:
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||
capabilities:
|
||||
- gpu
|
||||
|
||||
celery_beat:
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: ${SURFSENSE_GPU_DRIVER:-nvidia}
|
||||
count: ${SURFSENSE_GPU_COUNT:-1}
|
||||
capabilities:
|
||||
- gpu
|
||||
|
|
@ -29,12 +29,11 @@ services:
|
|||
|
||||
# Short-lived schema runner. Executes `alembic upgrade head` and verifies
|
||||
# that the `zero_publication` Postgres logical-replication publication
|
||||
# exists, then exits 0. Downstream services (backend, celery_*, zero-cache)
|
||||
# gate on this with `condition: service_completed_successfully` so a failed
|
||||
# migration halts the whole stack instead of silently producing a half-built
|
||||
# system that crash-loops zero-cache on missing publications.
|
||||
# matches the canonical shape, then exits 0. Downstream services gate on this
|
||||
# with `condition: service_completed_successfully` so a failed migration halts
|
||||
# the whole stack instead of booting zero-cache against a drifted publication.
|
||||
migrations:
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
|
|
@ -42,8 +41,6 @@ services:
|
|||
PYTHONPATH: /app
|
||||
SERVICE_ROLE: migrate
|
||||
MIGRATION_TIMEOUT: ${MIGRATION_TIMEOUT:-900}
|
||||
volumes:
|
||||
- zero_init:/zero-init
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -61,28 +58,28 @@ services:
|
|||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.152.1
|
||||
profiles:
|
||||
- observability
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
volumes:
|
||||
- ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro
|
||||
environment:
|
||||
GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-}
|
||||
GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-}
|
||||
GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-}
|
||||
ports:
|
||||
- "${OTEL_GRPC_PORT:-4317}:4317"
|
||||
- "${OTEL_HTTP_PORT:-4318}:4318"
|
||||
- "${OTEL_HEALTH_PORT:-13133}:13133"
|
||||
mem_limit: 2g
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "/otelcol-contrib", "--version"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
# otel-collector:
|
||||
# image: otel/opentelemetry-collector-contrib:0.152.1
|
||||
# profiles:
|
||||
# - observability
|
||||
# command: ["--config=/etc/otelcol/config.yaml"]
|
||||
# volumes:
|
||||
# - ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro
|
||||
# environment:
|
||||
# GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-}
|
||||
# GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-}
|
||||
# GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-}
|
||||
# ports:
|
||||
# - "${OTEL_GRPC_PORT:-4317}:4317"
|
||||
# - "${OTEL_HTTP_PORT:-4318}:4318"
|
||||
# - "${OTEL_HEALTH_PORT:-13133}:13133"
|
||||
# mem_limit: 2g
|
||||
# restart: unless-stopped
|
||||
# healthcheck:
|
||||
# test: ["CMD", "/otelcol-contrib", "--version"]
|
||||
# interval: 30s
|
||||
# timeout: 5s
|
||||
# retries: 3
|
||||
|
||||
searxng:
|
||||
image: searxng/searxng:2026.3.13-3c1f68c59
|
||||
|
|
@ -98,7 +95,7 @@ services:
|
|||
retries: 5
|
||||
|
||||
backend:
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||
ports:
|
||||
- "${BACKEND_PORT:-8929}:8000"
|
||||
volumes:
|
||||
|
|
@ -144,28 +141,28 @@ services:
|
|||
retries: 30
|
||||
start_period: 200s
|
||||
|
||||
whatsapp-bridge:
|
||||
build: ../surfsense_backend/scripts/whatsapp-bridge
|
||||
profiles:
|
||||
- whatsapp
|
||||
expose:
|
||||
- "9929"
|
||||
volumes:
|
||||
- whatsapp_sessions:/data/sessions
|
||||
environment:
|
||||
PORT: 9929
|
||||
WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat}
|
||||
WHATSAPP_SESSION_DIR: /data/sessions
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
# whatsapp-bridge:
|
||||
# build: ../surfsense_backend/scripts/whatsapp-bridge
|
||||
# profiles:
|
||||
# - whatsapp
|
||||
# expose:
|
||||
# - "9929"
|
||||
# volumes:
|
||||
# - whatsapp_sessions:/data/sessions
|
||||
# environment:
|
||||
# PORT: 9929
|
||||
# WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat}
|
||||
# WHATSAPP_SESSION_DIR: /data/sessions
|
||||
# mem_limit: 512m
|
||||
# restart: unless-stopped
|
||||
# healthcheck:
|
||||
# test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"]
|
||||
# interval: 30s
|
||||
# timeout: 5s
|
||||
# retries: 5
|
||||
|
||||
celery_worker:
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||
volumes:
|
||||
- shared_temp:/shared_tmp
|
||||
env_file:
|
||||
|
|
@ -195,7 +192,7 @@ services:
|
|||
restart: unless-stopped
|
||||
|
||||
celery_beat:
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
||||
image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}}
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
|
|
@ -218,22 +215,6 @@ services:
|
|||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
restart: unless-stopped
|
||||
|
||||
# flower:
|
||||
# image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}
|
||||
# ports:
|
||||
# - "${FLOWER_PORT:-5555}:5555"
|
||||
# env_file:
|
||||
# - .env
|
||||
# environment:
|
||||
# CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0}
|
||||
# CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0}
|
||||
# PYTHONPATH: /app
|
||||
# command: celery -A app.celery_app flower --port=5555
|
||||
# depends_on:
|
||||
# - redis
|
||||
# - celery_worker
|
||||
# restart: unless-stopped
|
||||
|
||||
zero-cache:
|
||||
image: rocicorp/zero:1.4.0
|
||||
ports:
|
||||
|
|
@ -247,6 +228,7 @@ services:
|
|||
ZERO_REPLICA_FILE: /data/zero.db
|
||||
ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||
ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||
ZERO_AUTO_RESET: ${ZERO_AUTO_RESET:-true}
|
||||
ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4}
|
||||
ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||
ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30}
|
||||
|
|
@ -254,16 +236,8 @@ services:
|
|||
ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate}
|
||||
volumes:
|
||||
- zero_cache_data:/data
|
||||
- zero_init:/zero-init
|
||||
# Wrapper: if the migrations service flagged a publication change via
|
||||
# /zero-init/needs_reset, wipe the SQLite replica before starting so
|
||||
# zero-cache does a clean initial sync. Recovers from the half-built
|
||||
# replica state (`_zero.tableMetadata` missing) caused by earlier crashes.
|
||||
entrypoint: ["sh", "-c"]
|
||||
# Pass the script as a single list element so Compose does not tokenize it.
|
||||
command:
|
||||
- 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache'
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 300s
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -274,6 +248,7 @@ services:
|
|||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 600s
|
||||
|
||||
frontend:
|
||||
image: ghcr.io/modsetter/surfsense-web:${SURFSENSE_VERSION:-latest}
|
||||
|
|
@ -305,7 +280,5 @@ volumes:
|
|||
name: surfsense-shared-temp
|
||||
zero_cache_data:
|
||||
name: surfsense-zero-cache
|
||||
zero_init:
|
||||
name: surfsense-zero-init
|
||||
whatsapp_sessions:
|
||||
name: surfsense-whatsapp-sessions
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
# To pass flags, save and run locally:
|
||||
# .\install.ps1 -NoWatchtower
|
||||
# .\install.ps1 -WatchtowerInterval 3600
|
||||
# .\install.ps1 -Variant cuda
|
||||
# .\install.ps1 -Variant cuda -GpuCount all
|
||||
#
|
||||
# Handles two cases automatically:
|
||||
# 1. Fresh install — no prior SurfSense data detected
|
||||
|
|
@ -17,7 +19,11 @@
|
|||
|
||||
param(
|
||||
[switch]$NoWatchtower,
|
||||
[int]$WatchtowerInterval = 86400
|
||||
[int]$WatchtowerInterval = 86400,
|
||||
[ValidateSet("cpu", "cuda", "cuda126")]
|
||||
[string]$Variant,
|
||||
[string]$GpuCount,
|
||||
[switch]$Quiet
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
|
@ -34,6 +40,11 @@ $MigrationMode = $false
|
|||
$SetupWatchtower = -not $NoWatchtower
|
||||
$WatchtowerContainer = "watchtower"
|
||||
|
||||
if ($GpuCount -and $GpuCount -notmatch '^([0-9]+|all)$') {
|
||||
Write-Host "[SurfSense] ERROR: Invalid -GpuCount '$GpuCount'. Use a number or 'all'." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Output helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
function Write-Info { param([string]$Msg) Write-Host "[SurfSense] " -ForegroundColor Cyan -NoNewline; Write-Host $Msg }
|
||||
|
|
@ -42,6 +53,27 @@ function Write-Warn { param([string]$Msg) Write-Host "[SurfSense] " -Foregrou
|
|||
function Write-Step { param([string]$Msg) Write-Host "`n-- $Msg" -ForegroundColor Cyan }
|
||||
function Write-Err { param([string]$Msg) Write-Host "[SurfSense] ERROR: $Msg" -ForegroundColor Red; exit 1 }
|
||||
|
||||
function Show-Banner {
|
||||
Write-Host ""
|
||||
Write-Host @"
|
||||
|
||||
|
||||
███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗
|
||||
██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝
|
||||
███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗
|
||||
╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝
|
||||
███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗
|
||||
╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝
|
||||
|
||||
|
||||
"@ -ForegroundColor White
|
||||
Write-Host " OSS Alternative to NotebookLM for Teams" -ForegroundColor Yellow
|
||||
Write-Host ("=" * 62) -ForegroundColor Cyan
|
||||
Write-Info "This installer will create $InstallDir\ and start SurfSense with Docker Compose."
|
||||
}
|
||||
|
||||
Show-Banner
|
||||
|
||||
function Invoke-NativeSafe {
|
||||
param([scriptblock]$Command)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
|
|
@ -53,6 +85,28 @@ function Invoke-NativeSafe {
|
|||
}
|
||||
}
|
||||
|
||||
function Resolve-WatchtowerPreference {
|
||||
if ($NoWatchtower -or $Quiet -or -not [Environment]::UserInteractive) {
|
||||
return
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Automatic updates" -ForegroundColor Cyan
|
||||
$choice = Read-Host "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]"
|
||||
|
||||
switch ($choice) {
|
||||
"" { $script:SetupWatchtower = $true }
|
||||
{ $_ -match '^(?i)y(es)?$' } { $script:SetupWatchtower = $true }
|
||||
{ $_ -match '^(?i)n(o)?$' } { $script:SetupWatchtower = $false }
|
||||
default {
|
||||
Write-Warn "Unrecognized choice '$choice'; enabling Watchtower by default. Use -NoWatchtower to skip it."
|
||||
$script:SetupWatchtower = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Resolve-WatchtowerPreference
|
||||
|
||||
# ── Pre-flight checks ──────────────────────────────────────────────────────
|
||||
|
||||
Write-Step "Checking prerequisites"
|
||||
|
|
@ -97,143 +151,11 @@ function Wait-ForPostgres {
|
|||
Write-Ok "PostgreSQL is ready."
|
||||
}
|
||||
|
||||
# ── Stack health helpers ────────────────────────────────────────────────────
|
||||
|
||||
function Get-ComposeServices {
|
||||
Push-Location $InstallDir
|
||||
try {
|
||||
$raw = Invoke-NativeSafe { docker compose ps -a --format json 2>$null }
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($raw)) { return @() }
|
||||
|
||||
# Compose v2.21+ emits a JSON array; older versions emit one object per line.
|
||||
try {
|
||||
$parsed = $raw | ConvertFrom-Json
|
||||
if ($parsed -is [System.Collections.IEnumerable] -and -not ($parsed -is [string])) {
|
||||
return @($parsed)
|
||||
}
|
||||
return @($parsed)
|
||||
} catch {
|
||||
$services = @()
|
||||
foreach ($line in ($raw -split "`r?`n")) {
|
||||
$line = $line.Trim()
|
||||
if (-not $line) { continue }
|
||||
try { $services += ($line | ConvertFrom-Json) } catch { }
|
||||
}
|
||||
return $services
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-StackHealthy {
|
||||
param([int]$TimeoutSec = 300)
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
||||
$lastReport = ""
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$services = Get-ComposeServices
|
||||
if (-not $services -or $services.Count -eq 0) {
|
||||
Start-Sleep -Seconds 3
|
||||
continue
|
||||
}
|
||||
|
||||
$bad = @()
|
||||
$waiting = @()
|
||||
$good = @()
|
||||
|
||||
foreach ($svc in $services) {
|
||||
$name = $svc.Service
|
||||
$state = $svc.State
|
||||
$health = if ($svc.PSObject.Properties.Name -contains 'Health') { $svc.Health } else { '' }
|
||||
$exit = if ($svc.PSObject.Properties.Name -contains 'ExitCode') { $svc.ExitCode } else { $null }
|
||||
|
||||
if ($name -eq 'migrations') {
|
||||
if ($state -eq 'exited' -and $exit -eq 0) { $good += $name }
|
||||
elseif ($state -eq 'exited') { $bad += "${name} (exit=${exit})" }
|
||||
else { $waiting += "${name} (${state})" }
|
||||
continue
|
||||
}
|
||||
|
||||
if ($state -eq 'running') {
|
||||
if ([string]::IsNullOrEmpty($health) -or $health -eq 'healthy') {
|
||||
$good += $name
|
||||
} elseif ($health -eq 'starting') {
|
||||
$waiting += "${name} (starting)"
|
||||
} elseif ($health -eq 'unhealthy') {
|
||||
$bad += "${name} (unhealthy)"
|
||||
} else {
|
||||
$waiting += "${name} (${health})"
|
||||
}
|
||||
} elseif ($state -eq 'restarting') {
|
||||
$bad += "${name} (restarting)"
|
||||
} elseif ($state -eq 'exited') {
|
||||
$bad += "${name} (exited, code=${exit})"
|
||||
} else {
|
||||
$waiting += "${name} (${state})"
|
||||
}
|
||||
}
|
||||
|
||||
if ($bad.Count -gt 0) {
|
||||
return @{ Ok = $false; Reason = 'failure'; Bad = $bad; Waiting = $waiting; Good = $good }
|
||||
}
|
||||
if ($waiting.Count -eq 0) {
|
||||
return @{ Ok = $true; Reason = 'all_healthy'; Good = $good }
|
||||
}
|
||||
|
||||
$report = "Waiting on: " + ($waiting -join ', ')
|
||||
if ($report -ne $lastReport) {
|
||||
Write-Info $report
|
||||
$lastReport = $report
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
||||
return @{ Ok = $false; Reason = 'timeout'; Bad = $bad; Waiting = $waiting; Good = $good }
|
||||
}
|
||||
|
||||
function Test-StaleZeroCacheVolume {
|
||||
$raw = Invoke-NativeSafe { docker volume ls --format '{{.Name}}' 2>$null }
|
||||
if ([string]::IsNullOrWhiteSpace($raw)) { return $false }
|
||||
$names = $raw -split "`r?`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ }
|
||||
$hasZeroCache = $names -contains 'surfsense-zero-cache'
|
||||
$hasZeroInit = $names -contains 'surfsense-zero-init'
|
||||
# Pre-fix installs created surfsense-zero-cache but never surfsense-zero-init.
|
||||
# Such a volume may hold a half-initialized SQLite replica from an earlier
|
||||
# crash-loop. Wiping it forces zero-cache to do a fresh initial sync.
|
||||
return ($hasZeroCache -and -not $hasZeroInit)
|
||||
}
|
||||
|
||||
function Invoke-StaleZeroCacheCleanup {
|
||||
if (-not (Test-StaleZeroCacheVolume)) { return }
|
||||
|
||||
Write-Warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that"
|
||||
Write-Warn "predates the migrations-service fix. It may contain a half-initialized"
|
||||
Write-Warn "SQLite replica that would block zero-cache from starting."
|
||||
Write-Warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel."
|
||||
Start-Sleep -Seconds 5
|
||||
|
||||
Push-Location $InstallDir
|
||||
Invoke-NativeSafe { docker compose down --remove-orphans 2>$null } | Out-Null
|
||||
Pop-Location
|
||||
Invoke-NativeSafe { docker volume rm surfsense-zero-cache 2>$null } | Out-Null
|
||||
Write-Ok "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start."
|
||||
}
|
||||
|
||||
function Write-Err-NoExit {
|
||||
param([string]$Message)
|
||||
Write-Host "[ERROR] $Message" -ForegroundColor Red
|
||||
}
|
||||
# ── Stack startup helper ────────────────────────────────────────────────────
|
||||
|
||||
function Invoke-StackFailureReport {
|
||||
param([hashtable]$Result)
|
||||
|
||||
Write-Host ""
|
||||
Write-Err-NoExit "Stack did not reach a healthy state."
|
||||
if ($Result.Bad.Count -gt 0) { Write-Host (" Failed: " + ($Result.Bad -join ', ')) }
|
||||
if ($Result.Waiting.Count -gt 0) { Write-Host (" Stuck: " + ($Result.Waiting -join ', ')) }
|
||||
|
||||
Write-Host "[ERROR] Stack did not reach a healthy state." -ForegroundColor Red
|
||||
Write-Host ""
|
||||
Write-Info "Recent logs from migrations / zero-cache / backend:"
|
||||
Push-Location $InstallDir
|
||||
|
|
@ -247,11 +169,151 @@ function Invoke-StackFailureReport {
|
|||
Write-Host "Recovery hints:" -ForegroundColor Yellow
|
||||
Write-Host " 1. Inspect migrations: cd $InstallDir; docker compose logs migrations"
|
||||
Write-Host " 2. Verify publication: cd $InstallDir; docker compose exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
||||
Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d"
|
||||
Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d --wait"
|
||||
Write-Host ""
|
||||
exit 1
|
||||
}
|
||||
|
||||
function Invoke-ComposeUpWait {
|
||||
Push-Location $InstallDir
|
||||
try {
|
||||
Invoke-NativeSafe { docker compose up -d --wait }
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Invoke-StackFailureReport
|
||||
}
|
||||
}
|
||||
|
||||
# ── Variant and .env helpers ────────────────────────────────────────────────
|
||||
|
||||
function Set-EnvValue {
|
||||
param([string]$Path, [string]$Key, [string]$Value)
|
||||
$lines = @()
|
||||
if (Test-Path $Path) {
|
||||
$lines = @(Get-Content $Path)
|
||||
}
|
||||
$updated = $false
|
||||
$newLines = foreach ($line in $lines) {
|
||||
if ($line -match "^$([regex]::Escape($Key))=") {
|
||||
$updated = $true
|
||||
"$Key=$Value"
|
||||
} else {
|
||||
$line
|
||||
}
|
||||
}
|
||||
if (-not $updated) {
|
||||
$newLines += "$Key=$Value"
|
||||
}
|
||||
Set-Content -Path $Path -Value $newLines
|
||||
}
|
||||
|
||||
function Remove-EnvValue {
|
||||
param([string]$Path, [string]$Key)
|
||||
if (-not (Test-Path $Path)) { return }
|
||||
$newLines = Get-Content $Path | Where-Object { $_ -notmatch "^$([regex]::Escape($Key))=" }
|
||||
Set-Content -Path $Path -Value $newLines
|
||||
}
|
||||
|
||||
function Test-NvidiaGpu {
|
||||
if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) { return $false }
|
||||
Invoke-NativeSafe { nvidia-smi *>$null } | Out-Null
|
||||
return ($LASTEXITCODE -eq 0)
|
||||
}
|
||||
|
||||
function Test-NvidiaRuntime {
|
||||
$info = Invoke-NativeSafe { docker info 2>$null }
|
||||
if ($info -match 'nvidia') { return $true }
|
||||
if (Get-Command nvidia-ctk -ErrorAction SilentlyContinue) { return $true }
|
||||
if (Get-Command nvidia-container-runtime -ErrorAction SilentlyContinue) { return $true }
|
||||
return $false
|
||||
}
|
||||
|
||||
function Get-RecommendedVariant {
|
||||
$driver = (Invoke-NativeSafe { nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>$null } | Select-Object -First 1)
|
||||
$major = 0
|
||||
if ($driver -match '^(\d+)') {
|
||||
$major = [int]$Matches[1]
|
||||
}
|
||||
if ($major -gt 0 -and $major -lt 570) {
|
||||
return "cuda126"
|
||||
}
|
||||
return "cuda"
|
||||
}
|
||||
|
||||
function Resolve-Variant {
|
||||
$hasGpu = Test-NvidiaGpu
|
||||
$hasRuntime = $false
|
||||
$recommended = "cpu"
|
||||
|
||||
if ($hasGpu) {
|
||||
$recommended = Get-RecommendedVariant
|
||||
$hasRuntime = Test-NvidiaRuntime
|
||||
}
|
||||
|
||||
if ($Variant) {
|
||||
if ($Variant -eq "cpu") { return "cpu" }
|
||||
if (-not $hasGpu) {
|
||||
Write-Warn "No NVIDIA GPU detected; falling back to CPU variant."
|
||||
return "cpu"
|
||||
}
|
||||
if (-not $hasRuntime) {
|
||||
Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant."
|
||||
Write-Warn "Install the toolkit before enabling SurfSense GPU acceleration."
|
||||
return "cpu"
|
||||
}
|
||||
return $Variant
|
||||
}
|
||||
|
||||
if ($hasGpu -and -not $hasRuntime) {
|
||||
Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant."
|
||||
}
|
||||
|
||||
if ($hasGpu -and $hasRuntime -and -not $Quiet -and [Environment]::UserInteractive) {
|
||||
Write-Host ""
|
||||
Write-Host "SurfSense detected an NVIDIA GPU." -ForegroundColor Cyan
|
||||
$choice = Read-Host "Use GPU acceleration? [Y/n]"
|
||||
switch ($choice) {
|
||||
"" { return $recommended }
|
||||
{ $_ -match '^(?i)y(es)?$' } { return $recommended }
|
||||
{ $_ -match '^(?i)n(o)?$' } { return "cpu" }
|
||||
default {
|
||||
Write-Warn "Unrecognized choice '$choice'; using CPU variant."
|
||||
return "cpu"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "cpu"
|
||||
}
|
||||
|
||||
function Set-VariantEnv {
|
||||
param([string]$Path, [string]$SelectedVariant, [bool]$AllowExistingUpdate)
|
||||
|
||||
if ((Test-Path $Path) -and -not $AllowExistingUpdate) {
|
||||
Write-Warn ".env already exists - keeping your existing configuration."
|
||||
Write-Info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in $Path, then run docker compose up -d --wait."
|
||||
return
|
||||
}
|
||||
|
||||
if ($SelectedVariant -eq "cpu") {
|
||||
Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value ""
|
||||
Remove-EnvValue -Path $Path -Key "COMPOSE_FILE"
|
||||
Remove-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT"
|
||||
} else {
|
||||
Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value $SelectedVariant
|
||||
Set-EnvValue -Path $Path -Key "COMPOSE_FILE" -Value "docker-compose.yml;docker-compose.gpu.yml"
|
||||
if ($GpuCount) {
|
||||
Set-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT" -Value $GpuCount
|
||||
}
|
||||
}
|
||||
|
||||
Remove-EnvValue -Path $Path -Key "COMPOSE_PROFILES"
|
||||
}
|
||||
|
||||
$SelectedVariant = Resolve-Variant
|
||||
|
||||
# ── Download files ──────────────────────────────────────────────────────────
|
||||
|
||||
Write-Step "Downloading SurfSense files"
|
||||
|
|
@ -262,6 +324,7 @@ New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null
|
|||
|
||||
$Files = @(
|
||||
@{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" }
|
||||
@{ Src = "docker/docker-compose.gpu.yml"; Dest = "docker-compose.gpu.yml" }
|
||||
@{ Src = "docker/.env.example"; Dest = ".env.example" }
|
||||
@{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" }
|
||||
@{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" }
|
||||
|
|
@ -339,15 +402,19 @@ if (-not (Test-Path $envPath)) {
|
|||
$content = $content -replace 'SECRET_KEY=replace_me_with_a_random_string', "SECRET_KEY=$SecretKey"
|
||||
Set-Content -Path $envPath -Value $content -NoNewline
|
||||
|
||||
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false
|
||||
Write-Info "Created $envPath"
|
||||
} else {
|
||||
Write-Warn ".env already exists - keeping your existing configuration."
|
||||
if ($PSBoundParameters.ContainsKey('Variant')) {
|
||||
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $true
|
||||
Write-Info "Updated SurfSense image variant in existing $envPath"
|
||||
} else {
|
||||
Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false
|
||||
}
|
||||
}
|
||||
|
||||
# ── Start containers ────────────────────────────────────────────────────────
|
||||
|
||||
Invoke-StaleZeroCacheCleanup
|
||||
|
||||
if ($MigrationMode) {
|
||||
$envContent = Get-Content $envPath
|
||||
$DbUser = ($envContent | Select-String '^DB_USER=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||
|
|
@ -405,31 +472,15 @@ if ($MigrationMode) {
|
|||
}
|
||||
|
||||
Write-Step "Starting all SurfSense services"
|
||||
Push-Location $InstallDir
|
||||
Invoke-NativeSafe { docker compose up -d }
|
||||
Pop-Location
|
||||
Write-Ok "All containers started; waiting for stack to become healthy..."
|
||||
|
||||
$waitResult = Wait-StackHealthy -TimeoutSec 300
|
||||
if (-not $waitResult.Ok) {
|
||||
Invoke-StackFailureReport -Result $waitResult
|
||||
}
|
||||
Write-Ok "All services healthy."
|
||||
Invoke-ComposeUpWait
|
||||
Write-Ok "All services started and healthy."
|
||||
|
||||
Remove-Item $KeyFile -ErrorAction SilentlyContinue
|
||||
|
||||
} else {
|
||||
Write-Step "Starting SurfSense"
|
||||
Push-Location $InstallDir
|
||||
Invoke-NativeSafe { docker compose up -d }
|
||||
Pop-Location
|
||||
Write-Ok "All containers started; waiting for stack to become healthy..."
|
||||
|
||||
$waitResult = Wait-StackHealthy -TimeoutSec 300
|
||||
if (-not $waitResult.Ok) {
|
||||
Invoke-StackFailureReport -Result $waitResult
|
||||
}
|
||||
Write-Ok "All services healthy."
|
||||
Invoke-ComposeUpWait
|
||||
Write-Ok "All services started and healthy."
|
||||
}
|
||||
|
||||
# ── Watchtower (auto-update) ────────────────────────────────────────────────
|
||||
|
|
@ -461,7 +512,7 @@ if ($SetupWatchtower) {
|
|||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Ok "Watchtower started - labeled SurfSense containers will auto-update."
|
||||
} else {
|
||||
Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d"
|
||||
Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d --wait"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -471,39 +522,26 @@ if ($SetupWatchtower) {
|
|||
# ── Done ────────────────────────────────────────────────────────────────────
|
||||
|
||||
Write-Host ""
|
||||
Write-Host @"
|
||||
|
||||
|
||||
.d8888b. .d888 .d8888b.
|
||||
d88P Y88b d88P" d88P Y88b
|
||||
Y88b. 888 Y88b.
|
||||
"Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b.
|
||||
"Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b
|
||||
"888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888
|
||||
Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b.
|
||||
"Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888
|
||||
|
||||
|
||||
"@ -ForegroundColor White
|
||||
|
||||
$versionDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VERSION=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||
if (-not $versionDisplay) { $versionDisplay = "latest" }
|
||||
Write-Host " OSS Alternative to NotebookLM for Teams [$versionDisplay]" -ForegroundColor Yellow
|
||||
Write-Host ("=" * 62) -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
$variantDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VARIANT=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1
|
||||
if (-not $variantDisplay) { $variantDisplay = "cpu" }
|
||||
$wtHours = [math]::Floor($WatchtowerInterval / 3600)
|
||||
Write-Step "SurfSense is now installed [$versionDisplay]"
|
||||
|
||||
Write-Info " Frontend: http://localhost:3929"
|
||||
Write-Info " Backend: http://localhost:8929"
|
||||
Write-Info " API Docs: http://localhost:8929/docs"
|
||||
Write-Info ""
|
||||
Write-Info " Config: $InstallDir\.env"
|
||||
Write-Info " Variant: $variantDisplay"
|
||||
Write-Info " Logs: cd $InstallDir; docker compose logs -f"
|
||||
Write-Info " Stop: cd $InstallDir; docker compose down"
|
||||
Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d"
|
||||
Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d --wait"
|
||||
Write-Info ""
|
||||
|
||||
if ($SetupWatchtower) {
|
||||
Write-Info " Watchtower: auto-updates every ${wtHours}h (stop: docker rm -f $WatchtowerContainer)"
|
||||
Write-Info " Watchtower: auto-updates every ${wtHours}h (disable: docker rm -f $WatchtowerContainer)"
|
||||
} else {
|
||||
Write-Warn " Watchtower skipped. For auto-updates, re-run without -NoWatchtower."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@
|
|||
# Flags:
|
||||
# --no-watchtower Skip automatic Watchtower setup
|
||||
# --watchtower-interval=SECS Check interval in seconds (default: 86400 = 24h)
|
||||
# --variant=cpu|cuda|cuda126 Select backend image variant
|
||||
# --gpu Alias for --variant=cuda
|
||||
# --cpu Alias for --variant=cpu
|
||||
# --gpu-count=N|all Number of GPUs to reserve when GPU is enabled
|
||||
# --quiet Skip interactive prompts
|
||||
#
|
||||
# Handles two cases automatically:
|
||||
# 1. Fresh install — no prior SurfSense data detected
|
||||
|
|
@ -35,12 +40,22 @@ MIGRATION_MODE=false
|
|||
SETUP_WATCHTOWER=true
|
||||
WATCHTOWER_INTERVAL=86400
|
||||
WATCHTOWER_CONTAINER="watchtower"
|
||||
WATCHTOWER_EXPLICIT=false
|
||||
REQUESTED_VARIANT=""
|
||||
VARIANT_EXPLICIT=false
|
||||
GPU_COUNT=""
|
||||
QUIET=false
|
||||
|
||||
# ── Parse flags ─────────────────────────────────────────────────────────────
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--no-watchtower) SETUP_WATCHTOWER=false ;;
|
||||
--no-watchtower) SETUP_WATCHTOWER=false; WATCHTOWER_EXPLICIT=true ;;
|
||||
--watchtower-interval=*) WATCHTOWER_INTERVAL="${arg#*=}" ;;
|
||||
--variant=*) REQUESTED_VARIANT="${arg#*=}"; VARIANT_EXPLICIT=true ;;
|
||||
--gpu) REQUESTED_VARIANT="cuda"; VARIANT_EXPLICIT=true ;;
|
||||
--cpu) REQUESTED_VARIANT="cpu"; VARIANT_EXPLICIT=true ;;
|
||||
--gpu-count=*) GPU_COUNT="${arg#*=}" ;;
|
||||
--quiet) QUIET=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
|
@ -57,6 +72,57 @@ warn() { printf "${YELLOW}[SurfSense]${NC} %s\n" "$1"; }
|
|||
error() { printf "${RED}[SurfSense]${NC} ERROR: %s\n" "$1" >&2; exit 1; }
|
||||
step() { printf "\n${BOLD}${CYAN}── %s${NC}\n" "$1"; }
|
||||
|
||||
show_banner() {
|
||||
echo ""
|
||||
printf '\033[1;37m'
|
||||
cat << 'EOF'
|
||||
|
||||
|
||||
███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗
|
||||
██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝
|
||||
███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗
|
||||
╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝
|
||||
███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗
|
||||
╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝
|
||||
|
||||
|
||||
EOF
|
||||
printf "${YELLOW} OSS Alternative to NotebookLM for Teams${NC}\n"
|
||||
printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n"
|
||||
info "This installer will create ${INSTALL_DIR}/ and start SurfSense with Docker Compose."
|
||||
}
|
||||
|
||||
show_banner
|
||||
|
||||
case "${REQUESTED_VARIANT}" in
|
||||
""|cpu|cuda|cuda126) ;;
|
||||
*) error "Invalid --variant='${REQUESTED_VARIANT}'. Use cpu, cuda, or cuda126." ;;
|
||||
esac
|
||||
|
||||
if [[ -n "${GPU_COUNT}" && ! "${GPU_COUNT}" =~ ^([0-9]+|all)$ ]]; then
|
||||
error "Invalid --gpu-count='${GPU_COUNT}'. Use a number or 'all'."
|
||||
fi
|
||||
|
||||
resolve_watchtower_preference() {
|
||||
if $WATCHTOWER_EXPLICIT || $QUIET || [[ ! -r /dev/tty || ! -w /dev/tty ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local choice
|
||||
echo "" > /dev/tty
|
||||
printf "${BOLD}${CYAN}Automatic updates${NC}\n" > /dev/tty
|
||||
printf "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]: " > /dev/tty
|
||||
read -r choice < /dev/tty || choice=""
|
||||
|
||||
case "$choice" in
|
||||
""|[Yy]|[Yy][Ee][Ss]) SETUP_WATCHTOWER=true ;;
|
||||
[Nn]|[Nn][Oo]) SETUP_WATCHTOWER=false ;;
|
||||
*) warn "Unrecognized choice '${choice}', enabling Watchtower by default. Use --no-watchtower to skip it." >&2; SETUP_WATCHTOWER=true ;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_watchtower_preference
|
||||
|
||||
# ── Pre-flight checks ────────────────────────────────────────────────────────
|
||||
|
||||
step "Checking prerequisites"
|
||||
|
|
@ -97,126 +163,11 @@ wait_for_pg() {
|
|||
success "PostgreSQL is ready."
|
||||
}
|
||||
|
||||
# ── Stack health helpers ─────────────────────────────────────────────────────
|
||||
|
||||
# Enumerate compose services for project `surfsense` as `service|state|health|exitcode`
|
||||
# lines. Uses `docker inspect` so we don't depend on `jq`, `python3`, or the
|
||||
# exact ordering of fields in `docker compose ps --format json` output.
|
||||
get_compose_services() {
|
||||
local containers
|
||||
containers=$(docker ps -a --filter "label=com.docker.compose.project=surfsense" --format '{{.Names}}' 2>/dev/null) || true
|
||||
[[ -z "$containers" ]] && return 0
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local svc state health code
|
||||
svc=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$container" 2>/dev/null || echo "")
|
||||
state=$(docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null || echo "unknown")
|
||||
health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$container" 2>/dev/null || echo "")
|
||||
code=$(docker inspect -f '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "")
|
||||
[[ -z "$svc" ]] && continue
|
||||
printf '%s|%s|%s|%s\n' "$svc" "$state" "$health" "$code"
|
||||
done <<< "$containers"
|
||||
}
|
||||
|
||||
# Globals populated by wait_stack_healthy / consumed by stack_failure_report.
|
||||
STACK_BAD=()
|
||||
STACK_WAITING=()
|
||||
STACK_GOOD=()
|
||||
STACK_TIMEOUT=false
|
||||
|
||||
wait_stack_healthy() {
|
||||
local timeout_sec=${1:-300}
|
||||
local deadline=$(($(date +%s) + timeout_sec))
|
||||
local last_report=""
|
||||
local bad=()
|
||||
local waiting=()
|
||||
local good=()
|
||||
|
||||
while [[ $(date +%s) -lt $deadline ]]; do
|
||||
local lines
|
||||
lines=$(get_compose_services)
|
||||
if [[ -z "$lines" ]]; then
|
||||
sleep 3
|
||||
continue
|
||||
fi
|
||||
|
||||
bad=()
|
||||
waiting=()
|
||||
good=()
|
||||
|
||||
while IFS='|' read -r name state health code; do
|
||||
[[ -z "$name" ]] && continue
|
||||
if [[ "$name" == "migrations" ]]; then
|
||||
if [[ "$state" == "exited" && "$code" == "0" ]]; then
|
||||
good+=("$name")
|
||||
elif [[ "$state" == "exited" ]]; then
|
||||
bad+=("${name} (exit=${code})")
|
||||
else
|
||||
waiting+=("${name} (${state})")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$state" == "running" ]]; then
|
||||
if [[ -z "$health" || "$health" == "healthy" ]]; then
|
||||
good+=("$name")
|
||||
elif [[ "$health" == "starting" ]]; then
|
||||
waiting+=("${name} (starting)")
|
||||
elif [[ "$health" == "unhealthy" ]]; then
|
||||
bad+=("${name} (unhealthy)")
|
||||
else
|
||||
waiting+=("${name} (${health})")
|
||||
fi
|
||||
elif [[ "$state" == "restarting" ]]; then
|
||||
bad+=("${name} (restarting)")
|
||||
elif [[ "$state" == "exited" ]]; then
|
||||
bad+=("${name} (exited, code=${code})")
|
||||
else
|
||||
waiting+=("${name} (${state})")
|
||||
fi
|
||||
done <<< "$lines"
|
||||
|
||||
if (( ${#bad[@]} > 0 )); then
|
||||
STACK_BAD=("${bad[@]}")
|
||||
STACK_WAITING=("${waiting[@]}")
|
||||
STACK_GOOD=("${good[@]}")
|
||||
return 1
|
||||
fi
|
||||
if (( ${#waiting[@]} == 0 )); then
|
||||
STACK_GOOD=("${good[@]}")
|
||||
return 0
|
||||
fi
|
||||
|
||||
local report="Waiting on: ${waiting[*]}"
|
||||
if [[ "$report" != "$last_report" ]]; then
|
||||
info "$report"
|
||||
last_report="$report"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# bad/waiting/good are declared at function scope so referencing them is
|
||||
# safe even if the polling loop never executed its body.
|
||||
STACK_BAD=()
|
||||
[[ ${#bad[@]} -gt 0 ]] && STACK_BAD=("${bad[@]}")
|
||||
STACK_WAITING=()
|
||||
[[ ${#waiting[@]} -gt 0 ]] && STACK_WAITING=("${waiting[@]}")
|
||||
STACK_GOOD=()
|
||||
[[ ${#good[@]} -gt 0 ]] && STACK_GOOD=("${good[@]}")
|
||||
STACK_TIMEOUT=true
|
||||
return 1
|
||||
}
|
||||
# ── Stack startup helper ─────────────────────────────────────────────────────
|
||||
|
||||
stack_failure_report() {
|
||||
echo ""
|
||||
echo -e "\033[31m[ERROR]\033[0m Stack did not reach a healthy state."
|
||||
if (( ${#STACK_BAD[@]} > 0 )) && [[ -n "${STACK_BAD[0]}" ]]; then
|
||||
echo " Failed: ${STACK_BAD[*]}"
|
||||
fi
|
||||
if (( ${#STACK_WAITING[@]} > 0 )) && [[ -n "${STACK_WAITING[0]}" ]]; then
|
||||
echo " Stuck: ${STACK_WAITING[*]}"
|
||||
fi
|
||||
echo ""
|
||||
info "Recent logs from migrations / zero-cache / backend:"
|
||||
(cd "${INSTALL_DIR}" && ${DC} logs --tail=60 migrations zero-cache backend 2>&1) || true
|
||||
|
|
@ -224,36 +175,158 @@ stack_failure_report() {
|
|||
echo "Recovery hints:"
|
||||
echo " 1. Inspect migrations: cd ${INSTALL_DIR} && ${DC} logs migrations"
|
||||
echo " 2. Verify publication: cd ${INSTALL_DIR} && ${DC} exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'"
|
||||
echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d"
|
||||
echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d --wait"
|
||||
echo ""
|
||||
exit 1
|
||||
}
|
||||
|
||||
# True if `surfsense-zero-cache` exists but `surfsense-zero-init` does not.
|
||||
# That signals an install that predates the migrations-service fix; the old
|
||||
# replica may be half-initialized and would block zero-cache on next start.
|
||||
test_stale_zero_cache_volume() {
|
||||
local has_zc has_zi
|
||||
has_zc=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-cache' || true)
|
||||
has_zi=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-init' || true)
|
||||
[[ -n "$has_zc" && -z "$has_zi" ]]
|
||||
compose_up_wait() {
|
||||
local service="${1:-}"
|
||||
if [[ -n "$service" ]]; then
|
||||
(cd "${INSTALL_DIR}" && ${DC} up -d --wait "$service") < /dev/null
|
||||
else
|
||||
(cd "${INSTALL_DIR}" && ${DC} up -d --wait) < /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
invoke_stale_zero_cache_cleanup() {
|
||||
if ! test_stale_zero_cache_volume; then
|
||||
# ── Variant and .env helpers ─────────────────────────────────────────────────
|
||||
|
||||
set_env_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local value="$3"
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
|
||||
if grep -q "^${key}=" "$file" 2>/dev/null; then
|
||||
awk -v key="$key" -v value="$value" 'BEGIN { prefix = key "=" } $0 ~ "^" prefix { print prefix value; next } { print }' "$file" > "$tmp"
|
||||
else
|
||||
cp "$file" "$tmp"
|
||||
printf '\n%s=%s\n' "$key" "$value" >> "$tmp"
|
||||
fi
|
||||
mv "$tmp" "$file"
|
||||
}
|
||||
|
||||
remove_env_value() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
local tmp
|
||||
tmp=$(mktemp)
|
||||
awk -v key="$key" 'BEGIN { prefix = key "=" } $0 !~ "^" prefix { print }' "$file" > "$tmp"
|
||||
mv "$tmp" "$file"
|
||||
}
|
||||
|
||||
version_major() {
|
||||
printf '%s' "$1" | cut -d. -f1
|
||||
}
|
||||
|
||||
recommend_cuda_variant() {
|
||||
local driver_version driver_major
|
||||
driver_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n 1 | tr -d '[:space:]' || true)
|
||||
driver_major=$(version_major "$driver_version")
|
||||
|
||||
# CUDA 12.8 generally requires an R570+ driver. Use CUDA 12.6 as the
|
||||
# compatibility fallback for older 12.x driver stacks and GPUs.
|
||||
if [[ "$driver_major" =~ ^[0-9]+$ && "$driver_major" -lt 570 ]]; then
|
||||
printf 'cuda126'
|
||||
else
|
||||
printf 'cuda'
|
||||
fi
|
||||
}
|
||||
|
||||
gpu_runtime_available() {
|
||||
docker info 2>/dev/null | grep -qi 'nvidia' \
|
||||
|| command -v nvidia-ctk >/dev/null 2>&1 \
|
||||
|| command -v nvidia-container-runtime >/dev/null 2>&1
|
||||
}
|
||||
|
||||
host_has_nvidia_gpu() {
|
||||
command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1
|
||||
}
|
||||
|
||||
resolve_variant() {
|
||||
local detected_variant="cpu"
|
||||
local has_gpu=false
|
||||
local has_runtime=false
|
||||
|
||||
if host_has_nvidia_gpu; then
|
||||
has_gpu=true
|
||||
detected_variant=$(recommend_cuda_variant)
|
||||
if gpu_runtime_available; then
|
||||
has_runtime=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if $VARIANT_EXPLICIT; then
|
||||
if [[ "$REQUESTED_VARIANT" == "cpu" ]]; then
|
||||
printf 'cpu'
|
||||
return 0
|
||||
fi
|
||||
if ! $has_gpu; then
|
||||
warn "No NVIDIA GPU detected; falling back to CPU variant." >&2
|
||||
printf 'cpu'
|
||||
return 0
|
||||
fi
|
||||
if ! $has_runtime; then
|
||||
warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant." >&2
|
||||
warn "Install the toolkit before enabling SurfSense GPU acceleration." >&2
|
||||
printf 'cpu'
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "$REQUESTED_VARIANT"
|
||||
return 0
|
||||
fi
|
||||
warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that"
|
||||
warn "predates the migrations-service fix. It may contain a half-initialized"
|
||||
warn "SQLite replica that would block zero-cache from starting."
|
||||
warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel."
|
||||
sleep 5
|
||||
|
||||
(cd "${INSTALL_DIR}" && ${DC} down --remove-orphans 2>/dev/null) || true
|
||||
docker volume rm surfsense-zero-cache 2>/dev/null || true
|
||||
success "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start."
|
||||
if $has_gpu && ! $has_runtime; then
|
||||
warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant." >&2
|
||||
fi
|
||||
|
||||
if $has_gpu && $has_runtime && ! $QUIET && [[ -r /dev/tty && -w /dev/tty ]]; then
|
||||
local choice
|
||||
echo "" > /dev/tty
|
||||
printf "${BOLD}${CYAN}SurfSense detected an NVIDIA GPU.${NC}\n" > /dev/tty
|
||||
printf "Use GPU acceleration? [Y/n]: " > /dev/tty
|
||||
read -r choice < /dev/tty || choice=""
|
||||
case "$choice" in
|
||||
"") printf '%s' "$detected_variant" ;;
|
||||
[Yy]|[Yy][Ee][Ss]) printf '%s' "$detected_variant" ;;
|
||||
[Nn]|[Nn][Oo]) printf 'cpu' ;;
|
||||
*) warn "Unrecognized choice '${choice}', using CPU variant." >&2; printf 'cpu' ;;
|
||||
esac
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf 'cpu'
|
||||
}
|
||||
|
||||
apply_variant_env() {
|
||||
local env_file="$1"
|
||||
local variant="$2"
|
||||
local allow_existing_update="$3"
|
||||
|
||||
if [[ -f "$env_file" && "$allow_existing_update" != "true" ]]; then
|
||||
warn ".env already exists — keeping your existing configuration."
|
||||
info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in ${env_file}, then run ${DC} up -d --wait."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$variant" == "cpu" ]]; then
|
||||
set_env_value "$env_file" "SURFSENSE_VARIANT" ""
|
||||
remove_env_value "$env_file" "COMPOSE_FILE"
|
||||
remove_env_value "$env_file" "SURFSENSE_GPU_COUNT"
|
||||
else
|
||||
set_env_value "$env_file" "SURFSENSE_VARIANT" "$variant"
|
||||
set_env_value "$env_file" "COMPOSE_FILE" "docker-compose.yml:docker-compose.gpu.yml"
|
||||
if [[ -n "$GPU_COUNT" ]]; then
|
||||
set_env_value "$env_file" "SURFSENSE_GPU_COUNT" "$GPU_COUNT"
|
||||
fi
|
||||
fi
|
||||
|
||||
remove_env_value "$env_file" "COMPOSE_PROFILES"
|
||||
}
|
||||
|
||||
SELECTED_VARIANT=$(resolve_variant)
|
||||
|
||||
# ── Download files ───────────────────────────────────────────────────────────
|
||||
|
||||
step "Downloading SurfSense files"
|
||||
|
|
@ -263,6 +336,7 @@ mkdir -p "${INSTALL_DIR}/searxng"
|
|||
|
||||
FILES=(
|
||||
"docker/docker-compose.yml:docker-compose.yml"
|
||||
"docker/docker-compose.gpu.yml:docker-compose.gpu.yml"
|
||||
"docker/.env.example:.env.example"
|
||||
"docker/postgresql.conf:postgresql.conf"
|
||||
"docker/scripts/migrate-database.sh:scripts/migrate-database.sh"
|
||||
|
|
@ -336,15 +410,19 @@ if [ ! -f "${INSTALL_DIR}/.env" ]; then
|
|||
else
|
||||
sed -i "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env"
|
||||
fi
|
||||
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false"
|
||||
info "Created ${INSTALL_DIR}/.env"
|
||||
else
|
||||
warn ".env already exists — keeping your existing configuration."
|
||||
if $VARIANT_EXPLICIT; then
|
||||
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "true"
|
||||
info "Updated SurfSense image variant in existing ${INSTALL_DIR}/.env"
|
||||
else
|
||||
apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Start containers ─────────────────────────────────────────────────────────
|
||||
|
||||
invoke_stale_zero_cache_cleanup
|
||||
|
||||
if $MIGRATION_MODE; then
|
||||
# Read DB credentials from .env (fall back to defaults from docker-compose.yml)
|
||||
DB_USER=$(grep '^DB_USER=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||
|
|
@ -401,26 +479,20 @@ if $MIGRATION_MODE; then
|
|||
fi
|
||||
|
||||
step "Starting all SurfSense services"
|
||||
(cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
|
||||
success "All containers started; waiting for stack to become healthy..."
|
||||
|
||||
if ! wait_stack_healthy 300; then
|
||||
if ! compose_up_wait; then
|
||||
stack_failure_report
|
||||
fi
|
||||
success "All services healthy."
|
||||
success "All services started and healthy."
|
||||
|
||||
# Key file is no longer needed — SECRET_KEY is now in .env
|
||||
rm -f "${KEY_FILE}"
|
||||
|
||||
else
|
||||
step "Starting SurfSense"
|
||||
(cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null
|
||||
success "All containers started; waiting for stack to become healthy..."
|
||||
|
||||
if ! wait_stack_healthy 300; then
|
||||
if ! compose_up_wait; then
|
||||
stack_failure_report
|
||||
fi
|
||||
success "All services healthy."
|
||||
success "All services started and healthy."
|
||||
fi
|
||||
|
||||
# ── Watchtower (auto-update) ─────────────────────────────────────────────────
|
||||
|
|
@ -445,7 +517,7 @@ if $SETUP_WATCHTOWER; then
|
|||
--label-enable \
|
||||
--interval "${WATCHTOWER_INTERVAL}" >/dev/null 2>&1 < /dev/null \
|
||||
&& success "Watchtower started — labeled SurfSense containers will auto-update." \
|
||||
|| warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d"
|
||||
|| warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d --wait"
|
||||
fi
|
||||
else
|
||||
info "Skipping Watchtower setup (--no-watchtower flag)."
|
||||
|
|
@ -454,38 +526,25 @@ fi
|
|||
# ── Done ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
printf '\033[1;37m'
|
||||
cat << 'EOF'
|
||||
|
||||
|
||||
.d8888b. .d888 .d8888b.
|
||||
d88P Y88b d88P" d88P Y88b
|
||||
Y88b. 888 Y88b.
|
||||
"Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b.
|
||||
"Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b
|
||||
"888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888
|
||||
Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b.
|
||||
"Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888
|
||||
|
||||
|
||||
EOF
|
||||
_version_display=$(grep '^SURFSENSE_VERSION=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||
_version_display="${_version_display:-latest}"
|
||||
printf " OSS Alternative to NotebookLM for Teams ${YELLOW}[%s]${NC}\n" "${_version_display}"
|
||||
printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n\n"
|
||||
_variant_display=$(grep '^SURFSENSE_VARIANT=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true)
|
||||
_variant_display="${_variant_display:-cpu}"
|
||||
step "SurfSense is now installed [${_version_display}]"
|
||||
|
||||
info " Frontend: http://localhost:3929"
|
||||
info " Backend: http://localhost:8929"
|
||||
info " API Docs: http://localhost:8929/docs"
|
||||
info ""
|
||||
info " Config: ${INSTALL_DIR}/.env"
|
||||
info " Variant: ${_variant_display}"
|
||||
info " Logs: cd ${INSTALL_DIR} && ${DC} logs -f"
|
||||
info " Stop: cd ${INSTALL_DIR} && ${DC} down"
|
||||
info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d"
|
||||
info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d --wait"
|
||||
info ""
|
||||
|
||||
if $SETUP_WATCHTOWER; then
|
||||
info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (stop: docker rm -f ${WATCHTOWER_CONTAINER})"
|
||||
info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (disable: docker rm -f ${WATCHTOWER_CONTAINER})"
|
||||
else
|
||||
warn " Watchtower skipped. For auto-updates, re-run without --no-watchtower."
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# syntax=docker.io/docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# SurfSense Backend — Multi-stage Dockerfile
|
||||
# =============================================================================
|
||||
|
|
@ -61,15 +62,25 @@ COPY pyproject.toml uv.lock ./
|
|||
# Exporting the lock to requirements.txt and feeding it to `uv pip install`
|
||||
# pins every transitive package to the exact version captured in uv.lock.
|
||||
#
|
||||
# Note on torch/CUDA: we do NOT install torch from a separate cu* index here.
|
||||
# PyPI's torch wheels for Linux x86_64 already ship CUDA-enabled and pull
|
||||
# nvidia-cudnn-cu13, nvidia-nccl-cu13, triton, etc. as install deps (all
|
||||
# captured in uv.lock). If a specific CUDA version is needed, wire it through
|
||||
# [tool.uv.sources] in pyproject.toml so the lock stays the source of truth.
|
||||
# Note on torch/CUDA: the export must always select either the cpu or CUDA
|
||||
# extra declared in pyproject.toml. A no-extra export would resolve torch from
|
||||
# PyPI on Linux, which currently pulls CUDA-enabled wheels and nvidia-* deps.
|
||||
# Keep CUDA version selection in [tool.uv.sources] so uv.lock remains the
|
||||
# source of truth. The install step also needs the matching PyTorch index,
|
||||
# because requirements.txt preserves the +cpu/+cu wheel pins but not uv's
|
||||
# package source metadata.
|
||||
ARG USE_CUDA=false
|
||||
ARG CUDA_EXTRA=cu128
|
||||
RUN pip install --no-cache-dir uv && \
|
||||
if [ "$USE_CUDA" = "true" ]; then EXTRA="$CUDA_EXTRA"; else EXTRA="cpu"; fi && \
|
||||
TORCH_INDEX="https://download.pytorch.org/whl/${EXTRA}" && \
|
||||
uv export --frozen --no-dev --no-hashes --no-emit-project \
|
||||
--extra "$EXTRA" \
|
||||
--format requirements-txt -o /tmp/requirements.txt && \
|
||||
uv pip install --system --no-cache-dir -r /tmp/requirements.txt && \
|
||||
uv pip install --system --no-cache-dir \
|
||||
--index "$TORCH_INDEX" \
|
||||
--index-strategy unsafe-best-match \
|
||||
-r /tmp/requirements.txt && \
|
||||
rm /tmp/requirements.txt
|
||||
|
||||
|
||||
|
|
@ -94,7 +105,9 @@ RUN printf '%s\n' \
|
|||
| python || true
|
||||
|
||||
ARG EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
|
||||
RUN python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')"
|
||||
RUN --mount=type=secret,id=HF_TOKEN \
|
||||
HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
|
||||
python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')"
|
||||
|
||||
# Install Playwright browsers (the playwright python package itself is in deps)
|
||||
RUN playwright install chromium --with-deps
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import os
|
|||
import sys
|
||||
from logging.config import fileConfig
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
|
@ -36,6 +37,9 @@ if config.config_file_name is not None:
|
|||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = Base.metadata
|
||||
|
||||
MIGRATION_ADVISORY_LOCK_NAMESPACE = "surfsense"
|
||||
MIGRATION_ADVISORY_LOCK_NAME = "alembic_migrations"
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
|
|
@ -73,8 +77,22 @@ def do_run_migrations(connection: Connection) -> None:
|
|||
transaction_per_migration=True,
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
lock_params = {
|
||||
"namespace": MIGRATION_ADVISORY_LOCK_NAMESPACE,
|
||||
"name": MIGRATION_ADVISORY_LOCK_NAME,
|
||||
}
|
||||
connection.execute(
|
||||
sa.text("SELECT pg_advisory_lock(hashtext(:namespace), hashtext(:name))"),
|
||||
lock_params,
|
||||
)
|
||||
try:
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
finally:
|
||||
connection.execute(
|
||||
sa.text("SELECT pg_advisory_unlock(hashtext(:namespace), hashtext(:name))"),
|
||||
lock_params,
|
||||
)
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
|
|
|
|||
|
|
@ -47,7 +47,6 @@ depends_on: str | Sequence[str] | None = None
|
|||
|
||||
PUBLICATION_NAME = "zero_publication"
|
||||
|
||||
# Must stay in sync with the column lists in migrations 117 / 139 / 140.
|
||||
DOCUMENT_COLS = [
|
||||
"id",
|
||||
"title",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
"""reconcile zero_publication from canonical definition
|
||||
|
||||
Revision ID: 155
|
||||
Revises: 154
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
from app.zero_publication import apply_publication
|
||||
|
||||
revision: str = "155"
|
||||
down_revision: str | None = "154"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
apply_publication(op.get_bind())
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""No-op. Historical publication shapes are immutable."""
|
||||
229
surfsense_backend/app/zero_publication.py
Normal file
229
surfsense_backend/app/zero_publication.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
"""Canonical Zero publication definition for SurfSense.
|
||||
|
||||
This module is the single source of truth for ``zero_publication``. Future
|
||||
publication changes should update ``ZERO_PUBLICATION`` and call
|
||||
``apply_publication()`` from a migration instead of hand-copying table lists.
|
||||
|
||||
SurfSense runs Zero on Postgres with Zero's event triggers installed, so the
|
||||
official Zero path is a plain ``ALTER PUBLICATION ... SET TABLE``. If a future
|
||||
deployment cannot use event triggers, use Zero's documented
|
||||
``zero_0.update_schemas()`` hook as the fallback instead of COMMENT bookends.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
PUBLICATION_NAME = "zero_publication"
|
||||
|
||||
DOCUMENT_COLS = [
|
||||
"id",
|
||||
"title",
|
||||
"document_type",
|
||||
"search_space_id",
|
||||
"folder_id",
|
||||
"created_by_id",
|
||||
"status",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
|
||||
USER_COLS = [
|
||||
"id",
|
||||
"pages_limit",
|
||||
"pages_used",
|
||||
"premium_credit_micros_limit",
|
||||
"premium_credit_micros_used",
|
||||
]
|
||||
|
||||
AUTOMATION_RUN_COLS = [
|
||||
"id",
|
||||
"automation_id",
|
||||
"trigger_id",
|
||||
"status",
|
||||
"step_results",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"created_at",
|
||||
]
|
||||
|
||||
ZERO_PUBLICATION: Mapping[str, Sequence[str] | None] = {
|
||||
"notifications": None,
|
||||
"documents": DOCUMENT_COLS,
|
||||
"folders": None,
|
||||
"search_source_connectors": None,
|
||||
"new_chat_messages": None,
|
||||
"chat_comments": None,
|
||||
"chat_session_state": None,
|
||||
"user": USER_COLS,
|
||||
"automation_runs": AUTOMATION_RUN_COLS,
|
||||
}
|
||||
|
||||
|
||||
def _quote_identifier(identifier: str) -> str:
|
||||
return '"' + identifier.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _column_exists(conn: Connection, table: str, column: str) -> bool:
|
||||
return (
|
||||
conn.execute(
|
||||
text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_schema = current_schema() "
|
||||
"AND table_name = :table AND column_name = :column"
|
||||
),
|
||||
{"table": table, "column": column},
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _expected_columns(conn: Connection, table: str) -> list[str] | None:
|
||||
columns = ZERO_PUBLICATION[table]
|
||||
if columns is None:
|
||||
return None
|
||||
|
||||
expected = list(columns)
|
||||
if table in {"documents", "user"} and _column_exists(conn, table, "_0_version"):
|
||||
expected.append("_0_version")
|
||||
return expected
|
||||
|
||||
|
||||
def _format_table_entry(conn: Connection, table: str) -> str:
|
||||
columns = _expected_columns(conn, table)
|
||||
table_sql = _quote_identifier(table)
|
||||
if columns is None:
|
||||
return table_sql
|
||||
|
||||
column_sql = ", ".join(_quote_identifier(column) for column in columns)
|
||||
return f"{table_sql} ({column_sql})"
|
||||
|
||||
|
||||
def build_set_table_sql(conn: Connection) -> str:
|
||||
"""Build the canonical plain SET TABLE statement for Zero's event triggers."""
|
||||
|
||||
table_list = ", ".join(_format_table_entry(conn, table) for table in ZERO_PUBLICATION)
|
||||
return f"ALTER PUBLICATION {_quote_identifier(PUBLICATION_NAME)} SET TABLE {table_list}"
|
||||
|
||||
|
||||
def apply_publication(conn: Connection) -> None:
|
||||
"""Reconcile ``zero_publication`` to the canonical shape."""
|
||||
|
||||
exists = conn.execute(
|
||||
text("SELECT 1 FROM pg_publication WHERE pubname = :name"),
|
||||
{"name": PUBLICATION_NAME},
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return
|
||||
|
||||
conn.execute(text(build_set_table_sql(conn)))
|
||||
|
||||
|
||||
def _actual_publication_shape(conn: Connection) -> dict[str, list[str] | None]:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"SELECT pt.tablename, pr.prattrs IS NULL AS all_columns, pt.attnames "
|
||||
"FROM pg_publication_tables pt "
|
||||
"JOIN pg_publication p ON p.pubname = pt.pubname "
|
||||
"JOIN pg_class c ON c.relname = pt.tablename "
|
||||
"JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = pt.schemaname "
|
||||
"JOIN pg_publication_rel pr ON pr.prpubid = p.oid AND pr.prrelid = c.oid "
|
||||
"WHERE pt.pubname = :name AND pt.schemaname = current_schema() "
|
||||
"ORDER BY pt.tablename"
|
||||
),
|
||||
{"name": PUBLICATION_NAME},
|
||||
).mappings()
|
||||
|
||||
return {
|
||||
str(row["tablename"]): None
|
||||
if row["all_columns"]
|
||||
else list(row["attnames"] or [])
|
||||
for row in rows
|
||||
}
|
||||
|
||||
|
||||
def expected_publication_shape(conn: Connection) -> dict[str, list[str] | None]:
|
||||
return {table: _expected_columns(conn, table) for table in ZERO_PUBLICATION}
|
||||
|
||||
|
||||
def verify_publication(conn: Connection) -> list[str]:
|
||||
"""Return human-readable mismatches between Postgres and the canonical shape."""
|
||||
|
||||
publication_exists = conn.execute(
|
||||
text("SELECT 1 FROM pg_publication WHERE pubname = :name"),
|
||||
{"name": PUBLICATION_NAME},
|
||||
).fetchone()
|
||||
if not publication_exists:
|
||||
return [f"Publication {PUBLICATION_NAME!r} does not exist"]
|
||||
|
||||
actual = _actual_publication_shape(conn)
|
||||
expected = expected_publication_shape(conn)
|
||||
mismatches: list[str] = []
|
||||
|
||||
for table, expected_columns in expected.items():
|
||||
if table not in actual:
|
||||
mismatches.append(f"{table}: missing from publication")
|
||||
continue
|
||||
|
||||
actual_columns = actual[table]
|
||||
actual_key = sorted(actual_columns) if actual_columns is not None else None
|
||||
expected_key = sorted(expected_columns) if expected_columns is not None else None
|
||||
if actual_key != expected_key:
|
||||
mismatches.append(
|
||||
f"{table}: expected columns {expected_columns or 'ALL'}, "
|
||||
f"got {actual_columns or 'ALL'}"
|
||||
)
|
||||
|
||||
for table in sorted(set(actual) - set(expected)):
|
||||
mismatches.append(f"{table}: unexpected table in publication")
|
||||
|
||||
return mismatches
|
||||
|
||||
|
||||
async def _verify_cli() -> int:
|
||||
database_url = os.getenv("DATABASE_URL")
|
||||
if not database_url:
|
||||
print("DATABASE_URL is required to verify zero_publication.", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
engine = create_async_engine(database_url)
|
||||
async with engine.connect() as async_conn:
|
||||
def run_verify(sync_conn: Connection) -> list[str]:
|
||||
return verify_publication(sync_conn)
|
||||
|
||||
mismatches = await async_conn.run_sync(run_verify)
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
if mismatches:
|
||||
print("zero_publication shape mismatch:", file=sys.stderr)
|
||||
for mismatch in mismatches:
|
||||
print(f" - {mismatch}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("zero_publication shape verified.")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Manage SurfSense's Zero publication")
|
||||
parser.add_argument("--verify", action="store_true", help="verify zero_publication shape")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verify:
|
||||
return asyncio.run(_verify_cli())
|
||||
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -92,6 +92,11 @@ dependencies = [
|
|||
"croniter>=2.0.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
cpu = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||
cu126 = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||
cu128 = ["torch==2.11.0", "torchvision==0.26.0"]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ruff>=0.12.5",
|
||||
|
|
@ -101,6 +106,36 @@ dev = [
|
|||
"httpx>=0.28.1",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
conflicts = [[{ extra = "cpu" }, { extra = "cu126" }, { extra = "cu128" }]]
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [
|
||||
{ index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" },
|
||||
{ index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" },
|
||||
{ index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
torchvision = [
|
||||
{ index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" },
|
||||
{ index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" },
|
||||
{ index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu126"
|
||||
url = "https://download.pytorch.org/whl/cu126"
|
||||
explicit = true
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu128"
|
||||
url = "https://download.pytorch.org/whl/cu128"
|
||||
explicit = true
|
||||
|
||||
[tool.ruff]
|
||||
# Exclude a variety of commonly ignored directories.
|
||||
exclude = [
|
||||
|
|
|
|||
|
|
@ -49,10 +49,10 @@ trap cleanup SIGTERM SIGINT
|
|||
# ── Database migrations (only for migrate / all) ─────────────
|
||||
# Fail-fast contract:
|
||||
# - alembic upgrade head must succeed within ${MIGRATION_TIMEOUT:-900}s
|
||||
# - zero_publication must exist in pg_publication afterwards
|
||||
# - zero_publication must match the canonical app.zero_publication shape
|
||||
# Either failure exits non-zero so the dedicated `migrations` compose
|
||||
# service exits non-zero, halting the rest of the stack instead of
|
||||
# silently producing a half-built system that crash-loops zero-cache.
|
||||
# silently producing a drifted Zero publication.
|
||||
run_migrations() {
|
||||
echo "Running database migrations..."
|
||||
for i in {1..30}; do
|
||||
|
|
@ -73,58 +73,13 @@ run_migrations() {
|
|||
fi
|
||||
echo "Migrations completed successfully."
|
||||
|
||||
echo "Verifying zero_publication exists in Postgres..."
|
||||
local pub_oid
|
||||
pub_oid=$(python <<'PY' 2>/dev/null || true
|
||||
import asyncio
|
||||
import sys
|
||||
from sqlalchemy import text
|
||||
from app.db import engine
|
||||
|
||||
|
||||
async def get_oid():
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(
|
||||
text("SELECT oid FROM pg_publication WHERE pubname = 'zero_publication'")
|
||||
)
|
||||
row = result.first()
|
||||
if row is None:
|
||||
sys.exit(1)
|
||||
print(int(row[0]))
|
||||
|
||||
|
||||
asyncio.run(get_oid())
|
||||
PY
|
||||
)
|
||||
if [ -z "${pub_oid}" ]; then
|
||||
echo "ERROR: zero_publication is missing from Postgres after running alembic." >&2
|
||||
echo "This usually means migration 116 (or a later publication migration) did not run." >&2
|
||||
echo "Verifying zero_publication matches the canonical shape..."
|
||||
if ! python -m app.zero_publication --verify; then
|
||||
echo "ERROR: zero_publication does not match the canonical shape." >&2
|
||||
echo "Inspect alembic state with:" >&2
|
||||
echo " docker compose exec db psql -U \"\$DB_USER\" -d \"\$DB_NAME\" -c 'SELECT * FROM alembic_version;'" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "zero_publication verified (oid=${pub_oid})."
|
||||
|
||||
# Stale-replica safety net: if /zero-init is mounted (i.e. we are the
|
||||
# dedicated `migrations` compose service), drop a marker file when the
|
||||
# publication oid changed (or on first run) so the wrapped zero-cache
|
||||
# entrypoint can wipe /data/zero.db before starting. This recovers from
|
||||
# the case where a previous zero-cache crashed mid-init and left a
|
||||
# half-built SQLite replica without a `_zero.tableMetadata` table.
|
||||
if [ -d /zero-init ]; then
|
||||
local stored_oid=""
|
||||
[ -f /zero-init/last_pub_oid ] && stored_oid=$(cat /zero-init/last_pub_oid 2>/dev/null || true)
|
||||
if [ -z "${stored_oid}" ] || [ "${stored_oid}" != "${pub_oid}" ]; then
|
||||
echo "Publication oid changed (stored=${stored_oid:-<none>}, current=${pub_oid}); writing /zero-init/needs_reset."
|
||||
: > /zero-init/needs_reset
|
||||
chmod 666 /zero-init/needs_reset 2>/dev/null || true
|
||||
fi
|
||||
echo "${pub_oid}" > /zero-init/last_pub_oid
|
||||
chmod 666 /zero-init/last_pub_oid 2>/dev/null || true
|
||||
# World-writable dir so the (possibly non-root) zero-cache container
|
||||
# can `rm -f /zero-init/needs_reset` after acting on the marker.
|
||||
chmod 777 /zero-init 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Service starters ─────────────────────────────────────────
|
||||
|
|
|
|||
1279
surfsense_backend/uv.lock
generated
1279
surfsense_backend/uv.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,12 +1,16 @@
|
|||
import { mustGetQuery } from "@rocicorp/zero";
|
||||
import { handleQueryRequest } from "@rocicorp/zero/server";
|
||||
import { NextResponse } from "next/server";
|
||||
import { BACKEND_URL } from "@/lib/env-config";
|
||||
import type { Context } from "@/types/zero";
|
||||
import { queries } from "@/zero/queries";
|
||||
import { schema } from "@/zero/schema";
|
||||
|
||||
const backendURL = BACKEND_URL;
|
||||
function getBackendBaseUrl() {
|
||||
const base = process.env.FASTAPI_BACKEND_INTERNAL_URL || "http://localhost:8000";
|
||||
return base.endsWith("/") ? base.slice(0, -1) : base;
|
||||
}
|
||||
|
||||
const backendURL = getBackendBaseUrl();
|
||||
|
||||
async function authenticateRequest(
|
||||
request: Request
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com
|
|||
import { useFolderSync } from "@/hooks/use-folder-sync";
|
||||
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
||||
import { useElectronAPI } from "@/hooks/use-platform";
|
||||
import { isLlmOnboardingComplete } from "@/lib/onboarding";
|
||||
|
||||
export function DashboardClientLayout({
|
||||
children,
|
||||
|
|
@ -47,9 +48,8 @@ export function DashboardClientLayout({
|
|||
const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
|
||||
|
||||
const isOnboardingComplete = useCallback(() => {
|
||||
// Check that the Agent LLM ID is set, including 0 for Auto mode.
|
||||
return preferences.agent_llm_id !== null && preferences.agent_llm_id !== undefined;
|
||||
}, [preferences.agent_llm_id]);
|
||||
return isLlmOnboardingComplete(preferences.agent_llm_id, globalConfigs.length > 0);
|
||||
}, [preferences.agent_llm_id, globalConfigs.length]);
|
||||
|
||||
const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom);
|
||||
const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import { Button } from "@/components/ui/button";
|
|||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
|
||||
import { getBearerToken, redirectToLogin } from "@/lib/auth-utils";
|
||||
import { isLlmOnboardingComplete } from "@/lib/onboarding";
|
||||
|
||||
export default function OnboardPage() {
|
||||
const router = useRouter();
|
||||
|
|
@ -52,15 +53,16 @@ export default function OnboardPage() {
|
|||
}
|
||||
}, []);
|
||||
|
||||
// Check if onboarding is already complete (including 0 for Auto mode)
|
||||
const isOnboardingComplete =
|
||||
preferences.agent_llm_id !== null && preferences.agent_llm_id !== undefined;
|
||||
const isOnboardingComplete = isLlmOnboardingComplete(
|
||||
preferences.agent_llm_id,
|
||||
globalConfigs.length > 0
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!preferencesLoading && isOnboardingComplete) {
|
||||
if (!preferencesLoading && globalConfigsLoaded && isOnboardingComplete) {
|
||||
router.push(`/dashboard/${searchSpaceId}/new-chat`);
|
||||
}
|
||||
}, [preferencesLoading, isOnboardingComplete, router, searchSpaceId]);
|
||||
}, [preferencesLoading, globalConfigsLoaded, isOnboardingComplete, router, searchSpaceId]);
|
||||
|
||||
useEffect(() => {
|
||||
const autoConfigureWithGlobal = async () => {
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y
|
|||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` |
|
||||
| `SURFSENSE_VARIANT` | Backend image variant. Leave empty for CPU, set `cuda` for CUDA 12.8, or `cuda126` for CUDA 12.6. | *(empty)* |
|
||||
| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` |
|
||||
| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` |
|
||||
| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` |
|
||||
|
|
@ -42,6 +43,62 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y
|
|||
| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` |
|
||||
| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` |
|
||||
|
||||
### Image Variants
|
||||
|
||||
SurfSense publishes CPU and CUDA backend image variants. The frontend image is not variant-specific.
|
||||
|
||||
| Backend tag | Use case | `SURFSENSE_VARIANT` |
|
||||
|-------------|----------|---------------------|
|
||||
| `:latest` | CPU-only default | *(empty)* |
|
||||
| `:latest-cuda` | NVIDIA CUDA 12.8 backend image | `cuda` |
|
||||
| `:latest-cuda126` | NVIDIA CUDA 12.6 backend image for older driver stacks | `cuda126` |
|
||||
|
||||
All backend variants are published for `linux/amd64` and `linux/arm64`. CUDA on `linux/arm64` is best-effort.
|
||||
|
||||
<Callout type="info">
|
||||
GPU acceleration needs two settings: `SURFSENSE_VARIANT` selects the CUDA image, and `COMPOSE_FILE` enables the GPU device overlay. The host must have the NVIDIA Container Toolkit installed.
|
||||
</Callout>
|
||||
|
||||
### NVIDIA GPU Acceleration
|
||||
|
||||
For most NVIDIA systems, add these values to `.env` to use the CUDA 12.8 image:
|
||||
|
||||
```dotenv
|
||||
SURFSENSE_VARIANT=cuda
|
||||
COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml
|
||||
SURFSENSE_GPU_COUNT=1
|
||||
```
|
||||
|
||||
Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks or older GPUs that need the CUDA 12.6 fallback image.
|
||||
|
||||
On Windows, use `;` instead of `:` in `COMPOSE_FILE` inside `.env`:
|
||||
|
||||
```dotenv
|
||||
COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml
|
||||
```
|
||||
|
||||
To switch variants later, edit `SURFSENSE_VARIANT` and `COMPOSE_FILE` in `.env`, then run:
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
||||
### Automatic Updates
|
||||
|
||||
Manual Docker Compose installs do not start Watchtower automatically. To enable external automatic updates, run Watchtower separately:
|
||||
|
||||
```bash
|
||||
docker run -d --name watchtower \
|
||||
--restart unless-stopped \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
nickfedor/watchtower \
|
||||
--label-enable \
|
||||
--interval 86400
|
||||
```
|
||||
|
||||
SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services.
|
||||
|
||||
### Ports
|
||||
|
||||
| Variable | Description | Default |
|
||||
|
|
@ -270,11 +327,13 @@ Symptom (in `docker compose logs zero-cache`):
|
|||
Error: Unknown or invalid publications. Specified: [zero_publication]. Found: []
|
||||
```
|
||||
|
||||
This means `zero-cache` started before `zero_publication` was created. With
|
||||
the current compose files this should be impossible. The `migrations`
|
||||
service blocks `zero-cache` from starting. If you see it, your stack
|
||||
predates the fix or you brought up `zero-cache` manually with `docker
|
||||
compose up zero-cache` before the migrations service ran.
|
||||
This means `zero-cache` started before `zero_publication` was created or the
|
||||
publication does not match SurfSense's canonical Zero shape. With the current
|
||||
compose files this should be impossible: the `migrations` service blocks
|
||||
`zero-cache` from starting and verifies the publication before exiting
|
||||
successfully. If you see it, your stack predates the fix or you brought up
|
||||
`zero-cache` manually with `docker compose up zero-cache` before the migrations
|
||||
service ran.
|
||||
|
||||
Recovery:
|
||||
|
||||
|
|
@ -284,18 +343,13 @@ docker volume rm surfsense-zero-cache # wipe half-built SQLite replica
|
|||
docker compose up -d # migrations runs first, then zero-cache
|
||||
```
|
||||
|
||||
The install script (`install.ps1` / `install.sh`) detects this case
|
||||
automatically: if it finds a `surfsense-zero-cache` volume from a previous
|
||||
install with no matching `surfsense-zero-init` volume, it removes the stale
|
||||
volume before bringing the stack up.
|
||||
|
||||
### Zero-cache crashes with `_zero.tableMetadata` errors
|
||||
|
||||
This indicates a half-initialized SQLite replica left behind by a previous
|
||||
crash. The `migrations` service writes a marker file on a shared volume
|
||||
(`surfsense-zero-init`) when the publication oid changes; zero-cache wipes
|
||||
its replica and re-syncs on next start. If the marker mechanism somehow did
|
||||
not trigger, run the recovery one-liner above.
|
||||
crash. Zero's own event triggers and `ZERO_AUTO_RESET` handle schema and
|
||||
replication halts automatically. If the local SQLite replica is wedged, run the
|
||||
recovery one-liner above to wipe `surfsense-zero-cache`; zero-cache will
|
||||
re-sync from Postgres on the next start.
|
||||
|
||||
### Ensuring `wal_level = logical`
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ title: One-Line Install Script
|
|||
description: One-command installation of SurfSense using Docker
|
||||
---
|
||||
|
||||
Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates.
|
||||
Downloads the compose files, generates a `SECRET_KEY`, starts all services with `docker compose up -d --wait`, and starts [Watchtower](https://github.com/nicholas-fedor/watchtower) as an external updater for automatic daily updates.
|
||||
|
||||
**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running.
|
||||
|
||||
|
|
@ -19,9 +19,38 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr
|
|||
irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex
|
||||
```
|
||||
|
||||
This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`.
|
||||
This creates a `./surfsense/` directory with `docker-compose.yml`, `docker-compose.gpu.yml`, and `.env`, then runs `docker compose up -d --wait`.
|
||||
|
||||
To skip Watchtower (e.g. in production where you manage updates yourself):
|
||||
If an NVIDIA GPU and NVIDIA Container Toolkit are detected, the installer asks whether to use GPU acceleration and chooses the compatible backend image automatically. Non-interactive installs default to CPU unless you pass an explicit flag.
|
||||
|
||||
Interactive installs also ask whether to enable automatic daily updates with Watchtower, noting that updates may download several GB in the background.
|
||||
|
||||
### GPU options
|
||||
|
||||
Linux/macOS:
|
||||
|
||||
```bash
|
||||
# CUDA 12.8
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda
|
||||
|
||||
# CUDA 12.6 fallback for older driver stacks
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda126
|
||||
|
||||
# Reserve all available GPUs
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --gpu --gpu-count=all
|
||||
```
|
||||
|
||||
PowerShell:
|
||||
|
||||
```powershell
|
||||
# Save the script locally first when passing PowerShell parameters.
|
||||
.\install.ps1 -Variant cuda
|
||||
.\install.ps1 -Variant cuda126 -GpuCount all
|
||||
```
|
||||
|
||||
The installer writes the same `.env` settings you would configure manually: `SURFSENSE_VARIANT` selects the backend image and `COMPOSE_FILE` enables the GPU overlay.
|
||||
|
||||
To skip Watchtower (e.g. in production where you manage updates yourself, or to avoid large background image downloads):
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower
|
||||
|
|
@ -29,6 +58,16 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr
|
|||
|
||||
To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`.
|
||||
|
||||
Manual updates use the same compose state stored in `.env`, so GPU overlays and variants are preserved:
|
||||
|
||||
```bash
|
||||
cd surfsense
|
||||
docker compose pull
|
||||
docker compose up -d --wait
|
||||
```
|
||||
|
||||
If Watchtower is enabled, it preserves the running image variant tag automatically. Because SurfSense images are large, use `--no-watchtower` when you prefer to manage update timing yourself.
|
||||
|
||||
---
|
||||
|
||||
## Access SurfSense
|
||||
|
|
|
|||
8
surfsense_web/lib/onboarding.ts
Normal file
8
surfsense_web/lib/onboarding.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
export function isLlmOnboardingComplete(
|
||||
agentLlmId: number | null | undefined,
|
||||
hasGlobalConfigs: boolean
|
||||
): boolean {
|
||||
if (agentLlmId === null || agentLlmId === undefined) return false;
|
||||
if (agentLlmId === 0) return hasGlobalConfigs;
|
||||
return true;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue