diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 224591d1f..8da56fc33 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -5,6 +5,9 @@ on: branches: - main - dev + tags: + - 'v*' + - 'beta-v*' paths: - 'surfsense_backend/**' - 'surfsense_web/**' @@ -24,11 +27,13 @@ permissions: packages: write jobs: - tag_release: + compute_version: runs-on: ubuntu-latest - if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch' + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/beta-v') outputs: new_tag: ${{ steps.tag_version.outputs.next_version }} + commit_sha: ${{ steps.tag_version.outputs.commit_sha }} + is_release_tag: ${{ steps.tag_version.outputs.is_release_tag }} steps: - name: Checkout code uses: actions/checkout@v6 @@ -37,57 +42,65 @@ jobs: ref: ${{ github.event.inputs.branch }} token: ${{ secrets.GITHUB_TOKEN }} + # Compute-only: tag is pushed by finalize_release after everything succeeds. - name: Read app version and calculate next Docker build version id: tag_version run: | - APP_VERSION=$(tr -d '[:space:]' < VERSION) - echo "App version from VERSION file: $APP_VERSION" + if [[ "$GITHUB_REF" == refs/tags/beta-v* ]]; then + VERSION="${GITHUB_REF#refs/tags/beta-v}" + NEXT_VERSION="beta-${VERSION}" + IS_RELEASE_TAG="true" - if [ -z "$APP_VERSION" ]; then - echo "Error: Could not read version from VERSION file" - exit 1 - fi + if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then + echo "::error::Version '$VERSION' is not valid semver (expected X.Y.Z). Fix your tag name." + exit 1 + fi - git fetch --tags + echo "Docker beta release version from git tag: $NEXT_VERSION" + elif [[ "$GITHUB_REF" == refs/tags/v* ]]; then + NEXT_VERSION="${GITHUB_REF#refs/tags/v}" + IS_RELEASE_TAG="true" - LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1) + if ! echo "$NEXT_VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$'; then + echo "::error::Version '$NEXT_VERSION' is not valid semver (expected X.Y.Z). Fix your tag name." + exit 1 + fi - if [ -z "$LATEST_BUILD_TAG" ]; then - echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1" - NEXT_VERSION="${APP_VERSION}.1" + echo "Docker release version from git tag: $NEXT_VERSION" else - echo "Latest Docker build tag found: $LATEST_BUILD_TAG" - BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev) - NEXT_BUILD=$((BUILD_NUMBER + 1)) - NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}" + APP_VERSION=$(tr -d '[:space:]' < VERSION) + echo "App version from VERSION file: $APP_VERSION" + + if [ -z "$APP_VERSION" ]; then + echo "Error: Could not read version from VERSION file" + exit 1 + fi + + git fetch --tags + + LATEST_BUILD_TAG=$(git tag --list "${APP_VERSION}.*" --sort='-v:refname' | head -n 1) + + if [ -z "$LATEST_BUILD_TAG" ]; then + echo "No previous Docker build tag found for version ${APP_VERSION}. Starting with ${APP_VERSION}.1" + NEXT_VERSION="${APP_VERSION}.1" + else + echo "Latest Docker build tag found: $LATEST_BUILD_TAG" + BUILD_NUMBER=$(echo "$LATEST_BUILD_TAG" | rev | cut -d. -f1 | rev) + NEXT_BUILD=$((BUILD_NUMBER + 1)) + NEXT_VERSION="${APP_VERSION}.${NEXT_BUILD}" + fi + + IS_RELEASE_TAG="false" + echo "Calculated next Docker version: $NEXT_VERSION" fi - echo "Calculated next Docker version: $NEXT_VERSION" echo "next_version=$NEXT_VERSION" >> $GITHUB_OUTPUT - - - name: Create and Push Tag - run: | - git config --global user.name 'github-actions[bot]' - git config --global user.email 'github-actions[bot]@users.noreply.github.com' - - NEXT_TAG="${{ steps.tag_version.outputs.next_version }}" - COMMIT_SHA=$(git rev-parse HEAD) - echo "Tagging commit $COMMIT_SHA with $NEXT_TAG" - - git tag -a "$NEXT_TAG" -m "Docker build $NEXT_TAG" - echo "Pushing tag $NEXT_TAG to origin" - git push origin "$NEXT_TAG" - - - name: Verify Tag Push - run: | - echo "Checking if tag ${{ steps.tag_version.outputs.next_version }} exists remotely..." - sleep 5 - git ls-remote --tags origin | grep "refs/tags/${{ steps.tag_version.outputs.next_version }}" || (echo "Tag push verification failed!" && exit 1) - echo "Tag successfully pushed." + echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "is_release_tag=$IS_RELEASE_TAG" >> $GITHUB_OUTPUT build: - needs: tag_release - if: always() && (needs.tag_release.result == 'success' || needs.tag_release.result == 'skipped') + needs: compute_version + if: always() && (needs.compute_version.result == 'success' || needs.compute_version.result == 'skipped') runs-on: ${{ matrix.os }} permissions: packages: write @@ -97,6 +110,12 @@ jobs: matrix: platform: [linux/amd64, linux/arm64] image: [backend, web] + variant: [cpu, cuda, cuda126] + exclude: + - image: web + variant: cuda + - image: web + variant: cuda126 include: - platform: linux/amd64 suffix: amd64 @@ -114,6 +133,18 @@ jobs: context: ./surfsense_web file: ./surfsense_web/Dockerfile target: runner + - variant: cpu + tag_suffix: "" + use_cuda: "false" + cuda_extra: cpu + - variant: cuda + tag_suffix: "-cuda" + use_cuda: "true" + cuda_extra: cu128 + - variant: cuda126 + tag_suffix: "-cuda126" + use_cuda: "true" + cuda_extra: cu126 env: REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }} @@ -149,7 +180,7 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true docker system prune -af - - name: Build and push by digest ${{ matrix.name }} (${{ matrix.suffix }}) + - name: Build and push by digest ${{ matrix.name }} (${{ matrix.variant }}, ${{ matrix.suffix }}) id: build uses: docker/build-push-action@v7 with: @@ -160,10 +191,14 @@ jobs: tags: ${{ steps.image.outputs.name }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true platforms: ${{ matrix.platform }} - cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.suffix }} - cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.suffix }} + cache-from: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }} + cache-to: type=registry,ref=${{ steps.image.outputs.name }}:buildcache-${{ matrix.variant }}-${{ matrix.suffix }},mode=max,image-manifest=true,oci-mediatypes=true + secrets: | + HF_TOKEN=${{ secrets.HF_TOKEN }} provenance: false build-args: | + ${{ matrix.image == 'backend' && format('USE_CUDA={0}', matrix.use_cuda) || '' }} + ${{ matrix.image == 'backend' && format('CUDA_EXTRA={0}', matrix.cuda_extra) || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__' || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__' || '' }} ${{ matrix.image == 'web' && 'NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__' || '' }} @@ -179,15 +214,47 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v7 with: - name: digests-${{ matrix.image }}-${{ matrix.suffix }} + name: digests-${{ matrix.image }}-${{ matrix.variant }}-${{ matrix.suffix }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 + # Release gate: require both arches for every variant, else block publishing. + # Release-only; skipped on dev so the tolerant create_manifest path is kept. + verify_digests: + runs-on: ubuntu-latest + needs: [compute_version, build] + if: ${{ always() && needs.compute_version.result == 'success' && needs.compute_version.outputs.new_tag != '' }} + steps: + - name: Download all digests + uses: actions/download-artifact@v8 + with: + pattern: digests-* + path: /tmp/digests + merge-multiple: false + + - name: Require both arches for every required variant + run: | + fail=0 + check() { + c=$(find /tmp/digests -type f -path "*/digests-$1-*/*" 2>/dev/null | wc -l | tr -d ' ') + if [ "$c" -lt 2 ]; then + echo "::error::$1 has $c/2 arch digests — blocking release" + fail=1 + else + echo "OK: $1 ($c/2)" + fi + } + check backend-cpu + check backend-cuda + check backend-cuda126 + check web-cpu + [ "$fail" -eq 0 ] || exit 1 + create_manifest: runs-on: ubuntu-latest - needs: [tag_release, build] - if: always() && needs.build.result == 'success' + needs: [compute_version, build, verify_digests] + if: ${{ !cancelled() && needs.verify_digests.result != 'failure' }} permissions: packages: write contents: read @@ -197,8 +264,20 @@ jobs: include: - name: surfsense-backend image: backend + variant: cpu + tag_suffix: "" + - name: surfsense-backend + image: backend + variant: cuda + tag_suffix: "-cuda" + - name: surfsense-backend + image: backend + variant: cuda126 + tag_suffix: "-cuda126" - name: surfsense-web image: web + variant: cpu + tag_suffix: "" env: REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ matrix.name }} @@ -207,22 +286,33 @@ jobs: id: image run: echo "name=${REGISTRY_IMAGE,,}" >> $GITHUB_OUTPUT - - name: Download amd64 digest + - name: Download digests + id: download uses: actions/download-artifact@v8 with: - name: digests-${{ matrix.image }}-amd64 + pattern: digests-${{ matrix.image }}-${{ matrix.variant }}-* path: /tmp/digests + merge-multiple: true + continue-on-error: true - - name: Download arm64 digest - uses: actions/download-artifact@v8 - with: - name: digests-${{ matrix.image }}-arm64 - path: /tmp/digests + - name: Check digests + id: check + run: | + count=$(find /tmp/digests -type f 2>/dev/null | wc -l | tr -d ' ') + echo "digest_count=$count" >> $GITHUB_OUTPUT + if [ "$count" -lt 2 ]; then + echo "::warning::${{ matrix.variant }}: $count/2 digests, skipping merge" + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + fi - name: Set up Docker Buildx + if: steps.check.outputs.skip != 'true' uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry + if: steps.check.outputs.skip != 'true' uses: docker/login-action@v4 with: registry: ghcr.io @@ -230,9 +320,10 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Compute app version + if: steps.check.outputs.skip != 'true' id: appver run: | - VERSION_TAG="${{ needs.tag_release.outputs.new_tag }}" + VERSION_TAG="${{ needs.compute_version.outputs.new_tag }}" if [ -n "$VERSION_TAG" ]; then APP_VERSION=$(echo "$VERSION_TAG" | rev | cut -d. -f2- | rev) else @@ -241,29 +332,69 @@ jobs: echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT - name: Docker meta + if: steps.check.outputs.skip != 'true' id: meta uses: docker/metadata-action@v6 with: images: ${{ steps.image.outputs.name }} tags: | - type=raw,value=${{ needs.tag_release.outputs.new_tag }},enable=${{ needs.tag_release.outputs.new_tag != '' }} - type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.tag_release.outputs.new_tag != '' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }} + type=raw,value=${{ needs.compute_version.outputs.new_tag }},enable=${{ needs.compute_version.outputs.new_tag != '' }} + type=raw,value=${{ steps.appver.outputs.app_version }},enable=${{ needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch) }} type=ref,event=branch type=sha,prefix=git- flavor: | - latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch }} + latest=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/v') }} + ${{ matrix.tag_suffix != '' && format('suffix={0},onlatest=true', matrix.tag_suffix) || '' }} - name: Create manifest list and push + if: steps.check.outputs.skip != 'true' working-directory: /tmp/digests run: | docker buildx imagetools create \ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf '${{ steps.image.outputs.name }}@sha256:%s ' *) - name: Inspect image + if: steps.check.outputs.skip != 'true' run: | docker buildx imagetools inspect ${{ steps.image.outputs.name }}:${{ steps.meta.outputs.version }} - name: Summary + if: steps.check.outputs.skip != 'true' run: | echo "Multi-arch manifest created for ${{ matrix.name }}!" echo "Tags: $(jq -cr '.tags | join(", ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")" + + # Push the git tag only after build, gate, and manifest publish all succeed. + finalize_release: + runs-on: ubuntu-latest + needs: [compute_version, create_manifest] + if: ${{ success() && needs.compute_version.outputs.new_tag != '' && needs.compute_version.outputs.is_release_tag != 'true' }} + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.branch }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push git tag + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + NEXT_TAG="${{ needs.compute_version.outputs.new_tag }}" + COMMIT_SHA="${{ needs.compute_version.outputs.commit_sha }}" + echo "Tagging commit $COMMIT_SHA with $NEXT_TAG" + + git tag -a "$NEXT_TAG" "$COMMIT_SHA" -m "Docker build $NEXT_TAG" + echo "Pushing tag $NEXT_TAG to origin" + git push origin "$NEXT_TAG" + + - name: Verify tag push + run: | + echo "Checking if tag ${{ needs.compute_version.outputs.new_tag }} exists remotely..." + sleep 5 + git ls-remote --tags origin | grep "refs/tags/${{ needs.compute_version.outputs.new_tag }}" || (echo "Tag push verification failed!" && exit 1) + echo "Tag successfully pushed." diff --git a/VERSION b/VERSION index c4475d3bb..24ff85581 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.26 +0.0.27 diff --git a/docker/.env.example b/docker/.env.example index 748f03048..cafc74af9 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -7,6 +7,16 @@ # SurfSense version (use "latest" or a specific version like "0.0.14") SURFSENSE_VERSION=latest +# Image variant: empty = CPU (default), "cuda" = CUDA 12.8, "cuda126" = CUDA 12.6. +# GPU acceleration also requires the NVIDIA Container Toolkit on the host and +# the GPU overlay in COMPOSE_FILE. Linux/macOS use ":"; Windows uses ";". +# Example Linux/macOS: COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml +# Example Windows: COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml +# Use "cuda126" for older NVIDIA driver stacks; use "cuda" for newer drivers. +SURFSENSE_VARIANT= +# COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml +# SURFSENSE_GPU_COUNT=1 + # Deployment environment: dev or production SURFSENSE_ENV=production @@ -55,6 +65,9 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # -- Redis exposed port (dev only; Redis is internal-only in prod) -- # REDIS_PORT=6379 +# -- WhatsApp bridge exposed port (dev/hybrid only; prod keeps it Docker-internal) -- +# WHATSAPP_BRIDGE_PORT=9929 + # -- Frontend Build Args -- # In dev, the frontend is built from source and these are passed as build args. # In prod, they are automatically derived from AUTH_TYPE, ETL_SERVICE, and the port settings above. @@ -67,7 +80,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # ------------------------------------------------------------------------------ # ONLY set these if you are serving SurfSense on a real domain via a reverse # proxy (e.g. Caddy, Nginx, Cloudflare Tunnel). -# For standard localhost deployments, leave all of these commented out — +# For standard localhost deployments, leave all of these commented out. # they are automatically derived from the port settings above. # # NEXT_FRONTEND_URL=https://app.yourdomain.com @@ -89,7 +102,11 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # Only change this if you manage publications manually. # ZERO_APP_PUBLICATIONS=zero_publication -# Sync worker tuning — zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number +# Keep Zero's documented halt safety net enabled. If replication halts, Zero +# can wipe and re-sync its local SQLite replica without touching Postgres. +# ZERO_AUTO_RESET=true + +# Sync worker tuning. zero-cache defaults ZERO_NUM_SYNC_WORKERS to the number # of CPU cores, which can exceed the connection pool limits on high-core machines. # Each sync worker needs at least 1 connection from both the UPSTREAM and CVR # pools, so these constraints must hold: @@ -134,7 +151,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # SSL mode for database connections: disable, require, verify-ca, verify-full # DB_SSLMODE=disable -# Full DATABASE_URL override — when set, takes precedence over the individual +# Full DATABASE_URL override. When set, this takes precedence over the individual # DB_USER / DB_PASSWORD / DB_NAME / DB_HOST / DB_PORT settings above. # Use this for managed databases (AWS RDS, GCP Cloud SQL, Supabase, etc.) # DATABASE_URL=postgresql+asyncpg://user:password@your-rds-host:5432/surfsense?sslmode=require @@ -149,7 +166,7 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # REDIS_URL=redis://redis:6379/0 # ------------------------------------------------------------------------------ -# Stripe (pay-as-you-go page packs — disabled by default) +# Stripe (pay-as-you-go page packs, disabled by default) # ------------------------------------------------------------------------------ # Set TRUE to allow users to buy additional page packs via Stripe Checkout @@ -168,7 +185,6 @@ STRIPE_PAGE_BUYING_ENABLED=FALSE # STRIPE_TOKEN_BUYING_ENABLED=FALSE # STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... # STRIPE_CREDIT_MICROS_PER_UNIT=1000000 -# DEPRECATED — STRIPE_TOKENS_PER_UNIT=1000000 # ------------------------------------------------------------------------------ # TTS & STT (Text-to-Speech / Speech-to-Text) @@ -263,7 +279,44 @@ STT_SERVICE=local/base # COMPOSIO_REDIRECT_URI=http://localhost:8000/api/v1/auth/composio/connector/callback # ------------------------------------------------------------------------------ -# SearXNG (bundled web search — works out of the box, no config needed) +# Messaging Channels (optional) +# ------------------------------------------------------------------------------ +# Configure only the external chat channels you want to use. + +# -- Telegram -- +# TELEGRAM_SHARED_BOT_TOKEN= +# TELEGRAM_SHARED_BOT_USERNAME= +# TELEGRAM_WEBHOOK_SECRET= +# GATEWAY_BASE_URL=http://localhost:8929 +# GATEWAY_TELEGRAM_INTAKE_MODE=webhook + +# -- WhatsApp -- +# GATEWAY_WHATSAPP_INTAKE_MODE=disabled +# WHATSAPP_SHARED_BUSINESS_TOKEN= +# WHATSAPP_SHARED_PHONE_NUMBER_ID= +# WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER= +# WHATSAPP_SHARED_WABA_ID= +# WHATSAPP_GRAPH_API_VERSION=v25.0 +# WHATSAPP_WEBHOOK_VERIFY_TOKEN= +# WHATSAPP_WEBHOOK_APP_SECRET= +# WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 + +# -- Slack -- +# Uses SLACK_CLIENT_ID and SLACK_CLIENT_SECRET from the Slack connector section. +# +# GATEWAY_SLACK_ENABLED=FALSE +# GATEWAY_SLACK_SIGNING_SECRET= +# GATEWAY_SLACK_REDIRECT_URI=http://localhost:8929/api/v1/gateway/slack/callback + +# -- Discord -- +# Uses DISCORD_CLIENT_ID, DISCORD_CLIENT_SECRET, and DISCORD_BOT_TOKEN from the +# Discord connector section. +# +# GATEWAY_DISCORD_ENABLED=FALSE +# GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8929/api/v1/gateway/discord/callback + +# ------------------------------------------------------------------------------ +# SearXNG (bundled web search, works out of the box with no config needed) # ------------------------------------------------------------------------------ # SearXNG provides web search to all search spaces automatically. # To access the SearXNG UI directly: http://localhost:8888 @@ -273,7 +326,7 @@ STT_SERVICE=local/base # SEARXNG_SECRET=surfsense-searxng-secret # ------------------------------------------------------------------------------ -# Daytona Sandbox (optional — cloud code execution for the deep agent) +# Daytona Sandbox (optional cloud code execution for the deep agent) # ------------------------------------------------------------------------------ # Set DAYTONA_SANDBOX_ENABLED=TRUE and provide credentials to give the agent # an isolated code execution environment via the Daytona cloud API. @@ -286,9 +339,6 @@ STT_SERVICE=local/base # External API Keys (optional) # ------------------------------------------------------------------------------ -# Firecrawl (web scraping) -# FIRECRAWL_API_KEY= - # Unstructured (if ETL_SERVICE=UNSTRUCTURED) # UNSTRUCTURED_API_KEY= @@ -364,7 +414,6 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # Premium turns are debited at the actual per-call provider cost reported # by LiteLLM. Only applies to models with billing_tier=premium. # PREMIUM_CREDIT_MICROS_LIMIT=5000000 -# DEPRECATED — PREMIUM_TOKEN_LIMIT=5000000 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 @@ -376,10 +425,10 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # QUOTA_DEFAULT_PODCAST_RESERVE_MICROS=200000 # Per-video-presentation reservation for the video Celery task ($1.00 default). -# Override path bypasses QUOTA_MAX_RESERVE_MICROS clamp — raise with care. +# Override path bypasses QUOTA_MAX_RESERVE_MICROS clamp. Raise with care. # QUOTA_DEFAULT_VIDEO_PRESENTATION_RESERVE_MICROS=1000000 -# No-login (anonymous) mode — public users can chat without an account +# No-login (anonymous) mode. Public users can chat without an account # Set TRUE to enable /free pages and anonymous chat API NOLOGIN_MODE_ENABLED=FALSE # ANON_TOKEN_LIMIT=1000000 diff --git a/docker/docker-compose.deps-only.yml b/docker/docker-compose.deps-only.yml index 2be0bfe6e..ad4cc3127 100644 --- a/docker/docker-compose.deps-only.yml +++ b/docker/docker-compose.deps-only.yml @@ -114,6 +114,7 @@ services: - ZERO_REPLICA_FILE=/data/zero.db - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} - ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication} + - ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true} - ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4} - ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20} - ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30} @@ -122,11 +123,30 @@ services: volumes: - zero_cache_data:/data restart: unless-stopped + stop_grace_period: 300s healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] interval: 10s timeout: 5s retries: 5 + start_period: 600s + + # OPTIONAL — Azurite emulates Azure Blob Storage for testing the Azure + # original-file backend. The default filesystem backend needs none of this. + # To exercise it, set in surfsense_backend/.env: + # FILE_STORAGE_BACKEND=azure + # AZURE_STORAGE_CONTAINER=surfsense-documents + # AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:${AZURITE_BLOB_PORT:-10000}/devstoreaccount1; + # The backend creates blobs on upload; create the container once first + # (Azure CLI / Storage Explorer), then upload a document. + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.33.0 + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 + ports: + - "${AZURITE_BLOB_PORT:-10000}:10000" + volumes: + - azurite_data:/data + restart: unless-stopped volumes: postgres_data: @@ -137,3 +157,5 @@ volumes: name: surfsense-deps-redis zero_cache_data: name: surfsense-deps-zero-cache + azurite_data: + name: surfsense-deps-azurite diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 58cb7b42f..35effefc0 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -46,8 +46,6 @@ services: - PYTHONPATH=/app - SERVICE_ROLE=migrate - MIGRATION_TIMEOUT=${MIGRATION_TIMEOUT:-900} - volumes: - - zero_init:/zero-init depends_on: db: condition: service_healthy @@ -126,6 +124,7 @@ services: - AUTH_TYPE=${AUTH_TYPE:-LOCAL} - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} - SEARXNG_DEFAULT_HOST=${SEARXNG_DEFAULT_HOST:-http://searxng:8080} + - WHATSAPP_BRIDGE_URL=${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # - DAYTONA_SANDBOX_ENABLED=TRUE # - DAYTONA_API_KEY=${DAYTONA_API_KEY:-} @@ -148,6 +147,25 @@ services: retries: 30 start_period: 200s + whatsapp-bridge: + build: ../surfsense_backend/scripts/whatsapp-bridge + profiles: + - whatsapp + ports: + - "127.0.0.1:${WHATSAPP_BRIDGE_PORT:-9929}:9929" + volumes: + - whatsapp_sessions:/data/sessions + environment: + - PORT=9929 + - WHATSAPP_MODE=${WHATSAPP_MODE:-self-chat} + - WHATSAPP_SESSION_DIR=/data/sessions + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"] + interval: 30s + timeout: 5s + retries: 5 + celery_worker: build: *backend-build volumes: @@ -197,21 +215,6 @@ services: celery_worker: condition: service_started - # flower: - # build: *backend-build - # ports: - # - "${FLOWER_PORT:-5555}:5555" - # env_file: - # - ../surfsense_backend/.env - # environment: - # - CELERY_BROKER_URL=${REDIS_URL:-redis://redis:6379/0} - # - CELERY_RESULT_BACKEND=${REDIS_URL:-redis://redis:6379/0} - # - PYTHONPATH=/app - # command: celery -A app.celery_app flower --port=5555 - # depends_on: - # - redis - # - celery_worker - zero-cache: image: rocicorp/zero:1.4.0 ports: @@ -230,6 +233,7 @@ services: - ZERO_REPLICA_FILE=/data/zero.db - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} - ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication} + - ZERO_AUTO_RESET=${ZERO_AUTO_RESET:-true} - ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4} - ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20} - ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30} @@ -237,18 +241,14 @@ services: - ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: - zero_cache_data:/data - - zero_init:/zero-init - # Wrapper: see docker/docker-compose.yml `zero-cache` for rationale. - entrypoint: ["sh", "-c"] - # Pass the script as a single list element so Compose does not tokenize it. - command: - - 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache' restart: unless-stopped + stop_grace_period: 300s healthcheck: test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"] interval: 10s timeout: 5s retries: 5 + start_period: 600s frontend: build: @@ -280,5 +280,5 @@ volumes: name: surfsense-dev-shared-temp zero_cache_data: name: surfsense-dev-zero-cache - zero_init: - name: surfsense-dev-zero-init + whatsapp_sessions: + name: surfsense-dev-whatsapp-sessions diff --git a/docker/docker-compose.gpu.yml b/docker/docker-compose.gpu.yml new file mode 100644 index 000000000..a40aeb32f --- /dev/null +++ b/docker/docker-compose.gpu.yml @@ -0,0 +1,30 @@ +services: + backend: + deploy: + resources: + reservations: + devices: + - driver: ${SURFSENSE_GPU_DRIVER:-nvidia} + count: ${SURFSENSE_GPU_COUNT:-1} + capabilities: + - gpu + + celery_worker: + deploy: + resources: + reservations: + devices: + - driver: ${SURFSENSE_GPU_DRIVER:-nvidia} + count: ${SURFSENSE_GPU_COUNT:-1} + capabilities: + - gpu + + celery_beat: + deploy: + resources: + reservations: + devices: + - driver: ${SURFSENSE_GPU_DRIVER:-nvidia} + count: ${SURFSENSE_GPU_COUNT:-1} + capabilities: + - gpu diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 06a3ac79a..9bbf28ffd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -29,12 +29,11 @@ services: # Short-lived schema runner. Executes `alembic upgrade head` and verifies # that the `zero_publication` Postgres logical-replication publication - # exists, then exits 0. Downstream services (backend, celery_*, zero-cache) - # gate on this with `condition: service_completed_successfully` so a failed - # migration halts the whole stack instead of silently producing a half-built - # system that crash-loops zero-cache on missing publications. + # matches the canonical shape, then exits 0. Downstream services gate on this + # with `condition: service_completed_successfully` so a failed migration halts + # the whole stack instead of booting zero-cache against a drifted publication. migrations: - image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} + image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}} env_file: - .env environment: @@ -42,8 +41,6 @@ services: PYTHONPATH: /app SERVICE_ROLE: migrate MIGRATION_TIMEOUT: ${MIGRATION_TIMEOUT:-900} - volumes: - - zero_init:/zero-init depends_on: db: condition: service_healthy @@ -61,28 +58,28 @@ services: timeout: 5s retries: 5 - otel-collector: - image: otel/opentelemetry-collector-contrib:0.152.1 - profiles: - - observability - command: ["--config=/etc/otelcol/config.yaml"] - volumes: - - ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro - environment: - GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-} - GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-} - GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-} - ports: - - "${OTEL_GRPC_PORT:-4317}:4317" - - "${OTEL_HTTP_PORT:-4318}:4318" - - "${OTEL_HEALTH_PORT:-13133}:13133" - mem_limit: 2g - restart: unless-stopped - healthcheck: - test: ["CMD", "/otelcol-contrib", "--version"] - interval: 30s - timeout: 5s - retries: 3 + # otel-collector: + # image: otel/opentelemetry-collector-contrib:0.152.1 + # profiles: + # - observability + # command: ["--config=/etc/otelcol/config.yaml"] + # volumes: + # - ./otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + # environment: + # GRAFANA_CLOUD_OTLP_ENDPOINT: ${GRAFANA_CLOUD_OTLP_ENDPOINT:-} + # GRAFANA_CLOUD_INSTANCE_ID: ${GRAFANA_CLOUD_INSTANCE_ID:-} + # GRAFANA_CLOUD_API_KEY: ${GRAFANA_CLOUD_API_KEY:-} + # ports: + # - "${OTEL_GRPC_PORT:-4317}:4317" + # - "${OTEL_HTTP_PORT:-4318}:4318" + # - "${OTEL_HEALTH_PORT:-13133}:13133" + # mem_limit: 2g + # restart: unless-stopped + # healthcheck: + # test: ["CMD", "/otelcol-contrib", "--version"] + # interval: 30s + # timeout: 5s + # retries: 3 searxng: image: searxng/searxng:2026.3.13-3c1f68c59 @@ -98,7 +95,7 @@ services: retries: 5 backend: - image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} + image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}} ports: - "${BACKEND_PORT:-8929}:8000" volumes: @@ -118,6 +115,7 @@ services: UNSTRUCTURED_HAS_PATCHED_LOOP: "1" NEXT_FRONTEND_URL: ${NEXT_FRONTEND_URL:-http://localhost:${FRONTEND_PORT:-3929}} SEARXNG_DEFAULT_HOST: ${SEARXNG_DEFAULT_HOST:-http://searxng:8080} + WHATSAPP_BRIDGE_URL: ${WHATSAPP_BRIDGE_URL:-http://whatsapp-bridge:9929} # Daytona Sandbox – uncomment and set credentials to enable cloud code execution # DAYTONA_SANDBOX_ENABLED: "TRUE" # DAYTONA_API_KEY: ${DAYTONA_API_KEY:-} @@ -143,8 +141,28 @@ services: retries: 30 start_period: 200s + # whatsapp-bridge: + # build: ../surfsense_backend/scripts/whatsapp-bridge + # profiles: + # - whatsapp + # expose: + # - "9929" + # volumes: + # - whatsapp_sessions:/data/sessions + # environment: + # PORT: 9929 + # WHATSAPP_MODE: ${WHATSAPP_MODE:-self-chat} + # WHATSAPP_SESSION_DIR: /data/sessions + # mem_limit: 512m + # restart: unless-stopped + # healthcheck: + # test: ["CMD", "wget", "-qO-", "http://localhost:9929/health"] + # interval: 30s + # timeout: 5s + # retries: 5 + celery_worker: - image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} + image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}} volumes: - shared_temp:/shared_tmp env_file: @@ -174,7 +192,7 @@ services: restart: unless-stopped celery_beat: - image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} + image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest}${SURFSENSE_VARIANT:+-${SURFSENSE_VARIANT}} env_file: - .env environment: @@ -197,22 +215,6 @@ services: - "com.centurylinklabs.watchtower.enable=true" restart: unless-stopped - # flower: - # image: ghcr.io/modsetter/surfsense-backend:${SURFSENSE_VERSION:-latest} - # ports: - # - "${FLOWER_PORT:-5555}:5555" - # env_file: - # - .env - # environment: - # CELERY_BROKER_URL: ${REDIS_URL:-redis://redis:6379/0} - # CELERY_RESULT_BACKEND: ${REDIS_URL:-redis://redis:6379/0} - # PYTHONPATH: /app - # command: celery -A app.celery_app flower --port=5555 - # depends_on: - # - redis - # - celery_worker - # restart: unless-stopped - zero-cache: image: rocicorp/zero:1.4.0 ports: @@ -226,6 +228,7 @@ services: ZERO_REPLICA_FILE: /data/zero.db ZERO_ADMIN_PASSWORD: ${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin} ZERO_APP_PUBLICATIONS: ${ZERO_APP_PUBLICATIONS:-zero_publication} + ZERO_AUTO_RESET: ${ZERO_AUTO_RESET:-true} ZERO_NUM_SYNC_WORKERS: ${ZERO_NUM_SYNC_WORKERS:-4} ZERO_UPSTREAM_MAX_CONNS: ${ZERO_UPSTREAM_MAX_CONNS:-20} ZERO_CVR_MAX_CONNS: ${ZERO_CVR_MAX_CONNS:-30} @@ -233,16 +236,8 @@ services: ZERO_MUTATE_URL: ${ZERO_MUTATE_URL:-http://frontend:3000/api/zero/mutate} volumes: - zero_cache_data:/data - - zero_init:/zero-init - # Wrapper: if the migrations service flagged a publication change via - # /zero-init/needs_reset, wipe the SQLite replica before starting so - # zero-cache does a clean initial sync. Recovers from the half-built - # replica state (`_zero.tableMetadata` missing) caused by earlier crashes. - entrypoint: ["sh", "-c"] - # Pass the script as a single list element so Compose does not tokenize it. - command: - - 'if [ -f /zero-init/needs_reset ]; then echo "[zero-init] publication change detected; wiping replica file(s) under /data" && rm -f /data/zero.db /data/zero.db-shm /data/zero.db-wal && rm -f /zero-init/needs_reset; fi; exec zero-cache' restart: unless-stopped + stop_grace_period: 300s depends_on: db: condition: service_healthy @@ -253,6 +248,7 @@ services: interval: 10s timeout: 5s retries: 5 + start_period: 600s frontend: image: ghcr.io/modsetter/surfsense-web:${SURFSENSE_VERSION:-latest} @@ -264,6 +260,7 @@ services: NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${AUTH_TYPE:-LOCAL} NEXT_PUBLIC_ETL_SERVICE: ${ETL_SERVICE:-DOCLING} NEXT_PUBLIC_DEPLOYMENT_MODE: ${DEPLOYMENT_MODE:-self-hosted} + NEXT_PUBLIC_WHATSAPP_DISPLAY_PHONE_NUMBER: ${WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER:-} FASTAPI_BACKEND_INTERNAL_URL: ${FASTAPI_BACKEND_INTERNAL_URL:-http://backend:8000} labels: - "com.centurylinklabs.watchtower.enable=true" @@ -283,5 +280,5 @@ volumes: name: surfsense-shared-temp zero_cache_data: name: surfsense-zero-cache - zero_init: - name: surfsense-zero-init + whatsapp_sessions: + name: surfsense-whatsapp-sessions diff --git a/docker/scripts/install.ps1 b/docker/scripts/install.ps1 index 60c4fd5df..6e973a520 100644 --- a/docker/scripts/install.ps1 +++ b/docker/scripts/install.ps1 @@ -7,6 +7,8 @@ # To pass flags, save and run locally: # .\install.ps1 -NoWatchtower # .\install.ps1 -WatchtowerInterval 3600 +# .\install.ps1 -Variant cuda +# .\install.ps1 -Variant cuda -GpuCount all # # Handles two cases automatically: # 1. Fresh install — no prior SurfSense data detected @@ -17,7 +19,11 @@ param( [switch]$NoWatchtower, - [int]$WatchtowerInterval = 86400 + [int]$WatchtowerInterval = 86400, + [ValidateSet("cpu", "cuda", "cuda126")] + [string]$Variant, + [string]$GpuCount, + [switch]$Quiet ) $ErrorActionPreference = 'Stop' @@ -34,6 +40,11 @@ $MigrationMode = $false $SetupWatchtower = -not $NoWatchtower $WatchtowerContainer = "watchtower" +if ($GpuCount -and $GpuCount -notmatch '^([0-9]+|all)$') { + Write-Host "[SurfSense] ERROR: Invalid -GpuCount '$GpuCount'. Use a number or 'all'." -ForegroundColor Red + exit 1 +} + # ── Output helpers ────────────────────────────────────────────────────────── function Write-Info { param([string]$Msg) Write-Host "[SurfSense] " -ForegroundColor Cyan -NoNewline; Write-Host $Msg } @@ -42,6 +53,27 @@ function Write-Warn { param([string]$Msg) Write-Host "[SurfSense] " -Foregrou function Write-Step { param([string]$Msg) Write-Host "`n-- $Msg" -ForegroundColor Cyan } function Write-Err { param([string]$Msg) Write-Host "[SurfSense] ERROR: $Msg" -ForegroundColor Red; exit 1 } +function Show-Banner { + Write-Host "" + Write-Host @" + + +███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗ +██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝ +███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗ +╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝ +███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗ +╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝ + + +"@ -ForegroundColor White + Write-Host " OSS Alternative to NotebookLM for Teams" -ForegroundColor Yellow + Write-Host ("=" * 62) -ForegroundColor Cyan + Write-Info "This installer will create $InstallDir\ and start SurfSense with Docker Compose." +} + +Show-Banner + function Invoke-NativeSafe { param([scriptblock]$Command) $previousErrorActionPreference = $ErrorActionPreference @@ -53,6 +85,28 @@ function Invoke-NativeSafe { } } +function Resolve-WatchtowerPreference { + if ($NoWatchtower -or $Quiet -or -not [Environment]::UserInteractive) { + return + } + + Write-Host "" + Write-Host "Automatic updates" -ForegroundColor Cyan + $choice = Read-Host "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]" + + switch ($choice) { + "" { $script:SetupWatchtower = $true } + { $_ -match '^(?i)y(es)?$' } { $script:SetupWatchtower = $true } + { $_ -match '^(?i)n(o)?$' } { $script:SetupWatchtower = $false } + default { + Write-Warn "Unrecognized choice '$choice'; enabling Watchtower by default. Use -NoWatchtower to skip it." + $script:SetupWatchtower = $true + } + } +} + +Resolve-WatchtowerPreference + # ── Pre-flight checks ────────────────────────────────────────────────────── Write-Step "Checking prerequisites" @@ -97,143 +151,11 @@ function Wait-ForPostgres { Write-Ok "PostgreSQL is ready." } -# ── Stack health helpers ──────────────────────────────────────────────────── - -function Get-ComposeServices { - Push-Location $InstallDir - try { - $raw = Invoke-NativeSafe { docker compose ps -a --format json 2>$null } - } finally { - Pop-Location - } - if ([string]::IsNullOrWhiteSpace($raw)) { return @() } - - # Compose v2.21+ emits a JSON array; older versions emit one object per line. - try { - $parsed = $raw | ConvertFrom-Json - if ($parsed -is [System.Collections.IEnumerable] -and -not ($parsed -is [string])) { - return @($parsed) - } - return @($parsed) - } catch { - $services = @() - foreach ($line in ($raw -split "`r?`n")) { - $line = $line.Trim() - if (-not $line) { continue } - try { $services += ($line | ConvertFrom-Json) } catch { } - } - return $services - } -} - -function Wait-StackHealthy { - param([int]$TimeoutSec = 300) - - $deadline = (Get-Date).AddSeconds($TimeoutSec) - $lastReport = "" - - while ((Get-Date) -lt $deadline) { - $services = Get-ComposeServices - if (-not $services -or $services.Count -eq 0) { - Start-Sleep -Seconds 3 - continue - } - - $bad = @() - $waiting = @() - $good = @() - - foreach ($svc in $services) { - $name = $svc.Service - $state = $svc.State - $health = if ($svc.PSObject.Properties.Name -contains 'Health') { $svc.Health } else { '' } - $exit = if ($svc.PSObject.Properties.Name -contains 'ExitCode') { $svc.ExitCode } else { $null } - - if ($name -eq 'migrations') { - if ($state -eq 'exited' -and $exit -eq 0) { $good += $name } - elseif ($state -eq 'exited') { $bad += "${name} (exit=${exit})" } - else { $waiting += "${name} (${state})" } - continue - } - - if ($state -eq 'running') { - if ([string]::IsNullOrEmpty($health) -or $health -eq 'healthy') { - $good += $name - } elseif ($health -eq 'starting') { - $waiting += "${name} (starting)" - } elseif ($health -eq 'unhealthy') { - $bad += "${name} (unhealthy)" - } else { - $waiting += "${name} (${health})" - } - } elseif ($state -eq 'restarting') { - $bad += "${name} (restarting)" - } elseif ($state -eq 'exited') { - $bad += "${name} (exited, code=${exit})" - } else { - $waiting += "${name} (${state})" - } - } - - if ($bad.Count -gt 0) { - return @{ Ok = $false; Reason = 'failure'; Bad = $bad; Waiting = $waiting; Good = $good } - } - if ($waiting.Count -eq 0) { - return @{ Ok = $true; Reason = 'all_healthy'; Good = $good } - } - - $report = "Waiting on: " + ($waiting -join ', ') - if ($report -ne $lastReport) { - Write-Info $report - $lastReport = $report - } - Start-Sleep -Seconds 5 - } - - return @{ Ok = $false; Reason = 'timeout'; Bad = $bad; Waiting = $waiting; Good = $good } -} - -function Test-StaleZeroCacheVolume { - $raw = Invoke-NativeSafe { docker volume ls --format '{{.Name}}' 2>$null } - if ([string]::IsNullOrWhiteSpace($raw)) { return $false } - $names = $raw -split "`r?`n" | ForEach-Object { $_.Trim() } | Where-Object { $_ } - $hasZeroCache = $names -contains 'surfsense-zero-cache' - $hasZeroInit = $names -contains 'surfsense-zero-init' - # Pre-fix installs created surfsense-zero-cache but never surfsense-zero-init. - # Such a volume may hold a half-initialized SQLite replica from an earlier - # crash-loop. Wiping it forces zero-cache to do a fresh initial sync. - return ($hasZeroCache -and -not $hasZeroInit) -} - -function Invoke-StaleZeroCacheCleanup { - if (-not (Test-StaleZeroCacheVolume)) { return } - - Write-Warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that" - Write-Warn "predates the migrations-service fix. It may contain a half-initialized" - Write-Warn "SQLite replica that would block zero-cache from starting." - Write-Warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel." - Start-Sleep -Seconds 5 - - Push-Location $InstallDir - Invoke-NativeSafe { docker compose down --remove-orphans 2>$null } | Out-Null - Pop-Location - Invoke-NativeSafe { docker volume rm surfsense-zero-cache 2>$null } | Out-Null - Write-Ok "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start." -} - -function Write-Err-NoExit { - param([string]$Message) - Write-Host "[ERROR] $Message" -ForegroundColor Red -} +# ── Stack startup helper ──────────────────────────────────────────────────── function Invoke-StackFailureReport { - param([hashtable]$Result) - Write-Host "" - Write-Err-NoExit "Stack did not reach a healthy state." - if ($Result.Bad.Count -gt 0) { Write-Host (" Failed: " + ($Result.Bad -join ', ')) } - if ($Result.Waiting.Count -gt 0) { Write-Host (" Stuck: " + ($Result.Waiting -join ', ')) } - + Write-Host "[ERROR] Stack did not reach a healthy state." -ForegroundColor Red Write-Host "" Write-Info "Recent logs from migrations / zero-cache / backend:" Push-Location $InstallDir @@ -247,11 +169,151 @@ function Invoke-StackFailureReport { Write-Host "Recovery hints:" -ForegroundColor Yellow Write-Host " 1. Inspect migrations: cd $InstallDir; docker compose logs migrations" Write-Host " 2. Verify publication: cd $InstallDir; docker compose exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'" - Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d" + Write-Host " 3. Hard reset zero db: cd $InstallDir; docker compose down; docker volume rm surfsense-zero-cache; docker compose up -d --wait" Write-Host "" exit 1 } +function Invoke-ComposeUpWait { + Push-Location $InstallDir + try { + Invoke-NativeSafe { docker compose up -d --wait } + } finally { + Pop-Location + } + if ($LASTEXITCODE -ne 0) { + Invoke-StackFailureReport + } +} + +# ── Variant and .env helpers ──────────────────────────────────────────────── + +function Set-EnvValue { + param([string]$Path, [string]$Key, [string]$Value) + $lines = @() + if (Test-Path $Path) { + $lines = @(Get-Content $Path) + } + $updated = $false + $newLines = foreach ($line in $lines) { + if ($line -match "^$([regex]::Escape($Key))=") { + $updated = $true + "$Key=$Value" + } else { + $line + } + } + if (-not $updated) { + $newLines += "$Key=$Value" + } + Set-Content -Path $Path -Value $newLines +} + +function Remove-EnvValue { + param([string]$Path, [string]$Key) + if (-not (Test-Path $Path)) { return } + $newLines = Get-Content $Path | Where-Object { $_ -notmatch "^$([regex]::Escape($Key))=" } + Set-Content -Path $Path -Value $newLines +} + +function Test-NvidiaGpu { + if (-not (Get-Command nvidia-smi -ErrorAction SilentlyContinue)) { return $false } + Invoke-NativeSafe { nvidia-smi *>$null } | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Test-NvidiaRuntime { + $info = Invoke-NativeSafe { docker info 2>$null } + if ($info -match 'nvidia') { return $true } + if (Get-Command nvidia-ctk -ErrorAction SilentlyContinue) { return $true } + if (Get-Command nvidia-container-runtime -ErrorAction SilentlyContinue) { return $true } + return $false +} + +function Get-RecommendedVariant { + $driver = (Invoke-NativeSafe { nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>$null } | Select-Object -First 1) + $major = 0 + if ($driver -match '^(\d+)') { + $major = [int]$Matches[1] + } + if ($major -gt 0 -and $major -lt 570) { + return "cuda126" + } + return "cuda" +} + +function Resolve-Variant { + $hasGpu = Test-NvidiaGpu + $hasRuntime = $false + $recommended = "cpu" + + if ($hasGpu) { + $recommended = Get-RecommendedVariant + $hasRuntime = Test-NvidiaRuntime + } + + if ($Variant) { + if ($Variant -eq "cpu") { return "cpu" } + if (-not $hasGpu) { + Write-Warn "No NVIDIA GPU detected; falling back to CPU variant." + return "cpu" + } + if (-not $hasRuntime) { + Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant." + Write-Warn "Install the toolkit before enabling SurfSense GPU acceleration." + return "cpu" + } + return $Variant + } + + if ($hasGpu -and -not $hasRuntime) { + Write-Warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant." + } + + if ($hasGpu -and $hasRuntime -and -not $Quiet -and [Environment]::UserInteractive) { + Write-Host "" + Write-Host "SurfSense detected an NVIDIA GPU." -ForegroundColor Cyan + $choice = Read-Host "Use GPU acceleration? [Y/n]" + switch ($choice) { + "" { return $recommended } + { $_ -match '^(?i)y(es)?$' } { return $recommended } + { $_ -match '^(?i)n(o)?$' } { return "cpu" } + default { + Write-Warn "Unrecognized choice '$choice'; using CPU variant." + return "cpu" + } + } + } + + return "cpu" +} + +function Set-VariantEnv { + param([string]$Path, [string]$SelectedVariant, [bool]$AllowExistingUpdate) + + if ((Test-Path $Path) -and -not $AllowExistingUpdate) { + Write-Warn ".env already exists - keeping your existing configuration." + Write-Info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in $Path, then run docker compose up -d --wait." + return + } + + if ($SelectedVariant -eq "cpu") { + Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value "" + Remove-EnvValue -Path $Path -Key "COMPOSE_FILE" + Remove-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT" + } else { + Set-EnvValue -Path $Path -Key "SURFSENSE_VARIANT" -Value $SelectedVariant + Set-EnvValue -Path $Path -Key "COMPOSE_FILE" -Value "docker-compose.yml;docker-compose.gpu.yml" + if ($GpuCount) { + Set-EnvValue -Path $Path -Key "SURFSENSE_GPU_COUNT" -Value $GpuCount + } + } + + Remove-EnvValue -Path $Path -Key "COMPOSE_PROFILES" +} + +$SelectedVariant = Resolve-Variant + # ── Download files ────────────────────────────────────────────────────────── Write-Step "Downloading SurfSense files" @@ -262,6 +324,7 @@ New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null $Files = @( @{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" } + @{ Src = "docker/docker-compose.gpu.yml"; Dest = "docker-compose.gpu.yml" } @{ Src = "docker/.env.example"; Dest = ".env.example" } @{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" } @{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" } @@ -339,15 +402,19 @@ if (-not (Test-Path $envPath)) { $content = $content -replace 'SECRET_KEY=replace_me_with_a_random_string', "SECRET_KEY=$SecretKey" Set-Content -Path $envPath -Value $content -NoNewline + Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false Write-Info "Created $envPath" } else { - Write-Warn ".env already exists - keeping your existing configuration." + if ($PSBoundParameters.ContainsKey('Variant')) { + Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $true + Write-Info "Updated SurfSense image variant in existing $envPath" + } else { + Set-VariantEnv -Path $envPath -SelectedVariant $SelectedVariant -AllowExistingUpdate $false + } } # ── Start containers ──────────────────────────────────────────────────────── -Invoke-StaleZeroCacheCleanup - if ($MigrationMode) { $envContent = Get-Content $envPath $DbUser = ($envContent | Select-String '^DB_USER=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1 @@ -405,31 +472,15 @@ if ($MigrationMode) { } Write-Step "Starting all SurfSense services" - Push-Location $InstallDir - Invoke-NativeSafe { docker compose up -d } - Pop-Location - Write-Ok "All containers started; waiting for stack to become healthy..." - - $waitResult = Wait-StackHealthy -TimeoutSec 300 - if (-not $waitResult.Ok) { - Invoke-StackFailureReport -Result $waitResult - } - Write-Ok "All services healthy." + Invoke-ComposeUpWait + Write-Ok "All services started and healthy." Remove-Item $KeyFile -ErrorAction SilentlyContinue } else { Write-Step "Starting SurfSense" - Push-Location $InstallDir - Invoke-NativeSafe { docker compose up -d } - Pop-Location - Write-Ok "All containers started; waiting for stack to become healthy..." - - $waitResult = Wait-StackHealthy -TimeoutSec 300 - if (-not $waitResult.Ok) { - Invoke-StackFailureReport -Result $waitResult - } - Write-Ok "All services healthy." + Invoke-ComposeUpWait + Write-Ok "All services started and healthy." } # ── Watchtower (auto-update) ──────────────────────────────────────────────── @@ -461,7 +512,7 @@ if ($SetupWatchtower) { if ($LASTEXITCODE -eq 0) { Write-Ok "Watchtower started - labeled SurfSense containers will auto-update." } else { - Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d" + Write-Warn "Could not start Watchtower. You can set it up manually or use: docker compose pull; docker compose up -d --wait" } } } else { @@ -471,39 +522,26 @@ if ($SetupWatchtower) { # ── Done ──────────────────────────────────────────────────────────────────── Write-Host "" -Write-Host @" - - - .d8888b. .d888 .d8888b. -d88P Y88b d88P" d88P Y88b -Y88b. 888 Y88b. - "Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b. - "Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b - "888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888 -Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b. - "Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888 - - -"@ -ForegroundColor White - $versionDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VERSION=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1 if (-not $versionDisplay) { $versionDisplay = "latest" } -Write-Host " OSS Alternative to NotebookLM for Teams [$versionDisplay]" -ForegroundColor Yellow -Write-Host ("=" * 62) -ForegroundColor Cyan -Write-Host "" +$variantDisplay = (Get-Content $envPath | Select-String '^SURFSENSE_VARIANT=' | ForEach-Object { ($_ -split '=',2)[1].Trim('"') }) | Select-Object -First 1 +if (-not $variantDisplay) { $variantDisplay = "cpu" } +$wtHours = [math]::Floor($WatchtowerInterval / 3600) +Write-Step "SurfSense is now installed [$versionDisplay]" Write-Info " Frontend: http://localhost:3929" Write-Info " Backend: http://localhost:8929" Write-Info " API Docs: http://localhost:8929/docs" Write-Info "" Write-Info " Config: $InstallDir\.env" +Write-Info " Variant: $variantDisplay" Write-Info " Logs: cd $InstallDir; docker compose logs -f" Write-Info " Stop: cd $InstallDir; docker compose down" -Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d" +Write-Info " Update: cd $InstallDir; docker compose pull; docker compose up -d --wait" Write-Info "" if ($SetupWatchtower) { - Write-Info " Watchtower: auto-updates every ${wtHours}h (stop: docker rm -f $WatchtowerContainer)" + Write-Info " Watchtower: auto-updates every ${wtHours}h (disable: docker rm -f $WatchtowerContainer)" } else { Write-Warn " Watchtower skipped. For auto-updates, re-run without -NoWatchtower." } diff --git a/docker/scripts/install.sh b/docker/scripts/install.sh index db81f95eb..4df15fbd0 100644 --- a/docker/scripts/install.sh +++ b/docker/scripts/install.sh @@ -8,6 +8,11 @@ # Flags: # --no-watchtower Skip automatic Watchtower setup # --watchtower-interval=SECS Check interval in seconds (default: 86400 = 24h) +# --variant=cpu|cuda|cuda126 Select backend image variant +# --gpu Alias for --variant=cuda +# --cpu Alias for --variant=cpu +# --gpu-count=N|all Number of GPUs to reserve when GPU is enabled +# --quiet Skip interactive prompts # # Handles two cases automatically: # 1. Fresh install — no prior SurfSense data detected @@ -35,12 +40,22 @@ MIGRATION_MODE=false SETUP_WATCHTOWER=true WATCHTOWER_INTERVAL=86400 WATCHTOWER_CONTAINER="watchtower" +WATCHTOWER_EXPLICIT=false +REQUESTED_VARIANT="" +VARIANT_EXPLICIT=false +GPU_COUNT="" +QUIET=false # ── Parse flags ───────────────────────────────────────────────────────────── for arg in "$@"; do case "$arg" in - --no-watchtower) SETUP_WATCHTOWER=false ;; + --no-watchtower) SETUP_WATCHTOWER=false; WATCHTOWER_EXPLICIT=true ;; --watchtower-interval=*) WATCHTOWER_INTERVAL="${arg#*=}" ;; + --variant=*) REQUESTED_VARIANT="${arg#*=}"; VARIANT_EXPLICIT=true ;; + --gpu) REQUESTED_VARIANT="cuda"; VARIANT_EXPLICIT=true ;; + --cpu) REQUESTED_VARIANT="cpu"; VARIANT_EXPLICIT=true ;; + --gpu-count=*) GPU_COUNT="${arg#*=}" ;; + --quiet) QUIET=true ;; esac done @@ -57,6 +72,57 @@ warn() { printf "${YELLOW}[SurfSense]${NC} %s\n" "$1"; } error() { printf "${RED}[SurfSense]${NC} ERROR: %s\n" "$1" >&2; exit 1; } step() { printf "\n${BOLD}${CYAN}── %s${NC}\n" "$1"; } +show_banner() { + echo "" + printf '\033[1;37m' + cat << 'EOF' + + +███████╗██╗ ██╗██████╗ ███████╗███████╗███████╗███╗ ██╗███████╗███████╗ +██╔════╝██║ ██║██╔══██╗██╔════╝██╔════╝██╔════╝████╗ ██║██╔════╝██╔════╝ +███████╗██║ ██║██████╔╝█████╗ ███████╗█████╗ ██╔██╗ ██║███████╗█████╗ +╚════██║██║ ██║██╔══██╗██╔══╝ ╚════██║██╔══╝ ██║╚██╗██║╚════██║██╔══╝ +███████║╚██████╔╝██║ ██║██║ ███████║███████╗██║ ╚████║███████║███████╗ +╚══════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚══════╝╚══════╝╚═╝ ╚═══╝╚══════╝╚══════╝ + + +EOF + printf "${YELLOW} OSS Alternative to NotebookLM for Teams${NC}\n" + printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n" + info "This installer will create ${INSTALL_DIR}/ and start SurfSense with Docker Compose." +} + +show_banner + +case "${REQUESTED_VARIANT}" in + ""|cpu|cuda|cuda126) ;; + *) error "Invalid --variant='${REQUESTED_VARIANT}'. Use cpu, cuda, or cuda126." ;; +esac + +if [[ -n "${GPU_COUNT}" && ! "${GPU_COUNT}" =~ ^([0-9]+|all)$ ]]; then + error "Invalid --gpu-count='${GPU_COUNT}'. Use a number or 'all'." +fi + +resolve_watchtower_preference() { + if $WATCHTOWER_EXPLICIT || $QUIET || [[ ! -r /dev/tty || ! -w /dev/tty ]]; then + return 0 + fi + + local choice + echo "" > /dev/tty + printf "${BOLD}${CYAN}Automatic updates${NC}\n" > /dev/tty + printf "Enable automatic daily updates with Watchtower? (may download several GB in the background) [Y/n]: " > /dev/tty + read -r choice < /dev/tty || choice="" + + case "$choice" in + ""|[Yy]|[Yy][Ee][Ss]) SETUP_WATCHTOWER=true ;; + [Nn]|[Nn][Oo]) SETUP_WATCHTOWER=false ;; + *) warn "Unrecognized choice '${choice}', enabling Watchtower by default. Use --no-watchtower to skip it." >&2; SETUP_WATCHTOWER=true ;; + esac +} + +resolve_watchtower_preference + # ── Pre-flight checks ──────────────────────────────────────────────────────── step "Checking prerequisites" @@ -97,126 +163,11 @@ wait_for_pg() { success "PostgreSQL is ready." } -# ── Stack health helpers ───────────────────────────────────────────────────── - -# Enumerate compose services for project `surfsense` as `service|state|health|exitcode` -# lines. Uses `docker inspect` so we don't depend on `jq`, `python3`, or the -# exact ordering of fields in `docker compose ps --format json` output. -get_compose_services() { - local containers - containers=$(docker ps -a --filter "label=com.docker.compose.project=surfsense" --format '{{.Names}}' 2>/dev/null) || true - [[ -z "$containers" ]] && return 0 - - while IFS= read -r container; do - [[ -z "$container" ]] && continue - local svc state health code - svc=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$container" 2>/dev/null || echo "") - state=$(docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null || echo "unknown") - health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$container" 2>/dev/null || echo "") - code=$(docker inspect -f '{{.State.ExitCode}}' "$container" 2>/dev/null || echo "") - [[ -z "$svc" ]] && continue - printf '%s|%s|%s|%s\n' "$svc" "$state" "$health" "$code" - done <<< "$containers" -} - -# Globals populated by wait_stack_healthy / consumed by stack_failure_report. -STACK_BAD=() -STACK_WAITING=() -STACK_GOOD=() -STACK_TIMEOUT=false - -wait_stack_healthy() { - local timeout_sec=${1:-300} - local deadline=$(($(date +%s) + timeout_sec)) - local last_report="" - local bad=() - local waiting=() - local good=() - - while [[ $(date +%s) -lt $deadline ]]; do - local lines - lines=$(get_compose_services) - if [[ -z "$lines" ]]; then - sleep 3 - continue - fi - - bad=() - waiting=() - good=() - - while IFS='|' read -r name state health code; do - [[ -z "$name" ]] && continue - if [[ "$name" == "migrations" ]]; then - if [[ "$state" == "exited" && "$code" == "0" ]]; then - good+=("$name") - elif [[ "$state" == "exited" ]]; then - bad+=("${name} (exit=${code})") - else - waiting+=("${name} (${state})") - fi - continue - fi - - if [[ "$state" == "running" ]]; then - if [[ -z "$health" || "$health" == "healthy" ]]; then - good+=("$name") - elif [[ "$health" == "starting" ]]; then - waiting+=("${name} (starting)") - elif [[ "$health" == "unhealthy" ]]; then - bad+=("${name} (unhealthy)") - else - waiting+=("${name} (${health})") - fi - elif [[ "$state" == "restarting" ]]; then - bad+=("${name} (restarting)") - elif [[ "$state" == "exited" ]]; then - bad+=("${name} (exited, code=${code})") - else - waiting+=("${name} (${state})") - fi - done <<< "$lines" - - if (( ${#bad[@]} > 0 )); then - STACK_BAD=("${bad[@]}") - STACK_WAITING=("${waiting[@]}") - STACK_GOOD=("${good[@]}") - return 1 - fi - if (( ${#waiting[@]} == 0 )); then - STACK_GOOD=("${good[@]}") - return 0 - fi - - local report="Waiting on: ${waiting[*]}" - if [[ "$report" != "$last_report" ]]; then - info "$report" - last_report="$report" - fi - sleep 5 - done - - # bad/waiting/good are declared at function scope so referencing them is - # safe even if the polling loop never executed its body. - STACK_BAD=() - [[ ${#bad[@]} -gt 0 ]] && STACK_BAD=("${bad[@]}") - STACK_WAITING=() - [[ ${#waiting[@]} -gt 0 ]] && STACK_WAITING=("${waiting[@]}") - STACK_GOOD=() - [[ ${#good[@]} -gt 0 ]] && STACK_GOOD=("${good[@]}") - STACK_TIMEOUT=true - return 1 -} +# ── Stack startup helper ───────────────────────────────────────────────────── stack_failure_report() { echo "" echo -e "\033[31m[ERROR]\033[0m Stack did not reach a healthy state." - if (( ${#STACK_BAD[@]} > 0 )) && [[ -n "${STACK_BAD[0]}" ]]; then - echo " Failed: ${STACK_BAD[*]}" - fi - if (( ${#STACK_WAITING[@]} > 0 )) && [[ -n "${STACK_WAITING[0]}" ]]; then - echo " Stuck: ${STACK_WAITING[*]}" - fi echo "" info "Recent logs from migrations / zero-cache / backend:" (cd "${INSTALL_DIR}" && ${DC} logs --tail=60 migrations zero-cache backend 2>&1) || true @@ -224,36 +175,158 @@ stack_failure_report() { echo "Recovery hints:" echo " 1. Inspect migrations: cd ${INSTALL_DIR} && ${DC} logs migrations" echo " 2. Verify publication: cd ${INSTALL_DIR} && ${DC} exec db psql -U surfsense -d surfsense -c 'SELECT pubname FROM pg_publication;'" - echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d" + echo " 3. Hard reset zero db: cd ${INSTALL_DIR} && ${DC} down && docker volume rm surfsense-zero-cache && ${DC} up -d --wait" echo "" exit 1 } -# True if `surfsense-zero-cache` exists but `surfsense-zero-init` does not. -# That signals an install that predates the migrations-service fix; the old -# replica may be half-initialized and would block zero-cache on next start. -test_stale_zero_cache_volume() { - local has_zc has_zi - has_zc=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-cache' || true) - has_zi=$(docker volume ls --format '{{.Name}}' 2>/dev/null | grep -Fx 'surfsense-zero-init' || true) - [[ -n "$has_zc" && -z "$has_zi" ]] +compose_up_wait() { + local service="${1:-}" + if [[ -n "$service" ]]; then + (cd "${INSTALL_DIR}" && ${DC} up -d --wait "$service") < /dev/null + else + (cd "${INSTALL_DIR}" && ${DC} up -d --wait) < /dev/null + fi } -invoke_stale_zero_cache_cleanup() { - if ! test_stale_zero_cache_volume; then +# ── Variant and .env helpers ───────────────────────────────────────────────── + +set_env_value() { + local file="$1" + local key="$2" + local value="$3" + local tmp + tmp=$(mktemp) + + if grep -q "^${key}=" "$file" 2>/dev/null; then + awk -v key="$key" -v value="$value" 'BEGIN { prefix = key "=" } $0 ~ "^" prefix { print prefix value; next } { print }' "$file" > "$tmp" + else + cp "$file" "$tmp" + printf '\n%s=%s\n' "$key" "$value" >> "$tmp" + fi + mv "$tmp" "$file" +} + +remove_env_value() { + local file="$1" + local key="$2" + local tmp + tmp=$(mktemp) + awk -v key="$key" 'BEGIN { prefix = key "=" } $0 !~ "^" prefix { print }' "$file" > "$tmp" + mv "$tmp" "$file" +} + +version_major() { + printf '%s' "$1" | cut -d. -f1 +} + +recommend_cuda_variant() { + local driver_version driver_major + driver_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n 1 | tr -d '[:space:]' || true) + driver_major=$(version_major "$driver_version") + + # CUDA 12.8 generally requires an R570+ driver. Use CUDA 12.6 as the + # compatibility fallback for older 12.x driver stacks and GPUs. + if [[ "$driver_major" =~ ^[0-9]+$ && "$driver_major" -lt 570 ]]; then + printf 'cuda126' + else + printf 'cuda' + fi +} + +gpu_runtime_available() { + docker info 2>/dev/null | grep -qi 'nvidia' \ + || command -v nvidia-ctk >/dev/null 2>&1 \ + || command -v nvidia-container-runtime >/dev/null 2>&1 +} + +host_has_nvidia_gpu() { + command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi >/dev/null 2>&1 +} + +resolve_variant() { + local detected_variant="cpu" + local has_gpu=false + local has_runtime=false + + if host_has_nvidia_gpu; then + has_gpu=true + detected_variant=$(recommend_cuda_variant) + if gpu_runtime_available; then + has_runtime=true + fi + fi + + if $VARIANT_EXPLICIT; then + if [[ "$REQUESTED_VARIANT" == "cpu" ]]; then + printf 'cpu' + return 0 + fi + if ! $has_gpu; then + warn "No NVIDIA GPU detected; falling back to CPU variant." >&2 + printf 'cpu' + return 0 + fi + if ! $has_runtime; then + warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; falling back to CPU variant." >&2 + warn "Install the toolkit before enabling SurfSense GPU acceleration." >&2 + printf 'cpu' + return 0 + fi + printf '%s' "$REQUESTED_VARIANT" return 0 fi - warn "Detected pre-existing 'surfsense-zero-cache' volume from an install that" - warn "predates the migrations-service fix. It may contain a half-initialized" - warn "SQLite replica that would block zero-cache from starting." - warn "The volume will be removed in 5 seconds; press Ctrl+C to cancel." - sleep 5 - (cd "${INSTALL_DIR}" && ${DC} down --remove-orphans 2>/dev/null) || true - docker volume rm surfsense-zero-cache 2>/dev/null || true - success "Removed surfsense-zero-cache volume; zero-cache will re-sync on next start." + if $has_gpu && ! $has_runtime; then + warn "NVIDIA GPU detected, but NVIDIA Container Toolkit was not detected; using CPU variant." >&2 + fi + + if $has_gpu && $has_runtime && ! $QUIET && [[ -r /dev/tty && -w /dev/tty ]]; then + local choice + echo "" > /dev/tty + printf "${BOLD}${CYAN}SurfSense detected an NVIDIA GPU.${NC}\n" > /dev/tty + printf "Use GPU acceleration? [Y/n]: " > /dev/tty + read -r choice < /dev/tty || choice="" + case "$choice" in + "") printf '%s' "$detected_variant" ;; + [Yy]|[Yy][Ee][Ss]) printf '%s' "$detected_variant" ;; + [Nn]|[Nn][Oo]) printf 'cpu' ;; + *) warn "Unrecognized choice '${choice}', using CPU variant." >&2; printf 'cpu' ;; + esac + return 0 + fi + + printf 'cpu' } +apply_variant_env() { + local env_file="$1" + local variant="$2" + local allow_existing_update="$3" + + if [[ -f "$env_file" && "$allow_existing_update" != "true" ]]; then + warn ".env already exists — keeping your existing configuration." + info "To change variants later, edit SURFSENSE_VARIANT and COMPOSE_FILE in ${env_file}, then run ${DC} up -d --wait." + return 0 + fi + + if [[ "$variant" == "cpu" ]]; then + set_env_value "$env_file" "SURFSENSE_VARIANT" "" + remove_env_value "$env_file" "COMPOSE_FILE" + remove_env_value "$env_file" "SURFSENSE_GPU_COUNT" + else + set_env_value "$env_file" "SURFSENSE_VARIANT" "$variant" + set_env_value "$env_file" "COMPOSE_FILE" "docker-compose.yml:docker-compose.gpu.yml" + if [[ -n "$GPU_COUNT" ]]; then + set_env_value "$env_file" "SURFSENSE_GPU_COUNT" "$GPU_COUNT" + fi + fi + + remove_env_value "$env_file" "COMPOSE_PROFILES" +} + +SELECTED_VARIANT=$(resolve_variant) + # ── Download files ─────────────────────────────────────────────────────────── step "Downloading SurfSense files" @@ -263,6 +336,7 @@ mkdir -p "${INSTALL_DIR}/searxng" FILES=( "docker/docker-compose.yml:docker-compose.yml" + "docker/docker-compose.gpu.yml:docker-compose.gpu.yml" "docker/.env.example:.env.example" "docker/postgresql.conf:postgresql.conf" "docker/scripts/migrate-database.sh:scripts/migrate-database.sh" @@ -336,15 +410,19 @@ if [ ! -f "${INSTALL_DIR}/.env" ]; then else sed -i "s|SECRET_KEY=replace_me_with_a_random_string|SECRET_KEY=${SECRET_KEY}|" "${INSTALL_DIR}/.env" fi + apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false" info "Created ${INSTALL_DIR}/.env" else - warn ".env already exists — keeping your existing configuration." + if $VARIANT_EXPLICIT; then + apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "true" + info "Updated SurfSense image variant in existing ${INSTALL_DIR}/.env" + else + apply_variant_env "${INSTALL_DIR}/.env" "$SELECTED_VARIANT" "false" + fi fi # ── Start containers ───────────────────────────────────────────────────────── -invoke_stale_zero_cache_cleanup - if $MIGRATION_MODE; then # Read DB credentials from .env (fall back to defaults from docker-compose.yml) DB_USER=$(grep '^DB_USER=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true) @@ -401,26 +479,20 @@ if $MIGRATION_MODE; then fi step "Starting all SurfSense services" - (cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null - success "All containers started; waiting for stack to become healthy..." - - if ! wait_stack_healthy 300; then + if ! compose_up_wait; then stack_failure_report fi - success "All services healthy." + success "All services started and healthy." # Key file is no longer needed — SECRET_KEY is now in .env rm -f "${KEY_FILE}" else step "Starting SurfSense" - (cd "${INSTALL_DIR}" && ${DC} up -d) < /dev/null - success "All containers started; waiting for stack to become healthy..." - - if ! wait_stack_healthy 300; then + if ! compose_up_wait; then stack_failure_report fi - success "All services healthy." + success "All services started and healthy." fi # ── Watchtower (auto-update) ───────────────────────────────────────────────── @@ -445,7 +517,7 @@ if $SETUP_WATCHTOWER; then --label-enable \ --interval "${WATCHTOWER_INTERVAL}" >/dev/null 2>&1 < /dev/null \ && success "Watchtower started — labeled SurfSense containers will auto-update." \ - || warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d" + || warn "Could not start Watchtower. You can set it up manually or use: docker compose pull && docker compose up -d --wait" fi else info "Skipping Watchtower setup (--no-watchtower flag)." @@ -454,38 +526,25 @@ fi # ── Done ───────────────────────────────────────────────────────────────────── echo "" -printf '\033[1;37m' -cat << 'EOF' - - - .d8888b. .d888 .d8888b. -d88P Y88b d88P" d88P Y88b -Y88b. 888 Y88b. - "Y888b. 888 888 888d888 888888 "Y888b. .d88b. 88888b. .d8888b .d88b. - "Y88b. 888 888 888P" 888 "Y88b. d8P Y8b 888 "88b 88K d8P Y8b - "888 888 888 888 888 "888 88888888 888 888 "Y8888b. 88888888 -Y88b d88P Y88b 888 888 888 Y88b d88P Y8b. 888 888 X88 Y8b. - "Y8888P" "Y88888 888 888 "Y8888P" "Y8888 888 888 88888P' "Y8888 - - -EOF _version_display=$(grep '^SURFSENSE_VERSION=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true) _version_display="${_version_display:-latest}" -printf " OSS Alternative to NotebookLM for Teams ${YELLOW}[%s]${NC}\n" "${_version_display}" -printf "${CYAN}══════════════════════════════════════════════════════════════${NC}\n\n" +_variant_display=$(grep '^SURFSENSE_VARIANT=' "${INSTALL_DIR}/.env" 2>/dev/null | cut -d= -f2 | tr -d '"' | head -1 || true) +_variant_display="${_variant_display:-cpu}" +step "SurfSense is now installed [${_version_display}]" info " Frontend: http://localhost:3929" info " Backend: http://localhost:8929" info " API Docs: http://localhost:8929/docs" info "" info " Config: ${INSTALL_DIR}/.env" +info " Variant: ${_variant_display}" info " Logs: cd ${INSTALL_DIR} && ${DC} logs -f" info " Stop: cd ${INSTALL_DIR} && ${DC} down" -info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d" +info " Update: cd ${INSTALL_DIR} && ${DC} pull && ${DC} up -d --wait" info "" if $SETUP_WATCHTOWER; then - info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (stop: docker rm -f ${WATCHTOWER_CONTAINER})" + info " Watchtower: auto-updates every $((WATCHTOWER_INTERVAL / 3600))h (disable: docker rm -f ${WATCHTOWER_CONTAINER})" else warn " Watchtower skipped. For auto-updates, re-run without --no-watchtower." fi diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index 6638dbba1..ac03a7902 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -212,9 +212,9 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 | 字段 | 值 | 说明 | |------|-----|------| -| **Configuration Name** | `MiniMax M2.5` | 配置名称(自定义) | +| **Configuration Name** | `MiniMax M3` | 配置名称(自定义) | | **Provider** | `MINIMAX` | 选择 MiniMax | -| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | +| **Model Name** | `MiniMax-M3` | 推荐模型
其他选项: `MiniMax-M2.7`、`MiniMax-M2.7-highspeed` | | **API Key** | `eyJ...` | 你的 MiniMax API Key | | **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | | **Parameters** | `{"temperature": 1.0}` | 注意:temperature 必须在 (0.0, 1.0] 范围内,不能为 0 | @@ -222,22 +222,23 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 ### 示例配置 ``` -Configuration Name: MiniMax M2.5 +Configuration Name: MiniMax M3 Provider: MINIMAX -Model Name: MiniMax-M2.5 +Model Name: MiniMax-M3 API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx API Base URL: https://api.minimax.io/v1 ``` ### 可用模型 -- **MiniMax-M2.5**: 高性能通用模型,204K 上下文窗口(推荐) -- **MiniMax-M2.5-highspeed**: 高速推理版本,204K 上下文窗口 +- **MiniMax-M3**: 旗舰模型,512K 上下文窗口(推荐) +- **MiniMax-M2.7**: 上一代通用模型,204K 上下文窗口 +- **MiniMax-M2.7-highspeed**: 上一代高速推理版本,204K 上下文窗口 ### 注意事项 - **temperature 参数**: MiniMax 要求 temperature 必须在 (0.0, 1.0] 范围内,不能设置为 0。建议使用 1.0。 -- 两个模型都支持 204K 超长上下文窗口,适合处理长文本任务。 +- M3 支持 512K 超长上下文,M2.7 系列保留 204K,适合按需求选择。 ### 定价 - 请访问 [MiniMax 定价页面](https://platform.minimaxi.com/document/Price) 查看最新价格 @@ -315,8 +316,8 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K, MiniMax-M2.5 (204K) | 超长上下文 | -| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | +| **长文本处理** | Kimi 128K, MiniMax-M3 (512K) | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.7-highspeed | 速度优先 | ### 2. 成本优化 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 70cf687d8..6e49a7132 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -3,18 +3,46 @@ DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense # Deployment environment: dev or production SURFSENSE_ENV=dev -#Celery Config -CELERY_BROKER_URL=redis://localhost:6379/0 -CELERY_RESULT_BACKEND=redis://localhost:6379/0 +# Redis (single endpoint for Celery broker/result backend + app features) +REDIS_URL=redis://localhost:6379/0 +# Optional: override individually only to split Redis across instances. +# Each defaults to REDIS_URL when unset. +# CELERY_BROKER_URL=redis://localhost:6379/0 +# CELERY_RESULT_BACKEND=redis://localhost:6379/0 +# REDIS_APP_URL=redis://localhost:6379/0 # Optional: isolate queues when sharing Redis with other apps CELERY_TASK_DEFAULT_QUEUE=surfsense - -# Redis for app-level features (heartbeats, podcast markers) -# Defaults to CELERY_BROKER_URL when not set -REDIS_APP_URL=redis://localhost:6379/0 # Optional: TTL in seconds for connector indexing lock key # CONNECTOR_INDEXING_LOCK_TTL_SECONDS=28800 +# Messaging Gateway (global) +# GATEWAY_ENABLED: master switch for ALL messaging gateway channels (Telegram, WhatsApp, +# Slack, Discord). When FALSE, no gateway background workers/supervisors start and all +# gateway HTTP routes (webhooks, OAuth callbacks, pairing) return 404. Set per-channel +# flags below to control individual platforms once the gateway is enabled. +GATEWAY_ENABLED=TRUE + +# Telegram Gateway +# TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, _ or - +# GATEWAY_TELEGRAM_INTAKE_MODE: `webhook` for production, `longpoll` for single-replica self-host fallback, `disabled` to skip Telegram intake +TELEGRAM_SHARED_BOT_TOKEN= +TELEGRAM_SHARED_BOT_USERNAME= +TELEGRAM_WEBHOOK_SECRET= +GATEWAY_BASE_URL=http://localhost:8000 +GATEWAY_TELEGRAM_INTAKE_MODE=webhook + +# WhatsApp Gateway +# GATEWAY_WHATSAPP_INTAKE_MODE: `cloud` for Meta Cloud API, `baileys` for self-hosted bridge, `disabled` to skip WhatsApp intake +GATEWAY_WHATSAPP_INTAKE_MODE=disabled +WHATSAPP_SHARED_BUSINESS_TOKEN= +WHATSAPP_SHARED_PHONE_NUMBER_ID= +WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER= +WHATSAPP_SHARED_WABA_ID= +WHATSAPP_GRAPH_API_VERSION=v25.0 +WHATSAPP_WEBHOOK_VERIFY_TOKEN= +WHATSAPP_WEBHOOK_APP_SECRET= +WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 + # Platform Web Search (SearXNG) # Set this to enable built-in web search. Docker Compose sets it automatically. # Only uncomment if running the backend outside Docker (e.g. uvicorn on host). @@ -64,8 +92,6 @@ STRIPE_PAGE_BUYING_ENABLED=TRUE STRIPE_TOKEN_BUYING_ENABLED=FALSE STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... STRIPE_CREDIT_MICROS_PER_UNIT=1000000 -# DEPRECATED — use STRIPE_CREDIT_MICROS_PER_UNIT (1:1 numerical mapping): -# STRIPE_TOKENS_PER_UNIT=1000000 # Periodic Stripe safety net for purchases left in PENDING (minutes old) STRIPE_RECONCILIATION_LOOKBACK_MINUTES=10 @@ -98,11 +124,14 @@ CLICKUP_CLIENT_ID=your_clickup_client_id_here CLICKUP_CLIENT_SECRET=your_clickup_client_secret_here CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback -# Discord OAuth Configuration +# Discord OAuth / Gateway Configuration +# The Discord connector and Discord gateway use the same Discord application/bot. DISCORD_CLIENT_ID=your_discord_client_id_here DISCORD_CLIENT_SECRET=your_discord_client_secret_here DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/auth/discord/connector/callback DISCORD_BOT_TOKEN=your_bot_token_from_developer_portal +GATEWAY_DISCORD_ENABLED=FALSE +GATEWAY_DISCORD_REDIRECT_URI=http://localhost:8000/api/v1/gateway/discord/callback # Atlassian OAuth Configuration (Jira & Confluence) ATLASSIAN_CLIENT_ID=your_atlassian_client_id_here @@ -120,10 +149,14 @@ NOTION_CLIENT_ID=your_notion_client_id_here NOTION_CLIENT_SECRET=your_notion_client_secret_here NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback -# Slack OAuth Configuration +# Slack OAuth / Gateway Configuration +# The Slack connector and Slack gateway can use the same Slack app client ID/secret. SLACK_CLIENT_ID=your_slack_client_id_here SLACK_CLIENT_SECRET=your_slack_client_secret_here SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback +GATEWAY_SLACK_ENABLED=FALSE +GATEWAY_SLACK_SIGNING_SECRET=your_slack_signing_secret_here +GATEWAY_SLACK_REDIRECT_URI=http://localhost:8000/api/v1/gateway/slack/callback # Microsoft OAuth (Teams & OneDrive) MICROSOFT_CLIENT_ID=your_microsoft_client_id_here @@ -197,8 +230,6 @@ PAGES_LIMIT=500 # models bill proportionally. Applies only to models with # billing_tier=premium in global_llm_config.yaml. PREMIUM_CREDIT_MICROS_LIMIT=5000000 -# DEPRECATED — use PREMIUM_CREDIT_MICROS_LIMIT (1:1 numerical mapping): -# PREMIUM_TOKEN_LIMIT=5000000 # Safety ceiling on per-call premium reservation, in micro-USD. # stream_new_chat estimates an upper-bound cost from the model's @@ -246,17 +277,19 @@ TURNSTILE_ENABLED=FALSE TURNSTILE_SECRET_KEY= +# Proxy provider selection. Selects a ProxyProvider implementation registered in +# app/utils/proxy/registry.py. Default: "anonymous_proxies". Add new vendors there. +# PROXY_PROVIDER=anonymous_proxies + # Residential Proxy Configuration (anonymous-proxies.net) # Used for web crawling, link previews, and YouTube transcript fetching to avoid IP bans. -# Leave commented out to disable proxying. +# Consumed by the "anonymous_proxies" provider. Leave commented out to disable proxying. # RESIDENTIAL_PROXY_USERNAME=your_proxy_username # RESIDENTIAL_PROXY_PASSWORD=your_proxy_password # RESIDENTIAL_PROXY_HOSTNAME=rotating.dnsproxifier.com:31230 # RESIDENTIAL_PROXY_LOCATION= # RESIDENTIAL_PROXY_TYPE=1 -FIRECRAWL_API_KEY=fcr-01J0000000000000000000000 - # File Parser Service ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING UNSTRUCTURED_API_KEY=Tpu3P0U8iy @@ -265,6 +298,16 @@ LLAMA_CLOUD_API_KEY=llx-nnn # AZURE_DI_ENDPOINT=https://your-resource.cognitiveservices.azure.com/ # AZURE_DI_KEY=your-key +# Original File Storage +# Where to persist the original bytes of uploaded documents (for download today, +# redaction / form-filling later). "local" needs no cloud creds and is the dev default. +FILE_STORAGE_BACKEND=local +# Local backend: directory for stored files (defaults to surfsense_backend/.local_object_store) +# FILE_STORAGE_LOCAL_PATH=/var/lib/surfsense/object-store +# Azure Blob backend (set FILE_STORAGE_BACKEND=azure): +# AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net +# AZURE_STORAGE_CONTAINER=surfsense-documents + # Daytona Sandbox (isolated code execution) # DAYTONA_SANDBOX_ENABLED=FALSE # DAYTONA_API_KEY=your-daytona-api-key @@ -285,9 +328,6 @@ LANGSMITH_PROJECT=surfsense # ============================================================================= # OPTIONAL: New-chat agent feature flags # ============================================================================= -# Multi-agent orchestrator switch for authenticated chat streaming. -# MULTI_AGENT_CHAT_ENABLED=false - # Master kill-switch — when true, every flag below is forced OFF. # SURFSENSE_DISABLE_NEW_AGENT_STACK=false @@ -322,6 +362,13 @@ LANGSMITH_PROJECT=surfsense # SURFSENSE_ENABLE_SPECIALIZED_SUBAGENTS=false # SURFSENSE_ENABLE_KB_PLANNER_RUNNABLE=false +# KB retrieval mode (default OFF = lazy). When OFF, the main agent retrieves +# KB content on demand via the `search_knowledge_base` tool and skips the +# expensive per-turn pre-injection (planner LLM + embed + hybrid search, +# ~2.3s); explicit @-mentions are still surfaced cheaply. Set to true to +# restore the original eager `` pre-injection. +# SURFSENSE_ENABLE_KB_PRIORITY_PREINJECTION=false + # Snapshot / revert # SURFSENSE_ENABLE_ACTION_LOG=false # SURFSENSE_ENABLE_REVERT_ROUTE=false # Backend-only; flip when UI ships @@ -342,6 +389,15 @@ LANGSMITH_PROJECT=surfsense # rollback if you suspect cache-related staleness. # SURFSENSE_ENABLE_AGENT_CACHE=true +# Cross-thread reuse (default ON). Drops thread_id from the cache key so a +# returning user's NEW chats (same user + search space + config + visibility) +# hit the already-compiled graph instead of paying a fresh ~4-5s compile — +# turning a cold first turn into a warm one. Safe because ActionLog, +# KB-persistence, and the deliverables tools now resolve the chat thread from +# the live RunnableConfig at call time rather than a build-time closure. Flip +# OFF to fall back to a per-thread cache key (instant rollback). +# SURFSENSE_ENABLE_CROSS_THREAD_AGENT_CACHE=true + # Cache capacity (max number of compiled-agent entries kept in memory) # and TTL per entry (seconds). Working set is typically one entry per # active thread on this replica; tune up for very large deployments. diff --git a/surfsense_backend/.gitignore b/surfsense_backend/.gitignore index 47fd53aef..efc6c90d7 100644 --- a/surfsense_backend/.gitignore +++ b/surfsense_backend/.gitignore @@ -2,6 +2,7 @@ .venv venv/ data/ +.local_object_store/ __pycache__/ .flashrank_cache surf_new_backend.egg-info/ diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index 0c783f403..292cb2671 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker.io/docker/dockerfile:1 # ============================================================================= # SurfSense Backend — Multi-stage Dockerfile # ============================================================================= @@ -61,15 +62,25 @@ COPY pyproject.toml uv.lock ./ # Exporting the lock to requirements.txt and feeding it to `uv pip install` # pins every transitive package to the exact version captured in uv.lock. # -# Note on torch/CUDA: we do NOT install torch from a separate cu* index here. -# PyPI's torch wheels for Linux x86_64 already ship CUDA-enabled and pull -# nvidia-cudnn-cu13, nvidia-nccl-cu13, triton, etc. as install deps (all -# captured in uv.lock). If a specific CUDA version is needed, wire it through -# [tool.uv.sources] in pyproject.toml so the lock stays the source of truth. +# Note on torch/CUDA: the export must always select either the cpu or CUDA +# extra declared in pyproject.toml. A no-extra export would resolve torch from +# PyPI on Linux, which currently pulls CUDA-enabled wheels and nvidia-* deps. +# Keep CUDA version selection in [tool.uv.sources] so uv.lock remains the +# source of truth. The install step also needs the matching PyTorch index, +# because requirements.txt preserves the +cpu/+cu wheel pins but not uv's +# package source metadata. +ARG USE_CUDA=false +ARG CUDA_EXTRA=cu128 RUN pip install --no-cache-dir uv && \ + if [ "$USE_CUDA" = "true" ]; then EXTRA="$CUDA_EXTRA"; else EXTRA="cpu"; fi && \ + TORCH_INDEX="https://download.pytorch.org/whl/${EXTRA}" && \ uv export --frozen --no-dev --no-hashes --no-emit-project \ + --extra "$EXTRA" \ --format requirements-txt -o /tmp/requirements.txt && \ - uv pip install --system --no-cache-dir -r /tmp/requirements.txt && \ + uv pip install --system --no-cache-dir \ + --index "$TORCH_INDEX" \ + --index-strategy unsafe-best-match \ + -r /tmp/requirements.txt && \ rm /tmp/requirements.txt @@ -94,10 +105,14 @@ RUN printf '%s\n' \ | python || true ARG EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 -RUN python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')" +RUN --mount=type=secret,id=HF_TOKEN \ + HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \ + python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')" -# Install Playwright browsers (the playwright python package itself is in deps) -RUN playwright install chromium --with-deps +# Install Scrapling's browser engines (patchright Chromium + Camoufox). +# Scrapling pulls playwright/patchright via the `fetchers` extra; `scrapling install` +# downloads the matching browser binaries used by DynamicFetcher/StealthyFetcher. +RUN scrapling install # Shared temp directory for file uploads between API and Worker containers. # Python's tempfile module uses TMPDIR, so uploaded files land here. diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index 5354211aa..04a6b50ff 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -3,6 +3,7 @@ import os import sys from logging.config import fileConfig +import sqlalchemy as sa from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config @@ -36,6 +37,9 @@ if config.config_file_name is not None: # target_metadata = mymodel.Base.metadata target_metadata = Base.metadata +MIGRATION_ADVISORY_LOCK_NAMESPACE = "surfsense" +MIGRATION_ADVISORY_LOCK_NAME = "alembic_migrations" + # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") @@ -73,8 +77,22 @@ def do_run_migrations(connection: Connection) -> None: transaction_per_migration=True, ) - with context.begin_transaction(): - context.run_migrations() + lock_params = { + "namespace": MIGRATION_ADVISORY_LOCK_NAMESPACE, + "name": MIGRATION_ADVISORY_LOCK_NAME, + } + connection.execute( + sa.text("SELECT pg_advisory_lock(hashtext(:namespace), hashtext(:name))"), + lock_params, + ) + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.execute( + sa.text("SELECT pg_advisory_unlock(hashtext(:namespace), hashtext(:name))"), + lock_params, + ) async def run_async_migrations() -> None: diff --git a/surfsense_backend/alembic/versions/143_force_zero_publication_resync.py b/surfsense_backend/alembic/versions/143_force_zero_publication_resync.py index 147cbde56..193d51039 100644 --- a/surfsense_backend/alembic/versions/143_force_zero_publication_resync.py +++ b/surfsense_backend/alembic/versions/143_force_zero_publication_resync.py @@ -47,7 +47,6 @@ depends_on: str | Sequence[str] | None = None PUBLICATION_NAME = "zero_publication" -# Must stay in sync with the column lists in migrations 117 / 139 / 140. DOCUMENT_COLS = [ "id", "title", diff --git a/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py new file mode 100644 index 000000000..e23f3a371 --- /dev/null +++ b/surfsense_backend/alembic/versions/148_add_automation_runs_to_zero_publication.py @@ -0,0 +1,175 @@ +"""add automation_runs to zero_publication with thin column list + +Publishes ``automation_runs`` so the dashboard can replace polling with a +live run status + per-step ticker. Only the columns the list and ticker +read are exposed (``id, automation_id, trigger_id, status, step_results, +started_at, finished_at, created_at``); heavy JSONB +(``definition_snapshot``, ``inputs``, ``output``, ``artifacts``, ``error``) +stays on REST and is fetched lazily on detail expand. + +Uses the canonical ``ALTER PUBLICATION ... SET TABLE`` + ``COMMENT`` +bookend pattern (see migration 143) -- the shape Zero ``>=1.0`` requires +to fire its schema-change hook. Existing tables are re-emitted unchanged. + +Revision ID: 148 +Revises: 147 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "148" +down_revision: str | None = "147" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +# Mirrors migration 143. Kept in sync explicitly: any change to these lists +# must be re-emitted in a new resync migration with COMMENT bookends. +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +# Thin set: status + lightweight progress only. Heavy JSONB stays on REST. +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _has_zero_version(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = '_0_version'" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _build_set_table_ddl( + *, documents_has_zero_ver: bool, user_has_zero_ver: bool +) -> str: + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + doc_col_list = ", ".join(doc_cols) + user_col_list = ", ".join(user_cols) + run_col_list = ", ".join(AUTOMATION_RUN_COLS) + return ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({doc_col_list}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({user_col_list}), ' + f"automation_runs ({run_col_list})" + ) + + +def upgrade() -> None: + conn = op.get_bind() + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + + # COMMENT-ALTER-COMMENT trio must be one transaction so Zero observes + # them as one schema-change event. Matches the SAVEPOINT pattern used + # in migrations 117 / 139 / 140 / 143. + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-resync'") + ) + conn.execute( + sa.text( + _build_set_table_ddl( + documents_has_zero_ver=documents_has_zero_ver, + user_has_zero_ver=user_has_zero_ver, + ) + ) + ) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-148-resync'") + ) + + +def downgrade() -> None: + """Re-emit migration 143's shape (no automation_runs).""" + conn = op.get_bind() + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + doc_col_list = ", ".join(doc_cols) + user_col_list = ", ".join(user_cols) + ddl = ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({doc_col_list}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({user_col_list})' + ) + + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-148-downgrade'") + ) + conn.execute(sa.text(ddl)) + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-148-downgrade'" + ) + ) diff --git a/surfsense_backend/alembic/versions/149_add_gateway_tables.py b/surfsense_backend/alembic/versions/149_add_gateway_tables.py new file mode 100644 index 000000000..eee4a45b6 --- /dev/null +++ b/surfsense_backend/alembic/versions/149_add_gateway_tables.py @@ -0,0 +1,667 @@ +"""add external chat surface tables + +Revision ID: 149 +Revises: 148 +Create Date: 2026-05-27 + +Adds the lean external chat surface schema: + +* external_chat_accounts +* external_chat_bindings +* external_chat_inbound_events + +External chat surfaces store Telegram-originated conversations in the existing +chat tables. This migration adds ``source`` to ``new_chat_threads`` and +``new_chat_messages`` as UI metadata while publishing all chat-message sources +through Zero so a future SurfSense UI layer can render external chats. External +chat adapter tables are served through REST in v1, so they are intentionally not +added to ``zero_publication``. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision: str = "149" +down_revision: str | None = "148" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _has_zero_version(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = '_0_version'" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _cols(columns: list[str]) -> str: + return ", ".join(columns) + + +def _table_exists(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.tables " + "WHERE table_schema = current_schema() AND table_name = :tbl" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_schema = current_schema() " + "AND table_name = :tbl AND column_name = :col" + ), + {"tbl": table, "col": column}, + ).fetchone() + is not None + ) + + +def _index_exists(conn, index_name: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :name" + ), + {"name": index_name}, + ).fetchone() + is not None + ) + + +def _constraint_exists(conn, table: str, constraint_name: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.table_constraints " + "WHERE table_schema = current_schema() " + "AND table_name = :tbl AND constraint_name = :name" + ), + {"tbl": table, "name": constraint_name}, + ).fetchone() + is not None + ) + + +def _drop_index_if_exists(index_name: str, table_name: str) -> None: + if _index_exists(op.get_bind(), index_name): + op.drop_index(index_name, table_name=table_name) + + +def _drop_column_if_exists(table_name: str, column_name: str) -> None: + if _column_exists(op.get_bind(), table_name, column_name): + op.drop_column(table_name, column_name) + + +def _build_set_table_ddl( + *, documents_has_zero_ver: bool, user_has_zero_ver: bool +) -> str: + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + + return ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({_cols(doc_cols)}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({_cols(user_cols)}), ' + f"automation_runs ({_cols(AUTOMATION_RUN_COLS)})" + ) + + +def _create_enum(name: str, values: tuple[str, ...]) -> postgresql.ENUM: + enum = postgresql.ENUM(*values, name=name) + enum.create(op.get_bind(), checkfirst=True) + return postgresql.ENUM(*values, name=name, create_type=False) + + +def upgrade() -> None: + conn = op.get_bind() + external_chat_platform_enum = _create_enum( + "external_chat_platform", ("telegram", "whatsapp", "signal") + ) + external_chat_account_mode_enum = _create_enum( + "external_chat_account_mode", ("cloud_shared", "self_host_byo") + ) + external_chat_health_status_enum = _create_enum( + "external_chat_health_status", ("unknown", "ok", "failing") + ) + external_chat_binding_state_enum = _create_enum( + "external_chat_binding_state", ("pending", "bound", "revoked", "suspended") + ) + external_chat_peer_kind_enum = _create_enum( + "external_chat_peer_kind", ("direct", "group", "channel", "unknown") + ) + external_chat_event_kind_enum = _create_enum( + "external_chat_event_kind", + ("message", "edited_message", "callback_query", "other"), + ) + external_chat_event_status_enum = _create_enum( + "external_chat_event_status", + ("received", "processing", "processed", "ignored", "failed"), + ) + + if not _table_exists(conn, "external_chat_accounts"): + op.create_table( + "external_chat_accounts", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("platform", external_chat_platform_enum, nullable=False), + sa.Column("mode", external_chat_account_mode_enum, nullable=False), + sa.Column("owner_user_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("owner_search_space_id", sa.Integer(), nullable=True), + sa.Column( + "is_system_account", + sa.Boolean(), + nullable=False, + server_default="false", + ), + sa.Column("encrypted_credentials", sa.Text(), nullable=True), + sa.Column("bot_username", sa.String(255), nullable=True), + sa.Column("webhook_secret", sa.String(64), nullable=True), + sa.Column( + "cursor_state", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column( + "health_status", + external_chat_health_status_enum, + nullable=False, + server_default="unknown", + ), + sa.Column( + "last_health_check_at", sa.TIMESTAMP(timezone=True), nullable=True + ), + sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.CheckConstraint( + "(is_system_account = true AND owner_user_id IS NULL) OR " + "(is_system_account = false AND owner_user_id IS NOT NULL)", + name="ck_external_chat_accounts_owner_shape", + ), + sa.ForeignKeyConstraint(["owner_user_id"], ["user.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["owner_search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), + ) + op.create_index( + "uq_external_chat_accounts_owner_platform", + "external_chat_accounts", + ["owner_user_id", "platform"], + unique=True, + postgresql_where=sa.text("is_system_account = false"), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text("is_system_account = true"), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_webhook_secret", + "external_chat_accounts", + ["webhook_secret"], + unique=True, + postgresql_where=sa.text("webhook_secret IS NOT NULL"), + if_not_exists=True, + ) + + if not _table_exists(conn, "external_chat_bindings"): + op.create_table( + "external_chat_bindings", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("account_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("search_space_id", sa.Integer(), nullable=False), + sa.Column( + "state", + external_chat_binding_state_enum, + nullable=False, + server_default="pending", + ), + sa.Column("pairing_code", sa.Text(), nullable=True), + sa.Column( + "pairing_code_expires_at", sa.TIMESTAMP(timezone=True), nullable=True + ), + sa.Column("external_peer_id", sa.Text(), nullable=True), + sa.Column( + "external_peer_kind", + external_chat_peer_kind_enum, + nullable=False, + server_default="unknown", + ), + sa.Column( + "external_thread_id", + sa.Text(), + nullable=True, + comment="Reserved for Telegram message_thread_id when group/forum support lands.", + ), + sa.Column("external_display_name", sa.Text(), nullable=True), + sa.Column("external_username", sa.Text(), nullable=True), + sa.Column( + "external_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("new_chat_thread_id", sa.Integer(), nullable=True), + sa.Column("revoked_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("suspended_reason", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.ForeignKeyConstraint( + ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["new_chat_thread_id"], ["new_chat_threads.id"], ondelete="SET NULL" + ), + ) + op.create_index( + "uq_external_chat_bindings_account_peer_active", + "external_chat_bindings", + ["account_id", "external_peer_id"], + unique=True, + postgresql_where=sa.text( + "state IN ('bound', 'suspended') AND external_peer_id IS NOT NULL" + ), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_bindings_pairing_code_pending", + "external_chat_bindings", + ["pairing_code"], + unique=True, + postgresql_where=sa.text("state = 'pending'"), + if_not_exists=True, + ) + op.create_index( + "ix_external_chat_bindings_user_state", + "external_chat_bindings", + ["user_id", "state"], + if_not_exists=True, + ) + op.create_index( + "ix_external_chat_bindings_search_space_state", + "external_chat_bindings", + ["search_space_id", "state"], + if_not_exists=True, + ) + + if not _table_exists(conn, "external_chat_inbound_events"): + op.create_table( + "external_chat_inbound_events", + sa.Column("id", sa.BigInteger(), primary_key=True), + sa.Column("account_id", sa.BigInteger(), nullable=False), + sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True), + sa.Column("platform", external_chat_platform_enum, nullable=False), + sa.Column("event_dedupe_key", sa.Text(), nullable=False), + sa.Column("external_event_id", sa.Text(), nullable=True), + sa.Column("external_message_id", sa.Text(), nullable=True), + sa.Column("event_kind", external_chat_event_kind_enum, nullable=False), + sa.Column( + "raw_payload", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + sa.Column("request_id", sa.String(64), nullable=True), + sa.Column( + "status", + external_chat_event_status_enum, + nullable=False, + server_default="received", + ), + sa.Column( + "attempt_count", sa.Integer(), nullable=False, server_default="0" + ), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "received_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.Column("processed_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("(now() AT TIME ZONE 'utc')"), + ), + sa.ForeignKeyConstraint( + ["account_id"], ["external_chat_accounts.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["external_chat_binding_id"], + ["external_chat_bindings.id"], + ondelete="SET NULL", + ), + sa.UniqueConstraint( + "account_id", + "event_dedupe_key", + name="uq_external_chat_inbound_account_dedupe_key", + ), + ) + op.create_index( + "ix_external_chat_inbound_status_received_at", + "external_chat_inbound_events", + ["status", "received_at"], + if_not_exists=True, + ) + op.create_index( + "ix_external_chat_inbound_binding_received_at", + "external_chat_inbound_events", + ["external_chat_binding_id", "received_at"], + if_not_exists=True, + ) + op.create_index( + "ix_external_chat_inbound_request_id", + "external_chat_inbound_events", + ["request_id"], + postgresql_where=sa.text("request_id IS NOT NULL"), + if_not_exists=True, + ) + + if not _column_exists(conn, "new_chat_threads", "source"): + op.add_column( + "new_chat_threads", + sa.Column("source", sa.Text(), nullable=False, server_default="surfsense"), + ) + op.alter_column("new_chat_threads", "source", type_=sa.Text()) + if not _column_exists(conn, "new_chat_threads", "external_chat_binding_id"): + op.add_column( + "new_chat_threads", + sa.Column("external_chat_binding_id", sa.BigInteger(), nullable=True), + ) + if not _constraint_exists( + conn, + "new_chat_threads", + "fk_new_chat_threads_external_chat_external_chat_binding_id", + ): + op.create_foreign_key( + "fk_new_chat_threads_external_chat_external_chat_binding_id", + "new_chat_threads", + "external_chat_bindings", + ["external_chat_binding_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index( + "ix_new_chat_threads_source", "new_chat_threads", ["source"], if_not_exists=True + ) + op.create_index( + "ix_new_chat_threads_external_chat_binding_id", + "new_chat_threads", + ["external_chat_binding_id"], + if_not_exists=True, + ) + + if not _column_exists(conn, "new_chat_messages", "source"): + op.add_column( + "new_chat_messages", + sa.Column("source", sa.Text(), nullable=False, server_default="surfsense"), + ) + op.alter_column("new_chat_messages", "source", type_=sa.Text()) + if not _column_exists(conn, "new_chat_messages", "platform_metadata"): + op.add_column( + "new_chat_messages", + sa.Column( + "platform_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + ), + ) + op.create_index( + "ix_new_chat_messages_source", + "new_chat_messages", + ["source"], + if_not_exists=True, + ) + op.create_index( + "uq_new_chat_messages_inbound_platform", + "new_chat_messages", + [ + "thread_id", + sa.text("(platform_metadata->>'platform')"), + sa.text("(platform_metadata->>'external_message_id')"), + ], + unique=True, + postgresql_where=sa.text( + "platform_metadata IS NOT NULL " + "AND platform_metadata->>'direction' = 'inbound'" + ), + if_not_exists=True, + ) + op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL") + + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if exists: + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-external-chat'" + ) + ) + conn.execute( + sa.text( + _build_set_table_ddl( + documents_has_zero_ver=documents_has_zero_ver, + user_has_zero_ver=user_has_zero_ver, + ) + ) + ) + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-external-chat'" + ) + ) + + +def downgrade() -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if exists: + documents_has_zero_ver = _has_zero_version(conn, "documents") + user_has_zero_ver = _has_zero_version(conn, "user") + # Restore the publication shape from migration 148. + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) + user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) + ddl = ( + f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + f"notifications, " + f"documents ({_cols(doc_cols)}), " + f"folders, " + f"search_source_connectors, " + f"new_chat_messages, " + f"chat_comments, " + f"chat_session_state, " + f'"user" ({_cols(user_cols)}), ' + f"automation_runs ({_cols(AUTOMATION_RUN_COLS)})" + ) + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-144-downgrade'" + ) + ) + conn.execute(sa.text(ddl)) + conn.execute( + sa.text( + f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-144-downgrade'" + ) + ) + + if _column_exists(conn, "new_chat_messages", "source"): + op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY DEFAULT") + _drop_index_if_exists("uq_new_chat_messages_inbound_platform", "new_chat_messages") + _drop_index_if_exists("ix_new_chat_messages_source", "new_chat_messages") + _drop_column_if_exists("new_chat_messages", "platform_metadata") + _drop_column_if_exists("new_chat_messages", "source") + + _drop_index_if_exists( + "ix_new_chat_threads_external_chat_binding_id", "new_chat_threads" + ) + _drop_index_if_exists("ix_new_chat_threads_source", "new_chat_threads") + if _constraint_exists( + conn, + "new_chat_threads", + "fk_new_chat_threads_external_chat_external_chat_binding_id", + ): + op.drop_constraint( + "fk_new_chat_threads_external_chat_external_chat_binding_id", + "new_chat_threads", + type_="foreignkey", + ) + _drop_column_if_exists("new_chat_threads", "external_chat_binding_id") + _drop_column_if_exists("new_chat_threads", "source") + + _drop_index_if_exists( + "ix_external_chat_inbound_binding_received_at", "external_chat_inbound_events" + ) + _drop_index_if_exists( + "ix_external_chat_inbound_request_id", "external_chat_inbound_events" + ) + _drop_index_if_exists( + "ix_external_chat_inbound_status_received_at", "external_chat_inbound_events" + ) + if _table_exists(conn, "external_chat_inbound_events"): + op.drop_table("external_chat_inbound_events") + + _drop_index_if_exists( + "ix_external_chat_bindings_search_space_state", + "external_chat_bindings", + ) + _drop_index_if_exists( + "ix_external_chat_bindings_user_state", "external_chat_bindings" + ) + _drop_index_if_exists( + "uq_external_chat_bindings_pairing_code_pending", + "external_chat_bindings", + ) + _drop_index_if_exists( + "uq_external_chat_bindings_account_peer_active", + "external_chat_bindings", + ) + if _table_exists(conn, "external_chat_bindings"): + op.drop_table("external_chat_bindings") + + _drop_index_if_exists( + "uq_external_chat_accounts_system_platform", "external_chat_accounts" + ) + _drop_index_if_exists( + "uq_external_chat_accounts_owner_platform", "external_chat_accounts" + ) + _drop_index_if_exists( + "uq_external_chat_accounts_webhook_secret", "external_chat_accounts" + ) + if _table_exists(conn, "external_chat_accounts"): + op.drop_table("external_chat_accounts") + + for enum_name in ( + "external_chat_event_status", + "external_chat_event_kind", + "external_chat_peer_kind", + "external_chat_binding_state", + "external_chat_health_status", + "external_chat_account_mode", + "external_chat_platform", + ): + postgresql.ENUM(name=enum_name).drop(conn, checkfirst=True) diff --git a/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py new file mode 100644 index 000000000..388d8ef42 --- /dev/null +++ b/surfsense_backend/alembic/versions/150_add_slack_gateway_platform.py @@ -0,0 +1,102 @@ +"""add slack gateway platform + +Revision ID: 150 +Revises: 149 +Create Date: 2026-05-31 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "150" +down_revision: str | None = "149" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _enum_value_exists(enum_name: str, value: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_enum e " + "JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = :enum_name AND e.enumlabel = :value" + ), + {"enum_name": enum_name, "value": value}, + ).fetchone() + is not None + ) + + +def _index_exists(index_name: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :index_name" + ), + {"index_name": index_name}, + ).fetchone() + is not None + ) + + +def upgrade() -> None: + if not _enum_value_exists("external_chat_platform", "slack"): + op.execute("ALTER TYPE external_chat_platform ADD VALUE 'slack'") + + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND NOT (cursor_state ? 'team_id')" + ), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_slack_team", + "external_chat_accounts", + ["platform", sa.text("(cursor_state ->> 'team_id')")], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND cursor_state ? 'team_id'" + ), + if_not_exists=True, + ) + + +def downgrade() -> None: + if _index_exists("uq_external_chat_accounts_slack_team"): + op.drop_index( + "uq_external_chat_accounts_slack_team", + table_name="external_chat_accounts", + ) + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text("is_system_account = true"), + if_not_exists=True, + ) + # PostgreSQL enum values are intentionally not removed on downgrade. diff --git a/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py b/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py new file mode 100644 index 000000000..f91e71210 --- /dev/null +++ b/surfsense_backend/alembic/versions/151_add_discord_gateway_platform.py @@ -0,0 +1,106 @@ +"""add discord gateway platform + +Revision ID: 151 +Revises: 150 +Create Date: 2026-06-01 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "151" +down_revision: str | None = "150" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _enum_value_exists(enum_name: str, value: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_enum e " + "JOIN pg_type t ON t.oid = e.enumtypid " + "WHERE t.typname = :enum_name AND e.enumlabel = :value" + ), + {"enum_name": enum_name, "value": value}, + ).fetchone() + is not None + ) + + +def _index_exists(index_name: str) -> bool: + conn = op.get_bind() + return ( + conn.execute( + sa.text( + "SELECT 1 FROM pg_indexes " + "WHERE schemaname = current_schema() AND indexname = :index_name" + ), + {"index_name": index_name}, + ).fetchone() + is not None + ) + + +def upgrade() -> None: + if not _enum_value_exists("external_chat_platform", "discord"): + op.execute("ALTER TYPE external_chat_platform ADD VALUE 'discord'") + + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true " + "AND NOT (cursor_state ? 'team_id') " + "AND NOT (cursor_state ? 'guild_id')" + ), + if_not_exists=True, + ) + op.create_index( + "uq_external_chat_accounts_discord_guild", + "external_chat_accounts", + ["platform", sa.text("(cursor_state ->> 'guild_id')")], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND cursor_state ? 'guild_id'" + ), + if_not_exists=True, + ) + + +def downgrade() -> None: + if _index_exists("uq_external_chat_accounts_discord_guild"): + op.drop_index( + "uq_external_chat_accounts_discord_guild", + table_name="external_chat_accounts", + ) + if _index_exists("uq_external_chat_accounts_system_platform"): + op.drop_index( + "uq_external_chat_accounts_system_platform", + table_name="external_chat_accounts", + ) + op.create_index( + "uq_external_chat_accounts_system_platform", + "external_chat_accounts", + ["platform"], + unique=True, + postgresql_where=sa.text( + "is_system_account = true AND NOT (cursor_state ? 'team_id')" + ), + if_not_exists=True, + ) + # PostgreSQL enum values are intentionally not removed on downgrade. diff --git a/surfsense_backend/alembic/versions/152_add_document_files.py b/surfsense_backend/alembic/versions/152_add_document_files.py new file mode 100644 index 000000000..034399640 --- /dev/null +++ b/surfsense_backend/alembic/versions/152_add_document_files.py @@ -0,0 +1,85 @@ +"""add document_files table for stored original uploads + +Revision ID: 152 +Revises: 151 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "152" +down_revision: str | None = "151" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # The enum type must precede the table that references it. + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'document_file_kind' + ) THEN + CREATE TYPE document_file_kind AS ENUM ( + 'ORIGINAL', 'REDACTED', 'FILLED_FORM' + ); + END IF; + END + $$; + """ + ) + + op.execute( + """ + CREATE TABLE IF NOT EXISTS document_files ( + id SERIAL PRIMARY KEY, + document_id INTEGER NOT NULL + REFERENCES documents(id) ON DELETE CASCADE, + search_space_id INTEGER NOT NULL + REFERENCES searchspaces(id) ON DELETE CASCADE, + kind document_file_kind NOT NULL DEFAULT 'ORIGINAL', + storage_backend VARCHAR(32) NOT NULL, + storage_key TEXT NOT NULL, + original_filename TEXT NOT NULL, + mime_type TEXT, + size_bytes BIGINT NOT NULL, + checksum_sha256 VARCHAR(64), + created_by_id UUID + REFERENCES "user"(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ); + """ + ) + + op.execute( + "CREATE INDEX IF NOT EXISTS ix_document_files_document_id " + "ON document_files(document_id);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_document_files_search_space_id " + "ON document_files(search_space_id);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_document_files_kind ON document_files(kind);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_document_files_created_by_id " + "ON document_files(created_by_id);" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_document_files_created_at " + "ON document_files(created_at);" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_document_files_created_at;") + op.execute("DROP INDEX IF EXISTS ix_document_files_created_by_id;") + op.execute("DROP INDEX IF EXISTS ix_document_files_kind;") + op.execute("DROP INDEX IF EXISTS ix_document_files_search_space_id;") + op.execute("DROP INDEX IF EXISTS ix_document_files_document_id;") + op.execute("DROP TABLE IF EXISTS document_files;") + op.execute("DROP TYPE IF EXISTS document_file_kind;") diff --git a/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py b/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py new file mode 100644 index 000000000..320e75465 --- /dev/null +++ b/surfsense_backend/alembic/versions/153_restore_automation_runs_to_zero_publication.py @@ -0,0 +1,121 @@ +"""restore automation_runs to zero_publication + +Migration 149's ``SET TABLE`` dropped ``automation_runs`` (added in 148), +breaking the dashboard live run ticker with a SchemaVersionNotSupported +reload loop. Re-emit the publication with ``automation_runs`` using the +``COMMENT`` bookend pattern so zero-cache fires its schema-change hook. + +Revision ID: 153 +Revises: 152 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "153" +down_revision: str | None = "152" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _has_zero_version(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = '_0_version'" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _set_table_ddl(*, with_automation_runs: bool, conn) -> str: + doc_cols = DOCUMENT_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "documents") else [] + ) + user_cols = USER_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "user") else [] + ) + tables = [ + "notifications", + f"documents ({', '.join(doc_cols)})", + "folders", + "search_source_connectors", + "new_chat_messages", + "chat_comments", + "chat_session_state", + f'"user" ({", ".join(user_cols)})', + ] + if with_automation_runs: + tables.append(f"automation_runs ({', '.join(AUTOMATION_RUN_COLS)})") + return f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + ", ".join(tables) + + +def _resync(*, with_automation_runs: bool, tag: str) -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'") + ) + conn.execute( + sa.text( + _set_table_ddl(with_automation_runs=with_automation_runs, conn=conn) + ) + ) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'") + ) + + +def upgrade() -> None: + _resync(with_automation_runs=True, tag="153-resync") + + +def downgrade() -> None: + _resync(with_automation_runs=False, tag="153-downgrade") diff --git a/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py new file mode 100644 index 000000000..94f1bf4ce --- /dev/null +++ b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py @@ -0,0 +1,147 @@ +"""remove document summary llm settings + +Revision ID: 154 +Revises: 153 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "154" +down_revision: str | None = "153" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :table AND column_name = :column" + ), + {"table": table, "column": column}, + ).fetchone() + is not None + ) + + +def _has_zero_version(conn, table: str) -> bool: + return _column_exists(conn, table, "_0_version") + + +def _set_table_ddl(conn) -> str: + doc_cols = DOCUMENT_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "documents") else [] + ) + user_cols = USER_COLS + ( + ['"_0_version"'] if _has_zero_version(conn, "user") else [] + ) + tables = [ + "notifications", + f"documents ({', '.join(doc_cols)})", + "folders", + "search_source_connectors", + "new_chat_messages", + "chat_comments", + "chat_session_state", + f'"user" ({", ".join(user_cols)})', + f"automation_runs ({', '.join(AUTOMATION_RUN_COLS)})", + ] + return f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + ", ".join(tables) + + +def _resync_zero_publication(tag: str) -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'") + ) + conn.execute(sa.text(_set_table_ddl(conn))) + conn.execute( + sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'") + ) + + +def upgrade() -> None: + conn = op.get_bind() + + if _column_exists(conn, "searchspaces", "document_summary_llm_id"): + op.drop_column("searchspaces", "document_summary_llm_id") + + if _column_exists(conn, "search_source_connectors", "enable_summary"): + op.drop_column("search_source_connectors", "enable_summary") + + _resync_zero_publication("154-summary-removal") + + +def downgrade() -> None: + conn = op.get_bind() + + if not _column_exists(conn, "searchspaces", "document_summary_llm_id"): + op.add_column( + "searchspaces", + sa.Column( + "document_summary_llm_id", + sa.Integer(), + nullable=True, + server_default="0", + ), + ) + + if not _column_exists(conn, "search_source_connectors", "enable_summary"): + op.add_column( + "search_source_connectors", + sa.Column( + "enable_summary", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + + _resync_zero_publication("154-summary-removal-downgrade") diff --git a/surfsense_backend/alembic/versions/155_reconcile_zero_publication.py b/surfsense_backend/alembic/versions/155_reconcile_zero_publication.py new file mode 100644 index 000000000..1d2e6ed34 --- /dev/null +++ b/surfsense_backend/alembic/versions/155_reconcile_zero_publication.py @@ -0,0 +1,23 @@ +"""reconcile zero_publication from canonical definition + +Revision ID: 155 +Revises: 154 +""" + +from collections.abc import Sequence + +from alembic import op +from app.zero_publication import apply_publication + +revision: str = "155" +down_revision: str | None = "154" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + apply_publication(op.get_bind()) + + +def downgrade() -> None: + """No-op. Historical publication shapes are immutable.""" diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py deleted file mode 100644 index 890b3e06e..000000000 --- a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py +++ /dev/null @@ -1,557 +0,0 @@ -"""Vision autocomplete agent with scoped filesystem exploration. - -Converts the stateless single-shot vision autocomplete into an agent that -seeds a virtual filesystem from KB search results and lets the vision LLM -explore documents via ``ls``, ``read_file``, ``glob``, ``grep``, etc. -before generating the final completion. - -Performance: KB search and agent graph compilation run in parallel so -the only sequential latency is KB-search (or agent compile, whichever is -slower) + the agent's LLM turns. There is no separate "query extraction" -LLM call — the window title is used directly as the KB search query. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import re -import uuid -from collections.abc import AsyncGenerator -from typing import Any - -from deepagents.graph import BASE_AGENT_PROMPT -from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware -from langchain.agents import create_agent -from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware -from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, ToolMessage - -from app.agents.new_chat.document_xml import build_document_xml -from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware -from app.agents.new_chat.middleware.knowledge_search import ( - search_knowledge_base, -) -from app.agents.new_chat.path_resolver import ( - DOCUMENTS_ROOT, - build_path_index, - doc_to_virtual_path, -) -from app.db import shielded_async_session -from app.services.new_streaming_service import VercelStreamingService - -try: - from deepagents.backends.utils import create_file_data -except Exception: # pragma: no cover - defensive - - def create_file_data(content: str) -> dict[str, Any]: - return {"content": content.split("\n")} - - -async def _build_autocomplete_filesystem( - *, - documents: Any, - search_space_id: int, -) -> tuple[dict[str, Any], dict[int, str]]: - """Build a ``state['files']``-shaped dict from KB search results. - - This is the autocomplete-specific replacement for the previous - ``build_scoped_filesystem`` helper. It uses the canonical path resolver - so paths line up with the rest of the system, including collision - suffixes for duplicate titles. - """ - files: dict[str, Any] = {} - doc_id_to_path: dict[int, str] = {} - - if not documents: - return files, doc_id_to_path - - async with shielded_async_session() as session: - index = await build_path_index(session, search_space_id) - - for document in documents: - if not isinstance(document, dict): - continue - meta = document.get("document") or {} - doc_id = meta.get("id") - if not isinstance(doc_id, int): - continue - title = str(meta.get("title") or "untitled") - folder_id = meta.get("folder_id") - path = doc_to_virtual_path( - doc_id=doc_id, title=title, folder_id=folder_id, index=index - ) - chunk_ids = document.get("matched_chunk_ids") or [] - try: - matched_set = {int(c) for c in chunk_ids} - except (TypeError, ValueError): - matched_set = set() - xml = build_document_xml(document, matched_chunk_ids=matched_set) - files[path] = create_file_data(xml) - doc_id_to_path[doc_id] = path - - if not files: - # Ensure the synthetic /documents folder is visible even when empty. - files.setdefault(f"{DOCUMENTS_ROOT}/.placeholder", create_file_data("")) - - return files, doc_id_to_path - - -logger = logging.getLogger(__name__) - -KB_TOP_K = 10 - -# --------------------------------------------------------------------------- -# System prompt -# --------------------------------------------------------------------------- - -AUTOCOMPLETE_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text. - -You will receive a screenshot of the user's screen. Your PRIMARY source of truth is the screenshot itself — the visual context determines what to write. - -Your job: -1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.). -2. Identify the text area where the user will type. -3. Generate the text the user most likely wants to write based on the visual context. - -You also have access to the user's knowledge base documents via filesystem tools. However: -- ONLY consult the knowledge base if the screenshot clearly involves a topic where your KB documents are DIRECTLY relevant (e.g., the user is writing about a specific project/topic that matches a document title). -- Do NOT explore documents just because they exist. Most autocomplete requests can be answered purely from the screenshot. -- If you do read a document, only incorporate information that is 100% relevant to what the user is typing RIGHT NOW. Do not add extra details, background, or tangential information from the KB. -- Keep your output SHORT — autocomplete should feel like a natural continuation, not an essay. - -Key behavior: -- If the text area is EMPTY, draft a concise response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document). -- If the text area already has text, continue it naturally — typically just a sentence or two. - -Rules: -- Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft. -- Match the tone and formality of the surrounding context. -- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal. -- Do NOT describe the screenshot or explain your reasoning. -- Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally. -- If you cannot determine what to write, output an empty JSON array: [] - -## Output Format - -You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle. - -Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary. - -Example format: -["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."] - -## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep` - -All file paths must start with a `/`. -- ls: list files and directories at a given path. -- read_file: read a file from the filesystem. -- write_file: create a temporary file in the session (not persisted). -- edit_file: edit a file in the session (not persisted for /documents/ files). -- glob: find files matching a pattern (e.g., "**/*.xml"). -- grep: search for text within files. - -## When to Use Filesystem Tools - -BEFORE reaching for any tool, ask yourself: "Can I write a good completion purely from the screenshot?" If yes, just write it — do NOT explore the KB. - -Only use tools when: -- The user is clearly writing about a specific topic that likely has detailed information in their KB. -- You need a specific fact, name, number, or reference that the screenshot doesn't provide. - -When you do use tools, be surgical: -- Check the `ls` output first. If no document title looks relevant, stop — do not read files just to see what's there. -- If a title looks relevant, read only the `` (first ~20 lines) and jump to matched chunks. Do not read entire documents. -- Extract only the specific information you need and move on to generating the completion. - -## Reading Documents Efficiently - -Documents are formatted as XML. Each document contains: -- `` — title, type, URL, etc. -- `` — a table of every chunk with its **line range** and a - `matched="true"` flag for chunks that matched the search query. -- `` — the actual chunks in original document order. - -**Workflow**: read the first ~20 lines to see the ``, identify -chunks marked `matched="true"`, then use `read_file(path, offset=, -limit=)` to jump directly to those sections.""" - -APP_CONTEXT_BLOCK = """ - -The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly.""" - - -def _build_autocomplete_system_prompt(app_name: str, window_title: str) -> str: - prompt = AUTOCOMPLETE_SYSTEM_PROMPT - if app_name: - prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title) - return prompt - - -# --------------------------------------------------------------------------- -# Pre-compute KB filesystem (runs in parallel with agent compilation) -# --------------------------------------------------------------------------- - - -class _KBResult: - """Container for pre-computed KB filesystem results.""" - - __slots__ = ("files", "ls_ai_msg", "ls_tool_msg") - - def __init__( - self, - files: dict[str, Any] | None = None, - ls_ai_msg: AIMessage | None = None, - ls_tool_msg: ToolMessage | None = None, - ) -> None: - self.files = files - self.ls_ai_msg = ls_ai_msg - self.ls_tool_msg = ls_tool_msg - - @property - def has_documents(self) -> bool: - return bool(self.files) - - -async def precompute_kb_filesystem( - search_space_id: int, - query: str, - top_k: int = KB_TOP_K, -) -> _KBResult: - """Search the KB and build the scoped filesystem outside the agent. - - This is designed to be called via ``asyncio.gather`` alongside agent - graph compilation so the two run concurrently. - """ - if not query: - return _KBResult() - - try: - search_results = await search_knowledge_base( - query=query, - search_space_id=search_space_id, - top_k=top_k, - ) - - if not search_results: - return _KBResult() - - new_files, _ = await _build_autocomplete_filesystem( - documents=search_results, - search_space_id=search_space_id, - ) - - if not new_files: - return _KBResult() - - doc_paths = [ - p - for p, v in new_files.items() - if p.startswith("/documents/") and v is not None - ] - tool_call_id = f"auto_ls_{uuid.uuid4().hex[:12]}" - ai_msg = AIMessage( - content="", - tool_calls=[ - {"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id} - ], - ) - tool_msg = ToolMessage( - content=str(doc_paths) if doc_paths else "No documents found.", - tool_call_id=tool_call_id, - ) - return _KBResult(files=new_files, ls_ai_msg=ai_msg, ls_tool_msg=tool_msg) - - except Exception: - logger.warning( - "KB pre-computation failed, proceeding without KB", exc_info=True - ) - return _KBResult() - - -# --------------------------------------------------------------------------- -# Filesystem middleware — no save_document, no persistence -# --------------------------------------------------------------------------- - - -class AutocompleteFilesystemMiddleware(SurfSenseFilesystemMiddleware): - """Filesystem middleware for autocomplete — read-only exploration only. - - Passes ``search_space_id=None`` so the new persistence pipeline is - bypassed; the autocomplete flow only reads, never commits to Postgres. - """ - - def __init__(self) -> None: - super().__init__(search_space_id=None, created_by_id=None) - - -# --------------------------------------------------------------------------- -# Agent factory -# --------------------------------------------------------------------------- - - -async def _compile_agent( - llm: BaseChatModel, - app_name: str, - window_title: str, -) -> Any: - """Compile the agent graph (CPU-bound, runs in a thread).""" - system_prompt = _build_autocomplete_system_prompt(app_name, window_title) - final_system_prompt = system_prompt + "\n\n" + BASE_AGENT_PROMPT - - middleware = [ - AutocompleteFilesystemMiddleware(), - PatchToolCallsMiddleware(), - AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"), - ] - - agent = await asyncio.to_thread( - create_agent, - llm, - system_prompt=final_system_prompt, - tools=[], - middleware=middleware, - ) - return agent.with_config({"recursion_limit": 200}) - - -async def create_autocomplete_agent( - llm: BaseChatModel, - *, - search_space_id: int, - kb_query: str, - app_name: str = "", - window_title: str = "", -) -> tuple[Any, _KBResult]: - """Create the autocomplete agent and pre-compute KB in parallel. - - Returns ``(agent, kb_result)`` so the caller can inject the pre-computed - filesystem into the agent's initial state without any middleware delay. - """ - agent, kb = await asyncio.gather( - _compile_agent(llm, app_name, window_title), - precompute_kb_filesystem(search_space_id, kb_query), - ) - return agent, kb - - -# --------------------------------------------------------------------------- -# JSON suggestion parsing (with fallback) -# --------------------------------------------------------------------------- - - -def _parse_suggestions(raw: str) -> list[str]: - """Extract a list of suggestion strings from the agent's output. - - Tries, in order: - 1. Direct ``json.loads`` - 2. Extract content between ```json ... ``` fences - 3. Find the first ``[`` … ``]`` span - Falls back to wrapping the raw text as a single suggestion. - """ - text = raw.strip() - if not text: - return [] - - for candidate in _json_candidates(text): - try: - parsed = json.loads(candidate) - if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed): - return [s for s in parsed if s.strip()] - except (json.JSONDecodeError, ValueError): - continue - - return [text] - - -def _json_candidates(text: str) -> list[str]: - """Yield candidate JSON strings from raw text.""" - candidates = [text] - - fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL) - if fence: - candidates.append(fence.group(1).strip()) - - bracket = re.search(r"\[.*]", text, re.DOTALL) - if bracket: - candidates.append(bracket.group(0)) - - return candidates - - -# --------------------------------------------------------------------------- -# Streaming helper -# --------------------------------------------------------------------------- - - -async def stream_autocomplete_agent( - agent: Any, - input_data: dict[str, Any], - streaming_service: VercelStreamingService, - *, - emit_message_start: bool = True, -) -> AsyncGenerator[str, None]: - """Stream agent events as Vercel SSE, with thinking steps for tool calls. - - When ``emit_message_start`` is False the caller has already sent the - ``message_start`` event (e.g. to show preparation steps before the agent - runs). - """ - thread_id = uuid.uuid4().hex - config = {"configurable": {"thread_id": thread_id}} - - text_buffer: list[str] = [] - active_tool_depth = 0 - thinking_step_counter = 0 - tool_step_ids: dict[str, str] = {} - step_titles: dict[str, str] = {} - completed_step_ids: set[str] = set() - last_active_step_id: str | None = None - - def next_thinking_step_id() -> str: - nonlocal thinking_step_counter - thinking_step_counter += 1 - return f"autocomplete-step-{thinking_step_counter}" - - def complete_current_step() -> str | None: - nonlocal last_active_step_id - if last_active_step_id and last_active_step_id not in completed_step_ids: - completed_step_ids.add(last_active_step_id) - title = step_titles.get(last_active_step_id, "Done") - event = streaming_service.format_thinking_step( - step_id=last_active_step_id, - title=title, - status="complete", - ) - last_active_step_id = None - return event - return None - - if emit_message_start: - yield streaming_service.format_message_start() - - gen_step_id = next_thinking_step_id() - last_active_step_id = gen_step_id - step_titles[gen_step_id] = "Generating suggestions" - yield streaming_service.format_thinking_step( - step_id=gen_step_id, - title="Generating suggestions", - status="in_progress", - ) - - try: - async for event in agent.astream_events( - input_data, config=config, version="v2" - ): - event_type = event.get("event", "") - if event_type == "on_chat_model_stream": - if active_tool_depth > 0: - continue - if "surfsense:internal" in event.get("tags", []): - continue - chunk = event.get("data", {}).get("chunk") - if chunk and hasattr(chunk, "content"): - content = chunk.content - if content and isinstance(content, str): - text_buffer.append(content) - - elif event_type == "on_chat_model_end": - if active_tool_depth > 0: - continue - if "surfsense:internal" in event.get("tags", []): - continue - output = event.get("data", {}).get("output") - if output and hasattr(output, "content"): - if getattr(output, "tool_calls", None): - continue - content = output.content - if content and isinstance(content, str) and not text_buffer: - text_buffer.append(content) - - elif event_type == "on_tool_start": - active_tool_depth += 1 - tool_name = event.get("name", "unknown_tool") - run_id = event.get("run_id", "") - tool_input = event.get("data", {}).get("input", {}) - - step_event = complete_current_step() - if step_event: - yield step_event - - tool_step_id = next_thinking_step_id() - tool_step_ids[run_id] = tool_step_id - last_active_step_id = tool_step_id - - title, items = _describe_tool_call(tool_name, tool_input) - step_titles[tool_step_id] = title - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title=title, - status="in_progress", - items=items, - ) - - elif event_type == "on_tool_end": - active_tool_depth = max(0, active_tool_depth - 1) - run_id = event.get("run_id", "") - step_id = tool_step_ids.pop(run_id, None) - if step_id and step_id not in completed_step_ids: - completed_step_ids.add(step_id) - title = step_titles.get(step_id, "Done") - yield streaming_service.format_thinking_step( - step_id=step_id, - title=title, - status="complete", - ) - if last_active_step_id == step_id: - last_active_step_id = None - - step_event = complete_current_step() - if step_event: - yield step_event - - raw_text = "".join(text_buffer) - suggestions = _parse_suggestions(raw_text) - - yield streaming_service.format_data("suggestions", {"options": suggestions}) - - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True) - yield streaming_service.format_error("Autocomplete failed. Please try again.") - yield streaming_service.format_done() - - -def _describe_tool_call(tool_name: str, tool_input: Any) -> tuple[str, list[str]]: - """Return a human-readable (title, items) for a tool call thinking step.""" - inp = tool_input if isinstance(tool_input, dict) else {} - if tool_name == "ls": - path = inp.get("path", "/") - return "Listing files", [path] - if tool_name == "read_file": - fp = inp.get("file_path", "") - display = fp if len(fp) <= 80 else "…" + fp[-77:] - return "Reading file", [display] - if tool_name == "write_file": - fp = inp.get("file_path", "") - display = fp if len(fp) <= 80 else "…" + fp[-77:] - return "Writing file", [display] - if tool_name == "edit_file": - fp = inp.get("file_path", "") - display = fp if len(fp) <= 80 else "…" + fp[-77:] - return "Editing file", [display] - if tool_name == "glob": - pat = inp.get("pattern", "") - base = inp.get("path", "/") - return "Searching files", [f"{pat} in {base}"] - if tool_name == "grep": - pat = inp.get("pattern", "") - path = inp.get("path", "") - display_pat = pat[:60] + ("…" if len(pat) > 60 else "") - return "Searching content", [ - f'"{display_pat}"' + (f" in {path}" if path else "") - ] - return f"Using {tool_name}", [] diff --git a/surfsense_backend/app/agents/chat/__init__.py b/surfsense_backend/app/agents/chat/__init__.py new file mode 100644 index 000000000..4f6b7d07f --- /dev/null +++ b/surfsense_backend/app/agents/chat/__init__.py @@ -0,0 +1,5 @@ +"""Chat agents category. + +Groups the conversational agents that share a kernel: ``anonymous_chat`` and +``multi_agent_chat``. Code shared by *both* lives in ``chat/shared/``. +""" diff --git a/surfsense_backend/app/agents/chat/anonymous_chat/__init__.py b/surfsense_backend/app/agents/chat/anonymous_chat/__init__.py new file mode 100644 index 000000000..ba3b2a6f1 --- /dev/null +++ b/surfsense_backend/app/agents/chat/anonymous_chat/__init__.py @@ -0,0 +1,14 @@ +"""Anonymous / free-chat agent. + +The no-login chat experience: a deliberately minimal agent that bypasses the +full SurfSense deep-agent stack (filesystem, knowledge-base persistence, +subagents, skills, memory) and answers with an optional ``web_search`` tool and +an optional read-only uploaded document. See :mod:`.agent` for details. +""" + +from app.agents.chat.anonymous_chat.agent import ( + build_anonymous_system_prompt, + create_anonymous_chat_agent, +) + +__all__ = ["build_anonymous_system_prompt", "create_anonymous_chat_agent"] diff --git a/surfsense_backend/app/agents/new_chat/anonymous_agent.py b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py similarity index 97% rename from surfsense_backend/app/agents/new_chat/anonymous_agent.py rename to surfsense_backend/app/agents/chat/anonymous_chat/agent.py index c783d9a45..250b4c158 100644 --- a/surfsense_backend/app/agents/new_chat/anonymous_agent.py +++ b/surfsense_backend/app/agents/chat/anonymous_chat/agent.py @@ -27,12 +27,12 @@ from langchain.agents.middleware import ( from langchain_core.language_models import BaseChatModel from langgraph.types import Checkpointer -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.middleware import ( +from app.agents.chat.shared.context import SurfSenseContextSchema +from app.agents.chat.shared.middleware import ( RetryAfterMiddleware, create_surfsense_compaction_middleware, ) -from app.agents.new_chat.tools.web_search import create_web_search_tool +from app.agents.chat.shared.tools.web_search import create_web_search_tool # Cap how much of an uploaded document we inline into the system prompt. The # upload endpoint allows files up to several MB, but the doc is re-sent on diff --git a/surfsense_backend/app/agents/multi_agent_chat/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/constants.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/constants.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/constants.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/context_prune/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/context_prune/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/context_prune/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/context_prune/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/context_prune/prune_tool_names.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/context_prune/prune_tool_names.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/context_prune/prune_tool_names.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/context_prune/prune_tool_names.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/graph/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/graph/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/graph/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/graph/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/graph/compile_graph_sync.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/graph/compile_graph_sync.py similarity index 75% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/graph/compile_graph_sync.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/graph/compile_graph_sync.py index b86da932a..e3ab50e8c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/graph/compile_graph_sync.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/graph/compile_graph_sync.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from collections.abc import Sequence from typing import Any @@ -11,13 +12,16 @@ from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool from langgraph.types import Checkpointer -from app.agents.multi_agent_chat.middleware.stack import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.stack import ( build_main_agent_deepagent_middleware, ) -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.shared.context import SurfSenseContextSchema from app.db import ChatVisibility +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() def build_compiled_agent_graph_sync( @@ -43,6 +47,7 @@ def build_compiled_agent_graph_sync( disabled_tools: list[str] | None = None, ): """Sync compile: middleware + ``create_agent`` (run via ``asyncio.to_thread``).""" + mw_start = time.perf_counter() main_agent_middleware = build_main_agent_deepagent_middleware( llm=llm, tools=tools, @@ -63,7 +68,9 @@ def build_compiled_agent_graph_sync( mcp_tools_by_agent=mcp_tools_by_agent, disabled_tools=disabled_tools, ) + mw_elapsed = time.perf_counter() - mw_start + create_start = time.perf_counter() agent = create_agent( llm, system_prompt=final_system_prompt, @@ -72,6 +79,15 @@ def build_compiled_agent_graph_sync( context_schema=SurfSenseContextSchema, checkpointer=checkpointer, ) + create_elapsed = time.perf_counter() - create_start + _perf_log.info( + "[graph_compile] middleware_build=%.3fs main_create_agent=%.3fs " + "total=%.3fs mw_count=%d", + mw_elapsed, + create_elapsed, + mw_elapsed + create_elapsed, + len(main_agent_middleware), + ) return agent.with_config( { "recursion_limit": 10_000, diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/__init__.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/__init__.py new file mode 100644 index 000000000..46fa28009 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/__init__.py @@ -0,0 +1,10 @@ +"""Action-log middleware: audit row per tool call (impl + builder).""" + +from .builder import build_action_log_mw +from .middleware import ActionLogMiddleware, ToolDefinition + +__all__ = [ + "ActionLogMiddleware", + "ToolDefinition", + "build_action_log_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/action_log.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/builder.py similarity index 62% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/action_log.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/builder.py index c9f893d97..9213f1339 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/action_log.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/builder.py @@ -4,11 +4,10 @@ from __future__ import annotations import logging -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import ActionLogMiddleware -from app.agents.new_chat.tools.registry import BUILTIN_TOOLS +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import ActionLogMiddleware def build_action_log_mw( @@ -21,12 +20,13 @@ def build_action_log_mw( if not enabled(flags, "enable_action_log") or thread_id is None: return None try: - tool_defs_by_name = {td.name: td for td in BUILTIN_TOOLS} + # No built-in tool declares a ``reverse`` callable yet, so the action + # log runs without a tool_definitions map. Reversibility is opt-in per + # tool via ``ToolDefinition.reverse`` and can be wired here when used. return ActionLogMiddleware( thread_id=thread_id, search_space_id=search_space_id, user_id=user_id, - tool_definitions=tool_defs_by_name, ) except Exception: # pragma: no cover - defensive logging.warning( diff --git a/surfsense_backend/app/agents/new_chat/middleware/action_log.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/middleware.py similarity index 73% rename from surfsense_backend/app/agents/new_chat/middleware/action_log.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/middleware.py index 716a1616c..2cce7eb53 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/action_log.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/action_log/middleware.py @@ -1,25 +1,15 @@ """Append-only action-log middleware for the SurfSense agent. -Wraps every tool call via :meth:`AgentMiddleware.awrap_tool_call` and writes -a row to :class:`~app.db.AgentActionLog` after the tool returns. Tools opt -into reversibility by declaring a ``reverse`` callable on their -:class:`~app.agents.new_chat.tools.registry.ToolDefinition`; the rendered -descriptor is persisted in ``reverse_descriptor`` for use by +Wraps every tool call and writes a row to :class:`~app.db.AgentActionLog` +after the tool returns. Tools opt into reversibility via a ``reverse`` +callable on their :class:`ToolDefinition`; the rendered descriptor powers ``/api/threads/{thread_id}/revert/{action_id}``. -Design points: - -* **Defensive.** Logging never blocks the agent. We catch every exception - on the DB write path and emit a warning; the tool's ``ToolMessage`` - result is always returned untouched. -* **Lightweight payload.** Only the tool ``name`` + ``args`` (capped) + - ``result_id`` + ``reverse_descriptor`` are stored. Tool output text - remains in the LangGraph checkpoint / spilled tool-output files. -* **Best-effort reversibility.** We invoke ``reverse(args, result_obj)`` - with the parsed JSON result when the tool's content is a JSON object; - otherwise the raw text is passed. Exceptions in the reverse callable - are swallowed and logged — a failed descriptor render simply means the - action is NOT marked reversible. +Logging is fully defensive — DB-write failures are swallowed so the tool's +result is always returned untouched. Only metadata (name, capped args, +result_id, reverse_descriptor) is stored; tool output stays in the +checkpoint. Reversibility is best-effort: a reverse callable that raises +just leaves the action non-reversible. """ from __future__ import annotations @@ -27,14 +17,14 @@ from __future__ import annotations import json import logging from collections.abc import Awaitable, Callable +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from langchain.agents.middleware import AgentMiddleware from langchain_core.callbacks import adispatch_custom_event from langchain_core.messages import ToolMessage -from app.agents.new_chat.feature_flags import get_flags -from app.agents.new_chat.tools.registry import ToolDefinition +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags if TYPE_CHECKING: # pragma: no cover - type-only from langchain.agents.middleware.types import ToolCallRequest @@ -44,6 +34,31 @@ if TYPE_CHECKING: # pragma: no cover - type-only logger = logging.getLogger(__name__) +@dataclass +class ToolDefinition: + """Reversibility descriptor consumed by :class:`ActionLogMiddleware`. + + Only ``name`` and ``reverse`` are read by the middleware; the remaining + fields let callers and tests describe a tool declaratively. A tool is + marked reversible in the action log when ``reverse`` is set and renders a + descriptor without raising. + + Attributes: + name: Unique identifier for the tool. + description: Human-readable description of what the tool does. + factory: Optional callable that builds the tool (unused by the + middleware; retained for declarative call sites/tests). + reverse: Optional callable that, given the tool's ``(args, result)``, + returns a ``ReverseDescriptor`` describing the inverse invocation. + + """ + + name: str + description: str = "" + factory: Callable[[dict[str, Any]], Any] | None = None + reverse: Callable[[dict[str, Any], Any], dict[str, Any]] | None = None + + # Cap for the persisted ``args`` JSON to avoid bloating the action log with # accidentally-huge inputs. Values are truncated and a flag is set in the # stored payload so consumers can detect truncation. @@ -93,18 +108,32 @@ class ActionLogMiddleware(AgentMiddleware): self._user_id = user_id self._tool_definitions = dict(tool_definitions or {}) - def _enabled(self) -> bool: + def _enabled(self, thread_id: int | None) -> bool: flags = get_flags() if flags.disable_new_agent_stack: return False - return bool(flags.enable_action_log) and self._thread_id is not None + return bool(flags.enable_action_log) and thread_id is not None + + def _resolve_thread_id(self, request: ToolCallRequest) -> int | None: + """Resolve the live thread id, preferring the runtime config. + + Reading ``configurable.thread_id`` from the active ``RunnableConfig`` + (rather than the value captured at ``__init__``) lets a single cached + compiled graph safely serve many threads — without it, a cache hit + would attribute action-log rows to whichever thread first built the + graph. Falls back to the constructor value for legacy/test runtimes + that don't surface a config. + """ + resolved = _resolve_thread_id(request) + return resolved if resolved is not None else self._thread_id async def awrap_tool_call( self, request: ToolCallRequest, handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]], ) -> ToolMessage | Command[Any]: - if not self._enabled(): + thread_id = self._resolve_thread_id(request) + if not self._enabled(thread_id): return await handler(request) result: ToolMessage | Command[Any] @@ -119,10 +148,16 @@ class ActionLogMiddleware(AgentMiddleware): request=request, result=None, error_payload=error_payload, + thread_id=thread_id, ) raise - await self._record(request=request, result=result, error_payload=None) + await self._record( + request=request, + result=result, + error_payload=None, + thread_id=thread_id, + ) return result async def _record( @@ -131,6 +166,7 @@ class ActionLogMiddleware(AgentMiddleware): request: ToolCallRequest, result: ToolMessage | Command[Any] | None, error_payload: dict[str, Any] | None, + thread_id: int | None, ) -> None: """Persist one ``agent_action_log`` row. Defensive: never raises.""" try: @@ -149,7 +185,7 @@ class ActionLogMiddleware(AgentMiddleware): chat_turn_id = _resolve_chat_turn_id(request) row = AgentActionLog( - thread_id=self._thread_id, + thread_id=thread_id, user_id=self._user_id, search_space_id=self._search_space_id, # ``turn_id`` is the deprecated alias of ``tool_call_id`` @@ -178,11 +214,9 @@ class ActionLogMiddleware(AgentMiddleware): ) return - # Surface a side-channel SSE event so the chat tool card can - # render a Revert button immediately after the row is durable. - # ``stream_new_chat`` translates this into a - # ``data-action-log`` SSE event. We DO NOT include the - # ``reverse_descriptor`` payload here; only a presence flag. + # Side-channel event (relayed by ``stream_new_chat`` as a + # ``data-action-log`` SSE) so the tool card can show a Revert button + # once the row is durable. Carries a presence flag, not the descriptor. try: await adispatch_custom_event( "action_log", @@ -337,6 +371,36 @@ def _resolve_chat_turn_id(request: Any) -> str | None: return None +def _resolve_thread_id(request: Any) -> int | None: + """Return ``configurable.thread_id`` (as int) for this request, if accessible. + + Mirrors :func:`_resolve_chat_turn_id`: ``ToolRuntime.config`` is exposed by + LangGraph at ``request.runtime.config``, and the chat thread id lives at + ``configurable.thread_id`` (a stringified ``chat_id`` at the main-graph + level). Returns ``None`` when absent or unparseable so the caller can fall + back to the constructor value. + """ + try: + runtime = getattr(request, "runtime", None) + if runtime is None: + return None + config = getattr(runtime, "config", None) + if not isinstance(config, dict): + return None + configurable = config.get("configurable") + if not isinstance(configurable, dict): + return None + value = configurable.get("thread_id") + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + except Exception: # pragma: no cover - defensive + return None + + def _resolve_message_id(request: Any) -> str | None: """Tool-call IDs serve as best-available message correlator at this layer.""" return _resolve_tool_call_id(request) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/__init__.py new file mode 100644 index 000000000..5684a592c --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/__init__.py @@ -0,0 +1,9 @@ +"""Anonymous-document middleware: Redis hydration, cloud only (impl + builder).""" + +from .builder import build_anonymous_doc_mw +from .middleware import AnonymousDocumentMiddleware + +__all__ = [ + "AnonymousDocumentMiddleware", + "build_anonymous_doc_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/anonymous_doc.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/builder.py similarity index 73% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/anonymous_doc.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/builder.py index afd54a2d3..f03543124 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/anonymous_doc.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/builder.py @@ -2,8 +2,9 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import AnonymousDocumentMiddleware +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode + +from .middleware import AnonymousDocumentMiddleware def build_anonymous_doc_mw( diff --git a/surfsense_backend/app/agents/new_chat/middleware/anonymous_document.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/middleware.py similarity index 93% rename from surfsense_backend/app/agents/new_chat/middleware/anonymous_document.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/middleware.py index 2893d2e11..d29c31230 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/anonymous_document.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/anonymous_document/middleware.py @@ -24,8 +24,13 @@ from typing import Any from langchain.agents.middleware import AgentMiddleware, AgentState from langgraph.runtime import Runtime -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT, safe_filename +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import ( + DOCUMENTS_ROOT, + safe_filename, +) logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/__init__.py new file mode 100644 index 000000000..17c33b8ab --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/__init__.py @@ -0,0 +1,25 @@ +"""Per-turn cooperative busy-lock middleware + cancel primitives (main-agent).""" + +from .builder import build_busy_mutex_mw +from .middleware import ( + BusyMutexMiddleware, + end_turn, + get_cancel_event, + get_cancel_state, + is_cancel_requested, + manager, + request_cancel, + reset_cancel, +) + +__all__ = [ + "BusyMutexMiddleware", + "build_busy_mutex_mw", + "end_turn", + "get_cancel_event", + "get_cancel_state", + "is_cancel_requested", + "manager", + "request_cancel", + "reset_cancel", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/busy_mutex.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/builder.py similarity index 54% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/busy_mutex.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/builder.py index 0ea53bf16..0daf87e0b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/busy_mutex.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/builder.py @@ -2,10 +2,12 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import BusyMutexMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import ( + BusyMutexMiddleware, +) def build_busy_mutex_mw(flags: AgentFeatureFlags) -> BusyMutexMiddleware | None: diff --git a/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/middleware.py similarity index 83% rename from surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/middleware.py index e7d9b8f75..7a82196d9 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/busy_mutex/middleware.py @@ -1,32 +1,12 @@ -""" -BusyMutexMiddleware — per-thread asyncio lock + cancel token. +"""Per-thread asyncio lock + cooperative cancel token, keyed by ``thread_id``. -LangChain has no built-in concept of "this thread is already running a -turn — refuse the second concurrent request". Without it, a user -double-clicking "send" or refreshing the page mid-stream can spawn two -turns racing on the same checkpoint, producing duplicated tool calls -and mangled state. +Refuses a second concurrent turn on the same thread (e.g. double-clicked +"send") that would otherwise race on the same checkpoint and duplicate tool +calls. Also exposes a per-thread cancel event that long-running tools poll +via ``runtime.context.cancel_event.is_set()`` to abort cooperatively. -Ported from OpenCode's ``Stream.scoped(AbortController)`` pattern: a -single-process, in-memory lock + cooperative cancellation token keyed by -``thread_id``. For multi-worker deployments a distributed lock backend -(Redis or PostgreSQL advisory locks) is a phase-2 follow-up. - -What this provides: -- A ``WeakValueDictionary[str, asyncio.Lock]`` keyed by ``thread_id``; - acquiring the lock during ``before_agent`` blocks any concurrent - prompt on the same thread until release. -- A per-thread ``asyncio.Event`` (``cancel_event``) that long-running - tools can poll to abort cooperatively. The event is reset between - turns. Tools should check ``runtime.context.cancel_event.is_set()`` - in tight inner loops. -- A typed :class:`~app.agents.new_chat.errors.BusyError` raised when a - second turn arrives while the lock is held. - -Note: SurfSense's ``stream_new_chat`` is the call site that should -acquire/release. Wiring this as middleware means the contract is -explicit and the lock manager is shared with subagents that compile -their own ``create_agent`` runnables. +Process-local and in-memory; multi-worker deployments need a distributed lock +(Redis / PostgreSQL advisory locks) as a follow-up. """ from __future__ import annotations @@ -46,7 +26,7 @@ from langchain.agents.middleware.types import ( from langgraph.config import get_config from langgraph.runtime import Runtime -from app.agents.new_chat.errors import BusyError +from app.agents.chat.runtime.errors import BusyError logger = logging.getLogger(__name__) @@ -152,9 +132,8 @@ class _ThreadLockManager: return True -# Module-level singleton — process-local but reused across all agent -# instances built in this process. Subagents created in nested -# ``create_agent`` calls also get this so locks are coherent. +# Process-local singleton shared across all agents/subagents built in this +# process so per-thread locks stay coherent. manager = _ThreadLockManager() @@ -266,7 +245,6 @@ class BusyMutexMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respo await lock.acquire() epoch = manager.bump_turn_epoch(thread_id) self._held_locks[thread_id] = (lock, epoch) - # Reset the cancel event so this turn starts fresh reset_cancel(thread_id) return None @@ -289,17 +267,14 @@ class BusyMutexMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respo return None if lock.locked(): lock.release() - # Always clear cancel event between turns so a stale signal - # doesn't leak into the next request. + # Clear cancel event so a stale signal doesn't leak into the next turn. reset_cancel(thread_id) return None - # Provide sync no-ops because the middleware base class allows them def before_agent( # type: ignore[override] self, state: AgentState[Any], runtime: Runtime[ContextT] ) -> dict[str, Any] | None: - # Sync path: no asyncio.Lock to acquire. Best we can do is reject - # if anyone else is in flight. + # Sync path can't await an asyncio.Lock; only reject if one is in flight. thread_id = self._thread_id(runtime) if thread_id is None: if self._require_thread_id: diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/config.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/config.py similarity index 66% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/config.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/config.py index ad5b58607..72e2282ff 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/config.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/config.py @@ -1,7 +1,9 @@ -"""RunnableConfig wiring for nested subagent invocations. +"""HITL resume side-channel for nested subagent invocations. -Forwards the parent's ``runtime.config`` (thread_id, …) into the subagent and -exposes the side-channel ``stream_resume_chat`` uses to ferry resume payloads. +Exposes the configurable side-channel ``stream_resume_chat`` uses to ferry +resume payloads into a mid-flight subagent. The ``RunnableConfig`` builder and +state-key filter shared with subagents live in +``app.agents.chat.multi_agent_chat.subagents.shared.invocation``. """ from __future__ import annotations @@ -11,8 +13,6 @@ from typing import Any from langchain.tools import ToolRuntime -from .constants import DEFAULT_SUBAGENT_RECURSION_LIMIT - logger = logging.getLogger(__name__) # langgraph stores the parent task's scratchpad under this configurable key; @@ -20,39 +20,6 @@ logger = logging.getLogger(__name__) _LANGGRAPH_SCRATCHPAD_KEY = "__pregel_scratchpad" -def subagent_invoke_config(runtime: ToolRuntime) -> dict[str, Any]: - """RunnableConfig for the nested invoke; raises ``recursion_limit`` and isolates ``thread_id``. - - Each parallel subagent invocation lands in its own checkpoint slot keyed - by an extended ``thread_id`` of the form ``{parent_thread}::task:{tool_call_id}``. - The same call across the resume cycle keeps reading from the same snapshot - (``tool_call_id`` is stable per LLM-emitted call). - - We namespace via ``thread_id`` rather than ``checkpoint_ns`` because - langgraph's ``aget_state`` interprets a non-empty ``checkpoint_ns`` as a - subgraph path and raises ``ValueError("Subgraph X not found")``. - """ - merged: dict[str, Any] = dict(runtime.config) if runtime.config else {} - current_limit = merged.get("recursion_limit") - try: - current_int = int(current_limit) if current_limit is not None else 0 - except (TypeError, ValueError): - current_int = 0 - if current_int < DEFAULT_SUBAGENT_RECURSION_LIMIT: - merged["recursion_limit"] = DEFAULT_SUBAGENT_RECURSION_LIMIT - - configurable: dict[str, Any] = dict(merged.get("configurable") or {}) - parent_thread_id = configurable.get("thread_id") - per_call_suffix = f"task:{runtime.tool_call_id}" - configurable["thread_id"] = ( - f"{parent_thread_id}::{per_call_suffix}" - if parent_thread_id - else per_call_suffix - ) - merged["configurable"] = configurable - return merged - - def consume_surfsense_resume(runtime: ToolRuntime) -> Any: """Pop the resume payload for *this* call's ``tool_call_id``. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/constants.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/constants.py similarity index 85% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/constants.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/constants.py index e11f3c3ec..d6a328b2a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/constants.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/constants.py @@ -1,24 +1,14 @@ -"""Constants shared by the checkpointed subagent middleware.""" +"""Tuning constants for the checkpointed subagent middleware. + +``EXCLUDED_STATE_KEYS`` and ``DEFAULT_SUBAGENT_RECURSION_LIMIT`` are part of the +subagent-invocation contract shared with subagents and now live in +``app.agents.chat.multi_agent_chat.subagents.shared.invocation``. +""" from __future__ import annotations import os -# Mirror of deepagents.middleware.subagents._EXCLUDED_STATE_KEYS. -EXCLUDED_STATE_KEYS = frozenset( - { - "messages", - "todos", - "structured_response", - "skills_metadata", - "memory_contents", - } -) - -# Match the parent graph's budget; the LangGraph default of 25 trips on -# multi-step subagent runs. -DEFAULT_SUBAGENT_RECURSION_LIMIT = 10_000 - def _read_timeout_env(name: str, default: float) -> float: """Parse ``name`` from the environment; fall back to ``default`` on bad values. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/middleware.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/middleware.py new file mode 100644 index 000000000..ab6c3a1f5 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/middleware.py @@ -0,0 +1,188 @@ +"""SubAgent middleware that compiles each subagent against the parent checkpointer.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from typing import Any, cast + +from deepagents.backends.protocol import BackendFactory, BackendProtocol +from deepagents.middleware.subagents import ( + TASK_SYSTEM_PROMPT, + CompiledSubAgent, + SubAgent, + SubAgentMiddleware, +) +from langchain.agents import create_agent +from langchain.chat_models import init_chat_model +from langchain_core.runnables import Runnable +from langgraph.types import Checkpointer + +from app.agents.chat.multi_agent_chat.subagents.shared.spec import ( + SURF_CONTEXT_HINT_PROVIDER_KEY, + SURF_LAZY_SPEC_FACTORY_KEY, +) +from app.utils.perf import get_perf_logger + +from .task_tool import build_task_tool_with_parent_config + +_perf_log = get_perf_logger() + + +class SurfSenseCheckpointedSubAgentMiddleware(SubAgentMiddleware): + """``SubAgentMiddleware`` variant that compiles each subagent against the parent checkpointer.""" + + def __init__( + self, + *, + checkpointer: Checkpointer, + backend: BackendProtocol | BackendFactory, + subagents: list[SubAgent | CompiledSubAgent], + system_prompt: str | None = TASK_SYSTEM_PROMPT, + task_description: str | None = None, + search_space_id: int | None = None, + ) -> None: + self._surf_checkpointer = checkpointer + super(SubAgentMiddleware, self).__init__() + if not subagents: + raise ValueError( + "At least one subagent must be specified when using the new API" + ) + self._backend = backend + self._subagents = subagents + # Search-space id is captured at build time (the orchestrator runs in + # exactly one search space for its lifetime). The spawn-paused kill + # switch keys on it so an operator can quarantine one workspace + # without affecting the rest of the deployment. + self._search_space_id = search_space_id + + # Lazy subagent compilation. Compiling a subagent graph via + # ``create_agent`` is expensive (~250-400ms each) and there can be up + # to ~17 of them. Doing it all in ``__init__`` put the full cost on + # every cold ``agent_cache`` miss (i.e. on time-to-first-token), even + # though a turn usually invokes zero or one subagent. We instead index + # the raw specs here and compile each graph on first ``task(name)`` + # use, memoizing the result for the life of this (cached) instance. + self._compiled: dict[str, Runnable] = {} + self._lazy_specs: dict[str, dict[str, Any]] = {} + # Subagents whose *spec itself* is built lazily (not just compiled). + # Keyed by name → zero-arg factory returning the full spec dict. Used + # for the write knowledge_base subagent, whose filesystem middleware + # builds ~13 tool schemas (~2s) that almost never matter on turn 1. + self._lazy_spec_factories: dict[str, Callable[[], dict[str, Any]]] = {} + descriptors = self._build_subagent_registry() + + task_tool = build_task_tool_with_parent_config( + descriptors, + task_description, + search_space_id=search_space_id, + resolve_subagent=self._resolve_subagent, + ) + if system_prompt and descriptors: + agents_desc = "\n".join( + f"- {s['name']}: {s['description']}" for s in descriptors + ) + self.system_prompt = ( + system_prompt + "\n\nAvailable subagent types:\n" + agents_desc + ) + else: + self.system_prompt = system_prompt + self.tools = [task_tool] + + def _build_subagent_registry(self) -> list[dict[str, Any]]: + """Index subagents for lazy compilation; return lightweight descriptors. + + Pre-compiled specs (those carrying a ``runnable``) are seeded directly + into the memo. Lazy specs are stashed by name and compiled on first + ``task(...)`` use via :meth:`_resolve_subagent`. The returned + descriptors carry only ``name``/``description`` plus the optional + context-hint provider — everything the ``task`` tool needs to validate + names, render its catalog, and run hints, without paying the + ``create_agent`` cost up front. + """ + descriptors: list[dict[str, Any]] = [] + for spec in self._subagents: + # Provider may be ``None`` (no hint), in which case task_tool skips + # the prepend step. We forward the key unconditionally so the + # descriptor shape is uniform. + hint_provider = cast(dict, spec).get(SURF_CONTEXT_HINT_PROVIDER_KEY) + name = spec["name"] + spec_factory = cast(dict, spec).get(SURF_LAZY_SPEC_FACTORY_KEY) + if spec_factory is not None: + # Descriptor-only entry: the spec dict is built on first use. + self._lazy_spec_factories[name] = spec_factory + elif "runnable" in spec: + compiled = cast(CompiledSubAgent, spec) + self._compiled[name] = compiled["runnable"] + else: + if "model" not in spec: + msg = f"SubAgent '{name}' must specify 'model'" + raise ValueError(msg) + if "tools" not in spec: + msg = f"SubAgent '{name}' must specify 'tools'" + raise ValueError(msg) + self._lazy_specs[name] = cast(dict, spec) + descriptors.append( + { + "name": name, + "description": spec["description"], + SURF_CONTEXT_HINT_PROVIDER_KEY: hint_provider, + } + ) + return descriptors + + def _resolve_subagent(self, name: str) -> Runnable: + """Return the compiled subagent graph for ``name``, compiling on first use. + + Memoized: the ``create_agent`` cost is paid once per subagent per + cached middleware instance. Raises ``KeyError`` for unknown names + (callers in the ``task`` tool validate membership before resolving). + """ + cached = self._compiled.get(name) + if cached is not None: + return cached + spec = self._lazy_specs.get(name) + if spec is None: + factory = self._lazy_spec_factories.get(name) + if factory is None: + raise KeyError(name) + # Build the spec on first use (pays the deferred construction cost + # here, off the cold agent-build path), then compile and memoize. + build_start = time.perf_counter() + spec = factory() + _perf_log.info( + "[subagent_spec_lazy] name=%s (deferred spec build) in %.3fs", + name, + time.perf_counter() - build_start, + ) + runnable = self._compile_one(spec) + self._compiled[name] = runnable + return runnable + + def _compile_one(self, spec: dict[str, Any]) -> Runnable: + """Compile a single subagent graph against the parent checkpointer.""" + model = spec["model"] + if isinstance(model, str): + model = init_chat_model(model) + + middleware: list[Any] = list(spec.get("middleware", [])) + tools_count = len(spec.get("tools") or []) + mw_count = len(middleware) + + compile_start = time.perf_counter() + runnable = create_agent( + model, + system_prompt=spec["system_prompt"], + tools=spec["tools"], + middleware=middleware, + name=spec["name"], + checkpointer=self._surf_checkpointer, + ) + _perf_log.info( + "[subagent_compile_lazy] name=%s in %.3fs tools=%d mw=%d", + spec["name"], + time.perf_counter() - compile_start, + tools_count, + mw_count, + ) + return runnable diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/propagation.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/propagation.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/propagation.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/propagation.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/resume.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/resume.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/resume.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/resume.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/resume_routing.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/resume_routing.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/resume_routing.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/resume_routing.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/spawn_paused.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/spawn_paused.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/spawn_paused.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/spawn_paused.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_description.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_description.py index 73afa6823..3464b889a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_description.py @@ -6,7 +6,7 @@ and the ```` block render from the same source. from __future__ import annotations -from app.agents.multi_agent_chat.main_agent.system_prompt.builder.load_md import ( +from app.agents.chat.multi_agent_chat.main_agent.system_prompt.builder.load_md import ( read_prompt_md, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py similarity index 85% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_tool.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py index eaed9a55f..644d3ef82 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/task_tool.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/checkpointed_subagent_middleware/task_tool.py @@ -12,7 +12,7 @@ import asyncio import json import logging import time -from collections.abc import Awaitable +from collections.abc import Awaitable, Callable from typing import Annotated, Any, NoReturn, TypeVar from deepagents.middleware.subagents import TASK_TOOL_DESCRIPTION @@ -23,7 +23,11 @@ from langchain_core.tools import StructuredTool from langgraph.errors import GraphInterrupt from langgraph.types import Command, Interrupt -from app.agents.multi_agent_chat.subagents.shared.spec import ( +from app.agents.chat.multi_agent_chat.subagents.shared.invocation import ( + EXCLUDED_STATE_KEYS, + subagent_invoke_config, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import ( SURF_CONTEXT_HINT_PROVIDER_KEY, ContextHintProvider, ) @@ -34,13 +38,11 @@ from .config import ( consume_surfsense_resume, drain_parent_null_resume, has_surfsense_resume, - subagent_invoke_config, ) from .constants import ( DEFAULT_SUBAGENT_BATCH_CONCURRENCY, DEFAULT_SUBAGENT_BILLABLE_THRESHOLD, DEFAULT_SUBAGENT_INVOKE_TIMEOUT_SECONDS, - EXCLUDED_STATE_KEYS, MAX_SUBAGENT_BATCH_SIZE, ) from .propagation import wrap_with_tool_call_id @@ -80,13 +82,10 @@ _T = TypeVar("_T") async def _ainvoke_with_timeout[T]( coro: Awaitable[_T], *, subagent_type: str, started_at: float ) -> _T: - """Apply :data:`DEFAULT_SUBAGENT_INVOKE_TIMEOUT_SECONDS` to ``coro``. + """Apply the subagent invoke timeout to ``coro`` (non-positive disables it). - A non-positive timeout disables the cap (configurable via the - ``SURFSENSE_SUBAGENT_INVOKE_TIMEOUT_SECONDS`` env var). On expiry the - underlying task is cancelled and :class:`SubagentInvokeTimeoutError` is - raised — the caller wraps it into a synthetic ToolMessage so the - orchestrator can decide what to do. + On expiry the task is cancelled and :class:`SubagentInvokeTimeoutError` is + raised for the caller to turn into a synthetic ToolMessage. """ timeout = DEFAULT_SUBAGENT_INVOKE_TIMEOUT_SECONDS if timeout <= 0: @@ -144,17 +143,31 @@ def build_task_tool_with_parent_config( task_description: str | None = None, *, search_space_id: int | None = None, + resolve_subagent: Callable[[str], Runnable] | None = None, ) -> BaseTool: - """Upstream ``_build_task_tool`` + parent ``runtime.config`` propagation + resume bridging.""" - subagent_graphs: dict[str, Runnable] = { - spec["name"]: spec["runnable"] for spec in subagents - } - # Per-subagent context-hint providers (see ``SurfSenseSubagentSpec``). - # The mapping is sparse: only routes that opted in via ``pack_subagent`` - # appear here, and the value is invoked once per ``task(...)`` call to - # generate a short string prepended to the subagent's first - # ``HumanMessage``. Failures are logged and swallowed — a broken hint - # provider must never prevent the underlying task from running. + """Upstream ``_build_task_tool`` + parent ``runtime.config`` propagation + resume bridging. + + ``subagents`` are lightweight descriptors (``name``/``description`` + the + optional context-hint provider); the actual compiled graph is fetched + lazily via ``resolve_subagent(name)`` so subagent ``create_agent`` cost is + paid on first ``task(name)`` use rather than at graph-build time. + + For backward compatibility (and tests), ``resolve_subagent`` may be omitted + when every descriptor already carries a pre-compiled ``runnable``; in that + case a trivial dict-backed resolver is used. + """ + subagent_names: set[str] = {spec["name"] for spec in subagents} + if resolve_subagent is None: + _eager_graphs: dict[str, Runnable] = { + spec["name"]: spec["runnable"] for spec in subagents if "runnable" in spec + } + + def resolve_subagent(name: str) -> Runnable: + return _eager_graphs[name] + + # Sparse map of opt-in context-hint providers; each runs once per task() + # call to prepend a string to the subagent's first HumanMessage. Failures + # are swallowed so a broken hint never blocks the task. subagent_hint_providers: dict[str, ContextHintProvider] = { spec["name"]: provider for spec in subagents @@ -176,24 +189,18 @@ def build_task_tool_with_parent_config( def _billable_call_update( subagent_type: str, runtime: ToolRuntime ) -> dict[str, Any]: - """Build the per-call ``billable_calls`` delta + an optional warning. + """Build the per-call ``billable_calls`` delta plus an optional soft-cap warning. - The orchestrator's ``billable_calls`` map is summed by - :func:`_int_counter_merge_reducer`, so we always emit - ``{subagent_type: 1}`` and let the reducer accumulate. If the - cumulative count *after* this call would cross the configured - threshold, we also slip a soft ``messages`` entry into the update - so the orchestrator can read it on its next step and self-limit. - Returning a plain ``dict`` (vs. an extra :class:`Command`) keeps - the helper composable with the existing single/batch return paths. + Always emits ``{subagent_type: 1}`` (a reducer accumulates it); when this + call would cross the threshold, also adds a soft ``messages`` entry so the + orchestrator self-limits on its next step. """ delta: dict[str, Any] = {"billable_calls": {subagent_type: 1}} threshold = DEFAULT_SUBAGENT_BILLABLE_THRESHOLD if threshold <= 0: return delta prior = runtime.state.get("billable_calls") or {} - # ``prior`` may be a plain dict or a reducer-managed mapping; only - # int values are counted so a malformed checkpoint can't crash us. + # Count int values only so a malformed checkpoint can't crash us. prior_total = sum(v for v in prior.values() if isinstance(v, int)) new_total = prior_total + 1 if prior_total < threshold <= new_total: @@ -212,8 +219,7 @@ def build_task_tool_with_parent_config( """Merge the per-call billable counter (and warning) into ``cmd``.""" delta = _billable_call_update(subagent_type, runtime) warn_text = delta.pop("_billable_warn_text", None) - # ``cmd.update`` may be a dict or LangGraph ``UpdateDict``; defensively - # copy so we don't mutate state shared across other tool returns. + # Copy so we don't mutate state shared with other tool returns. update = dict(getattr(cmd, "update", {}) or {}) for key, value in delta.items(): update[key] = value @@ -226,14 +232,10 @@ def build_task_tool_with_parent_config( return Command(update=update) def _safe_message_text(msg: Any) -> str: - """Pull text out of a BaseMessage without trusting the ``.text`` property. + """Pull text out of a BaseMessage without using the ``.text`` property. - ``BaseMessage.text`` walks ``content_blocks`` and crashes with - ``TypeError: 'NoneType' object is not iterable`` when ``content`` is - ``None`` (common for tool-call AIMessages whose payload is purely - structured). ``getattr(msg, "text", None)`` does not catch this - because Python evaluates the property body before falling back to - the default. Read ``content`` directly and coerce defensively. + ``.text`` crashes when ``content`` is ``None`` (common for tool-call + AIMessages), and ``getattr`` won't catch it, so read ``content`` directly. """ try: content = getattr(msg, "content", None) @@ -256,23 +258,18 @@ def build_task_tool_with_parent_config( return str(content) def _build_tool_trace(messages: list[Any]) -> list[dict[str, Any]]: - """Compress the subagent's message stream into a compact tool trace. + """Compress the subagent's messages into a compact tool trace. - Each entry is ``{"tool": , "status": "ok"|"error", "preview": - <≤120 chars>}`` so the orchestrator can show "this is what your - specialist actually did" without dumping the full message stream - back through the prompt. The list is attached to the returned - ToolMessage's ``additional_kwargs`` (under ``"surf_tool_trace"``); - the LLM never sees it, but UI / observability code can pluck it - out of the checkpoint. + Entries (``{tool, status, preview}``) ride on the ToolMessage's + ``additional_kwargs["surf_tool_trace"]`` for UI/observability; the LLM + never sees them. """ trace: list[dict[str, Any]] = [] for msg in messages: tool_name = getattr(msg, "name", None) tool_call_id_attr = getattr(msg, "tool_call_id", None) if not tool_name and not tool_call_id_attr: - # Only ToolMessages have either field; skip AIMessage / - # HumanMessage / SystemMessage frames. + # Only ToolMessages carry either field. continue status = getattr(msg, "status", None) or "ok" preview = _safe_message_text(msg).strip().replace("\n", " ") @@ -306,8 +303,7 @@ def build_task_tool_with_parent_config( ) raise ValueError(msg) message_text = _safe_message_text(messages[-1]).rstrip() - # Tool-trace is purely observability — wrap defensively so a single - # malformed frame never bubbles up and kills the whole user turn. + # Trace is observability-only; never let a bad frame kill the turn. try: tool_trace = _build_tool_trace(messages) except Exception: @@ -318,10 +314,7 @@ def build_task_tool_with_parent_config( tool_trace = [] tool_msg = ToolMessage(message_text, tool_call_id=tool_call_id) if tool_trace: - # ``additional_kwargs`` is a free-form dict on BaseMessage; using - # a ``surf_`` prefix avoids collision with provider-specific keys - # (e.g. Anthropic's ``cache_control``). The LLM doesn't see it; - # consumers (UI, observability) read it off the checkpoint. + # surf_ prefix avoids collision with provider keys (e.g. cache_control). tool_msg.additional_kwargs["surf_tool_trace"] = tool_trace return Command( update={ @@ -353,15 +346,13 @@ def build_task_tool_with_parent_config( def _validate_and_prepare_state( subagent_type: str, description: str, runtime: ToolRuntime ) -> tuple[Runnable, dict]: - subagent = subagent_graphs[subagent_type] + subagent = resolve_subagent(subagent_type) subagent_state = { k: v for k, v in runtime.state.items() if k not in EXCLUDED_STATE_KEYS } hint = _resolve_context_hint(subagent_type, description, runtime) if hint: - # Prepend as a tagged block so the subagent prompt can pattern-match - # on the section (and a future change can lift it into its own - # ``SystemMessage`` if needed). + # Tagged block so the subagent prompt can pattern-match the section. payload = f"\n{hint}\n\n\n{description}" else: payload = description @@ -372,16 +363,12 @@ def build_task_tool_with_parent_config( results: list[tuple[int, str, dict | str, dict | None]], runtime: ToolRuntime, ) -> Command: - """Combine per-child results into one Command with a combined ToolMessage. + """Combine per-child results into one Command with an aggregate ToolMessage. - ``results`` is a list of ``(task_index, subagent_type, - payload_or_error_text, child_state_update)`` tuples — preserving the - input order so the orchestrator can map each block back to the task - it dispatched. State updates are merged by reducer for keys outside - :data:`EXCLUDED_STATE_KEYS`; everything else (``messages``, ``todos``, - etc.) is replaced by the synthesized aggregate ToolMessage. Every - child also contributes a ``billable_calls`` increment so cost - accounting matches single-mode dispatch. + ``results`` tuples are ``(task_index, subagent_type, payload_or_error, + child_state_update)``; output blocks are sorted by index so the LLM can + map them back to dispatch order, and each child contributes a + ``billable_calls`` increment to match single-mode accounting. """ results.sort(key=lambda r: r[0]) merged_state: dict[str, Any] = {} @@ -422,8 +409,8 @@ def build_task_tool_with_parent_config( } ) if state_update: - # Naive merge: later tasks win on scalar collisions; reducer-backed - # fields (``receipts``, ``files`` etc.) accumulate at apply time. + # Later tasks win on scalar collisions; reducer-backed fields + # accumulate at apply time. merged_state.update(state_update) aggregate = "\n\n".join(message_blocks) aggregate_msg = ToolMessage( @@ -467,15 +454,13 @@ def build_task_tool_with_parent_config( ) -> tuple[int, str, dict | str, dict | None]: """Run one child of a batched ``task`` call under the concurrency cap. - Errors are returned as plain text in slot 2 so a single child's - failure does not abort the whole batch. ``GraphInterrupt`` from a - batched child is currently treated as a hard failure for that child - only — batched HITL is intentionally out of scope for the v1 - rollout (see plan tier 2 item 4 risks). + Errors are returned as text (slot 2) so one child's failure doesn't abort + the batch. A child's ``GraphInterrupt`` is a hard failure for that child: + batched HITL is intentionally out of scope. """ async with semaphore: - if subagent_type not in subagent_graphs: - allowed_types = ", ".join([f"`{k}`" for k in subagent_graphs]) + if subagent_type not in subagent_names: + allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( task_index, subagent_type, @@ -505,8 +490,7 @@ def build_task_tool_with_parent_config( ) return (task_index, subagent_type, str(exc), None) except GraphInterrupt: - # Batched HITL is unsupported in v1 — surface as a failure - # for this child so the rest of the batch still completes. + # Batched HITL unsupported; fail this child so the batch finishes. logger.warning( "Batch child %d (%s) raised GraphInterrupt; batched HITL " "is not supported. Re-dispatch this task as a single " @@ -543,14 +527,11 @@ def build_task_tool_with_parent_config( return (task_index, subagent_type, result, child_state_update) def _coerce_batch_arg(tasks: Any) -> list[dict] | str: - """Rescue common LLM-side malformations of the ``tasks`` argument. + """Rescue common LLM malformations of the ``tasks`` argument. - Some providers serialise an array argument as a JSON-encoded string, - and small models occasionally hand back a single ``{description, - subagent_type}`` dict instead of a one-element array. Both are - recovered here with a WARN log so the issue is visible in metrics - but the user's turn still completes; truly broken shapes return a - plain string that the caller surfaces as the tool error. + Recovers a JSON-encoded array string and a single dict (instead of a + 1-element array), logging a WARN. Unrecoverable shapes return a string + the caller surfaces as the tool error. """ if isinstance(tasks, list): return tasks @@ -585,13 +566,10 @@ def build_task_tool_with_parent_config( async def _adispatch_batch( tasks: list[dict], runtime: ToolRuntime ) -> Command | str: - """Fan-out helper for the ``tasks`` array shape. + """Fan out the ``tasks`` array (size- and concurrency-capped). - Bounded by :data:`MAX_SUBAGENT_BATCH_SIZE` and concurrency-capped - at :data:`DEFAULT_SUBAGENT_BATCH_CONCURRENCY`. Returns a single - :class:`Command` that the LLM sees as one ToolMessage per child, - prefixed with ``[task ]`` so it can map back to the input - order. + Returns one Command; the LLM sees one ``[task ]``-prefixed block + per child, in input order. """ if not tasks: return "tasks: array is empty; nothing to dispatch." @@ -657,8 +635,8 @@ def build_task_tool_with_parent_config( "task: must provide either single-mode (`description`+`subagent_type`) " "or batch-mode (`tasks`)." ) - if subagent_type not in subagent_graphs: - allowed_types = ", ".join([f"`{k}`" for k in subagent_graphs]) + if subagent_type not in subagent_names: + allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( f"We cannot invoke subagent {subagent_type} because it does not exist, " f"the only allowed types are {allowed_types}" @@ -701,17 +679,16 @@ def build_task_tool_with_parent_config( if pending_value is not None: resume_value = consume_surfsense_resume(runtime) if resume_value is None: - # Bridge invariant: a queued resume must accompany any pending - # subagent interrupt. Fall-through replay would silently re-prompt - # the user; raise so the streaming layer surfaces a clear error. + # A pending interrupt must have a queued resume; otherwise replay + # would silently re-prompt the user. Raise instead. raise RuntimeError( f"Subagent {subagent_type!r} has a pending interrupt but no " "surfsense_resume_value on config; resume bridge is broken." ) expected = hitlrequest_action_count(pending_value) resume_value = fan_out_decisions_to_match(resume_value, expected) - # Prevent the parent's resume payload from leaking into subagent - # interrupts via langgraph's parent_scratchpad fallback. + # Stop the parent's resume leaking into subagent interrupts via + # langgraph's parent_scratchpad fallback. drain_parent_null_resume(runtime) with ot.subagent_invoke_span( subagent_type=subagent_type, path=invoke_path @@ -827,10 +804,8 @@ def build_task_tool_with_parent_config( ] = None, ) -> str | Command: atask_start = time.perf_counter() - # Kill switch: when ops flips the spawn-paused flag for this - # workspace, every ``task(...)`` invocation (single- or batch-mode) - # short-circuits with a clear ToolMessage so the orchestrator can - # tell the user what happened and stop hammering downstream APIs. + # Ops kill switch: short-circuit every task() call for this workspace + # so the orchestrator stops hammering downstream APIs. if await is_spawn_paused(search_space_id): logger.warning( "[hitl_route] atask SPAWN_PAUSED: search_space_id=%s tool_call_id=%s", @@ -869,8 +844,8 @@ def build_task_tool_with_parent_config( subagent_type, runtime.tool_call_id, ) - if subagent_type not in subagent_graphs: - allowed_types = ", ".join([f"`{k}`" for k in subagent_graphs]) + if subagent_type not in subagent_names: + allowed_types = ", ".join([f"`{k}`" for k in subagent_names]) return ( f"We cannot invoke subagent {subagent_type} because it does not exist, " f"the only allowed types are {allowed_types}" @@ -921,8 +896,8 @@ def build_task_tool_with_parent_config( ) expected = hitlrequest_action_count(pending_value) resume_value = fan_out_decisions_to_match(resume_value, expected) - # Prevent the parent's resume payload from leaking into subagent - # interrupts via langgraph's parent_scratchpad fallback. + # Stop the parent's resume leaking into subagent interrupts via + # langgraph's parent_scratchpad fallback. drain_parent_null_resume(runtime) with ot.subagent_invoke_span( subagent_type=subagent_type, path=invoke_path diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/__init__.py new file mode 100644 index 000000000..0c86c8cbd --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/__init__.py @@ -0,0 +1,15 @@ +"""Context-editing middleware: spill + clear-tool-uses passes (impl + builder).""" + +from .builder import build_context_editing_mw +from .middleware import ( + ClearToolUsesEdit, + SpillingContextEditingMiddleware, + SpillToBackendEdit, +) + +__all__ = [ + "ClearToolUsesEdit", + "SpillToBackendEdit", + "SpillingContextEditingMiddleware", + "build_context_editing_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/context_editing.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/builder.py similarity index 82% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/context_editing.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/builder.py index e8f99933e..1d7a2f47f 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/context_editing.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/builder.py @@ -7,18 +7,18 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.main_agent.context_prune.prune_tool_names import ( +from app.agents.chat.multi_agent_chat.main_agent.context_prune.prune_tool_names import ( safe_exclude_tools, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled + +from .middleware import ( ClearToolUsesEdit, SpillingContextEditingMiddleware, SpillToBackendEdit, ) -from ..shared.flags import enabled - def build_context_editing_mw( *, diff --git a/surfsense_backend/app/agents/new_chat/middleware/context_editing.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/middleware.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/context_editing.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/context_editing/middleware.py diff --git a/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/dedup_hitl.py similarity index 63% rename from surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/dedup_hitl.py index a6d2ce310..7710731ab 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/dedup_hitl.py @@ -1,4 +1,4 @@ -"""Middleware that deduplicates HITL tool calls within a single LLM response. +"""Drop duplicate HITL tool calls before execution. When the LLM emits multiple calls to the same HITL tool with the same primary argument (e.g. two ``delete_calendar_event("Doctor Appointment")``), @@ -9,72 +9,33 @@ the duplicate call is stripped from the AIMessage that gets checkpointed. That means it is also safe across LangGraph ``interrupt()`` boundaries: the removed call will never appear on graph resume. -Dedup-key resolution order: +Dedup-key resolution order (read from each tool's own ``metadata``): -1. :class:`ToolDefinition.dedup_key` — callable provided by the registry - entry. This is the canonical mechanism. -2. ``tool.metadata["hitl_dedup_key"]`` — string with a primary arg name; - used by MCP / Composio tools whose schemas the registry doesn't see. +1. ``tool.metadata["dedup_key"]`` — callable mapping the args dict to a + stable signature string. This is the canonical mechanism. +2. ``tool.metadata["hitl_dedup_key"]`` — string naming a primary arg; + used by MCP / Composio tools that only expose a single key field. A tool with no resolver from either path simply opts out of dedup. """ from __future__ import annotations -import json import logging -from collections.abc import Callable +from collections.abc import Sequence from typing import Any from langchain.agents.middleware import AgentMiddleware, AgentState +from langchain_core.tools import BaseTool from langgraph.runtime import Runtime +from app.agents.chat.multi_agent_chat.shared.middleware.dedup_tool_calls import ( + DedupResolver, + wrap_dedup_key_by_arg_name, +) + logger = logging.getLogger(__name__) -# Resolver type — given the tool ``args`` dict returns a stable -# string used to dedupe consecutive calls. ``None`` means no dedup. -DedupResolver = Callable[[dict[str, Any]], str] - - -def wrap_dedup_key_by_arg_name(arg_name: str) -> DedupResolver: - """Adapt a string-arg name into a :data:`DedupResolver`. - - Convenience helper used by registry entries that just want to dedupe - on a single arg's lowercased value (the most common case for native - HITL tools like ``send_gmail_email`` keyed on ``subject``). - - Example:: - - ToolDefinition( - name="send_gmail_email", - ..., - dedup_key=wrap_dedup_key_by_arg_name("subject"), - ) - """ - - def _resolver(args: dict[str, Any]) -> str: - return str(args.get(arg_name, "")).lower() - - return _resolver - - -def dedup_key_full_args(args: dict[str, Any]) -> str: - """Resolver that collapses calls only when **every** argument is identical. - - Safe default for tools where no single field uniquely identifies a call - (e.g. MCP tools whose first required field is a shared workspace id). - """ - - try: - return json.dumps(args, sort_keys=True, default=str) - except (TypeError, ValueError): - return repr(sorted(args.items())) if isinstance(args, dict) else repr(args) - - -# Backwards-compatible alias for code that imported the original -# private name. New callers should use :func:`wrap_dedup_key_by_arg_name`. -_wrap_string_key = wrap_dedup_key_by_arg_name - class DedupHITLToolCallsMiddleware(AgentMiddleware): # type: ignore[type-arg] """Remove duplicate HITL tool calls from a single LLM response. @@ -84,9 +45,8 @@ class DedupHITLToolCallsMiddleware(AgentMiddleware): # type: ignore[type-arg] The dedup-resolver map is built from two sources, in priority order: - 1. ``tool.metadata["dedup_key"]`` — callable provided by the registry's - ``ToolDefinition.dedup_key``. Receives the args dict and returns - a string signature. This is the canonical mechanism. + 1. ``tool.metadata["dedup_key"]`` — callable that receives the args dict + and returns a string signature. This is the canonical mechanism. 2. ``tool.metadata["hitl_dedup_key"]`` — string with a primary arg name; primarily used by MCP / Composio tools. """ @@ -162,3 +122,7 @@ class DedupHITLToolCallsMiddleware(AgentMiddleware): # type: ignore[type-arg] updated_msg = last_msg.model_copy(update={"tool_calls": deduped}) return {"messages": [updated_msg]} + + +def build_dedup_hitl_mw(tools: Sequence[BaseTool]) -> DedupHITLToolCallsMiddleware: + return DedupHITLToolCallsMiddleware(agent_tools=list(tools)) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/__init__.py new file mode 100644 index 000000000..d0a1126a5 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/__init__.py @@ -0,0 +1,9 @@ +"""Doom-loop middleware: detect repeated identical tool calls (impl + builder).""" + +from .builder import build_doom_loop_mw +from .middleware import DoomLoopMiddleware + +__all__ = [ + "DoomLoopMiddleware", + "build_doom_loop_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/doom_loop.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/builder.py similarity index 58% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/doom_loop.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/builder.py index d67b8d518..96024adfd 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/doom_loop.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/builder.py @@ -2,10 +2,10 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import DoomLoopMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import DoomLoopMiddleware def build_doom_loop_mw(flags: AgentFeatureFlags) -> DoomLoopMiddleware | None: diff --git a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/middleware.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/middleware/doom_loop.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/middleware.py index a7901c010..4f9b4af1c 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/doom_loop/middleware.py @@ -16,7 +16,7 @@ This ships **OFF by default** until the frontend explicitly handles ``context.permission == "doom_loop"`` interrupts. Wire format: uses SurfSense's existing ``interrupt()`` payload shape -(see ``app/agents/new_chat/tools/hitl.py``): +(see ``app/agents/shared/tools/hitl.py``): { "type": "permission_ask", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/__init__.py new file mode 100644 index 000000000..b5b0267ff --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/__init__.py @@ -0,0 +1,13 @@ +"""End-of-turn KB persistence middleware (main-agent only).""" + +from .builder import build_kb_persistence_mw +from .middleware import ( + KnowledgeBasePersistenceMiddleware, + commit_staged_filesystem_state, +) + +__all__ = [ + "KnowledgeBasePersistenceMiddleware", + "build_kb_persistence_mw", + "commit_staged_filesystem_state", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/kb_persistence.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/builder.py similarity index 78% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/kb_persistence.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/builder.py index 4b27581e7..7e8e06570 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/kb_persistence.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/builder.py @@ -2,8 +2,11 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import KnowledgeBasePersistenceMiddleware +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode + +from .middleware import ( + KnowledgeBasePersistenceMiddleware, +) def build_kb_persistence_mw( diff --git a/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py similarity index 81% rename from surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py index c88dced85..ef86eaddd 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py @@ -1,33 +1,19 @@ """End-of-turn persistence for the cloud-mode SurfSense filesystem. -This middleware runs ``aafter_agent`` once per turn (cloud only). It commits -all staged folder creations, file moves, content writes/edits, file deletes -(``rm``), and directory deletes (``rmdir``) to Postgres in a single ordered -pass: +Runs ``aafter_agent`` once per turn (cloud only), committing staged folder +creates, moves, writes/edits, and ``rm``/``rmdir`` to Postgres in one ordered +pass. Order matters: moves resolve before writes (so write-then-move lands at +the final path), and file deletes run before directory deletes (so a same-turn +``rm /a/x.md`` + ``rmdir /a`` works). -1. Materialize ``staged_dirs`` into ``Folder`` rows. -2. Apply ``pending_moves`` in order (chained moves resolved via - ``doc_id_by_path``). -3. Normalize ``dirty_paths`` through ``pending_moves`` so write-then-move - sequences commit at the final path. Paths queued for ``rm`` this turn - are dropped here so a write+rm sequence doesn't recreate the doc. -4. Commit content writes / edits for ``/documents/*`` paths, skipping - ``temp_*`` basenames. -5. Apply ``pending_deletes`` (``rm``) — file deletes run BEFORE directory - deletes so a same-turn ``rm /a/x.md`` + ``rmdir /a`` sequence works. -6. Apply ``pending_dir_deletes`` (``rmdir``); re-verifies emptiness against - the post-step-5 DB state. +When ``flags.enable_action_log`` is on, each destructive op also snapshots a +``DocumentRevision`` / ``FolderRevision`` for revert. For ``rm``/``rmdir`` the +snapshot and DELETE share a SAVEPOINT, so a failed snapshot aborts the delete +rather than making the data silently irreversible. -When ``flags.enable_action_log`` is on every destructive op also writes a -``DocumentRevision`` / ``FolderRevision`` snapshot bound to the -originating ``AgentActionLog`` row via ``tool_call_id``. ``rm``/``rmdir`` -share a single ``SAVEPOINT`` with their snapshot — if the snapshot fails -the DELETE rolls back and we surface the error rather than silently -making the data irreversible. - -The commit body is exposed as a free function ``commit_staged_filesystem_state`` -so the optional stream-task fallback (``stream_new_chat.py``) can call the -exact same routine when ``aafter_agent`` was skipped (e.g. client disconnect). +The commit body is a free function (``commit_staged_filesystem_state``) so the +stream-task fallback can run the identical routine when ``aafter_agent`` was +skipped (e.g. client disconnect). """ from __future__ import annotations @@ -40,22 +26,28 @@ from typing import Any from fractional_indexing import generate_key_between from langchain.agents.middleware import AgentMiddleware, AgentState from langchain_core.callbacks import adispatch_custom_event, dispatch_custom_event +from langgraph.config import get_config from langgraph.runtime import Runtime from sqlalchemy import delete, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.feature_flags import get_flags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import ( + Receipt, + make_receipt, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, parse_documents_path, safe_folder_segment, virtual_path_to_doc, ) -from app.agents.new_chat.state_reducers import _CLEAR -from app.agents.shared.receipt import Receipt, make_receipt from app.db import ( AgentActionLog, Chunk, @@ -211,11 +203,9 @@ async def _create_document( virtual_path, search_space_id, ) - # Filesystem-parity invariant: the only thing that *must* be unique is - # the path. Two notes can legitimately share content (e.g. ``cp a b``). - # Guard against the path-derived ``unique_identifier_hash`` constraint - # so we surface a clean ValueError instead of letting the INSERT poison - # the session with an IntegrityError. + # Pre-check the path-derived unique_identifier_hash so a duplicate path + # surfaces as a clean ValueError instead of an INSERT IntegrityError that + # poisons the session. Content is intentionally not unique (cp a b). path_collision = await session.execute( select(Document.id).where( Document.search_space_id == search_space_id, @@ -227,13 +217,6 @@ async def _create_document( f"a document already exists at path '{virtual_path}' " "(unique_identifier_hash collision)" ) - # ``content_hash`` is intentionally NOT checked for uniqueness here. - # In a real filesystem two files at different paths can hold identical - # bytes, and the agent's ``write_file`` path needs that semantic to - # support copy/duplicate operations. The hash remains useful as a - # change-detection hint for connector indexers, which still consult it - # via :func:`check_duplicate_document` but do so with a non-unique - # lookup (``.first()``). content_hash = generate_content_hash(content, search_space_id) doc = Document( title=title, @@ -430,15 +413,9 @@ async def _mark_action_reversible( ) -> None: """Flip ``agent_action_log.reversible = TRUE`` for ``action_id``. - Best-effort: caller may invoke from inside a SAVEPOINT and treat - failure as a soft demotion (snapshot persists, just no Revert button). - - Callers should also call ``_dispatch_reversibility_update`` (defined - below) AFTER the enclosing SAVEPOINT block exits successfully so the - chat tool card can light up its Revert button without - re-fetching ``GET /threads/.../actions``. Dispatching from inside the - SAVEPOINT would risk emitting "reversible=true" for rows whose - update gets rolled back if the surrounding destructive op fails. + Pair with ``_dispatch_reversibility_update`` *after* the enclosing + SAVEPOINT commits, so the UI never sees ``reversible=true`` for a row whose + update later rolls back. """ if action_id is None: return @@ -450,22 +427,11 @@ async def _mark_action_reversible( async def _dispatch_reversibility_update(action_id: int | None) -> None: - """Best-effort dispatch of an ``action_log_updated`` custom event. + """Emit an ``action_log_updated`` SSE event so the Revert button lights up. - Surfaces the post-SAVEPOINT reversibility flip to the SSE layer so - the chat tool card can flip its Revert button live. Defensive: - failures are logged at debug level and swallowed; the - REST endpoint ``GET /threads/.../actions`` is still authoritative. - - .. warning:: - Inside :func:`commit_staged_filesystem_state` we DEFER all - dispatches until the outer ``session.commit()`` succeeds — see - the ``deferred_dispatches`` queue in that function. Dispatching - from inside a SAVEPOINT block while the outer transaction is - still pending would emit ``reversible=true`` for rows whose - snapshots get rolled back if the outer commit fails. Direct - callers (e.g. the optional stream-task fallback) that own the - full session lifetime can still call this helper inline. + Best-effort (failures swallowed; the REST actions endpoint is + authoritative). Inside :func:`commit_staged_filesystem_state` this is + deferred until after the outer commit via ``deferred_dispatches``. """ if action_id is None: return @@ -484,12 +450,9 @@ async def _dispatch_reversibility_update(action_id: int | None) -> None: # --------------------------------------------------------------------------- # Snapshot helpers # --------------------------------------------------------------------------- -# -# Best-effort helpers swallow + log so a snapshot failure can never break -# the destructive op for non-destructive tools (write/edit/move/mkdir). -# Strict helpers run inside the SAME ``begin_nested()`` SAVEPOINT as the -# destructive DELETE — failure aborts the savepoint and leaves the doc / -# folder intact, so revertable ops never become irreversible silently. +# Best-effort variants (write/edit/move/mkdir) swallow failures. Strict +# variants (rm/rmdir) share the destructive op's SAVEPOINT so a snapshot +# failure aborts the delete instead of making it silently irreversible. def _doc_revision_payload( @@ -699,15 +662,9 @@ async def commit_staged_filesystem_state( ) -> dict[str, Any] | None: """Commit all staged filesystem changes; return the state delta for reducers. - Shared between :class:`KnowledgeBasePersistenceMiddleware.aafter_agent` - and the optional stream-task fallback. - - When ``flags.enable_action_log`` is on every destructive op also writes - a ``DocumentRevision`` / ``FolderRevision`` snapshot bound to the - originating ``AgentActionLog`` row via ``tool_call_id``. Snapshot - durability is best-effort for non-destructive ops and STRICT for - ``rm``/``rmdir`` (snapshot + DELETE share a SAVEPOINT — snapshot - failure aborts the delete). + Shared between :class:`KnowledgeBasePersistenceMiddleware.aafter_agent` and + the stream-task fallback. See the module docstring for ordering and the + action-log snapshot/revert semantics. """ if filesystem_mode != FilesystemMode.CLOUD: return None @@ -766,8 +723,7 @@ async def commit_staged_filesystem_state( flags = get_flags() snapshot_enabled = flags.enable_action_log - # De-duplicate pending deletes per-path while preserving the latest - # tool_call_id (the one the user is most likely to revert via the UI). + # De-dup deletes per-path, keeping the latest tool_call_id (likeliest revert). file_delete_paths: dict[str, str] = {} for entry in pending_deletes: if not isinstance(entry, dict): @@ -791,22 +747,14 @@ async def commit_staged_filesystem_state( applied_moves: list[dict[str, Any]] = [] doc_id_path_tombstones: dict[str, int | None] = {} tree_changed = False - # Reversibility-flip dispatches are deferred until AFTER the outer - # ``session.commit()`` succeeds. Dispatching from inside the - # SAVEPOINT chain while the outer transaction is still pending - # would emit ``reversible=true`` for rows whose snapshots get rolled - # back if the final commit raises. Snapshot helpers append on - # success; we drain this list after commit and silently abandon it - # on rollback so the UI stays consistent with durable state. + # Reversibility-flip dispatches are drained only after the outer commit + # succeeds (and abandoned on rollback), so the UI never sees reversible=true + # for a snapshot that didn't durably land. deferred_dispatches: list[int] = [] try: async with shielded_async_session() as session: - # ------------------------------------------------------------------ - # Resolve action-id bindings up front. One SELECT per turn for all - # tool_call_ids, NOT one per op — important because a turn that - # touches 50 paths would otherwise issue 50 lookups. - # ------------------------------------------------------------------ + # Resolve all action-id bindings in one SELECT per turn, not per op. action_id_by_call: dict[str, int] = {} if snapshot_enabled and thread_id is not None: tool_call_ids: set[str] = set() @@ -839,10 +787,7 @@ async def commit_staged_filesystem_state( next(iter(action_id_by_call), None) if action_id_by_call else None ) - # ------------------------------------------------------------------ - # 1. staged_dirs -> Folder rows. Snapshot post-flush so the new - # folder_id is available for the FK. - # ------------------------------------------------------------------ + # 1. staged_dirs -> Folder rows (snapshot post-flush for the FK). for folder_path in staged_dirs: if not isinstance(folder_path, str): continue @@ -863,7 +808,6 @@ async def commit_staged_filesystem_state( tcid = staged_dir_tool_calls.get(folder_path) action_id = _action_id_for(tcid) if action_id is not None: - # Re-read the folder for the snapshot. result = await session.execute( select(Folder).where(Folder.id == folder_id) ) @@ -878,16 +822,13 @@ async def commit_staged_filesystem_state( deferred_dispatches=deferred_dispatches, ) - # ------------------------------------------------------------------ - # 2. pending_moves. Snapshot pre-move (in-place restore on revert). - # ------------------------------------------------------------------ + # 2. pending_moves (snapshot pre-move for in-place restore on revert). for move in pending_moves: source = str(move.get("source") or "") if snapshot_enabled and source: tcid = str(move.get("tool_call_id") or "") action_id = _action_id_for(tcid) if action_id is not None: - # Resolve the doc to snapshot BEFORE we mutate it. doc_id_pre = doc_id_by_path.get(source) document_pre: Document | None = None if doc_id_pre is not None: @@ -937,10 +878,8 @@ async def commit_staged_filesystem_state( path = move_alias[path] return path - # ------------------------------------------------------------------ - # 3. dirty_paths -> writes/edits. Skip any path queued for ``rm`` - # this turn so a write+rm sequence doesn't recreate the doc. - # ------------------------------------------------------------------ + # 3. dirty_paths -> writes/edits. Paths queued for rm this turn are + # skipped so a write+rm sequence doesn't recreate the doc. kb_dirty_seen: set[str] = set() kb_dirty: list[str] = [] kb_dirty_origin: dict[str, str] = {} @@ -969,9 +908,7 @@ async def commit_staged_filesystem_state( continue content = "\n".join(file_data.get("content") or []) doc_id = doc_id_by_path.get(path) - # Path ↔ tool_call_id binding: the dirty_paths list dedupes via - # _add_unique_reducer, so we look up the latest tool_call_id by - # path (or by the un-renamed origin). + # Look up tool_call_id by final path or its pre-rename origin. origin = kb_dirty_origin.get(path, path) tcid = dirty_path_tool_calls.get(path) or dirty_path_tool_calls.get( origin @@ -979,12 +916,9 @@ async def commit_staged_filesystem_state( action_id = _action_id_for(tcid) if doc_id is None: - # The in-memory ``doc_id_by_path`` is per-thread and starts - # empty in every new chat. If the agent writes to a path - # that already exists in the DB (e.g. a previous chat's - # ``notes.md``), we must NOT try to INSERT — it would hit - # ``unique_identifier_hash`` (path-derived). Look up the - # existing doc and update it in place instead. + # doc_id_by_path is per-thread and empty in a new chat, so a + # write to a path already in the DB must update in place, not + # INSERT (which would hit the path-derived unique hash). existing = await virtual_path_to_doc( session, search_space_id=search_space_id, @@ -1033,12 +967,9 @@ async def commit_staged_filesystem_state( } ) else: - # Fresh create. Wrap each create in a SAVEPOINT so a - # residual ``IntegrityError`` (e.g. a deployment that - # hasn't run migration 133 yet, where - # ``documents.content_hash`` still carries its legacy - # global UNIQUE constraint) rolls back only this one - # create instead of poisoning the whole turn. + # Fresh create, wrapped in a SAVEPOINT so a residual + # IntegrityError (e.g. pre-migration-133 content_hash UNIQUE) + # rolls back only this create, not the whole turn. placeholder_revision_id: int | None = None if snapshot_enabled and action_id is not None: placeholder_revision_id = await _snapshot_document_pre_create( @@ -1061,8 +992,7 @@ async def commit_staged_filesystem_state( logger.warning( "kb_persistence: skipping %s create: %s", path, exc ) - # Roll back the placeholder revision since the create - # never happened. + # Create never happened; drop its placeholder revision. if placeholder_revision_id is not None: await session.execute( delete(DocumentRevision).where( @@ -1109,19 +1039,14 @@ async def commit_staged_filesystem_state( ) tree_changed = True - # ------------------------------------------------------------------ - # 4. pending_deletes -> ``rm``. STRICT durability: snapshot + DELETE - # share a SAVEPOINT. If the snapshot insert fails, the DELETE - # rolls back too and we surface the error rather than silently - # making the data irreversible. - # ------------------------------------------------------------------ + # 4. pending_deletes -> rm. Strict: snapshot + DELETE share a + # SAVEPOINT, so a failed snapshot rolls the delete back too. for raw_path, tcid in file_delete_paths.items(): final = _final_path(raw_path) if not final.startswith(DOCUMENTS_ROOT + "/"): continue action_id = _action_id_for(tcid) - # Resolve the doc. doc_id_for_delete = doc_id_by_path.get(final) document_to_delete: Document | None = None if doc_id_for_delete is not None: @@ -1150,7 +1075,6 @@ async def commit_staged_filesystem_state( try: async with session.begin_nested(): - # Strict: snapshot first; failure aborts the delete. if snapshot_enabled and action_id is not None: chunks = await _load_chunks_for_snapshot( session, doc_id=doc_pk @@ -1179,10 +1103,7 @@ async def commit_staged_filesystem_state( ) continue - # B1 — SAVEPOINT released. Defer the reversibility-flip - # dispatch until AFTER the outer commit succeeds so we - # never tell the UI a row is reversible if its snapshot - # gets rolled back. + # Defer the reversibility flip until after the outer commit. if snapshot_enabled and action_id is not None: deferred_dispatches.append(int(action_id)) @@ -1201,11 +1122,8 @@ async def commit_staged_filesystem_state( ) tree_changed = True - # ------------------------------------------------------------------ - # 5. pending_dir_deletes -> ``rmdir``. STRICT durability + final - # emptiness check (after step 4's deletes have run, an "empty - # mid-turn" directory really IS empty in DB now). - # ------------------------------------------------------------------ + # 5. pending_dir_deletes -> rmdir. Strict, and re-checks emptiness + # against post-step-4 DB state. for raw_path, tcid in dir_delete_paths.items(): final = _final_path(raw_path) if not final.startswith(DOCUMENTS_ROOT + "/"): @@ -1226,7 +1144,6 @@ async def commit_staged_filesystem_state( ) continue - # Re-check emptiness against in-DB state. docs_in_folder = await session.execute( select(Document.id) .where(Document.folder_id == folder_id) @@ -1291,10 +1208,7 @@ async def commit_staged_filesystem_state( ) continue - # B1 — SAVEPOINT released. Defer the reversibility-flip - # dispatch until AFTER the outer commit succeeds so we - # never tell the UI a row is reversible if its snapshot - # gets rolled back. + # Defer the reversibility flip until after the outer commit. if snapshot_enabled and action_id is not None: deferred_dispatches.append(int(action_id)) @@ -1314,18 +1228,13 @@ async def commit_staged_filesystem_state( logger.exception( "kb_persistence: commit failed (search_space=%s)", search_space_id ) - # Outer commit raised — every SAVEPOINT-released change above - # (snapshots + reversibility flips) is now rolled back. Drop - # the deferred SSE dispatches so the UI stays consistent with - # durable state. + # Outer commit raised: everything above rolled back, so drop the + # deferred dispatches. deferred_dispatches.clear() return None - # Outer commit succeeded; flush deferred reversibility-flip - # dispatches now so the chat tool card can light up its Revert - # button without re-fetching ``GET /threads/.../actions``. De-dup - # to avoid emitting the same id twice (e.g. write-then-rm in the - # same turn dispatches once for each snapshot site). + # Commit succeeded; flush deferred reversibility flips (de-duped, since + # write-then-rm in one turn appends an id per snapshot site). if deferred_dispatches and dispatch_events: for action_id in dict.fromkeys(deferred_dispatches): try: @@ -1371,9 +1280,8 @@ async def commit_staged_filesystem_state( p for p in files if isinstance(p, str) and _basename(p).startswith(_TEMP_PREFIX) ] - # Tombstone every committed-delete path so a stale ``state["files"]`` entry - # (which als_info would otherwise interpret as content) cannot survive into - # the next turn and make a now-empty folder look non-empty. + # Tombstone committed-delete paths so a stale state["files"] entry can't + # survive into the next turn and make a now-empty folder look non-empty. deleted_file_paths = [ str(payload.get("virtualPath") or "") for payload in committed_deletes @@ -1394,11 +1302,8 @@ async def commit_staged_filesystem_state( "dirty_path_tool_calls": {_CLEAR: True}, } - # Emit one Receipt per committed mutation, folded into ``state['receipts']`` - # via ``_list_append_reducer``. The receipts surface what actually committed - # (post-savepoint) rather than what the LLM intended; the orchestrator uses - # them as ground truth in the ```` teaching. KB writes do not - # have public verifiable URLs, so ``verifiable_url`` stays unset. + # One Receipt per committed mutation: ground truth (post-savepoint) for the + # orchestrator's teaching. KB writes have no public URL. receipts: list[Receipt] = [] def _kb_receipt( @@ -1439,8 +1344,6 @@ async def commit_staged_filesystem_state( external_id=payload.get("id"), ) for payload in applied_moves: - # ``applied_moves`` rows carry the destination ``virtualPath`` because - # the move has already landed in the DB by the time we reach this code. path = str(payload.get("virtualPath") or "") _kb_receipt( type="file", @@ -1480,9 +1383,7 @@ async def commit_staged_filesystem_state( if tree_changed: delta["tree_version"] = int(state_dict.get("tree_version") or 0) + 1 - # Avoid 'unused' lint when turn_id_for_revision was only useful for - # diagnostic purposes inside the SAVEPOINT chain above. - _ = turn_id_for_revision + _ = turn_id_for_revision # diagnostic-only; silence unused lint logger.info( "kb_persistence: commit (search_space=%s) creates=%d updates=%d " @@ -1536,9 +1437,33 @@ class KnowledgeBasePersistenceMiddleware(AgentMiddleware): # type: ignore[type- search_space_id=self.search_space_id, created_by_id=self.created_by_id, filesystem_mode=self.filesystem_mode, - thread_id=self.thread_id, + thread_id=self._resolve_thread_id(), ) + def _resolve_thread_id(self) -> int | None: + """Resolve the live thread id from the active ``RunnableConfig``. + + ``aafter_agent`` only receives a ``Runtime`` (which does NOT carry the + config), so we read ``configurable.thread_id`` via + :func:`langgraph.config.get_config` — the same node-context pattern used + by ``BusyMutexMiddleware``. Resolving at runtime (rather than using the + value captured at ``__init__``) lets one cached compiled graph commit + staged writes against the correct thread across many chats. Falls back + to the constructor value for legacy/test runtimes. + """ + try: + config = get_config() + except Exception: + config = None + if isinstance(config, dict): + value = (config.get("configurable") or {}).get("thread_id") + if value is not None: + try: + return int(value) + except (TypeError, ValueError): + return None + return self.thread_id + __all__ = [ "KnowledgeBasePersistenceMiddleware", diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_priority.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_priority.py similarity index 56% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_priority.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_priority.py index 27cee8b37..787dbe402 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_priority.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_priority.py @@ -4,8 +4,10 @@ from __future__ import annotations from langchain_core.language_models import BaseChatModel -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import KnowledgePriorityMiddleware +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.knowledge_search import ( + KnowledgePriorityMiddleware, +) from app.services.llm_service import get_planner_llm @@ -17,7 +19,16 @@ def build_knowledge_priority_mw( available_connectors: list[str] | None, available_document_types: list[str] | None, mentioned_document_ids: list[int] | None, + preinjection_enabled: bool = True, ) -> KnowledgePriorityMiddleware: + """Build the KB priority middleware. + + When ``preinjection_enabled`` is False (the lazy default), the middleware + runs in mentions-only mode: it skips the expensive planner LLM + embedding + + hybrid search and only surfaces explicit @-mentions. The main agent is + expected to pull relevant KB content on demand via the + ``search_knowledge_base`` tool instead. + """ return KnowledgePriorityMiddleware( llm=llm, planner_llm=get_planner_llm(), @@ -27,4 +38,5 @@ def build_knowledge_priority_mw( available_document_types=available_document_types, mentioned_document_ids=mentioned_document_ids, inject_system_message=False, + mentions_only=not preinjection_enabled, ) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/__init__.py new file mode 100644 index 000000000..f2d456b34 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/__init__.py @@ -0,0 +1,9 @@ +"""Knowledge-tree middleware: injection, cloud only (impl + builder).""" + +from .builder import build_knowledge_tree_mw +from .middleware import KnowledgeTreeMiddleware + +__all__ = [ + "KnowledgeTreeMiddleware", + "build_knowledge_tree_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_tree.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/builder.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_tree.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/builder.py index fb4511067..644d1e55a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/knowledge_tree.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/builder.py @@ -4,8 +4,9 @@ from __future__ import annotations from langchain_core.language_models import BaseChatModel -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import KnowledgeTreeMiddleware +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode + +from .middleware import KnowledgeTreeMiddleware def build_knowledge_tree_mw( diff --git a/surfsense_backend/app/agents/new_chat/middleware/knowledge_tree.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/middleware.py similarity index 97% rename from surfsense_backend/app/agents/new_chat/middleware/knowledge_tree.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/middleware.py index 6bd6430d1..a0c62834a 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/knowledge_tree.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/middleware.py @@ -33,9 +33,11 @@ from langchain_core.messages import SystemMessage from langgraph.runtime import Runtime from sqlalchemy import select -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, PathIndex, build_path_index, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/__init__.py new file mode 100644 index 000000000..0106234c0 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/__init__.py @@ -0,0 +1,5 @@ +"""User/team memory injection middleware (main-agent only).""" + +from .builder import build_memory_mw + +__all__ = ["build_memory_mw"] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/memory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/builder.py similarity index 86% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/memory.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/builder.py index 9316b3e21..4ea171e13 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/memory.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/builder.py @@ -2,9 +2,10 @@ from __future__ import annotations -from app.agents.new_chat.middleware import MemoryInjectionMiddleware from app.db import ChatVisibility +from .middleware import MemoryInjectionMiddleware + def build_memory_mw( *, diff --git a/surfsense_backend/app/agents/new_chat/middleware/memory_injection.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/memory_injection.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/memory/middleware.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/__init__.py new file mode 100644 index 000000000..c4c004618 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/__init__.py @@ -0,0 +1,9 @@ +"""Noop-injection middleware: provider-compat _noop tool (impl + builder).""" + +from .builder import build_noop_injection_mw +from .middleware import NoopInjectionMiddleware + +__all__ = [ + "NoopInjectionMiddleware", + "build_noop_injection_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/noop_injection.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/builder.py similarity index 59% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/noop_injection.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/builder.py index 6e6467ad0..774cb0f46 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/noop_injection.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/builder.py @@ -2,10 +2,10 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import NoopInjectionMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import NoopInjectionMiddleware def build_noop_injection_mw(flags: AgentFeatureFlags) -> NoopInjectionMiddleware | None: diff --git a/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/middleware.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/noop_injection.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/noop_injection/middleware.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/__init__.py new file mode 100644 index 000000000..801d08962 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/__init__.py @@ -0,0 +1,9 @@ +"""OTel-span middleware: spans on model and tool calls (impl + builder).""" + +from .builder import build_otel_mw +from .middleware import OtelSpanMiddleware + +__all__ = [ + "OtelSpanMiddleware", + "build_otel_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/otel.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/builder.py similarity index 53% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/otel.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/builder.py index bd7516e65..fe3bce4c5 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/otel.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/builder.py @@ -2,10 +2,10 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import OtelSpanMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import OtelSpanMiddleware def build_otel_mw(flags: AgentFeatureFlags) -> OtelSpanMiddleware | None: diff --git a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/middleware.py similarity index 91% rename from surfsense_backend/app/agents/new_chat/middleware/otel_span.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/middleware.py index ecaa042a9..834227666 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/otel_span/middleware.py @@ -24,6 +24,7 @@ from langchain.agents.middleware import AgentMiddleware from langchain_core.messages import AIMessage, ToolMessage from app.observability import metrics as ot_metrics, otel as ot +from app.utils.perf import get_perf_logger if TYPE_CHECKING: # pragma: no cover — type-only from langchain.agents.middleware.types import ( @@ -34,6 +35,7 @@ if TYPE_CHECKING: # pragma: no cover — type-only from langgraph.types import Command logger = logging.getLogger(__name__) +_perf_log = get_perf_logger() class OtelSpanMiddleware(AgentMiddleware): @@ -60,7 +62,23 @@ class OtelSpanMiddleware(AgentMiddleware): handler: Callable[[ModelRequest], Awaitable[ModelResponse | AIMessage | Any]], ) -> ModelResponse | AIMessage | Any: if not ot.is_enabled(): - return await handler(request) + # Always emit a [PERF] line for the model step even when OTel is + # disabled. This isolates provider/model latency from the agent's + # pre-flight (before_agent KB-priority/memory/tree) work, which is + # the usual culprit when the multi-agent path feels slow to start. + # ``perf_counter`` at entry doubles as the "before_agent finished / + # model call started" marker on the first step of a turn. + model_id, _provider = _resolve_model_attrs(request) + _t0 = time.perf_counter() + _perf_log.info("[model_call] start model=%s", model_id) + try: + return await handler(request) + finally: + _perf_log.info( + "[model_call] done model=%s elapsed=%.3fs", + model_id, + time.perf_counter() - _t0, + ) model_id, provider = _resolve_model_attrs(request) t0 = time.perf_counter() diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/plugins.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/plugins.py similarity index 86% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/plugins.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/plugins.py index 4418e3806..43f4136ec 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/plugins.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/plugins.py @@ -7,15 +7,15 @@ from typing import Any from langchain_core.language_models import BaseChatModel -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.plugin_loader import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled +from app.db import ChatVisibility + +from ..plugins.loader import ( PluginContext, load_allowed_plugin_names_from_env, load_plugin_middlewares, ) -from app.db import ChatVisibility - -from ..shared.flags import enabled def build_plugin_middlewares( diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/skills.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/skills.py similarity index 71% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/skills.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/skills.py index 63a57c5a0..13c62e817 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/skills.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/skills.py @@ -6,14 +6,11 @@ import logging from deepagents.middleware.skills import SkillsMiddleware -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import ( - build_skills_backend_factory, - default_skills_sources, -) +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from ..skills.backends import build_skills_backend_factory, default_skills_sources def build_skills_mw( diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/stack.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/stack.py new file mode 100644 index 000000000..675898d4c --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/stack.py @@ -0,0 +1,314 @@ +"""Main-agent middleware list assembly: one line per slot. + +The main agent is a pure router — filesystem reads/writes are owned by the +``knowledge_base`` subagent and delegated via the ``task`` tool. The stack +here only renders KB context (workspace tree + priority docs), projects it +into system messages, and commits any subagent-side staged writes at end of +turn (cloud mode). +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Sequence +from typing import Any, cast + +from deepagents import SubAgent +from deepagents.backends import StateBackend +from langchain.agents import create_agent +from langchain_core.language_models import BaseChatModel +from langchain_core.runnables import Runnable +from langchain_core.tools import BaseTool +from langgraph.types import Checkpointer + +from app.agents.chat.multi_agent_chat.main_agent.middleware.memory import ( + build_memory_mw, +) +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.anthropic_cache import ( + build_anthropic_cache_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.compaction import ( + build_compaction_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.kb_context_projection import ( + build_kb_context_projection_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.patch_tool_calls import ( + build_patch_tool_calls_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.resilience import ( + build_resilience_middlewares, +) +from app.agents.chat.multi_agent_chat.shared.middleware.todos import build_todos_mw +from app.agents.chat.multi_agent_chat.shared.permissions import ( + build_permission_mw, +) +from app.agents.chat.multi_agent_chat.subagents import ( + build_subagents, + get_subagents_to_exclude, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( + NAME as KB_WRITE_NAME, + READONLY_NAME as KB_READONLY_NAME, + build_readonly_subagent as build_kb_readonly_subagent, + build_subagent as build_kb_write_subagent, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.ask_knowledge_base_tool import ( + build_ask_knowledge_base_tool, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.prompts import ( + load_description as load_kb_write_description, +) +from app.agents.chat.multi_agent_chat.subagents.middleware_stack import ( + build_subagent_middleware_stack, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import ( + SURF_LAZY_SPEC_FACTORY_KEY, +) +from app.db import ChatVisibility +from app.utils.perf import get_perf_logger + +from .action_log import build_action_log_mw +from .anonymous_document import build_anonymous_doc_mw +from .busy_mutex import build_busy_mutex_mw +from .checkpointed_subagent_middleware import ( + SurfSenseCheckpointedSubAgentMiddleware, +) +from .checkpointed_subagent_middleware.task_description import ( + TASK_TOOL_DESCRIPTION, +) +from .context_editing import build_context_editing_mw +from .dedup_hitl import build_dedup_hitl_mw +from .doom_loop import build_doom_loop_mw +from .kb_persistence import build_kb_persistence_mw +from .knowledge_priority import build_knowledge_priority_mw +from .knowledge_tree import build_knowledge_tree_mw +from .noop_injection import build_noop_injection_mw +from .otel_span import build_otel_mw +from .plugins import build_plugin_middlewares +from .skills import build_skills_mw +from .tool_call_repair import build_repair_mw + +_perf_log = get_perf_logger() + + +def build_main_agent_deepagent_middleware( + *, + llm: BaseChatModel, + tools: Sequence[BaseTool], + backend_resolver: Any, + filesystem_mode: FilesystemMode, + search_space_id: int, + user_id: str | None, + thread_id: int | None, + visibility: ChatVisibility, + anon_session_id: str | None, + available_connectors: list[str] | None, + available_document_types: list[str] | None, + mentioned_document_ids: list[int] | None, + max_input_tokens: int | None, + flags: AgentFeatureFlags, + subagent_dependencies: dict[str, Any], + checkpointer: Checkpointer, + mcp_tools_by_agent: dict[str, list[BaseTool]] | None = None, + disabled_tools: list[str] | None = None, +) -> list[Any]: + """Ordered middleware for ``create_agent`` (None entries already stripped).""" + stack_build_start = time.perf_counter() + resilience = build_resilience_middlewares(flags) + + memory_mw = build_memory_mw( + user_id=user_id, + search_space_id=search_space_id, + visibility=visibility, + ) + + subagent_dependencies = { + **subagent_dependencies, + "backend_resolver": backend_resolver, + "filesystem_mode": filesystem_mode, + "flags": flags, + } + shared_mw_start = time.perf_counter() + shared_subagent_middleware = build_subagent_middleware_stack( + resilience=resilience, + flags=flags, + ) + shared_mw_elapsed = time.perf_counter() - shared_mw_start + + def _compile_kb_readonly() -> Runnable: + """Build *and* compile the read-only KB graph on first ``ask_knowledge_base`` use. + + Both the spec build (``build_kb_readonly_subagent`` — middleware + + tool-schema construction, ~the same cost as one regular subagent) and + the ``create_agent`` compile are deferred here (memoized by + ``build_ask_knowledge_base_tool``) so neither is paid on the cold + agent-build / TTFT path; most first turns never call a subagent. + """ + build_start = time.perf_counter() + kb_readonly_spec = build_kb_readonly_subagent( + dependencies=subagent_dependencies, + model=llm, + middleware_stack=shared_subagent_middleware, + ).spec + runnable = create_agent( + llm, + system_prompt=kb_readonly_spec["system_prompt"], + tools=kb_readonly_spec["tools"], + middleware=kb_readonly_spec["middleware"], + name=KB_READONLY_NAME, + checkpointer=checkpointer, + ) + _perf_log.info( + "[subagent_compile_lazy] name=%s (spec+compile) in %.3fs", + KB_READONLY_NAME, + time.perf_counter() - build_start, + ) + return runnable + + ask_kb_tool = build_ask_knowledge_base_tool(_compile_kb_readonly) + + def _build_kb_write_spec() -> dict[str, Any]: + """Build the *write* knowledge_base subagent spec on first ``task`` use. + + The KB filesystem middleware builds ~13 tool schemas at ~150ms each + (~2s total), all of which used to land on the cold agent-build / TTFT + path even though ``task("knowledge_base")`` is essentially never the + first thing a turn does. Deferring the whole spec build here (memoized + by the checkpointed subagent middleware) moves that cost to the first + actual KB-write delegation. Captures the same ``subagent_dependencies`` + the eager build would have used, so cross-thread cache behaviour is + unchanged. + """ + spec = build_kb_write_subagent( + dependencies=subagent_dependencies, + model=llm, + middleware_stack=shared_subagent_middleware, + ).spec + if disabled_tools: + disabled = frozenset(disabled_tools) + tools = spec.get("tools") # type: ignore[typeddict-item] + if isinstance(tools, list): + spec["tools"] = [ # type: ignore[typeddict-unknown-key] + t for t in tools if getattr(t, "name", None) not in disabled + ] + return cast(dict[str, Any], spec) + + subagents_start = time.perf_counter() + # The write knowledge_base subagent is excluded from the eager build and + # registered as a lazy descriptor (name + description cheap; spec built on + # first ``task("knowledge_base")`` use) — see ``_build_kb_write_spec``. + exclude_names = [*get_subagents_to_exclude(available_connectors), KB_WRITE_NAME] + subagents: list[SubAgent] = build_subagents( + dependencies=subagent_dependencies, + model=llm, + middleware_stack=shared_subagent_middleware, + mcp_tools_by_agent=mcp_tools_by_agent or {}, + exclude=exclude_names, + disabled_tools=disabled_tools, + ask_kb_tool=ask_kb_tool, + ) + kb_write_descriptor = cast( + SubAgent, + { + "name": KB_WRITE_NAME, + "description": load_kb_write_description(), + SURF_LAZY_SPEC_FACTORY_KEY: _build_kb_write_spec, + }, + ) + subagents.append(kb_write_descriptor) + subagents_elapsed = time.perf_counter() - subagents_start + logging.debug("Subagents registry: %s", [s["name"] for s in subagents]) + + assembly_start = time.perf_counter() + stack: list[Any] = [ + build_busy_mutex_mw(flags), + build_otel_mw(flags), + build_todos_mw(system_prompt=""), + memory_mw, + build_anonymous_doc_mw( + filesystem_mode=filesystem_mode, anon_session_id=anon_session_id + ), + build_knowledge_tree_mw( + filesystem_mode=filesystem_mode, + search_space_id=search_space_id, + llm=llm, + ), + build_knowledge_priority_mw( + llm=llm, + search_space_id=search_space_id, + filesystem_mode=filesystem_mode, + available_connectors=available_connectors, + available_document_types=available_document_types, + mentioned_document_ids=mentioned_document_ids, + preinjection_enabled=flags.enable_kb_priority_preinjection, + ), + build_kb_context_projection_mw(), + build_kb_persistence_mw( + filesystem_mode=filesystem_mode, + search_space_id=search_space_id, + user_id=user_id, + thread_id=thread_id, + ), + build_skills_mw( + flags=flags, + filesystem_mode=filesystem_mode, + search_space_id=search_space_id, + ), + SurfSenseCheckpointedSubAgentMiddleware( + checkpointer=checkpointer, + backend=StateBackend, + subagents=subagents, + system_prompt=None, + task_description=TASK_TOOL_DESCRIPTION, + search_space_id=search_space_id, + ), + resilience.model_call_limit, + resilience.tool_call_limit, + build_context_editing_mw( + flags=flags, + max_input_tokens=max_input_tokens, + tools=tools, + backend_resolver=backend_resolver, + ), + build_compaction_mw(llm), + build_noop_injection_mw(flags), + resilience.retry, + resilience.fallback, + build_repair_mw(flags=flags, tools=tools), + build_permission_mw(flags=flags), + build_doom_loop_mw(flags), + build_action_log_mw( + flags=flags, + thread_id=thread_id, + search_space_id=search_space_id, + user_id=user_id, + ), + build_patch_tool_calls_mw(), + build_dedup_hitl_mw(tools), + *build_plugin_middlewares( + flags=flags, + search_space_id=search_space_id, + user_id=user_id, + visibility=visibility, + llm=llm, + ), + build_anthropic_cache_mw(), + ] + result = [m for m in stack if m is not None] + assembly_elapsed = time.perf_counter() - assembly_start + _perf_log.info( + "[stack_build] total=%.3fs shared_subagent_mw=%.3fs " + "build_subagents=%.3fs stack_assembly=%.3fs subagents=%d mw=%d " + "(kb_readonly deferred to first ask_knowledge_base)", + time.perf_counter() - stack_build_start, + shared_mw_elapsed, + subagents_elapsed, + assembly_elapsed, + len(subagents), + len(result), + ) + return result diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/__init__.py new file mode 100644 index 000000000..1e6d93750 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/__init__.py @@ -0,0 +1,9 @@ +"""Tool-call-repair middleware: fix miscased/unknown tool names (impl + builder).""" + +from .builder import build_repair_mw +from .middleware import ToolCallNameRepairMiddleware + +__all__ = [ + "ToolCallNameRepairMiddleware", + "build_repair_mw", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/repair.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/builder.py similarity index 83% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/repair.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/builder.py index 378b61be1..a1cc558b2 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/repair.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/builder.py @@ -6,10 +6,10 @@ from collections.abc import Sequence from langchain_core.tools import BaseTool -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import ToolCallNameRepairMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.flags import enabled -from ..shared.flags import enabled +from .middleware import ToolCallNameRepairMiddleware # deepagents-built-in tool names the repair pass treats as known. _DEEPAGENT_BUILTIN_TOOL_NAMES: frozenset[str] = frozenset( diff --git a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/middleware.py similarity index 96% rename from surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/middleware.py index 9f81a168b..260e5cbd4 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/tool_call_repair/middleware.py @@ -34,8 +34,6 @@ from langchain.agents.middleware.types import ( from langchain_core.messages import AIMessage from langgraph.runtime import Runtime -from app.agents.new_chat.tools.invalid_tool import INVALID_TOOL_NAME - logger = logging.getLogger(__name__) @@ -120,6 +118,12 @@ class ToolCallNameRepairMiddleware( return call # Stage 2 — invalid fallback + # Local import keeps the middleware module import-light and avoids any + # tools <-> middleware import-order coupling at module scope. + from app.agents.chat.multi_agent_chat.main_agent.tools.invalid_tool import ( + INVALID_TOOL_NAME, + ) + if INVALID_TOOL_NAME in registered: original_args = call.get("args") or {} error_msg = ( diff --git a/surfsense_backend/app/agents/new_chat/plugins/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/plugins/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/plugin_loader.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/loader.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/plugin_loader.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/loader.py diff --git a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/year_substituter.py similarity index 95% rename from surfsense_backend/app/agents/new_chat/plugins/year_substituter.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/year_substituter.py index 2b7781b90..f6564fe6e 100644 --- a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/plugins/year_substituter.py @@ -17,7 +17,7 @@ Wire-up in ``pyproject.toml`` (illustrative; the in-repo plugin doesn't need this -- it's already on the import path):: [project.entry-points."surfsense.plugins"] - year_substituter = "app.agents.new_chat.plugins.year_substituter:make_middleware" + year_substituter = "app.agents.chat.multi_agent_chat.main_agent.plugins.year_substituter:make_middleware" """ from __future__ import annotations @@ -34,7 +34,7 @@ if TYPE_CHECKING: # pragma: no cover - type-only from langchain_core.messages import ToolMessage from langgraph.types import Command - from app.agents.new_chat.plugin_loader import PluginContext + from .loader import PluginContext logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/agent_cache.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache.py similarity index 82% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/agent_cache.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache.py index df1ee1b4c..6ac22e575 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/agent_cache.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache.py @@ -10,18 +10,18 @@ from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool from langgraph.types import Checkpointer -from app.agents.new_chat.agent_cache import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.db import ChatVisibility + +from ..graph.compile_graph_sync import build_compiled_agent_graph_sync +from .agent_cache_store import ( flags_signature, get_cache, stable_hash, system_prompt_hash, tools_signature, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.db import ChatVisibility - -from ..graph.compile_graph_sync import build_compiled_agent_graph_sync def mcp_signature(mcp_tools_by_agent: dict[str, list[BaseTool]]) -> str: @@ -91,10 +91,18 @@ async def build_agent_with_cache( # Every per-request value any middleware closes over at __init__ must be in # the key, otherwise a hit will leak state across threads. Bump the schema # version when the component list changes shape. + # + # Cross-thread reuse: when enabled, ``thread_id`` is dropped from the key so + # one compiled graph serves all of a user's (same space/config/visibility) + # chats. This is only safe because ActionLog, KB-persistence, and the + # deliverables tools now resolve the chat thread from the live + # RunnableConfig instead of a constructor closure; the schema tag is bumped + # so v2 (per-thread) entries are never confused with v3 (shared) ones. + cross_thread = flags.enable_cross_thread_agent_cache cache_key = stable_hash( - "multi-agent-v2", + "multi-agent-v3" if cross_thread else "multi-agent-v2", config_id, - thread_id, + None if cross_thread else thread_id, user_id, search_space_id, visibility, diff --git a/surfsense_backend/app/agents/new_chat/agent_cache.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache_store.py similarity index 95% rename from surfsense_backend/app/agents/new_chat/agent_cache.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache_store.py index fa8e6fb72..00866adf9 100644 --- a/surfsense_backend/app/agents/new_chat/agent_cache.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/agent_cache_store.py @@ -67,13 +67,13 @@ from __future__ import annotations import asyncio import hashlib import logging -import os import time from collections import OrderedDict from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any +from app.config import config from app.utils.perf import get_perf_logger logger = logging.getLogger(__name__) @@ -113,12 +113,11 @@ def tools_signature( MCP tools loaded for the user changes, gating rules flip, etc.). * The available connectors / document types for the search space change (new connector added, last connector removed, new document - type indexed). Because :func:`get_connector_gated_tools` derives - ``modified_disabled_tools`` from ``available_connectors``, the - tool surface is technically already covered — but we hash the - connector list separately so an empty-list "no tools changed" - situation still rotates the key when, say, the user re-adds a - connector that gates a tool we were already not exposing. + type indexed). Connector gating derives disabled tools from + ``available_connectors``, so the tool surface is technically already + covered — but we hash the connector list separately so an empty-list + "no tools changed" situation still rotates the key when, say, the user + re-adds a connector that gates a tool we were already not exposing. Stays stable across: @@ -329,8 +328,8 @@ def _short(key: str, n: int = 16) -> str: # Module-level singleton # --------------------------------------------------------------------------- -_DEFAULT_MAXSIZE = int(os.getenv("SURFSENSE_AGENT_CACHE_MAXSIZE", "256")) -_DEFAULT_TTL = float(os.getenv("SURFSENSE_AGENT_CACHE_TTL_SECONDS", "1800")) +_DEFAULT_MAXSIZE = config.AGENT_CACHE_MAXSIZE +_DEFAULT_TTL = config.AGENT_CACHE_TTL_SECONDS _cache: _AgentCache = _AgentCache(maxsize=_DEFAULT_MAXSIZE, ttl_seconds=_DEFAULT_TTL) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py new file mode 100644 index 000000000..be193be04 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/connector_searchable_types.py @@ -0,0 +1,100 @@ +"""Map configured connectors to the searchable document/connector types. + +This is agent-agnostic infrastructure shared by every agent factory (single- +and multi-agent). It translates the connectors a search space has enabled into +the set of searchable type strings that pre-search middleware and ``web_search`` +understand, and always layers in the document types that exist independently of +any connector (uploads, notes, extension captures, YouTube). + +It lives in its own module — rather than inside a specific agent factory — so +that retiring or moving any single agent never disturbs the others' access to +this mapping. +""" + +from __future__ import annotations + +from typing import Any + +# Maps SearchSourceConnectorType enum values to the searchable document/connector types +# used by pre-search middleware and web_search. +# Live search connectors (TAVILY_API, LINKUP_API, BAIDU_SEARCH_API) are routed to +# the web_search tool; all others are considered local/indexed data. +_CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = { + # Live search connectors (handled by web_search tool) + "TAVILY_API": "TAVILY_API", + "LINKUP_API": "LINKUP_API", + "BAIDU_SEARCH_API": "BAIDU_SEARCH_API", + # Local/indexed connectors (handled by KB pre-search middleware) + "SLACK_CONNECTOR": "SLACK_CONNECTOR", + "TEAMS_CONNECTOR": "TEAMS_CONNECTOR", + "NOTION_CONNECTOR": "NOTION_CONNECTOR", + "GITHUB_CONNECTOR": "GITHUB_CONNECTOR", + "LINEAR_CONNECTOR": "LINEAR_CONNECTOR", + "DISCORD_CONNECTOR": "DISCORD_CONNECTOR", + "JIRA_CONNECTOR": "JIRA_CONNECTOR", + "CONFLUENCE_CONNECTOR": "CONFLUENCE_CONNECTOR", + "CLICKUP_CONNECTOR": "CLICKUP_CONNECTOR", + "GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", + "GOOGLE_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", + "GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", # Connector type differs from document type + "AIRTABLE_CONNECTOR": "AIRTABLE_CONNECTOR", + "LUMA_CONNECTOR": "LUMA_CONNECTOR", + "ELASTICSEARCH_CONNECTOR": "ELASTICSEARCH_CONNECTOR", + "WEBCRAWLER_CONNECTOR": "CRAWLED_URL", # Maps to document type + "BOOKSTACK_CONNECTOR": "BOOKSTACK_CONNECTOR", + "CIRCLEBACK_CONNECTOR": "CIRCLEBACK", # Connector type differs from document type + "OBSIDIAN_CONNECTOR": "OBSIDIAN_CONNECTOR", + "DROPBOX_CONNECTOR": "DROPBOX_FILE", # Connector type differs from document type + "ONEDRIVE_CONNECTOR": "ONEDRIVE_FILE", # Connector type differs from document type + # Composio connectors (unified to native document types). + # Reverse of NATIVE_TO_LEGACY_DOCTYPE in app.db. + "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", + "COMPOSIO_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", + "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", +} + +# Document types that don't come from SearchSourceConnector but should always be searchable +_ALWAYS_AVAILABLE_DOC_TYPES: list[str] = [ + "EXTENSION", # Browser extension data + "FILE", # Uploaded files + "NOTE", # User notes + "YOUTUBE_VIDEO", # YouTube videos +] + + +def map_connectors_to_searchable_types( + connector_types: list[Any], +) -> list[str]: + """ + Map SearchSourceConnectorType enums to searchable document/connector types. + + This function: + 1. Converts connector type enums to their searchable counterparts + 2. Includes always-available document types (EXTENSION, FILE, NOTE, YOUTUBE_VIDEO) + 3. Deduplicates while preserving order + + Args: + connector_types: List of SearchSourceConnectorType enum values + + Returns: + List of searchable connector/document type strings + """ + result_set: set[str] = set() + result_list: list[str] = [] + + # Add always-available document types first + for doc_type in _ALWAYS_AVAILABLE_DOC_TYPES: + if doc_type not in result_set: + result_set.add(doc_type) + result_list.append(doc_type) + + # Map each connector type to its searchable equivalent + for ct in connector_types: + # Handle both enum and string types + ct_str = ct.value if hasattr(ct, "value") else str(ct) + searchable = _CONNECTOR_TYPE_TO_SEARCHABLE.get(ct_str) + if searchable and searchable not in result_set: + result_set.add(searchable) + result_list.append(searchable) + + return result_list diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/factory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/factory.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py index 44529d243..adb1bc1ed 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/runtime/factory.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py @@ -12,21 +12,28 @@ from langchain_core.tools import BaseTool from langgraph.types import Checkpointer from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import ( + AgentFeatureFlags, + get_flags, +) +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.resolver import ( + build_backend_resolver, +) +from app.agents.chat.multi_agent_chat.subagents import ( get_subagents_to_exclude, main_prompt_registry_subagent_lines, ) -from app.agents.multi_agent_chat.subagents.mcp_tools.index import ( +from app.agents.chat.multi_agent_chat.subagents.mcp_tools.index import ( load_mcp_tools_by_connector, ) -from app.agents.new_chat.chat_deepagent import _map_connectors_to_searchable_types -from app.agents.new_chat.feature_flags import AgentFeatureFlags, get_flags -from app.agents.new_chat.filesystem_backends import build_backend_resolver -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.llm_config import AgentConfig -from app.agents.new_chat.prompt_caching import apply_litellm_prompt_caching -from app.agents.new_chat.tools.invalid_tool import INVALID_TOOL_NAME, invalid_tool -from app.agents.new_chat.tools.registry import build_tools_async +from app.agents.chat.runtime.llm_config import AgentConfig +from app.agents.chat.runtime.prompt_caching import ( + apply_litellm_prompt_caching, +) from app.db import ChatVisibility from app.services.connector_service import ConnectorService from app.services.user_tool_allowlist import ( @@ -40,7 +47,10 @@ from ..tools import ( MAIN_AGENT_SURFSENSE_TOOL_NAMES, MAIN_AGENT_SURFSENSE_TOOL_NAMES_ORDERED, ) +from ..tools.invalid_tool import INVALID_TOOL_NAME, invalid_tool +from ..tools.registry import build_main_agent_tools from .agent_cache import build_agent_with_cache +from .connector_searchable_types import map_connectors_to_searchable_types _perf_log = get_perf_logger() @@ -90,7 +100,7 @@ async def create_multi_agent_chat_deep_agent( connector_types = await connector_service.get_available_connectors( search_space_id ) - available_connectors = _map_connectors_to_searchable_types(connector_types) + available_connectors = map_connectors_to_searchable_types(connector_types) available_document_types = await connector_service.get_available_document_types( search_space_id @@ -199,9 +209,6 @@ async def create_multi_agent_chat_deep_agent( modified_disabled_tools = list(disabled_tools) if disabled_tools else [] - if "search_knowledge_base" not in modified_disabled_tools: - modified_disabled_tools.append("search_knowledge_base") - if enabled_tools is not None: main_agent_enabled_tools = [ n for n in enabled_tools if n in MAIN_AGENT_SURFSENSE_TOOL_NAMES @@ -210,12 +217,14 @@ async def create_multi_agent_chat_deep_agent( main_agent_enabled_tools = list(MAIN_AGENT_SURFSENSE_TOOL_NAMES_ORDERED) _t0 = time.perf_counter() - tools = await build_tools_async( + # Main agent builds only its own small SurfSense toolset via the SRP + # main-agent registry; connectors/MCP/deliverables are delegated to + # subagents, so no MCP loading or connector construction happens here. + tools = build_main_agent_tools( dependencies=dependencies, enabled_tools=main_agent_enabled_tools, disabled_tools=modified_disabled_tools, additional_tools=list(additional_tools) if additional_tools else None, - include_mcp_tools=False, ) _flags: AgentFeatureFlags = get_flags() diff --git a/surfsense_backend/app/agents/new_chat/skills/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/backends.py similarity index 95% rename from surfsense_backend/app/agents/new_chat/middleware/skills_backends.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/backends.py index 072d73401..31620fe9b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/backends.py @@ -16,7 +16,7 @@ prompt at agent build time, not edited at runtime. Two backends are provided: * :class:`BuiltinSkillsBackend` — disk-backed read of bundled skills from - ``app/agents/new_chat/skills/builtin/``. + ``app/agents/shared/skills/builtin/``. * :class:`SearchSpaceSkillsBackend` — a thin read-only wrapper over :class:`KBPostgresBackend` that filters notes under the privileged folder ``/documents/_skills/``. @@ -47,7 +47,9 @@ from deepagents.backends.state import StateBackend if TYPE_CHECKING: from langchain.tools import ToolRuntime - from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, + ) logger = logging.getLogger(__name__) @@ -59,9 +61,10 @@ _MAX_SKILL_FILE_SIZE = 10 * 1024 * 1024 def _default_builtin_root() -> Path: """Return the absolute path to the bundled builtin skills directory. - Located at ``app/agents/new_chat/skills/builtin/`` relative to this module. + Located at ``builtin/`` next to this module (this module lives at + ``app/agents/multi_agent_chat/main_agent/skills/backends.py``). """ - return (Path(__file__).resolve().parent.parent / "skills" / "builtin").resolve() + return (Path(__file__).resolve().parent / "builtin").resolve() class BuiltinSkillsBackend(BackendProtocol): @@ -121,6 +124,8 @@ class BuiltinSkillsBackend(BackendProtocol): else ("/" + str(target.relative_to(self.root)).replace("\\", "/")) ) for child in sorted(target.iterdir()): + if child.name == "__pycache__" or child.name.startswith("."): + continue child_virtual = ( target_virtual.rstrip("/") + "/" + child.name if target_virtual != "/" @@ -305,7 +310,7 @@ def build_skills_backend_factory( # Imported lazily to avoid a hard dependency at module import time: # ``KBPostgresBackend`` pulls in DB models, which are unnecessary for # the unit-tested builtin path. - from app.agents.new_chat.middleware.kb_postgres_backend import ( + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( KBPostgresBackend, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/email-drafting/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/email-drafting/SKILL.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/email-drafting/SKILL.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/email-drafting/SKILL.md diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/kb-research/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/kb-research/SKILL.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/kb-research/SKILL.md diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/meeting-prep/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/meeting-prep/SKILL.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/meeting-prep/SKILL.md diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/report-writing/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/report-writing/SKILL.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/report-writing/SKILL.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/report-writing/SKILL.md diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/slack-summary/SKILL.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/slack-summary/SKILL.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/slack-summary/SKILL.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/skills/builtin/slack-summary/SKILL.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/compose.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/compose.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/compose.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/compose.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/load_md.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/load_md.py similarity index 85% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/load_md.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/load_md.py index 61e30b1c7..fae45f520 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/load_md.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/load_md.py @@ -4,7 +4,7 @@ from __future__ import annotations from importlib import resources -_PROMPTS_PACKAGE = "app.agents.multi_agent_chat.main_agent.system_prompt.prompts" +_PROMPTS_PACKAGE = "app.agents.chat.multi_agent_chat.main_agent.system_prompt.prompts" def read_prompt_md(filename: str) -> str: diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/citations.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/citations.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/citations.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/citations.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/dynamic_context.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/dynamic_context.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/dynamic_context.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/dynamic_context.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/identity.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/identity.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/identity.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/identity.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/memory_protocol.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/memory_protocol.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/memory_protocol.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/memory_protocol.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/specialists.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/specialists.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/specialists.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/specialists.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/tools.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/tools.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/tools.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/sections/tools.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/tool_instruction_block.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/tool_instruction_block.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/tool_instruction_block.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/builder/tool_instruction_block.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/off.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/off.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/off.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/off.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/citations/on.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/core_behavior.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/core_behavior.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/core_behavior.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/core_behavior.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/private.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/private.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/private.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/team.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/team.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/dynamic_context/team.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/private.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/identity/team.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md similarity index 64% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md index 80fa4bf8f..065b72983 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/kb_first.md @@ -1,9 +1,17 @@ CRITICAL — ground factual answers in what you actually receive this turn: +- the user's knowledge base via `search_knowledge_base` (your PRIMARY source + for anything about their documents, notes, or connected data — the + `` only lists what exists, so call the tool to read the + actual content before answering), - injected workspace context (see ``), -- results from your own tool calls (`web_search`, `scrape_webpage`), +- results from your other tool calls (`web_search`, `scrape_webpage`), - or substantive summaries returned by a `task` specialist you invoked. +For questions about the user's own workspace, call `search_knowledge_base` +first rather than answering from the tree or from memory. Use +`task(knowledge_base)` when you need a document's full text or deeper reads. + Do **not** answer factual or informational questions from general knowledge unless the user explicitly authorises it after you say you couldn't find enough in those sources. The flow when nothing is found: diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/private.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/private.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/private.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/private.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/team.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/team.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/team.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/memory_protocol/team.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/output_format.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/output_format.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/output_format.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/output_format.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/anthropic.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/deepseek.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/deepseek.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/deepseek.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/deepseek.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/default.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/default.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/default.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/default.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/google.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/google.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/google.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/google.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/grok.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/grok.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/grok.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/grok.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/kimi.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/kimi.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/kimi.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/kimi.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_classic.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_codex.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_codex.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_codex.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_codex.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_reasoning.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_reasoning.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_reasoning.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/providers/openai_reasoning.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/refusal_and_limits.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/refusal_and_limits.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/refusal_and_limits.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/refusal_and_limits.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/reminder.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/reminder.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/reminder.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/reminder.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/routing.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/routing.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/routing.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/create_automation/example.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/scrape_webpage/example.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/description.md new file mode 100644 index 000000000..a4854dfff --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/description.md @@ -0,0 +1,19 @@ +- `search_knowledge_base` — Search the user's own knowledge base (their + indexed documents, notes, files, and connected sources) with hybrid + semantic + keyword retrieval. + - This is your PRIMARY way to ground factual answers about the user's + workspace. The `` shows what files exist; this tool pulls + the actual relevant content. Call it BEFORE answering any question about + the user's documents, notes, or connected data — don't answer from the + tree alone or from memory. + - Each hit returns the document's virtual path, a relevance score, and the + matched snippets. The snippets are often enough to answer directly with a + citation. + - When you need a document's full text (not just snippets), delegate a read + to the `knowledge_base` specialist via `task`, passing the path from the + results. + - Args: `query` (focused; include concrete entities, acronyms, people, + projects, or terms), `top_k` (default 5, max 20). + - If nothing relevant comes back, tell the user you couldn't find it in + their workspace before offering to search the web or answer from general + knowledge. diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/example.md new file mode 100644 index 000000000..2d9ec61eb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/search_knowledge_base/example.md @@ -0,0 +1,13 @@ + +user: "What did our Q3 planning doc say about hiring?" +→ search_knowledge_base(query="Q3 planning hiring headcount plan") +(Answer from the returned snippets with a citation; if you need the full +document, task the knowledge_base specialist with the returned path.) + + + +user: "Summarize my notes on the Acme migration." +→ search_knowledge_base(query="Acme migration notes") +→ task(subagent_type="knowledge_base", description="Read and return a +detailed summary of the Acme migration plan, risks, and timeline.") + diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/task/example.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/private/example.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/update_memory/team/example.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/system_prompt/prompts/tools/web_search/example.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/create.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/create.py index 62d39fcf2..4472a11ac 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/create.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/create.py @@ -27,7 +27,7 @@ from langchain_core.messages import HumanMessage from langchain_core.tools import tool from pydantic import ValidationError -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.automations.schemas.api import AutomationCreate diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/prompt.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/prompt.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/automation/prompt.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/automation/prompt.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py index 70fb42c0d..40c6f08de 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/index.py @@ -6,6 +6,7 @@ Connector integrations, MCP, deliverables, etc. are delegated via ``task`` subag from __future__ import annotations MAIN_AGENT_SURFSENSE_TOOL_NAMES_ORDERED: tuple[str, ...] = ( + "search_knowledge_base", "web_search", "scrape_webpage", "update_memory", diff --git a/surfsense_backend/app/agents/new_chat/tools/invalid_tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/invalid_tool.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/tools/invalid_tool.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/invalid_tool.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py new file mode 100644 index 000000000..f04d7cdec --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py @@ -0,0 +1,146 @@ +"""SRP main-agent tool registry. + +The main agent exposes only a small, fixed set of SurfSense tools to its LLM; +connector integrations, MCP, and deliverables are delegated to ``task`` +subagents (see :mod:`app.agents.chat.multi_agent_chat.main_agent.tools.index`). + +This module is the *building* counterpart to that name list: it owns the +factories for those few tools and nothing else, so the main agent's tool +surface stays self-contained and connector-free. + +Tool *display* metadata for the whole app (the ``/agent/tools`` listing +endpoint) lives separately in :mod:`app.agents.chat.multi_agent_chat.shared.tools.catalog`, a +pure-data module that imports no connectors. This registry only governs what +the main agent actually builds and binds. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from langchain_core.tools import BaseTool + +from app.agents.chat.shared.tools.web_search import create_web_search_tool +from app.db import ChatVisibility + +from .scrape_webpage import create_scrape_webpage_tool +from .search_knowledge_base import create_search_knowledge_base_tool +from .update_memory import ( + create_update_memory_tool, + create_update_team_memory_tool, +) + + +def _build_scrape_webpage_tool(deps: dict[str, Any]) -> BaseTool: + return create_scrape_webpage_tool(firecrawl_api_key=deps.get("firecrawl_api_key")) + + +def _build_search_knowledge_base_tool(deps: dict[str, Any]) -> BaseTool: + return create_search_knowledge_base_tool( + search_space_id=deps["search_space_id"], + available_connectors=deps.get("available_connectors"), + available_document_types=deps.get("available_document_types"), + ) + + +def _build_web_search_tool(deps: dict[str, Any]) -> BaseTool: + return create_web_search_tool( + search_space_id=deps.get("search_space_id"), + available_connectors=deps.get("available_connectors"), + ) + + +def _build_create_automation_tool(deps: dict[str, Any]) -> BaseTool: + # Deferred import: the automation package is a sibling under ``main_agent`` + # and is only needed at build time, mirroring the shared registry's + # call-time import to keep module import order robust. + from .automation import create_create_automation_tool + + return create_create_automation_tool( + search_space_id=deps["search_space_id"], + user_id=deps["user_id"], + llm=deps["llm"], + ) + + +def _build_update_memory_tool(deps: dict[str, Any]) -> BaseTool: + if deps["thread_visibility"] == ChatVisibility.SEARCH_SPACE: + return create_update_team_memory_tool( + search_space_id=deps["search_space_id"], + db_session=deps["db_session"], + llm=deps.get("llm"), + ) + return create_update_memory_tool( + user_id=deps["user_id"], + db_session=deps["db_session"], + llm=deps.get("llm"), + ) + + +# Ordered to match the historical main-agent binding order: +# scrape_webpage, web_search, create_automation, update_memory. +# Each entry is ``(factory, required_dependency_names)``. +_MAIN_AGENT_TOOL_FACTORIES: dict[ + str, tuple[Callable[[dict[str, Any]], BaseTool], tuple[str, ...]] +] = { + "search_knowledge_base": ( + _build_search_knowledge_base_tool, + ("search_space_id",), + ), + "scrape_webpage": (_build_scrape_webpage_tool, ()), + "web_search": (_build_web_search_tool, ()), + "create_automation": ( + _build_create_automation_tool, + ("search_space_id", "user_id", "llm"), + ), + "update_memory": ( + _build_update_memory_tool, + ("user_id", "search_space_id", "db_session", "thread_visibility", "llm"), + ), +} + + +def build_main_agent_tools( + dependencies: dict[str, Any], + enabled_tools: list[str] | None = None, + disabled_tools: list[str] | None = None, + additional_tools: list[BaseTool] | None = None, +) -> list[BaseTool]: + """Build the main agent's tool instances. + + Args: + dependencies: Dependency bag passed to each tool factory. + enabled_tools: Explicit allow-list of tool names. When ``None``, all + main-agent tools are enabled. Names not owned by this registry are + ignored. + disabled_tools: Names to drop after the enabled set is resolved. + additional_tools: Extra tools appended verbatim (e.g. custom tools). + + Returns: + Tool instances in the registry's declaration order, with any + ``additional_tools`` appended. + """ + if enabled_tools is None: + names = list(_MAIN_AGENT_TOOL_FACTORIES) + else: + wanted = set(enabled_tools) + names = [n for n in _MAIN_AGENT_TOOL_FACTORIES if n in wanted] + + if disabled_tools: + disabled = set(disabled_tools) + names = [n for n in names if n not in disabled] + + tools: list[BaseTool] = [] + for name in names: + factory, requires = _MAIN_AGENT_TOOL_FACTORIES[name] + missing = [dep for dep in requires if dep not in dependencies] + if missing: + msg = f"Tool '{name}' requires dependencies: {missing}" + raise ValueError(msg) + tools.append(factory(dependencies)) + + if additional_tools: + tools.extend(additional_tools) + + return tools diff --git a/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py similarity index 87% rename from surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py index 014126927..c66506ca7 100644 --- a/surfsense_backend/app/agents/new_chat/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py @@ -8,18 +8,19 @@ transcript directly via the YouTubeTranscriptApi instead of crawling the page. import hashlib import logging +import time from typing import Any from urllib.parse import urlparse -import aiohttp from fake_useragent import UserAgent from langchain_core.tools import tool from requests import Session +from scrapling.fetchers import AsyncFetcher from youtube_transcript_api import YouTubeTranscriptApi from app.connectors.webcrawler_connector import WebCrawlerConnector from app.tasks.document_processors.youtube_processor import get_youtube_video_id -from app.utils.proxy_config import get_requests_proxies +from app.utils.proxy import get_proxy_url, get_requests_proxies logger = logging.getLogger(__name__) @@ -29,7 +30,6 @@ def extract_domain(url: str) -> str: try: parsed = urlparse(url) domain = parsed.netloc - # Remove 'www.' prefix if present if domain.startswith("www."): domain = domain[4:] return domain @@ -53,14 +53,13 @@ def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]: if len(content) <= max_length: return content, False - # Try to truncate at a sentence boundary + # Prefer truncating at a sentence/paragraph boundary. truncated = content[:max_length] last_period = truncated.rfind(".") last_newline = truncated.rfind("\n\n") - # Use the later of the two boundaries, or just truncate boundary = max(last_period, last_newline) - if boundary > max_length * 0.8: # Only use boundary if it's not too far back + if boundary > max_length * 0.8: # only if the boundary isn't too far back truncated = content[: boundary + 1] return truncated + "\n\n[Content truncated...]", True @@ -87,15 +86,20 @@ async def _scrape_youtube_video( oembed_url = "https://www.youtube.com/oembed" try: - async with ( - aiohttp.ClientSession() as http_session, - http_session.get( - oembed_url, - params=params, - proxy=residential_proxies["http"] if residential_proxies else None, - ) as response, - ): - video_data = await response.json() + oembed_fetch_start = time.perf_counter() + oembed_page = await AsyncFetcher.get( + oembed_url, + params=params, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + logger.info( + "[scrape_webpage][perf] source=oembed video=%s status=%s fetch_ms=%.1f", + video_id, + getattr(oembed_page, "status", None), + (time.perf_counter() - oembed_fetch_start) * 1000, + ) + video_data = oembed_page.json() except Exception: video_data = {} @@ -104,6 +108,7 @@ async def _scrape_youtube_video( # --- Transcript via YouTubeTranscriptApi --- try: + transcript_fetch_start = time.perf_counter() ua = UserAgent() http_client = Session() http_client.headers.update({"User-Agent": ua.random}) @@ -111,12 +116,17 @@ async def _scrape_youtube_video( http_client.proxies.update(residential_proxies) ytt_api = YouTubeTranscriptApi(http_client=http_client) - # List all available transcripts and pick the first one - # (the video's primary language) instead of defaulting to English + # Pick the first transcript (video's primary language) rather than + # defaulting to English. transcript_list = ytt_api.list(video_id) transcript = next(iter(transcript_list)) captions = transcript.fetch() + logger.info( + "[scrape_webpage][perf] source=transcript video=%s fetch_ms=%.1f", + video_id, + (time.perf_counter() - transcript_fetch_start) * 1000, + ) logger.info( f"[scrape_webpage] Fetched transcript for {video_id} " f"in {transcript.language} ({transcript.language_code})" @@ -134,10 +144,8 @@ async def _scrape_youtube_video( logger.warning(f"[scrape_webpage] No transcript for video {video_id}: {e}") transcript_text = f"No captions available for this video. Error: {e!s}" - # Build combined content content = f"# {title}\n\n**Author:** {author}\n**Video ID:** {video_id}\n\n## Transcript\n\n{transcript_text}" - # Truncate if needed content, was_truncated = truncate_content(content, max_length) word_count = len(content.split()) @@ -212,20 +220,16 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): scrape_id = generate_scrape_id(url) domain = extract_domain(url) - # Validate and normalize URL if not url.startswith(("http://", "https://")): url = f"https://{url}" try: - # Check if this is a YouTube URL and use transcript API instead + # YouTube URLs use the transcript API instead of crawling. video_id = get_youtube_video_id(url) if video_id: return await _scrape_youtube_video(url, video_id, max_length) - # Create webcrawler connector connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key) - - # Crawl the URL result, error = await connector.crawl_url(url, formats=["markdown"]) if error: @@ -250,28 +254,21 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): "error": "No content returned from crawler", } - # Extract content and metadata content = result.get("content", "") metadata = result.get("metadata", {}) - # Get title from metadata title = metadata.get("title", "") if not title: title = domain or url.split("/")[-1] or "Webpage" - # Get description from metadata description = metadata.get("description", "") if not description and content: - # Use first paragraph as description first_para = content.split("\n\n")[0] if content else "" description = ( first_para[:300] + "..." if len(first_para) > 300 else first_para ) - # Truncate content if needed content, was_truncated = truncate_content(content, max_length) - - # Calculate word count word_count = len(content.split()) return { diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/search_knowledge_base.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/search_knowledge_base.py new file mode 100644 index 000000000..9236e9121 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/search_knowledge_base.py @@ -0,0 +1,232 @@ +"""On-demand ``search_knowledge_base`` main-agent tool (OpenCode-style lazy RAG). + +The main agent no longer receives eagerly pre-injected KB context on every +turn (see :class:`KnowledgePriorityMiddleware`, now gated off by default). +Instead it calls this tool only when it decides it needs knowledge-base +content. The tool runs a single hybrid search (embed + DB search, ~0.5s), +formats the top matches for the model, and writes ``kb_matched_chunk_ids`` +into graph state so matched-section highlighting is preserved when the agent +later reads a document via ``task(knowledge_base)``. +""" + +from __future__ import annotations + +import time +from typing import Annotated, Any + +from langchain.tools import ToolRuntime +from langchain_core.messages import ToolMessage +from langchain_core.tools import BaseTool, StructuredTool +from langgraph.types import Command +from sqlalchemy import select + +from app.agents.chat.multi_agent_chat.shared.middleware.knowledge_search import ( + search_knowledge_base as _hybrid_search_kb, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import ( + PathIndex, + build_path_index, + doc_to_virtual_path, +) +from app.db import Document, shielded_async_session +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() + +_DEFAULT_TOP_K = 5 +_MAX_TOP_K = 20 +_PER_DOC_SNIPPET_CHARS = 1200 +_MAX_TOTAL_CHARS = 16_000 + +_TOOL_DESCRIPTION = ( + "Search the user's knowledge base (their indexed documents, files, and " + "connector content) for passages relevant to a query, using hybrid " + "semantic + keyword retrieval.\n\n" + "Use this FIRST to ground any factual or informational answer about the " + "user's own documents, notes, or connected sources. The workspace tree " + "shows which files exist; this tool pulls the actual relevant content. " + "Each hit returns the document's virtual path, a relevance score, and the " + "matched snippets. If you need a document's full text, delegate a read to " + "the knowledge_base specialist via `task` using the returned path.\n\n" + "Write a focused, specific query containing the concrete entities, " + "acronyms, people, projects, or terms you are looking for." +) + + +async def _resolve_virtual_paths( + results: list[dict[str, Any]], + *, + search_space_id: int, +) -> dict[int, str]: + """Resolve ``Document.id`` -> canonical virtual path for the search hits.""" + doc_ids = [ + doc_id + for doc_id in ( + (doc.get("document") or {}).get("id") + for doc in results + if isinstance(doc, dict) + ) + if isinstance(doc_id, int) + ] + if not doc_ids: + return {} + + async with shielded_async_session() as session: + index: PathIndex = await build_path_index(session, search_space_id) + folder_rows = await session.execute( + select(Document.id, Document.folder_id).where( + Document.search_space_id == search_space_id, + Document.id.in_(doc_ids), + ) + ) + folder_by_doc_id = {row.id: row.folder_id for row in folder_rows.all()} + + paths: dict[int, str] = {} + for doc in results: + doc_meta = doc.get("document") or {} + doc_id = doc_meta.get("id") + if not isinstance(doc_id, int): + continue + folder_id = folder_by_doc_id.get(doc_id, doc_meta.get("folder_id")) + paths[doc_id] = doc_to_virtual_path( + doc_id=doc_id, + title=str(doc_meta.get("title") or "untitled"), + folder_id=folder_id if isinstance(folder_id, int) else None, + index=index, + ) + return paths + + +def _format_hits( + results: list[dict[str, Any]], + *, + paths: dict[int, str], + query: str, +) -> str: + """Render search hits as a compact, model-readable block.""" + if not results: + return ( + f"No knowledge-base matches found for query: {query!r}.\n" + "Tell the user nothing relevant was found in their workspace, or " + "try a different query." + ) + + lines: list[str] = [f""] + total = len(lines[0]) + for rank, doc in enumerate(results, start=1): + doc_meta = doc.get("document") or {} + doc_id = doc_meta.get("id") + title = str(doc_meta.get("title") or "untitled") + doc_type = doc_meta.get("document_type") or doc.get("source") or "document" + score = doc.get("score") + score_str = f"{score:.3f}" if isinstance(score, int | float) else "n/a" + path = paths.get(doc_id) if isinstance(doc_id, int) else None + + header = f"\n{rank}. {title} (type={doc_type}, score={score_str})" + ( + f"\n path: {path}" if path else "" + ) + + content = (doc.get("content") or "").strip() + if content: + snippet = content[:_PER_DOC_SNIPPET_CHARS].strip() + if len(content) > _PER_DOC_SNIPPET_CHARS: + snippet += " ..." + body = "\n " + snippet.replace("\n", "\n ") + else: + body = "\n (no preview available; read the document for details)" + + entry = header + body + if total + len(entry) > _MAX_TOTAL_CHARS: + lines.append("\n") + break + lines.append(entry) + total += len(entry) + + lines.append( + "\n\nTo read a full document, delegate to the knowledge_base specialist " + "with `task`, referencing the path above." + ) + lines.append("\n") + return "".join(lines) + + +def _matched_chunk_ids(results: list[dict[str, Any]]) -> dict[int, list[int]]: + """Extract ``Document.id`` -> matched chunk ids for state hand-off.""" + matched: dict[int, list[int]] = {} + for doc in results: + doc_id = (doc.get("document") or {}).get("id") + if not isinstance(doc_id, int): + continue + chunk_ids = doc.get("matched_chunk_ids") or [] + normalized = [int(cid) for cid in chunk_ids if isinstance(cid, int | str)] + if normalized: + matched[doc_id] = normalized + return matched + + +def create_search_knowledge_base_tool( + *, + search_space_id: int, + available_connectors: list[str] | None = None, + available_document_types: list[str] | None = None, +) -> BaseTool: + """Factory for the on-demand ``search_knowledge_base`` tool.""" + + _space_id = search_space_id + _connectors = available_connectors + _doc_types = available_document_types + + async def _impl( + query: Annotated[ + str, + "Focused search query with the concrete entities/terms to look for.", + ], + runtime: ToolRuntime[None, SurfSenseFilesystemState], + top_k: Annotated[ + int, + "Maximum number of documents to return (default 5).", + ] = _DEFAULT_TOP_K, + ) -> Command | str: + cleaned_query = (query or "").strip() + if not cleaned_query: + return "Error: provide a non-empty search query." + + clamped_top_k = min(max(1, top_k), _MAX_TOP_K) + t0 = time.perf_counter() + results = await _hybrid_search_kb( + query=cleaned_query, + search_space_id=_space_id, + available_connectors=_connectors, + available_document_types=_doc_types, + top_k=clamped_top_k, + ) + + paths = await _resolve_virtual_paths(results, search_space_id=_space_id) + rendered = _format_hits(results, paths=paths, query=cleaned_query) + matched = _matched_chunk_ids(results) + + _perf_log.info( + "[search_knowledge_base] tool query=%r results=%d chars=%d in %.3fs", + cleaned_query[:60], + len(results), + len(rendered), + time.perf_counter() - t0, + ) + + update: dict[str, Any] = { + "messages": [ + ToolMessage(content=rendered, tool_call_id=runtime.tool_call_id) + ], + } + if matched: + update["kb_matched_chunk_ids"] = matched + return Command(update=update) + + return StructuredTool.from_function( + name="search_knowledge_base", + description=_TOOL_DESCRIPTION, + coroutine=_impl, + ) diff --git a/surfsense_backend/app/agents/new_chat/tools/update_memory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/update_memory.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/tools/update_memory.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/update_memory.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/utils.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/date_filters.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/utils.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/date_filters.py diff --git a/surfsense_backend/app/agents/new_chat/feature_flags.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/feature_flags.py similarity index 68% rename from surfsense_backend/app/agents/new_chat/feature_flags.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/feature_flags.py index 27188fac3..f5233c7d3 100644 --- a/surfsense_backend/app/agents/new_chat/feature_flags.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/feature_flags.py @@ -1,37 +1,9 @@ -""" -Feature flags for the SurfSense new_chat agent stack. +"""Feature flags for the SurfSense new_chat agent stack. -These flags gate the newer agent middleware (some ported from OpenCode, -some sourced from ``langchain.agents.middleware`` / ``deepagents``, some -SurfSense-native). Most shipped agent-stack upgrades default ON so Docker -image updates work even when older installs do not have newly introduced -environment variables. Risky/experimental integrations stay default OFF, -and the master kill-switch can still disable everything new. - -All new middleware checks its flag at agent build time. If the master -kill-switch ``SURFSENSE_DISABLE_NEW_AGENT_STACK`` is set, every new -middleware is disabled regardless of its individual flag. This gives -operators a single switch to revert to pre-port behavior. - -Examples --------- - -Defaults: - - SURFSENSE_ENABLE_CONTEXT_EDITING=true - SURFSENSE_ENABLE_COMPACTION_V2=true - SURFSENSE_ENABLE_RETRY_AFTER=true - SURFSENSE_ENABLE_MODEL_FALLBACK=false - SURFSENSE_ENABLE_MODEL_CALL_LIMIT=true - SURFSENSE_ENABLE_TOOL_CALL_LIMIT=true - SURFSENSE_ENABLE_TOOL_CALL_REPAIR=true - SURFSENSE_ENABLE_PERMISSION=true - SURFSENSE_ENABLE_DOOM_LOOP=true - SURFSENSE_ENABLE_LLM_TOOL_SELECTOR=false # adds a per-turn LLM call - -Master kill-switch (overrides everything else): - - SURFSENSE_DISABLE_NEW_AGENT_STACK=true +Flags are resolved at agent build time. Most upgrades default ON so Docker +updates work without operators adding new env vars; risky integrations stay +OFF. The master kill-switch ``SURFSENSE_DISABLE_NEW_AGENT_STACK`` forces every +flag below to False for a one-switch rollback to pre-port behavior. """ from __future__ import annotations @@ -83,6 +55,13 @@ class AgentFeatureFlags: enable_specialized_subagents: bool = True enable_kb_planner_runnable: bool = True + # KB retrieval mode — when False (default), the main agent retrieves KB + # content lazily via the on-demand ``search_knowledge_base`` tool and the + # expensive per-turn pre-injection (planner LLM + embed + hybrid search, + # ~2.3s) is skipped; explicit @-mentions are still surfaced cheaply. Set + # True to restore the original eager ```` pre-injection. + enable_kb_priority_preinjection: bool = False + # Snapshot / revert enable_action_log: bool = True enable_revert_route: bool = True @@ -93,39 +72,22 @@ class AgentFeatureFlags: # Observability — OTel (orthogonal; also requires OTEL_EXPORTER_OTLP_ENDPOINT) enable_otel: bool = False - # Performance — compiled-agent cache (Phase 1 + Phase 2). - # When ON, ``create_surfsense_deep_agent`` reuses a previously-compiled - # graph if the cache key matches (LLM config + thread + tool surface + - # flags + system prompt + filesystem mode). Cuts per-turn agent-build - # wall clock from ~4-5s to <50µs on cache hits. - # - # SAFETY (Phase 2 unblocked this default-on): - # All connector mutation tools (``tools/notion``, ``tools/gmail``, - # ``tools/google_drive``, ``tools/dropbox``, ``tools/onedrive``, - # ``tools/google_calendar``, ``tools/confluence``, ``tools/discord``, - # ``tools/teams``, ``tools/luma``, ``connected_accounts``, - # ``update_memory``) now acquire fresh - # short-lived ``AsyncSession`` instances per call via - # :data:`async_session_maker`. The factory still accepts ``db_session`` - # for registry compatibility but ``del``'s it immediately — see any - # of those files' factory docstrings for the rationale. The ``llm`` - # closure is per-(provider, model, config_id) which is already in - # the cache key, so the LLM is safe to share across cached hits of - # the same key. The KB priority middleware reads - # ``mentioned_document_ids`` from ``runtime.context`` (Phase 1.5), - # not its constructor closure, so the same compiled agent serves - # turns with different mention lists correctly. - # - # Rollback: set ``SURFSENSE_ENABLE_AGENT_CACHE=false`` in the - # environment if a regression surfaces. The path is exercised by - # the ``tests/unit/agents/new_chat/test_agent_cache_*`` suite. + # Performance — reuse a compiled agent graph when the cache key matches + # (~4-5s -> <50µs per turn). Safe to default-on because mutation tools take + # fresh short-lived sessions per call and per-turn context (mentions, etc.) + # is read from runtime.context, not the constructor closure. Rollback via + # SURFSENSE_ENABLE_AGENT_CACHE=false. enable_agent_cache: bool = True - # Phase 1 (deferred — measure first): pre-build & share the - # general-purpose subagent ``CompiledSubAgent`` across cold-cache - # misses. Only helps when the outer cache MISSES (cache hits already - # reuse the entire SubAgentMiddleware-compiled graph). Off by default - # until we have data showing cold misses are frequent enough to - # justify the extra global state. + # Reuse one compiled graph across a returning user's *new* chats by dropping + # ``thread_id`` from the agent_cache key. Safe because every middleware/tool + # that needs the chat thread now resolves it from the live RunnableConfig + # (ActionLog, KB-persistence, deliverables) rather than a constructor + # closure, and mutation tools open fresh per-call sessions. Turns a + # returning user's cold first turn into a cache hit (cold == warm). + # Rollback via SURFSENSE_ENABLE_CROSS_THREAD_AGENT_CACHE=false. + enable_cross_thread_agent_cache: bool = True + # Deferred: only helps on outer-cache MISSES, so off until data shows cold + # misses are frequent enough to justify the extra global state. enable_agent_cache_share_gp_subagent: bool = False @classmethod @@ -157,11 +119,14 @@ class AgentFeatureFlags: enable_skills=False, enable_specialized_subagents=False, enable_kb_planner_runnable=False, + # Full rollback restores the original eager KB pre-injection. + enable_kb_priority_preinjection=True, enable_action_log=False, enable_revert_route=False, enable_plugin_loader=False, enable_otel=False, enable_agent_cache=False, + enable_cross_thread_agent_cache=False, enable_agent_cache_share_gp_subagent=False, ) @@ -194,6 +159,9 @@ class AgentFeatureFlags: enable_kb_planner_runnable=_env_bool( "SURFSENSE_ENABLE_KB_PLANNER_RUNNABLE", True ), + enable_kb_priority_preinjection=_env_bool( + "SURFSENSE_ENABLE_KB_PRIORITY_PREINJECTION", False + ), # Snapshot / revert enable_action_log=_env_bool("SURFSENSE_ENABLE_ACTION_LOG", True), enable_revert_route=_env_bool("SURFSENSE_ENABLE_REVERT_ROUTE", True), @@ -203,6 +171,9 @@ class AgentFeatureFlags: enable_otel=_env_bool("SURFSENSE_ENABLE_OTEL", False), # Performance enable_agent_cache=_env_bool("SURFSENSE_ENABLE_AGENT_CACHE", True), + enable_cross_thread_agent_cache=_env_bool( + "SURFSENSE_ENABLE_CROSS_THREAD_AGENT_CACHE", True + ), enable_agent_cache_share_gp_subagent=_env_bool( "SURFSENSE_ENABLE_AGENT_CACHE_SHARE_GP_SUBAGENT", False ), diff --git a/surfsense_backend/app/agents/new_chat/filesystem_selection.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/filesystem_selection.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/filesystem_selection.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/filesystem_selection.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/anthropic_cache.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/anthropic_cache.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/anthropic_cache.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/anthropic_cache.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/compaction.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/compaction.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/compaction.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/compaction.py index b59e7d2c4..c1d26429e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/compaction.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/compaction.py @@ -7,7 +7,7 @@ from typing import Any from deepagents.backends import StateBackend from langchain_core.language_models import BaseChatModel -from app.agents.new_chat.middleware import create_surfsense_compaction_middleware +from app.agents.chat.shared.middleware import create_surfsense_compaction_middleware def build_compaction_mw(llm: BaseChatModel) -> Any: diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/dedup_tool_calls.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/dedup_tool_calls.py new file mode 100644 index 000000000..087a69ae6 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/dedup_tool_calls.py @@ -0,0 +1,59 @@ +"""Dedup-key resolvers for tool-call deduplication. + +A *resolver* maps a tool's ``args`` dict to a stable signature string used to +collapse duplicate calls. These helpers are shared: the MCP tool layer uses +:func:`dedup_key_full_args` as a safe default, and the main-agent +``DedupHITLToolCallsMiddleware`` builds its resolver map from them. + +Resolver resolution order (read from each tool's own ``metadata``): + +1. ``tool.metadata["dedup_key"]`` — callable mapping the args dict to a + stable signature string. This is the canonical mechanism. +2. ``tool.metadata["hitl_dedup_key"]`` — string naming a primary arg; + used by MCP / Composio tools that only expose a single key field. + +A tool with no resolver from either path simply opts out of dedup. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import Any + +# Resolver type — given the tool ``args`` dict returns a stable +# string used to dedupe consecutive calls. ``None`` means no dedup. +DedupResolver = Callable[[dict[str, Any]], str] + + +def wrap_dedup_key_by_arg_name(arg_name: str) -> DedupResolver: + """Adapt a string-arg name into a :data:`DedupResolver`. + + Convenience helper for tools that just want to dedupe on a single arg's + lowercased value (the most common case for HITL tools like + ``send_gmail_email`` keyed on ``subject``). Set the result on the tool's + ``metadata["dedup_key"]``. + """ + + def _resolver(args: dict[str, Any]) -> str: + return str(args.get(arg_name, "")).lower() + + return _resolver + + +def dedup_key_full_args(args: dict[str, Any]) -> str: + """Resolver that collapses calls only when **every** argument is identical. + + Safe default for tools where no single field uniquely identifies a call + (e.g. MCP tools whose first required field is a shared workspace id). + """ + + try: + return json.dumps(args, sort_keys=True, default=str) + except (TypeError, ValueError): + return repr(sorted(args.items())) if isinstance(args, dict) else repr(args) + + +# Backwards-compatible alias for code that imported the original +# private name. New callers should use :func:`wrap_dedup_key_by_arg_name`. +_wrap_string_key = wrap_dedup_key_by_arg_name diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/document_xml.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/document_xml.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/document_xml.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/document_xml.py diff --git a/surfsense_backend/app/agents/new_chat/middleware/kb_postgres_backend.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/kb_postgres.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/middleware/kb_postgres_backend.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/kb_postgres.py index 7cf3bf8cd..7b8aaf2b0 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/kb_postgres_backend.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/kb_postgres.py @@ -42,8 +42,10 @@ from langchain.tools import ToolRuntime from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.document_xml import build_document_xml -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.document_xml import ( + build_document_xml, +) +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, build_path_index, doc_to_virtual_path, diff --git a/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/local_folder.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/local_folder.py diff --git a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/multi_root_local_folder.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/multi_root_local_folder.py index a5add6248..db84a17eb 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/multi_root_local_folder.py @@ -15,7 +15,9 @@ from deepagents.backends.protocol import ( WriteResult, ) -from app.agents.new_chat.middleware.local_folder_backend import LocalFolderBackend +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.local_folder import ( + LocalFolderBackend, +) _INVALID_PATH = "invalid_path" _FILE_NOT_FOUND = "file_not_found" diff --git a/surfsense_backend/app/agents/new_chat/filesystem_backends.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/resolver.py similarity index 84% rename from surfsense_backend/app/agents/new_chat/filesystem_backends.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/resolver.py index c8288be71..6c35f369f 100644 --- a/surfsense_backend/app/agents/new_chat/filesystem_backends.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/resolver.py @@ -9,9 +9,14 @@ from deepagents.backends.protocol import BackendProtocol from deepagents.backends.state import StateBackend from langgraph.prebuilt.tool_node import ToolRuntime -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.multi_root_local_folder import ( MultiRootLocalFolderBackend, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/index.py similarity index 88% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/index.py index fb8dbe209..91bc4db7c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/index.py @@ -4,7 +4,7 @@ from __future__ import annotations from typing import Any -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode from .middleware import SurfSenseFilesystemMiddleware diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/async_dispatch.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/async_dispatch.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/async_dispatch.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/async_dispatch.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/index.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/index.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/middleware.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/middleware.py similarity index 78% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/middleware.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/middleware.py index c32e14438..c3b06ff12 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/middleware.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/middleware.py @@ -2,14 +2,20 @@ from __future__ import annotations +import time as _perf_time from typing import Any from deepagents import FilesystemMiddleware from langchain_core.tools import BaseTool -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.sandbox import is_sandbox_enabled +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( + is_sandbox_enabled, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.utils.perf import get_perf_logger from ..system_prompt import build_system_prompt from ..tools import ( @@ -30,6 +36,8 @@ from ..tools.glob.description import select_description as glob_description from ..tools.grep.description import select_description as grep_description from .read_only_policy import READ_ONLY_TOOL_NAMES +_perf_log = get_perf_logger() + class SurfSenseFilesystemMiddleware(FilesystemMiddleware): """SurfSense-specific filesystem middleware (cloud + desktop).""" @@ -56,16 +64,22 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): is_sandbox_enabled() and thread_id is not None and not read_only ) + _t0 = _perf_time.perf_counter() system_prompt = build_system_prompt( filesystem_mode, sandbox_available=self._sandbox_available, ) + _t_prompt = _perf_time.perf_counter() - _t0 + _t0 = _perf_time.perf_counter() super().__init__( backend=backend, system_prompt=system_prompt, tool_token_limit_before_evict=tool_token_limit_before_evict, ) + _t_super = _perf_time.perf_counter() - _t0 + + _t0 = _perf_time.perf_counter() self.tools = [t for t in self.tools if t.name != "execute"] self.tools.append(create_mkdir_tool(self)) self.tools.append(create_cd_tool(self)) @@ -79,6 +93,15 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): if read_only: self.tools = [t for t in self.tools if t.name in READ_ONLY_TOOL_NAMES] + _t_tools = _perf_time.perf_counter() - _t0 + _perf_log.info( + "[fs_middleware_init] ro=%s system_prompt=%.3fs super_init=%.3fs " + "surf_tools=%.3fs", + read_only, + _t_prompt, + _t_super, + _t_tools, + ) # ----------------------------------------- base-class tool overrides diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/mode.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/mode.py similarity index 70% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/mode.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/mode.py index a23d77535..44d69a50a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/mode.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/mode.py @@ -2,8 +2,8 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT def is_cloud(mode: FilesystemMode) -> bool: diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/namespace_policy.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/namespace_policy.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/namespace_policy.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/namespace_policy.py index 539050414..1eced41d7 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/namespace_policy.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/namespace_policy.py @@ -11,8 +11,10 @@ from typing import TYPE_CHECKING from langchain.tools import ToolRuntime -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT from ..shared.paths import TEMP_PREFIX, basename from .mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/path_resolution.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/path_resolution.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/path_resolution.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/path_resolution.py index 2c8ec6b4d..2650d9c34 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/path_resolution.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/path_resolution.py @@ -7,11 +7,13 @@ from typing import TYPE_CHECKING from langchain.tools import ToolRuntime -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.multi_root_local_folder import ( MultiRootLocalFolderBackend, ) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ..shared.paths import ( extract_mount_from_path, diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/read_only_policy.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/read_only_policy.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/middleware/read_only_policy.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/middleware/read_only_policy.py diff --git a/surfsense_backend/app/agents/new_chat/sandbox.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/sandbox.py similarity index 96% rename from surfsense_backend/app/agents/new_chat/sandbox.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/sandbox.py index efac7aae8..338a188d6 100644 --- a/surfsense_backend/app/agents/new_chat/sandbox.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/sandbox.py @@ -14,7 +14,6 @@ from __future__ import annotations import asyncio import contextlib import logging -import os import shutil import threading from pathlib import Path @@ -29,6 +28,10 @@ from daytona.common.errors import DaytonaError from deepagents.backends.protocol import ExecuteResponse from langchain_daytona import DaytonaSandbox +# Aliased to avoid clashing with the local ``config = DaytonaConfig(...)`` +# variable used inside ``_get_client``. +from app.config import config as app_config + logger = logging.getLogger(__name__) @@ -73,7 +76,7 @@ SANDBOX_DOCUMENTS_ROOT = "/home/daytona/documents" def is_sandbox_enabled() -> bool: - return os.environ.get("DAYTONA_SANDBOX_ENABLED", "FALSE").upper() == "TRUE" + return app_config.DAYTONA_SANDBOX_ENABLED def _get_client() -> Daytona: @@ -81,9 +84,9 @@ def _get_client() -> Daytona: with _client_lock: if _daytona_client is None: config = DaytonaConfig( - api_key=os.environ.get("DAYTONA_API_KEY", ""), - api_url=os.environ.get("DAYTONA_API_URL", "https://app.daytona.io/api"), - target=os.environ.get("DAYTONA_TARGET", "us"), + api_key=app_config.DAYTONA_API_KEY, + api_url=app_config.DAYTONA_API_URL, + target=app_config.DAYTONA_TARGET, ) _daytona_client = Daytona(config) return _daytona_client @@ -92,7 +95,7 @@ def _get_client() -> Daytona: def _sandbox_create_params( labels: dict[str, str], ) -> CreateSandboxFromSnapshotParams: - snapshot_id = os.environ.get("DAYTONA_SNAPSHOT_ID") or None + snapshot_id = app_config.DAYTONA_SNAPSHOT_ID return CreateSandboxFromSnapshotParams( language="python", labels=labels, @@ -302,7 +305,7 @@ async def delete_sandbox(thread_id: int | str) -> None: def _get_sandbox_files_dir() -> Path: - return Path(os.environ.get("SANDBOX_FILES_DIR", "sandbox_files")) + return Path(app_config.SANDBOX_FILES_DIR) def _local_path_for(thread_id: int | str, sandbox_path: str) -> Path: diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/shared/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/shared/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/shared/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/shared/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/shared/paths.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/shared/paths.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/shared/paths.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/shared/paths.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/cloud.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/cloud.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/cloud.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/cloud.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/common.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/common.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/common.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/common.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/desktop.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/desktop.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/desktop.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/desktop.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/index.py similarity index 86% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/index.py index 9d3cdbae3..491b5a762 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/system_prompt/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/system_prompt/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode from .cloud import BODY as CLOUD_BODY from .common import HEADER, SANDBOX_ADDENDUM diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/description.py similarity index 83% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/description.py index 6d7b987c8..bc106efcf 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _DESCRIPTION = """Changes the current working directory (cwd). diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/index.py index 8df6b9edb..0e78e8640 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/cd/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/cd/index.py @@ -10,8 +10,10 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT from ...middleware.async_dispatch import run_async_blocking from ...middleware.path_resolution import resolve_relative diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/description.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/description.py index de2a47648..5c474e2f8 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Performs exact string replacements in files. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/index.py similarity index 95% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/index.py index 324ef09b0..775469531 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/edit_file/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/edit_file/index.py @@ -11,8 +11,12 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/description.py similarity index 86% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/description.py index 89415c2f3..ae19b977e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _DESCRIPTION = """Executes Python code in an isolated sandbox environment. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/helpers.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/helpers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/helpers.py index cda9f535d..2c3293e14 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/helpers.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/helpers.py @@ -14,12 +14,14 @@ from typing import TYPE_CHECKING from daytona.common.errors import DaytonaError from langchain.tools import ToolRuntime -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.sandbox import ( +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( _evict_sandbox_cache, delete_sandbox, get_or_create_sandbox, ) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) if TYPE_CHECKING: from ...middleware import SurfSenseFilesystemMiddleware diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/index.py similarity index 95% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/index.py index 2711636e4..b530c91f2 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/execute_code/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/execute_code/index.py @@ -7,7 +7,9 @@ from typing import TYPE_CHECKING, Annotated from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from .description import select_description diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/glob/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/glob/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/glob/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/glob/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/glob/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/glob/description.py similarity index 77% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/glob/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/glob/description.py index d022f9a7a..7c9fafa36 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/glob/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/glob/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _DESCRIPTION = """Find files matching a glob pattern. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/grep/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/grep/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/grep/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/grep/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/grep/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/grep/description.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/grep/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/grep/description.py index 5d7c393a9..4b34ac60b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/grep/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/grep/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Search for a literal text pattern across files. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/description.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/description.py index a24230fb0..619a639d1 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Lists files/folders recursively in a single bounded call. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/index.py index 8bad88a74..21bba1fc3 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/list_tree/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/list_tree/index.py @@ -9,8 +9,12 @@ from deepagents.backends.utils import validate_path from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.path_resolution import resolve_list_target_path diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/description.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/description.py index 8c7e301dc..f49a64772 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Lists files and directories at the given path. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/index.py index 70f31dd04..e45a279d7 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/ls/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/ls/index.py @@ -8,8 +8,12 @@ from deepagents.backends.utils import validate_path from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import paginate_listing +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + paginate_listing, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.path_resolution import resolve_list_target_path diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/description.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/description.py index 1c86e72f7..94eb49d2d 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Creates a directory under `/documents/`. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/index.py index 788381faa..3ea38f525 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/mkdir/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/mkdir/index.py @@ -11,8 +11,10 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/description.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/description.py index fdba40b29..520692697 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Moves or renames a file or folder. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/helpers.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/helpers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/helpers.py index 7613f62f1..ded4701f9 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/helpers.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/helpers.py @@ -8,10 +8,14 @@ from langchain.tools import ToolRuntime from langchain_core.messages import ToolMessage from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT -from app.agents.new_chat.state_reducers import _CLEAR +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT if TYPE_CHECKING: from ...middleware import SurfSenseFilesystemMiddleware diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/index.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/index.py index d90535990..b7345b1a0 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/move_file/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/move_file/index.py @@ -11,7 +11,9 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/description.py similarity index 72% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/description.py index 594a38843..11f0b9f91 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _DESCRIPTION = """Prints the current working directory.""" diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/index.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/index.py index c15b67114..2c220efca 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/pwd/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/pwd/index.py @@ -7,7 +7,9 @@ from typing import TYPE_CHECKING from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.path_resolution import current_cwd from .description import select_description diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/description.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/description.py index 9b5d7623f..b10ca4acc 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _DESCRIPTION = """Reads a file from the filesystem. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/index.py index 8b0a1a1c8..5c20619d6 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/read_file/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/read_file/index.py @@ -10,8 +10,12 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.path_resolution import resolve_relative diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/description.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/description.py index a9e120e7c..7a8e96c09 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Deletes a single file under `/documents/`. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/helpers.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/helpers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/helpers.py index 8a02544d8..e2e445d08 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/helpers.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/helpers.py @@ -12,10 +12,14 @@ from langchain.tools import ToolRuntime from langchain_core.messages import ToolMessage from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT -from app.agents.new_chat.state_reducers import _CLEAR +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT if TYPE_CHECKING: from ...middleware import SurfSenseFilesystemMiddleware diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/index.py index 0c4e2fc71..099079476 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rm/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rm/index.py @@ -9,7 +9,9 @@ from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/description.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/description.py index 2b72f815b..0880b4d22 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Deletes an empty directory under `/documents/`. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/helpers.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/helpers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/helpers.py index de5afe722..b511a8d79 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/helpers.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/helpers.py @@ -13,10 +13,14 @@ from langchain.tools import ToolRuntime from langchain_core.messages import ToolMessage from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT -from app.agents.new_chat.state_reducers import _CLEAR +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT from ...middleware.path_resolution import current_cwd from ...shared.paths import is_ancestor_of diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/index.py index cdf057353..4c52f68ae 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/rmdir/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/rmdir/index.py @@ -9,7 +9,9 @@ from langchain.tools import ToolRuntime from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/description.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/description.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/description.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/description.py index 223cc3f26..933ba2caf 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/description.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/description.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode _CLOUD_DESCRIPTION = """Writes a new text file to the workspace. diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/index.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/index.py index a42f7ed62..5aa250143 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/filesystem/tools/write_file/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/tools/write_file/index.py @@ -11,7 +11,9 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool, StructuredTool from langgraph.types import Command -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from ...middleware.async_dispatch import run_async_blocking from ...middleware.mode import is_cloud diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/flags.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/flags.py similarity index 78% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/flags.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/flags.py index 69994ae00..dfbf3e6ee 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/flags.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/flags.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags def enabled(flags: AgentFeatureFlags, attr: str) -> bool: diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/kb_context_projection.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/kb_context_projection.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/kb_context_projection.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/kb_context_projection.py index 2685d8a9b..4667441ab 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/kb_context_projection.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/kb_context_projection.py @@ -9,10 +9,13 @@ from langchain.agents.middleware import AgentMiddleware, AgentState from langchain_core.messages import SystemMessage from langgraph.runtime import Runtime -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.knowledge_search import _render_priority_message +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) from app.utils.perf import get_perf_logger +from .knowledge_search import _render_priority_message + _perf_log = get_perf_logger() diff --git a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/knowledge_search.py similarity index 84% rename from surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/knowledge_search.py index 77b413940..681e80b0e 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/knowledge_search.py @@ -27,6 +27,7 @@ import asyncio import json import logging import re +import time from collections.abc import Sequence from datetime import UTC, datetime from typing import Any @@ -41,15 +42,20 @@ from litellm import token_counter from pydantic import BaseModel, Field, ValidationError from sqlalchemy import select -from app.agents.new_chat.feature_flags import get_flags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.multi_agent_chat.shared.date_filters import ( + parse_date_or_datetime, + resolve_date_range, +) +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import ( + SurfSenseFilesystemState, +) +from app.agents.chat.runtime.path_resolver import ( PathIndex, build_path_index, doc_to_virtual_path, ) -from app.agents.new_chat.utils import parse_date_or_datetime, resolve_date_range from app.db import ( NATIVE_TO_LEGACY_DOCTYPE, Chunk, @@ -341,6 +347,7 @@ async def browse_recent_documents( from app.db import DocumentType + _t0 = time.perf_counter() async with shielded_async_session() as session: base_conditions = [ Document.search_space_id == search_space_id, @@ -440,6 +447,12 @@ async def browse_recent_documents( ), } ) + _perf_log.info( + "[kb_priority.recent] db=%.3fs docs=%d space=%d", + time.perf_counter() - _t0, + len(results), + search_space_id, + ) return results @@ -457,10 +470,18 @@ async def search_knowledge_base( if not query: return [] + # ``embed_texts`` serializes behind a global embedding lock and, for API + # models, makes a network round-trip — so this can stall while another + # turn is embedding. Timed separately from the DB search to tell the two + # apart when debugging slow time-to-first-token. + _t_embed = time.perf_counter() [embedding] = await asyncio.to_thread(embed_texts, [query]) + _embed_elapsed = time.perf_counter() - _t_embed + doc_types = _resolve_search_types(available_connectors, available_document_types) retriever_top_k = min(top_k * 3, 30) + _t_search = time.perf_counter() async with shielded_async_session() as session: retriever = ChucksHybridSearchRetriever(session) results = await retriever.hybrid_search( @@ -472,7 +493,16 @@ async def search_knowledge_base( end_date=end_date, query_embedding=embedding.tolist(), ) + _search_elapsed = time.perf_counter() - _t_search + _perf_log.info( + "[kb_priority.search] embed=%.3fs hybrid_search=%.3fs results=%d space=%d query=%r", + _embed_elapsed, + _search_elapsed, + len(results), + search_space_id, + query[:80], + ) return results[:top_k] @@ -485,6 +515,7 @@ async def fetch_mentioned_documents( if not document_ids: return [] + _t0 = time.perf_counter() async with shielded_async_session() as session: doc_result = await session.execute( select(Document).where( @@ -541,6 +572,12 @@ async def fetch_mentioned_documents( "_user_mentioned": True, } ) + _perf_log.info( + "[kb_priority.mentioned] db=%.3fs requested=%d resolved=%d", + time.perf_counter() - _t0, + len(document_ids), + len(results), + ) return results @@ -587,16 +624,12 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] top_k: int = 10, mentioned_document_ids: list[int] | None = None, inject_system_message: bool = True, # For backwards compatibility + mentions_only: bool = False, ) -> None: self.llm = llm - # The planner LLM handles short, structured internal tasks (query - # rewriting, date extraction, recency classification). When an - # operator marks a global config ``is_planner: true`` we route - # those calls to a cheap/fast model (e.g. gpt-4o-mini, Haiku, Azure - # gpt-5.x-nano) instead of the user's chat LLM — those classification - # tasks don't need frontier-tier capability. Falls back to the chat - # LLM when no planner config is wired up so deployments without one - # keep working unchanged. + # Cheap model for structured internal tasks (query rewrite, date + # extraction, recency classification) when one is configured; falls back + # to the chat LLM otherwise. self.planner_llm = planner_llm or llm self.search_space_id = search_space_id self.filesystem_mode = filesystem_mode @@ -605,26 +638,21 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] self.top_k = top_k self.mentioned_document_ids = mentioned_document_ids or [] self.inject_system_message = inject_system_message - # Build the kb-planner private Runnable ONCE here so we don't pay - # the ``create_agent`` compile cost (50-200ms) on every turn. - # Disabled by default behind ``enable_kb_planner_runnable``; when - # off the planner falls back to the legacy ``planner_llm.ainvoke`` - # path. + # Lazy mode: skip the planner LLM + embedding + hybrid search and only + # surface explicit @-mentions. The agent retrieves topical KB content on + # demand via the ``search_knowledge_base`` tool instead. + self.mentions_only = mentions_only + # Compiled lazily and memoized to avoid the per-turn create_agent cost. self._planner: Runnable | None = None self._planner_compile_failed = False def _build_kb_planner_runnable(self) -> Runnable | None: - """Compile the kb-planner private :class:`Runnable` once. + """Lazily compile and memoize the kb-planner Runnable. - Returns ``None`` when the feature flag is disabled, when the LLM is - unavailable, or when ``create_agent`` raises (we fall back to the - legacy ``planner_llm.ainvoke`` path in that case). Compilation happens - lazily on first call, then memoized via ``self._planner``. - - The compiled agent is constructed without tools — the planner's - contract is "answer with structured JSON" — but it inherits the - :class:`RetryAfterMiddleware` so transient rate-limit errors - from the planner LLM call don't fail the whole turn. + Returns ``None`` (and the caller falls back to ``planner_llm.ainvoke``) + when the flag is off, the LLM is missing, or ``create_agent`` raises. + Built without tools but with RetryAfterMiddleware so a transient + rate-limit on the planner call doesn't fail the whole turn. """ if self._planner is not None or self._planner_compile_failed: return self._planner @@ -634,7 +662,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] if not flags.enable_kb_planner_runnable or flags.disable_new_agent_stack: return None - from app.agents.new_chat.middleware.retry_after import RetryAfterMiddleware + from app.agents.chat.shared.middleware.retry_after import RetryAfterMiddleware try: self._planner = create_agent( @@ -672,10 +700,8 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] loop = asyncio.get_running_loop() t0 = loop.time() - # Prefer the compiled-once planner Runnable when enabled; otherwise - # fall back to ``planner_llm.ainvoke``. The ``surfsense:internal`` - # tag is preserved on both paths so ``_stream_agent_events`` still - # suppresses the planner's intermediate events from the UI. + # Both paths tag surfsense:internal so the planner's intermediate + # events stay suppressed from the UI. planner = self._build_kb_planner_runnable() try: if planner is not None: @@ -804,42 +830,17 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] runtime: Runtime[Any] | None = None, ) -> dict[str, Any]: t0 = asyncio.get_event_loop().time() - ( - planned_query, - start_date, - end_date, - is_recency, - ) = await self._plan_search_inputs( - messages=messages, - user_text=user_text, - ) - # Per-turn ``mentioned_document_ids`` flow: - # 1. Preferred path (Phase 1.5+): read from ``runtime.context`` — the - # streaming task supplies a fresh :class:`SurfSenseContextSchema` - # on every ``astream_events`` call, so this list is naturally - # scoped to the current turn. Allows cross-turn graph reuse via - # ``agent_cache``. - # 2. Legacy fallback (cache disabled / context not propagated): the - # constructor-injected ``self.mentioned_document_ids`` list. We - # drain it after the first read so a cached graph (no Phase 1.5 - # wiring) doesn't keep replaying the same mentions on every - # turn. + # Prefer per-turn mentions from runtime.context (lets a cached graph + # serve different turns); fall back to the constructor closure, draining + # it after one read so stale mentions can't replay. # - # CRITICAL: distinguish "context absent" (legacy caller, no field at - # all) from "context provided but empty" (turn with no mentions). - # ``ctx_mentions`` is a ``list[int]``; an empty list is falsy in - # Python, so a naive ``if ctx_mentions:`` would fall through to the - # legacy closure on every no-mention follow-up turn — replaying the - # mentions baked in by turn 1's cache-miss build. Always drain the - # closure once the runtime path has fired so a cached middleware - # instance can never resurrect stale state. + # CRITICAL: test ``ctx_mentions is not None``, not truthiness — an empty + # list means "this turn has no mentions", not "use the closure". mention_ids: list[int] = [] ctx = getattr(runtime, "context", None) if runtime is not None else None ctx_mentions = getattr(ctx, "mentioned_document_ids", None) if ctx else None if ctx_mentions is not None: - # Runtime path is authoritative — even an empty list means - # "this turn has no mentions", NOT "look at the closure". mention_ids = list(ctx_mentions) if self.mentioned_document_ids: self.mentioned_document_ids = [] @@ -847,12 +848,8 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] mention_ids = list(self.mentioned_document_ids) self.mentioned_document_ids = [] - # Folder mentions live alongside doc mentions on the runtime - # context. They never feed hybrid search (folders aren't - # embedded) — they're surfaced purely as ``[USER-MENTIONED]`` - # priority entries so the agent walks the folder with ``ls`` / - # ``find_documents`` instead of ignoring it. Cloud filesystem - # mode only. + # Folder mentions aren't embedded, so they skip hybrid search and are + # surfaced only as [USER-MENTIONED] entries. Cloud mode only. folder_mention_ids: list[int] = [] if ( ctx is not None @@ -863,6 +860,52 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] if ctx_folders: folder_mention_ids = list(ctx_folders) + # Lazy mode: skip the planner LLM + embedding + hybrid search entirely. + # With no explicit mentions there is nothing cheap to surface, so we bail + # out early and let the agent decide to call ``search_knowledge_base``. + if self.mentions_only: + if not mention_ids and not folder_mention_ids: + return None + planned_query = user_text + start_date = end_date = None + is_recency = False + search_results: list[dict[str, Any]] = [] + _search_phase_elapsed = 0.0 + else: + ( + planned_query, + start_date, + end_date, + is_recency, + ) = await self._plan_search_inputs( + messages=messages, + user_text=user_text, + ) + + _t_search_phase = time.perf_counter() + if is_recency: + doc_types = _resolve_search_types( + self.available_connectors, self.available_document_types + ) + search_results = await browse_recent_documents( + search_space_id=self.search_space_id, + document_type=doc_types, + top_k=self.top_k, + start_date=start_date, + end_date=end_date, + ) + else: + search_results = await search_knowledge_base( + query=planned_query, + search_space_id=self.search_space_id, + available_connectors=self.available_connectors, + available_document_types=self.available_document_types, + top_k=self.top_k, + start_date=start_date, + end_date=end_date, + ) + _search_phase_elapsed = time.perf_counter() - _t_search_phase + mentioned_results: list[dict[str, Any]] = [] if mention_ids: mentioned_results = await fetch_mentioned_documents( @@ -870,28 +913,6 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] search_space_id=self.search_space_id, ) - if is_recency: - doc_types = _resolve_search_types( - self.available_connectors, self.available_document_types - ) - search_results = await browse_recent_documents( - search_space_id=self.search_space_id, - document_type=doc_types, - top_k=self.top_k, - start_date=start_date, - end_date=end_date, - ) - else: - search_results = await search_knowledge_base( - query=planned_query, - search_space_id=self.search_space_id, - available_connectors=self.available_connectors, - available_document_types=self.available_document_types, - top_k=self.top_k, - start_date=start_date, - end_date=end_date, - ) - seen_doc_ids: set[int] = set() merged: list[dict[str, Any]] = [] for doc in mentioned_results: @@ -905,15 +926,26 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] continue merged.append(doc) + _t_materialize = time.perf_counter() priority, matched_chunk_ids = await self._materialize_priority(merged) if folder_mention_ids: folder_entries = await self._materialize_folder_priority(folder_mention_ids) priority = folder_entries + priority + _materialize_elapsed = time.perf_counter() - _t_materialize + # ``recency=...`` reflects which retrieval path ran (recency browse vs + # hybrid search). The planner phase is logged separately by + # ``_plan_search_inputs``; here ``search_phase`` and ``materialize`` + # break down the remaining DB-bound work so a slow turn can be + # attributed to planner / search / materialize at a glance. _perf_log.info( - "[kb_priority] completed in %.3fs query=%r priority=%d mentioned=%d folders=%d", + "[kb_priority] completed in %.3fs (search_phase=%.3fs materialize=%.3fs " + "recency=%s) query=%r priority=%d mentioned=%d folders=%d", asyncio.get_event_loop().time() - t0, + _search_phase_elapsed, + _materialize_elapsed, + is_recency, user_text[:80], len(priority), len(mentioned_results), @@ -934,14 +966,10 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] async def _materialize_folder_priority( self, folder_ids: list[int] ) -> list[dict[str, Any]]: - """Resolve user-mentioned folder ids to ```` entries. + """Resolve mentioned folder ids to canonical-path priority entries. - Each entry uses the canonical ``/documents/Folder/Sub/`` virtual - path (matching ``KnowledgeTreeMiddleware`` and the agent's - ``ls`` adapter) and is flagged ``mentioned=True`` so the - rendered line carries ``[USER-MENTIONED]``. ``score`` is left - ``None`` so the renderer prints ``n/a`` — folders aren't - ranked, the agent decides which children to read. + Flagged ``mentioned=True`` with ``score=None`` (folders aren't ranked; + the agent decides which children to read). """ if not folder_ids: return [] @@ -993,6 +1021,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] if not merged: return priority, matched_chunk_ids + _t0 = time.perf_counter() async with shielded_async_session() as session: index: PathIndex = await build_path_index(session, self.search_space_id) doc_ids = [ @@ -1041,15 +1070,15 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] matched_chunk_ids[doc_id] = [ int(cid) for cid in chunk_ids if isinstance(cid, int | str) ] + _perf_log.info( + "[kb_priority.materialize] db=%.3fs docs=%d", + time.perf_counter() - _t0, + len(merged), + ) return priority, matched_chunk_ids -# Backwards-compatible alias for any external imports. -KnowledgeBaseSearchMiddleware = KnowledgePriorityMiddleware - - __all__ = [ - "KnowledgeBaseSearchMiddleware", "KnowledgePriorityMiddleware", "browse_recent_documents", "fetch_mentioned_documents", diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/patch_tool_calls.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/patch_tool_calls.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/patch_tool_calls.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/patch_tool_calls.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/bundle.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/bundle.py similarity index 88% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/bundle.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/bundle.py index 111244784..8b83c9b27 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/bundle.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/bundle.py @@ -10,15 +10,15 @@ from langchain.agents.middleware import ( ToolCallLimitMiddleware, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import RetryAfterMiddleware -from app.agents.new_chat.middleware.scoped_model_fallback import ( - ScopedModelFallbackMiddleware, -) +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.shared.middleware import RetryAfterMiddleware from .fallback import build_fallback_mw from .model_call_limit import build_model_call_limit_mw from .retry import build_retry_mw +from .scoped_model_fallback import ( + ScopedModelFallbackMiddleware, +) from .tool_call_limit import build_tool_call_limit_mw diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/fallback.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/fallback.py similarity index 82% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/fallback.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/fallback.py index ea68a764e..5b7dcc6ce 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/fallback.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/fallback.py @@ -4,12 +4,12 @@ from __future__ import annotations import logging -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware.scoped_model_fallback import ( - ScopedModelFallbackMiddleware, -) +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags from ..flags import enabled +from .scoped_model_fallback import ( + ScopedModelFallbackMiddleware, +) def build_fallback_mw( diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/model_call_limit.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/model_call_limit.py similarity index 85% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/model_call_limit.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/model_call_limit.py index 85707a385..2565a4b13 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/model_call_limit.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/model_call_limit.py @@ -4,7 +4,7 @@ from __future__ import annotations from langchain.agents.middleware import ModelCallLimitMiddleware -from app.agents.new_chat.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags from ..flags import enabled diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/retry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/retry.py similarity index 69% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/retry.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/retry.py index c98fc4083..b0ce3e324 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/retry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/retry.py @@ -2,8 +2,8 @@ from __future__ import annotations -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware import RetryAfterMiddleware +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.shared.middleware import RetryAfterMiddleware from ..flags import enabled diff --git a/surfsense_backend/app/agents/new_chat/middleware/scoped_model_fallback.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/scoped_model_fallback.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/scoped_model_fallback.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/scoped_model_fallback.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/tool_call_limit.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/tool_call_limit.py similarity index 85% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/tool_call_limit.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/tool_call_limit.py index dcde81f37..0e4708849 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/resilience/tool_call_limit.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/resilience/tool_call_limit.py @@ -4,7 +4,7 @@ from __future__ import annotations from langchain.agents.middleware import ToolCallLimitMiddleware -from app.agents.new_chat.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags from ..flags import enabled diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/todos.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/todos.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/todos.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/todos.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/__init__.py new file mode 100644 index 000000000..cad69379b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/__init__.py @@ -0,0 +1,41 @@ +"""Permissions vertical slice: rule model + allow/deny/ask enforcement. + +Self-contained subsystem combining the permission rule engine (:mod:`.model`) +with the pattern-based allow/deny/ask middleware and its HITL fallback +(:mod:`.middleware`, :mod:`.ask`, :mod:`.deny`). + +Public surface: +- rule model: ``Rule``, ``Ruleset``, ``RuleAction`` and the ``evaluate`` / + ``evaluate_many`` / ``aggregate_action`` / ``wildcard_match`` helpers. +- middleware: ``build_permission_mw`` — the construction recipe shared by + every agent stack. +""" + +# isort: off +# Import order matters: the rule model must be bound on this package before the +# middleware loads, because the middleware transitively imports consumers (e.g. +# app.services.user_tool_allowlist) that re-import ``Rule``/``Ruleset`` from this +# package root. Loading ``.model`` first avoids a partially-initialized cycle. +from .model import ( + Rule, + RuleAction, + Ruleset, + aggregate_action, + evaluate, + evaluate_many, + wildcard_match, +) +from .middleware.factory import build_permission_mw + +# isort: on + +__all__ = [ + "Rule", + "RuleAction", + "Ruleset", + "aggregate_action", + "build_permission_mw", + "evaluate", + "evaluate_many", + "wildcard_match", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/subagent/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/subagent/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/decision.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/decision.py similarity index 97% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/decision.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/decision.py index f507e85ff..e77f16c35 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/decision.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/decision.py @@ -17,7 +17,7 @@ from __future__ import annotations import logging from typing import Any -from app.agents.multi_agent_chat.subagents.shared.hitl.wire import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.wire import ( LC_DECISION_APPROVE, LC_DECISION_EDIT, LC_DECISION_REJECT, diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/edit/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/edit/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/edit/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/edit/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/edit/merge.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/edit/merge.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/edit/merge.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/edit/merge.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/payload.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/payload.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/payload.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/payload.py index 6c5d011df..c16b9072a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/payload.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/payload.py @@ -6,14 +6,14 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.hitl.wire import ( +from app.agents.chat.multi_agent_chat.shared.permissions.model import Rule +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.wire import ( LC_DECISION_APPROVE, LC_DECISION_EDIT, LC_DECISION_REJECT, SURFSENSE_DECISION_APPROVE_ALWAYS, build_lc_hitl_payload, ) -from app.agents.new_chat.permissions import Rule PERMISSION_ASK_INTERRUPT_TYPE = "permission_ask" diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/request.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/request.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/request.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/request.py index 3db51883d..7dc1e0a3c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/ask/request.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/ask/request.py @@ -16,7 +16,7 @@ from typing import Any from langchain_core.tools import BaseTool from langgraph.types import interrupt -from app.agents.new_chat.permissions import Rule +from app.agents.chat.multi_agent_chat.shared.permissions.model import Rule from app.observability import metrics as ot_metrics, otel as ot from .decision import normalize_permission_decision diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/deny.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/deny.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/deny.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/deny.py index 196c4040e..83677b4ca 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/deny.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/deny.py @@ -11,8 +11,8 @@ from typing import Any from langchain_core.messages import ToolMessage -from app.agents.new_chat.errors import StreamingError -from app.agents.new_chat.permissions import Rule +from app.agents.chat.multi_agent_chat.shared.permissions.model import Rule +from app.agents.chat.runtime.errors import StreamingError def build_deny_message(tool_call: dict[str, Any], rule: Rule) -> ToolMessage: diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/core.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/core.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/core.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/core.py index d2950c5b4..a97e32379 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/core.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/core.py @@ -26,8 +26,8 @@ from langchain_core.messages import AIMessage, ToolMessage from langchain_core.tools import BaseTool from langgraph.runtime import Runtime -from app.agents.new_chat.errors import CorrectedError, RejectedError -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions.model import Ruleset +from app.agents.chat.runtime.errors import CorrectedError, RejectedError from app.services.user_tool_allowlist import TrustedToolSaver from ..ask.edit import merge_edited_args diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/evaluation.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/evaluation.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/evaluation.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/evaluation.py index 51531c4eb..745c1d727 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/evaluation.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/evaluation.py @@ -16,7 +16,7 @@ from __future__ import annotations import logging from typing import Any -from app.agents.new_chat.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions.model import ( Rule, RuleAction, Ruleset, diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/factory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/factory.py similarity index 95% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/factory.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/factory.py index 3c061ded6..7f143d640 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/factory.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/factory.py @@ -27,8 +27,8 @@ from collections.abc import Sequence from langchain_core.tools import BaseTool -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.permissions.model import Rule, Ruleset from app.services.user_tool_allowlist import TrustedToolSaver from .core import PermissionMiddleware diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/pattern_resolver.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/pattern_resolver.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/pattern_resolver.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/pattern_resolver.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/ruleset_view.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/ruleset_view.py similarity index 87% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/ruleset_view.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/ruleset_view.py index fbb66d455..da089114e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/ruleset_view.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/ruleset_view.py @@ -9,7 +9,11 @@ newly-promoted rules apply to subsequent calls. from __future__ import annotations -from app.agents.new_chat.permissions import Ruleset, aggregate_action, evaluate_many +from app.agents.chat.multi_agent_chat.shared.permissions.model import ( + Ruleset, + aggregate_action, + evaluate_many, +) def all_rulesets( diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/runtime_promote.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/runtime_promote.py similarity index 88% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/runtime_promote.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/runtime_promote.py index afc65fdc0..2ae38db50 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/middleware/runtime_promote.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/middleware/runtime_promote.py @@ -7,7 +7,7 @@ is the streaming layer's job — this module keeps the in-memory copy only. from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions.model import Rule, Ruleset def persist_always( diff --git a/surfsense_backend/app/agents/new_chat/permissions.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/model.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/permissions.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/permissions/model.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/__init__.py diff --git a/surfsense_backend/app/agents/shared/receipt_command.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/command.py similarity index 88% rename from surfsense_backend/app/agents/shared/receipt_command.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/command.py index f1c269e90..d31df998c 100644 --- a/surfsense_backend/app/agents/shared/receipt_command.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/command.py @@ -6,7 +6,7 @@ participate in the verification teaching from ``multi_agent_chat/subagents/shared/snippets/verifiable_handle.md`` those tools now also need to write a :class:`Receipt` into the parent's ``state['receipts']`` list (declared on -:class:`~app.agents.new_chat.filesystem_state.SurfSenseFilesystemState` +:class:`~app.agents.chat.multi_agent_chat.shared.state.filesystem_state.SurfSenseFilesystemState` and backed by the append reducer). :func:`with_receipt` wraps both behaviours: it returns the tool payload as @@ -24,7 +24,7 @@ from typing import Any from langchain_core.messages import ToolMessage from langgraph.types import Command -from app.agents.shared.receipt import Receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import Receipt def _content_to_text(payload: dict[str, Any] | str) -> str: @@ -51,7 +51,7 @@ def with_receipt( """Return a Command that ships ``payload`` as a ToolMessage AND appends ``receipt``. The append happens via the ``_list_append_reducer`` on the ``receipts`` - field of :class:`~app.agents.new_chat.filesystem_state.SurfSenseFilesystemState`, + field of :class:`~app.agents.chat.multi_agent_chat.shared.state.filesystem_state.SurfSenseFilesystemState`, so concurrent subagent batches (item 4 in the plan) won't clobber each other's receipts. """ diff --git a/surfsense_backend/app/agents/shared/receipt.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py similarity index 96% rename from surfsense_backend/app/agents/shared/receipt.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py index 6f30067ee..b1986a224 100644 --- a/surfsense_backend/app/agents/shared/receipt.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/receipts/receipt.py @@ -5,7 +5,7 @@ delegate_tool.py:1663-1697``) for our 5 deliverable types + 15 connectors + KB writes. The supervisor reads the Receipt to verify what actually happened without round-tripping through LLM paraphrase. -**Why this lives under ``app.agents.shared`` and not under either of the +**Why this lives under ``app.agents.chat.shared`` and not under either of the two agent packages:** the Receipt is a *contract* shared between ``multi_agent_chat`` (where mutating tools emit it) and ``new_chat`` (where ``filesystem_state.SurfSenseFilesystemState`` declares the @@ -23,7 +23,7 @@ the receipt into the parent's ``receipts`` state via the append reducer. The KB write path is the one exception: file-tool calls cannot emit a durable receipt because the actual DB writes happen end-of-turn inside -:class:`app.agents.new_chat.middleware.kb_persistence.KnowledgeBasePersistenceMiddleware`. +:class:`app.agents.chat.multi_agent_chat.shared.middleware.kb_persistence.KnowledgeBasePersistenceMiddleware`. KB tools therefore emit a *provisional* receipt with ``status="pending"``; the persistence middleware flips it to ``"success"`` or ``"failed"`` before returning control to the parent. diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/filesystem_state.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/filesystem_state.py similarity index 96% rename from surfsense_backend/app/agents/new_chat/filesystem_state.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/filesystem_state.py index de2c94b41..41bed9d62 100644 --- a/surfsense_backend/app/agents/new_chat/filesystem_state.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/filesystem_state.py @@ -20,7 +20,7 @@ extra fields needed to implement Postgres-backed virtual filesystem semantics: * ``workspace_tree_text`` — pre-rendered ```` body for the turn. Tools mutate these fields ONLY via ``Command(update=...)`` returns; the -reducers in :mod:`app.agents.new_chat.state_reducers` handle merging. +reducers in :mod:`app.agents.chat.multi_agent_chat.shared.state.reducers` handle merging. """ from __future__ import annotations @@ -30,14 +30,14 @@ from typing import Annotated, Any, NotRequired from deepagents.middleware.filesystem import FilesystemState from typing_extensions import TypedDict -from app.agents.new_chat.state_reducers import ( +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import Receipt +from app.agents.chat.multi_agent_chat.shared.state.reducers import ( _add_unique_reducer, _dict_merge_with_tombstones_reducer, _int_counter_merge_reducer, _list_append_reducer, _replace_reducer, ) -from app.agents.shared.receipt import Receipt class PendingMove(TypedDict, total=False): @@ -190,7 +190,7 @@ class SurfSenseFilesystemState(FilesystemState): Each mutating tool (deliverables, every connector, KB writes via the persistence middleware) wraps its native return into a - :class:`~app.agents.shared.receipt.Receipt` + :class:`~app.agents.chat.multi_agent_chat.shared.receipts.receipt.Receipt` and returns it under the ``"receipt"`` key alongside its existing payload. The subagent's tool-call middleware folds the receipt into this list, and ``_return_command_with_state_update`` in diff --git a/surfsense_backend/app/agents/new_chat/state_reducers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/reducers.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/state_reducers.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/state/reducers.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/__init__.py new file mode 100644 index 000000000..a36be01eb --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/__init__.py @@ -0,0 +1 @@ +"""Tools shared across multi_agent_chat (main agent + subagents + boundary).""" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py new file mode 100644 index 000000000..63ce2ef1f --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/catalog.py @@ -0,0 +1,192 @@ +"""Pure-data catalog of built-in agent tools. + +This module advertises *what* tools exist and their display metadata. It is +intentionally free of any tool implementation imports (no connectors, no +factories) so it can be consumed without pulling the whole tool dependency +graph — and so connector packages stay independently deletable. + +The single live consumer is the ``GET /agent/tools`` endpoint, which renders +the tool picker in the web UI. Tool *construction* lives elsewhere: + +* main-agent tools -> ``app.agents.chat.multi_agent_chat.main_agent.tools.registry`` +* subagent / connector tools -> ``app.agents.chat.multi_agent_chat.subagents.*`` +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ToolMetadata: + """Display metadata for a single built-in tool. + + Attributes: + name: Unique identifier for the tool. + description: Human-readable description of what the tool does. + enabled_by_default: Whether the tool is on when no explicit config + is provided. + hidden: WIP tools that should be excluded from public listings. + + """ + + name: str + description: str + enabled_by_default: bool = True + hidden: bool = False + + +# Catalog of all built-in tools. Contributors: add new tools here so they show +# up in the UI tool picker. This list carries metadata only — wire the actual +# implementation in the relevant builder/registry module. +TOOL_CATALOG: list[ToolMetadata] = [ + ToolMetadata( + name="generate_podcast", + description="Generate an audio podcast from provided content", + ), + ToolMetadata( + name="generate_video_presentation", + description="Generate a video presentation with slides and narration from provided content", + ), + ToolMetadata( + name="generate_report", + description="Generate a structured report from provided content and export it", + ), + ToolMetadata( + name="generate_resume", + description="Generate a professional resume as a Typst document", + ), + ToolMetadata( + name="generate_image", + description="Generate images from text descriptions using AI image models", + ), + ToolMetadata( + name="search_knowledge_base", + description="Search the user's knowledge base with hybrid semantic + keyword retrieval", + ), + ToolMetadata( + name="scrape_webpage", + description="Scrape and extract the main content from a webpage", + ), + ToolMetadata( + name="web_search", + description="Search the web for real-time information using configured search engines", + ), + ToolMetadata( + name="create_automation", + description="Draft an automation from an NL intent; user approves the card; tool saves", + ), + ToolMetadata( + name="update_memory", + description="Save important long-term facts, preferences, and instructions to the (personal or team) memory", + ), + ToolMetadata( + name="create_notion_page", + description="Create a new page in the user's Notion workspace", + ), + ToolMetadata( + name="update_notion_page", + description="Append new content to an existing Notion page", + ), + ToolMetadata( + name="delete_notion_page", description="Delete an existing Notion page" + ), + ToolMetadata( + name="create_google_drive_file", + description="Create a new Google Doc or Google Sheet in Google Drive", + ), + ToolMetadata( + name="delete_google_drive_file", + description="Move an indexed Google Drive file to trash", + ), + ToolMetadata( + name="create_dropbox_file", description="Create a new file in Dropbox" + ), + ToolMetadata(name="delete_dropbox_file", description="Delete a file from Dropbox"), + ToolMetadata( + name="create_onedrive_file", + description="Create a new file in Microsoft OneDrive", + ), + ToolMetadata( + name="delete_onedrive_file", + description="Move a OneDrive file to the recycle bin", + ), + ToolMetadata( + name="search_calendar_events", + description="Search Google Calendar events within a date range", + ), + ToolMetadata( + name="create_calendar_event", + description="Create a new event on Google Calendar", + ), + ToolMetadata( + name="update_calendar_event", + description="Update an existing indexed Google Calendar event", + ), + ToolMetadata( + name="delete_calendar_event", + description="Delete an existing indexed Google Calendar event", + ), + ToolMetadata( + name="search_gmail", + description="Search emails in Gmail using Gmail search syntax", + ), + ToolMetadata( + name="read_gmail_email", + description="Read the full content of a specific Gmail email", + ), + ToolMetadata( + name="create_gmail_draft", description="Create a draft email in Gmail" + ), + ToolMetadata(name="send_gmail_email", description="Send an email via Gmail"), + ToolMetadata( + name="trash_gmail_email", description="Move an indexed email to trash in Gmail" + ), + ToolMetadata( + name="update_gmail_draft", description="Update an existing Gmail draft" + ), + ToolMetadata( + name="create_confluence_page", + description="Create a new page in the user's Confluence space", + ), + ToolMetadata( + name="update_confluence_page", + description="Update an existing indexed Confluence page", + ), + ToolMetadata( + name="delete_confluence_page", + description="Delete an existing indexed Confluence page", + ), + ToolMetadata( + name="list_discord_channels", + description="List text channels in the connected Discord server", + ), + ToolMetadata( + name="read_discord_messages", + description="Read recent messages from a Discord text channel", + ), + ToolMetadata( + name="send_discord_message", + description="Send a message to a Discord text channel", + ), + ToolMetadata( + name="list_teams_channels", + description="List Microsoft Teams and their channels", + ), + ToolMetadata( + name="read_teams_messages", + description="Read recent messages from a Microsoft Teams channel", + ), + ToolMetadata( + name="send_teams_message", + description="Send a message to a Microsoft Teams channel", + ), + ToolMetadata( + name="list_luma_events", description="List upcoming and recent Luma events" + ), + ToolMetadata( + name="read_luma_event", + description="Read detailed information about a specific Luma event", + ), + ToolMetadata(name="create_luma_event", description="Create a new event on Luma"), +] diff --git a/surfsense_backend/app/agents/new_chat/tools/hitl.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/hitl.py similarity index 81% rename from surfsense_backend/app/agents/new_chat/tools/hitl.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/hitl.py index 5b64929de..9b16e1a4c 100644 --- a/surfsense_backend/app/agents/new_chat/tools/hitl.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/hitl.py @@ -6,7 +6,7 @@ shared by every sensitive tool (native connectors and MCP tools alike). Usage inside a tool:: - from app.agents.new_chat.tools.hitl import request_approval + from app.agents.chat.multi_agent_chat.shared.tools.hitl import request_approval result = request_approval( action_type="gmail_email_send", @@ -30,22 +30,11 @@ from langgraph.types import interrupt logger = logging.getLogger(__name__) -# Tools that mirror the safety profile of ``write_file`` against the -# SurfSense KB: each call creates ONE artifact in the user's own workspace -# with no external visibility (drafts aren't sent; new files aren't shared -# unless the user shares them later). These are auto-approved by default -# so the agent can compose drafts and seed scratch files without a popup -# on every call. -# -# Members of this set still call ``request_approval`` exactly as before; -# the function returns immediately with ``decision_type="auto_approved"`` -# and the original params untouched. This preserves the call-site shape -# (logging, metadata fetching, account fallbacks) so the only behavior -# change is "no interrupt fires". -# -# To re-enable prompting, the future per-search-space rules table -# (``agent_permission_rules``) takes precedence — see the ``# (future)`` -# layer-3 comment in :mod:`app.agents.new_chat.chat_deepagent`. +# Low-stakes creation tools auto-approved by default: each creates one +# artifact in the user's own workspace with no external visibility (drafts +# aren't sent; new files aren't shared). They still call ``request_approval``, +# which returns ``decision_type="auto_approved"`` without firing an interrupt. +# Per-search-space ``agent_permission_rules`` can re-enable prompting. DEFAULT_AUTO_APPROVED_TOOLS: frozenset[str] = frozenset( { "create_gmail_draft", @@ -150,10 +139,6 @@ def request_approval( return HITLResult(rejected=False, decision_type="trusted", params=dict(params)) if tool_name in DEFAULT_AUTO_APPROVED_TOOLS: - # Default policy: low-stakes creation tools (drafts + new-file - # creates) skip HITL because they're as recoverable as a local - # ``write_file`` against the SurfSense KB. The user can still - # delete the artifact in <30s if it's wrong. logger.info( "Tool '%s' is in DEFAULT_AUTO_APPROVED_TOOLS — skipping HITL", tool_name, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/__init__.py new file mode 100644 index 000000000..07a5b02de --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/__init__.py @@ -0,0 +1,7 @@ +"""MCP (Model Context Protocol) integration: client, tool loading, and cache. + +Split by responsibility: +- ``client``: the low-level :class:`MCPClient` connection wrapper. +- ``tool``: discovery + LangChain tool construction and cache invalidation. +- ``cache``: the connector tool-cache refresh helpers. +""" diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_tools_cache.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/cache.py similarity index 94% rename from surfsense_backend/app/agents/new_chat/tools/mcp_tools_cache.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/cache.py index 81027e1c4..d088fac0b 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_tools_cache.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/cache.py @@ -112,7 +112,9 @@ def refresh_mcp_tools_cache_for_connector( when an event loop is available. Neither path raises. """ try: - from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + invalidate_mcp_tools_cache, + ) invalidate_mcp_tools_cache(search_space_id) except Exception: @@ -133,7 +135,9 @@ def refresh_mcp_tools_cache_for_connector( async def _run_connector_prefetch(connector_id: int) -> None: - from app.agents.new_chat.tools.mcp_tool import discover_single_mcp_connector + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + discover_single_mcp_connector, + ) try: await discover_single_mcp_connector(connector_id) diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_client.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/client.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/tools/mcp_client.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/client.py diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/tools/mcp_tool.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py index 6c4cfb6be..a1240391b 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py @@ -33,14 +33,16 @@ from sqlalchemy import cast, select from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.middleware.dedup_tool_calls import dedup_key_full_args -from app.agents.new_chat.tools.hitl import request_approval -from app.agents.new_chat.tools.mcp_client import MCPClient -from app.agents.new_chat.tools.mcp_tools_cache import ( +from app.agents.chat.multi_agent_chat.shared.middleware.dedup_tool_calls import ( + dedup_key_full_args, +) +from app.agents.chat.multi_agent_chat.shared.tools.hitl import request_approval +from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( CachedMCPTools, read_cached_tools, write_cached_tools, ) +from app.agents.chat.multi_agent_chat.shared.tools.mcp.client import MCPClient from app.db import SearchSourceConnector from app.services.mcp_oauth.registry import MCP_SERVICES, get_service_by_connector_type from app.utils.perf import get_perf_logger diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/agent.py index 396e0ec79..b483b8578 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/shared/deliverable_wait.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/deliverable_wait.py similarity index 92% rename from surfsense_backend/app/agents/shared/deliverable_wait.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/deliverable_wait.py index abaa017ea..2fcc98385 100644 --- a/surfsense_backend/app/agents/shared/deliverable_wait.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/deliverable_wait.py @@ -1,10 +1,10 @@ """Shared poll-until-terminal helper for Celery-backed deliverables. -Lives in ``app.agents.shared`` (neutral package, no dependencies on either -``new_chat`` or ``multi_agent_chat``) so both the flat single-agent tools -under ``app/agents/new_chat/tools/`` and the multi-agent subagent tools -under ``app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/`` -can import it without creating a circular dependency. +Lives in ``app.agents.chat.shared`` (neutral kernel package, no dependency on +``multi_agent_chat``) so both the shared tools under ``app/agents/shared/tools/`` +and the multi-agent subagent tools under +``app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/`` can import +it without creating a circular dependency. Background ---------- diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py index 094371760..7bb4a7c24 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/generate_image.py @@ -11,8 +11,8 @@ from litellm import aimage_generation from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt from app.config import config from app.db import ( ImageGeneration, @@ -25,6 +25,7 @@ from app.services.image_gen_router_service import ( ImageGenRouterService, is_image_gen_auto_mode, ) +from app.services.provider_api_base import resolve_api_base from app.utils.signed_image_urls import generate_image_token logger = logging.getLogger(__name__) @@ -43,13 +44,16 @@ _PROVIDER_MAP = { } +def _resolve_provider_prefix(provider: str, custom_provider: str | None) -> str: + if custom_provider: + return custom_provider + return _PROVIDER_MAP.get(provider.upper(), provider.lower()) + + def _build_model_string( provider: str, model_name: str, custom_provider: str | None ) -> str: - if custom_provider: - return f"{custom_provider}/{model_name}" - prefix = _PROVIDER_MAP.get(provider.upper(), provider.lower()) - return f"{prefix}/{model_name}" + return f"{_resolve_provider_prefix(provider, custom_provider)}/{model_name}" def _get_global_image_gen_config(config_id: int) -> dict | None: @@ -71,7 +75,7 @@ def create_generate_image_tool( captured model), use this config id instead of reading the search space's live ``image_generation_config_id``. """ - del db_session # use a fresh per-call session, see below + del db_session # tool uses a fresh per-call session instead @tool async def generate_image( @@ -136,17 +140,12 @@ def create_generate_image_tool( or IMAGE_GEN_AUTO_MODE_ID ) - # Build generation kwargs - # NOTE: size, quality, and style are intentionally NOT passed. - # Different models support different values for these params - # (e.g. DALL-E 3 wants "hd"/"standard" for quality while - # gpt-image-1 wants "high"/"medium"/"low"; size options also - # differ). Letting the model use its own defaults avoids errors. + # size/quality/style are intentionally omitted: valid values + # differ per model, so we let each model use its own defaults. gen_kwargs: dict[str, Any] = {} if n is not None and n > 1: gen_kwargs["n"] = n - # Call litellm based on config type if is_image_gen_auto_mode(config_id): if not ImageGenRouterService.is_initialized(): err = ( @@ -163,14 +162,20 @@ def create_generate_image_tool( err = f"Image generation config {config_id} not found" return _failed({"error": err}, error=err) - model_string = _build_model_string( - cfg.get("provider", ""), - cfg["model_name"], - cfg.get("custom_provider"), + provider_prefix = _resolve_provider_prefix( + cfg.get("provider", ""), cfg.get("custom_provider") ) + model_string = f"{provider_prefix}/{cfg['model_name']}" gen_kwargs["api_key"] = cfg.get("api_key") - if cfg.get("api_base"): - gen_kwargs["api_base"] = cfg["api_base"] + # Defense-in-depth: an empty ``api_base`` must not fall + # through to LiteLLM's global ``api_base`` (e.g. Azure). + api_base = resolve_api_base( + provider=cfg.get("provider"), + provider_prefix=provider_prefix, + config_api_base=cfg.get("api_base"), + ) + if api_base: + gen_kwargs["api_base"] = api_base if cfg.get("api_version"): gen_kwargs["api_version"] = cfg["api_version"] if cfg.get("litellm_params"): @@ -191,14 +196,20 @@ def create_generate_image_tool( err = f"Image generation config {config_id} not found" return _failed({"error": err}, error=err) - model_string = _build_model_string( - db_cfg.provider.value, - db_cfg.model_name, - db_cfg.custom_provider, + provider_prefix = _resolve_provider_prefix( + db_cfg.provider.value, db_cfg.custom_provider ) + model_string = f"{provider_prefix}/{db_cfg.model_name}" gen_kwargs["api_key"] = db_cfg.api_key - if db_cfg.api_base: - gen_kwargs["api_base"] = db_cfg.api_base + # Defense-in-depth: an empty ``api_base`` must not fall + # through to LiteLLM's global ``api_base`` (e.g. Azure). + api_base = resolve_api_base( + provider=db_cfg.provider.value, + provider_prefix=provider_prefix, + config_api_base=db_cfg.api_base, + ) + if api_base: + gen_kwargs["api_base"] = api_base if db_cfg.api_version: gen_kwargs["api_version"] = db_cfg.api_version if db_cfg.litellm_params: @@ -208,17 +219,13 @@ def create_generate_image_tool( prompt=prompt, model=model_string, **gen_kwargs ) - # Parse the response and store in DB response_dict = ( response.model_dump() if hasattr(response, "model_dump") else dict(response) ) - # Generate a random access token for this image access_token = generate_image_token() - - # Save to image_generations table for history db_image_gen = ImageGeneration( prompt=prompt, model=getattr(response, "_hidden_params", {}).get("model"), @@ -233,7 +240,6 @@ def create_generate_image_tool( await session.refresh(db_image_gen) db_image_gen_id = db_image_gen.id - # Extract image URLs from response images = response_dict.get("data", []) if not images: return _failed( @@ -244,11 +250,8 @@ def create_generate_image_tool( first_image = images[0] revised_prompt = first_image.get("revised_prompt", prompt) - # Resolve image URL: - # - If the API returned a URL, use it directly. - # - If the API returned b64_json (e.g. gpt-image-1), serve the - # image through our backend endpoint to avoid bloating the - # LLM context with megabytes of base64 data. + # b64_json (e.g. gpt-image-1) is served via our backend endpoint so + # megabytes of base64 don't bloat the LLM context. if first_image.get("url"): image_url = first_image["url"] elif first_image.get("b64_json"): diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/index.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/index.py index ddfcbd7fb..b968c1701 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .generate_image import create_generate_image_tool from .podcast import create_generate_podcast_tool diff --git a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/knowledge_base.py similarity index 87% rename from surfsense_backend/app/agents/new_chat/tools/knowledge_base.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/knowledge_base.py index c24497bfd..e99e0291a 100644 --- a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/knowledge_base.py @@ -241,23 +241,12 @@ def _normalize_connectors( connectors_to_search: list[str] | None, available_connectors: list[str] | None = None, ) -> list[str]: + """Normalize model-supplied connectors to canonical ConnectorService types. + + Maps user-facing aliases (e.g. WEBCRAWLER_CONNECTOR), drops unknowns, and + constrains to ``available_connectors`` when given. Empty input defaults to + all available connectors (minus live-search ones). """ - Normalize connectors provided by the model. - - - Accepts user-facing enums like WEBCRAWLER_CONNECTOR and maps them to canonical - ConnectorService types. - - Drops unknown values. - - If available_connectors is provided, only includes connectors from that list. - - If connectors_to_search is None/empty, defaults to available_connectors or all. - - Args: - connectors_to_search: List of connectors requested by the model - available_connectors: List of connectors actually available in the search space - - Returns: - List of normalized connector strings to search - """ - # Determine the set of valid connectors to consider valid_set = ( set(available_connectors) if available_connectors else set(_ALL_CONNECTORS) ) @@ -276,18 +265,16 @@ def _normalize_connectors( c = (raw or "").strip().upper() if not c: continue - # Map user-facing aliases to canonical names if c == "WEBCRAWLER_CONNECTOR": c = "CRAWLED_URL" normalized.append(c) - # de-dupe while preserving order + filter to valid connectors + # De-dupe (order-preserving), keeping only known + available connectors. seen: set[str] = set() out: list[str] = [] for c in normalized: if c in seen: continue - # Only include if it's a known connector AND available if c not in _ALL_CONNECTORS: continue if c not in valid_set: @@ -295,7 +282,7 @@ def _normalize_connectors( seen.add(c) out.append(c) - # Fallback to all available if nothing matched + # Nothing matched: fall back to all available. if not out: base = ( list(available_connectors) @@ -377,39 +364,17 @@ def format_documents_for_context( max_chunk_chars: int = _MAX_CHUNK_CHARS, max_chunks_per_doc: int = 0, ) -> str: - """ - Format retrieved documents into a readable context string for the LLM. + """Format retrieved documents into an XML context string for the LLM. - Documents are added in order (highest relevance first) until the character - budget is reached. Individual chunks are capped at ``max_chunk_chars`` and - each document is limited to a dynamically computed chunk cap so a single - large document cannot monopolize the output while still maximising the use - of available context space. - - Args: - documents: List of document dictionaries from connector search - max_chars: Approximate character budget for the entire output. - max_chunk_chars: Per-chunk character cap (content is tail-truncated). - max_chunks_per_doc: Maximum chunks per document. ``0`` (default) means - auto-compute per document using a rank-adaptive formula so - higher-ranked documents receive more chunks. - - Returns: - Formatted string with document contents and metadata + Documents are emitted highest-relevance first until ``max_chars`` is hit. + ``max_chunks_per_doc=0`` auto-computes a rank-adaptive cap so top results get + more chunks and no single large document monopolizes the budget. """ if not documents: return "" - # Group chunks by document id (preferred) to produce the XML structure. - # - # IMPORTANT: ConnectorService returns **document-grouped** results of the form: - # { - # "document": {...}, - # "chunks": [{"chunk_id": 123, "content": "..."}, ...], - # "source": "NOTION_CONNECTOR" | "FILE" | ... - # } - # - # We must preserve chunk_id so citations like [citation:123] are possible. + # Group chunks by document id, preserving chunk_id so [citation:123] works. + # ConnectorService returns document-grouped results ({document, chunks, source}). grouped: dict[str, dict[str, Any]] = {} for doc in documents: @@ -430,7 +395,7 @@ def format_documents_for_context( or "UNKNOWN" ) - # Document identity (prefer document_id; otherwise fall back to type+title+url) + # Identity: prefer document_id, else type+title+url. document_id_val = document_info.get("id") title = ( document_info.get("title") or metadata.get("title") or "Untitled Document" @@ -460,7 +425,7 @@ def format_documents_for_context( "chunks": [], } - # Prefer document-grouped chunks if available + # Prefer document-grouped chunks when present. chunks_list = doc.get("chunks") if isinstance(doc, dict) else None if isinstance(chunks_list, list) and chunks_list: for ch in chunks_list: @@ -492,7 +457,6 @@ def format_documents_for_context( "BAIDU_SEARCH_API", } - # Render XML expected by citation instructions, respecting the char budget. parts: list[str] = [] total_chars = 0 total_docs = len(grouped) @@ -594,30 +558,11 @@ async def search_knowledge_base_async( available_document_types: list[str] | None = None, max_input_tokens: int | None = None, ) -> str: - """ - Search the user's knowledge base for relevant documents. + """Search the knowledge base across connectors and return formatted results. - This is the async implementation that searches across multiple connectors. - - Args: - query: The search query - search_space_id: The user's search space ID - db_session: Database session - connector_service: Initialized connector service - connectors_to_search: Optional list of connector types to search. If omitted, searches all. - top_k: Number of results per connector - start_date: Optional start datetime (UTC) for filtering documents - end_date: Optional end datetime (UTC) for filtering documents - available_connectors: Optional list of connectors actually available in the search space. - If provided, only these connectors will be searched. - available_document_types: Optional list of document types that actually have indexed - data. When provided, local connectors whose document type is - absent are skipped entirely (no embedding / DB round-trip). - max_input_tokens: Model context window size (tokens). Used to dynamically - size the output so it fits within the model's limits. - - Returns: - Formatted string with search results + ``available_document_types`` lets local connectors with no indexed data be + skipped (no embedding / DB round-trip), and ``max_input_tokens`` sizes the + output to the model's context window. """ perf = get_perf_logger() t0 = time.perf_counter() @@ -692,7 +637,7 @@ async def search_knowledge_base_raw_async( # Preserve the public signature for compatibility even if values are unused. _ = (db_session, connector_service) - from app.agents.new_chat.utils import resolve_date_range + from app.agents.chat.multi_agent_chat.shared.date_filters import resolve_date_range resolved_start_date, resolved_end_date = resolve_date_range( start_date=start_date, diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py index 298257799..bfa3cc100 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/podcast.py @@ -16,9 +16,14 @@ from langchain_core.tools import tool from langgraph.types import Command from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.shared.deliverable_wait import wait_for_deliverable -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.deliverable_wait import ( + wait_for_deliverable, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.thread_resolver import ( + resolve_root_thread_id, +) from app.db import Podcast, PodcastStatus, shielded_async_session logger = logging.getLogger(__name__) @@ -69,7 +74,7 @@ def create_generate_podcast_tool( title=podcast_title, status=PodcastStatus.PENDING, search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), ) session.add(podcast) await session.commit() @@ -96,7 +101,7 @@ def create_generate_podcast_tool( # Wait until the Celery worker flips the row to a terminal # state. The wait is bounded only by the subagent invoke # timeout (multi-agent) or HTTP lifetime (single-agent) — - # see app.agents.shared.deliverable_wait for details. + # see app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.deliverable_wait for details. terminal_status, columns, elapsed = await wait_for_deliverable( model=Podcast, row_id=podcast_id, diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/report.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/report.py index f12ca8a90..ea831b891 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/report.py @@ -12,11 +12,14 @@ from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langgraph.types import Command -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.thread_resolver import ( + resolve_root_thread_id, +) from app.db import Report, shielded_async_session from app.services.connector_service import ConnectorService -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -196,13 +199,8 @@ def _strip_wrapping_code_fences(text: str) -> str: def _extract_metadata(content: str) -> dict[str, Any]: """Extract metadata from generated Markdown content.""" - # Count section headings headings = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE) - - # Word count word_count = len(content.split()) - - # Character count char_count = len(content) return { @@ -227,12 +225,11 @@ def _parse_sections(content: str) -> list[dict[str, str]]: in_code_block = False for line in lines: - # Track code blocks to avoid matching headings inside them + # Track fences so headings inside code blocks aren't treated as splits. stripped = line.strip() if stripped.startswith("```"): in_code_block = not in_code_block - # Only split on # or ## headings (not ### or deeper) and only outside code blocks is_section_heading = ( not in_code_block and re.match(r"^#{1,2}\s+", line) @@ -240,7 +237,6 @@ def _parse_sections(content: str) -> list[dict[str, str]]: ) if is_section_heading: - # Save previous section if current_heading or current_body_lines: sections.append( { @@ -253,7 +249,6 @@ def _parse_sections(content: str) -> list[dict[str, str]]: else: current_body_lines.append(line) - # Save last section if current_heading or current_body_lines: sections.append( { @@ -292,7 +287,6 @@ async def _revise_with_sections( Unchanged sections are kept byte-for-byte identical. Returns the revised content, or None to trigger full-document revision fallback. """ - # Parse report into sections sections = _parse_sections(parent_content) if len(sections) < 2: logger.info( @@ -300,7 +294,6 @@ async def _revise_with_sections( ) return None - # Build a sections listing for the LLM sections_listing = "" for i, sec in enumerate(sections): heading = sec["heading"] or "(preamble — content before first heading)" @@ -352,11 +345,9 @@ async def _revise_with_sections( ) return None - # Compute total operations for progress tracking total_ops = len(modify_indices) + len(add_sections) current_op = 0 - # Emit plan summary parts = [] if modify_indices: parts.append( @@ -394,7 +385,6 @@ async def _revise_with_sections( current_op += 1 sec = sections[idx] - # Extract plain section name (strip markdown heading markers) section_name = ( re.sub(r"^#+\s*", "", sec["heading"]).strip() if sec["heading"] @@ -412,7 +402,6 @@ async def _revise_with_sections( f"{sec['heading']}\n\n{sec['body']}" if sec["heading"] else sec["body"] ) - # Build context from surrounding sections context_parts = [] if idx > 0: prev = sections[idx - 1] @@ -442,7 +431,6 @@ async def _revise_with_sections( revised_text = resp.content if revised_text and isinstance(revised_text, str): revised_text = _strip_wrapping_code_fences(revised_text).strip() - # Parse the LLM output back into heading + body revised_parsed = _parse_sections(revised_text) if revised_parsed: revised_sections[idx] = revised_parsed[0] @@ -465,7 +453,6 @@ async def _revise_with_sections( heading = add_info.get("heading", "## New Section") description = add_info.get("description", "") - # Extract plain section name for progress display plain_heading = re.sub(r"^#+\s*", "", heading).strip() dispatch_custom_event( "report_progress", @@ -475,7 +462,6 @@ async def _revise_with_sections( }, ) - # Build context from the surrounding sections at the insertion point ctx_parts = [] if 0 <= after_idx < len(revised_sections): before_sec = revised_sections[after_idx] @@ -542,36 +528,13 @@ def create_generate_report_tool( available_connectors: list[str] | None = None, available_document_types: list[str] | None = None, ): - """ - Factory function to create the generate_report tool with injected dependencies. + """Create the generate_report tool with injected dependencies. - The tool generates a Markdown report inline using the search space's - document summary LLM, saves it to the database, and returns immediately. - - Uses short-lived database sessions for each DB operation so no connection - is held during the long LLM API call. - - Generation strategies: - - New reports: single-shot generation (1 LLM call) - - Revisions (targeted edits): section-level (unchanged sections preserved) - - Revisions (global changes): full-document revision fallback - - Source strategies: - - "provided"/"conversation": use only the supplied source_content - - "kb_search": search the knowledge base internally using targeted queries - - "auto": use source_content if sufficient, otherwise fall back to KB search - - Args: - search_space_id: The user's search space ID - thread_id: The chat thread ID for associating the report - connector_service: Optional connector service for internal KB search. - When provided, the tool can search the knowledge base internally - (used by the "kb_search" and "auto" source strategies). - available_connectors: Optional list of connector types available in the - search space (used to scope internal KB searches). - - Returns: - A configured tool function for generating reports + Uses short-lived DB sessions per operation so no connection is held during + the long LLM call. Generation: new reports are single-shot; revisions try + section-level first (unchanged sections preserved) and fall back to full-doc. + Source strategies: provided/conversation (use source_content), kb_search + (internal KB queries), auto (KB search only when source_content is thin). """ @tool @@ -693,7 +656,7 @@ def create_generate_report_tool( Returns: Dict with status, report_id, title, word_count, and message. """ - # Initialize version tracking variables (used by _save_failed_report closure) + # Shared with the _save_failed_report closure. parent_report_content: str | None = None report_group_id: int | None = None @@ -727,13 +690,13 @@ def create_generate_report_tool( }, report_style=report_style, search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), report_group_id=report_group_id, ) session.add(failed_report) await session.commit() await session.refresh(failed_report) - # If this is a new group (v1 failed), set group to self + # New group (v1 failed): point the group at itself. if not failed_report.report_group_id: failed_report.report_group_id = failed_report.id await session.commit() @@ -749,8 +712,8 @@ def create_generate_report_tool( try: # ── Phase 1: READ (short-lived session) ────────────────────── - # Fetch parent report and LLM config, then close the session - # so no DB connection is held during the long LLM call. + # Fetch parent report + LLM config, then release the connection + # before the long LLM call. async with shielded_async_session() as read_session: if parent_report_id: parent_report = await read_session.get(Report, parent_report_id) @@ -767,8 +730,7 @@ def create_generate_report_tool( "creating standalone report" ) - llm = await get_document_summary_llm(read_session, search_space_id) - # read_session closed — connection returned to pool + llm = await get_agent_llm(read_session, search_space_id) if not llm: error_msg = ( @@ -785,7 +747,6 @@ def create_generate_report_tool( error=error_msg, ) - # Build the user instructions string user_instructions_section = "" if user_instructions: user_instructions_section = ( @@ -829,7 +790,7 @@ def create_generate_report_tool( try: from .knowledge_base import search_knowledge_base_async - # Run all queries in parallel, each with its own session + # Each query gets its own short-lived session. async def _run_single_query(q: str) -> str: async with shielded_async_session() as kb_session: kb_connector_svc = ConnectorService( @@ -849,7 +810,6 @@ def create_generate_report_tool( *[_run_single_query(q) for q in search_queries[:5]] ) - # Merge non-empty results into source_content kb_text_parts = [r for r in kb_results if r and r.strip()] if kb_text_parts: kb_combined = "\n\n---\n\n".join(kb_text_parts) @@ -903,9 +863,9 @@ def create_generate_report_tool( "provided. Using source_content as-is." ) - capped_source = effective_source[:100000] # Cap source content + capped_source = effective_source[:100000] - # Length constraint — only when user explicitly asks for brevity + # Length constraint only when the user explicitly asked for brevity. length_instruction = "" if report_style == "brief": length_instruction = ( @@ -920,11 +880,8 @@ def create_generate_report_tool( report_content: str | None = None if parent_report_content: - # ─── REVISION MODE ─────────────────────────────────────── - # Strategy: Try section-level revision first (preserves - # unchanged sections byte-for-byte). Falls back to full- - # document revision if section identification fails or if - # all sections need changes. + # Revision mode: section-level first (preserves untouched + # sections), falling back to full-doc revision. dispatch_custom_event( "report_progress", { @@ -946,7 +903,6 @@ def create_generate_report_tool( ) if report_content is None: - # Fallback: full-document revision dispatch_custom_event( "report_progress", {"phase": "writing", "message": "Rewriting your full report"}, @@ -969,9 +925,7 @@ def create_generate_report_tool( report_content = response.content else: - # ─── NEW REPORT MODE ───────────────────────────────────── - # Single-shot generation: one LLM call produces the full - # report. Fast, globally coherent, and cost-efficient. + # New report: single-shot generation (one LLM call). dispatch_custom_event( "report_progress", {"phase": "writing", "message": "Writing your report"}, @@ -991,8 +945,6 @@ def create_generate_report_tool( response = await llm.ainvoke([HumanMessage(content=prompt)]) report_content = response.content - # ── Validate LLM output ────────────────────────────────────── - if not report_content or not isinstance(report_content, str): error_msg = "LLM returned empty or invalid content" report_id = await _save_failed_report(error_msg) @@ -1029,14 +981,12 @@ def create_generate_report_tool( if report_content.rstrip().endswith("---"): report_content = report_content.rstrip()[:-3].rstrip() - # Append exactly one standard disclaimer + # Append exactly one standard footer. report_content += "\n\n---\n\n" + _REPORT_FOOTER - # Extract metadata (includes "status": "ready") metadata = _extract_metadata(report_content) # ── Phase 3: WRITE (short-lived session) ───────────────────── - # Save the report to the database, then close the session. async with shielded_async_session() as write_session: report = Report( title=topic, @@ -1044,21 +994,20 @@ def create_generate_report_tool( report_metadata=metadata, report_style=report_style, search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), report_group_id=report_group_id, ) write_session.add(report) await write_session.commit() await write_session.refresh(report) - # If this is a brand-new report (v1), set report_group_id = own id + # Brand-new report (v1): point the group at itself. if not report.report_group_id: report.report_group_id = report.id await write_session.commit() saved_report_id = report.id saved_group_id = report.report_group_id - # write_session closed — connection returned to pool logger.info( f"[generate_report] Created report {saved_report_id} " diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py index ad16b7ba7..35dc996a1 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py @@ -14,10 +14,13 @@ from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langgraph.types import Command -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.thread_resolver import ( + resolve_root_thread_id, +) from app.db import Report, shielded_async_session -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -529,7 +532,7 @@ def create_generate_resume_tool( }, report_style="resume", search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), report_group_id=report_group_id, ) session.add(failed) @@ -578,7 +581,7 @@ def create_generate_resume_tool( f"(group {report_group_id})" ) - llm = await get_document_summary_llm(read_session, search_space_id) + llm = await get_agent_llm(read_session, search_space_id) if not llm: error_msg = ( @@ -817,7 +820,7 @@ def create_generate_resume_tool( report_metadata=metadata, report_style="resume", search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), report_group_id=report_group_id, ) write_session.add(report) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/thread_resolver.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/thread_resolver.py new file mode 100644 index 000000000..fcb0261bf --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/thread_resolver.py @@ -0,0 +1,39 @@ +"""Resolve the root chat ``thread_id`` from a deliverables tool's runtime. + +Deliverables tools run inside the ``deliverables`` subagent, which is invoked +with a *namespaced* ``thread_id`` of the form ``{chat_id}::task:{tool_call_id}`` +(see :func:`subagent_invoke_config`). To attribute a generated deliverable +(podcast / report / resume / video) to the correct chat, we parse the leading +segment of that namespaced id rather than trusting a ``thread_id`` captured at +tool-build time — the latter would be stale once a single compiled agent graph +is reused across chats (cross-thread ``agent_cache`` reuse). +""" + +from __future__ import annotations + +from langchain.tools import ToolRuntime + + +def resolve_root_thread_id(runtime: ToolRuntime, fallback: int | None) -> int | None: + """Return the root chat id from the live runtime config, else ``fallback``. + + The subagent's ``configurable.thread_id`` looks like ``"2099::task:call_x"``; + the chat id is the segment before the first ``"::"``. Returns ``fallback`` + when the config is absent or the leading segment is not an integer. + """ + try: + config = getattr(runtime, "config", None) + if not isinstance(config, dict): + return fallback + value = (config.get("configurable") or {}).get("thread_id") + if isinstance(value, int): + return value + if isinstance(value, str) and value: + root = value.split("::", 1)[0] + try: + return int(root) + except (TypeError, ValueError): + return fallback + except Exception: # pragma: no cover - defensive + return fallback + return fallback diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py index 5407c8834..f0fcf6e73 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/deliverables/tools/video_presentation.py @@ -17,9 +17,14 @@ from langchain_core.tools import tool from langgraph.types import Command from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.shared.deliverable_wait import wait_for_deliverable -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.deliverable_wait import ( + wait_for_deliverable, +) +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.thread_resolver import ( + resolve_root_thread_id, +) from app.db import VideoPresentation, VideoPresentationStatus, shielded_async_session logger = logging.getLogger(__name__) @@ -56,7 +61,7 @@ def create_generate_video_presentation_tool( title=video_title, status=VideoPresentationStatus.PENDING, search_space_id=search_space_id, - thread_id=thread_id, + thread_id=resolve_root_thread_id(runtime, thread_id), ) session.add(video_pres) await session.commit() @@ -83,7 +88,7 @@ def create_generate_video_presentation_tool( # Wait until the Celery worker flips the row to a terminal # state. The wait is bounded only by the subagent invoke # timeout (multi-agent) or HTTP lifetime (single-agent) — - # see app.agents.shared.deliverable_wait for details. + # see app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.deliverable_wait for details. terminal_status, _columns, elapsed = await wait_for_deliverable( model=VideoPresentation, row_id=video_pres_id, diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py index c6a0220ec..2720589ef 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py @@ -13,9 +13,9 @@ from deepagents import SubAgent from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec from .middleware_stack import build_kb_middleware from .prompts import load_description, load_readonly_system_prompt, load_system_prompt diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py similarity index 68% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py index 1708fe52f..2c81ca7c2 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/ask_knowledge_base_tool.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from typing import Annotated from langchain.tools import BaseTool, ToolRuntime @@ -10,11 +11,9 @@ from langchain_core.runnables import Runnable from langchain_core.tools import StructuredTool from langgraph.types import Command -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.config import ( - subagent_invoke_config, -) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.constants import ( +from app.agents.chat.multi_agent_chat.subagents.shared.invocation import ( EXCLUDED_STATE_KEYS, + subagent_invoke_config, ) from .prompts import load_readonly_description @@ -41,7 +40,28 @@ def _wrap_result(result: dict, tool_call_id: str) -> Command: ) -def build_ask_knowledge_base_tool(kb_readonly_runnable: Runnable) -> BaseTool: +def build_ask_knowledge_base_tool( + kb_readonly: Runnable | Callable[[], Runnable], +) -> BaseTool: + """Build the ``ask_knowledge_base`` tool backed by the read-only KB graph. + + ``kb_readonly`` may be a pre-compiled ``Runnable`` or a zero-arg factory + that compiles it on first use. Passing a factory defers the ~0.3-0.8s + ``create_agent`` cost of the read-only knowledge_base graph until a subagent + actually calls ``ask_knowledge_base``, keeping it off the cold agent-build + (time-to-first-token) path. The factory result is memoized. + """ + _cache: dict[str, Runnable] = {} + + def _resolve() -> Runnable: + if not callable(kb_readonly) or isinstance(kb_readonly, Runnable): + return kb_readonly # type: ignore[return-value] + cached = _cache.get("runnable") + if cached is None: + cached = kb_readonly() + _cache["runnable"] = cached + return cached + def ask_knowledge_base( query: Annotated[ str, @@ -54,7 +74,7 @@ def build_ask_knowledge_base_tool(kb_readonly_runnable: Runnable) -> BaseTool: raise ValueError("Tool call ID is required for ask_knowledge_base") sub_state = _forward_state(runtime, query) sub_config = subagent_invoke_config(runtime) - result = kb_readonly_runnable.invoke(sub_state, config=sub_config) + result = _resolve().invoke(sub_state, config=sub_config) return _wrap_result(result, runtime.tool_call_id) async def aask_knowledge_base( @@ -69,7 +89,7 @@ def build_ask_knowledge_base_tool(kb_readonly_runnable: Runnable) -> BaseTool: raise ValueError("Tool call ID is required for ask_knowledge_base") sub_state = _forward_state(runtime, query) sub_config = subagent_invoke_config(runtime) - result = await kb_readonly_runnable.ainvoke(sub_state, config=sub_config) + result = await _resolve().ainvoke(sub_state, config=sub_config) return _wrap_result(result, runtime.tool_call_id) return StructuredTool.from_function( diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/description_readonly.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/description_readonly.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/description_readonly.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/description_readonly.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py similarity index 58% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py index 778bb250c..2684e9db7 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/middleware_stack.py @@ -6,31 +6,35 @@ The KB-owned :class:`PermissionMiddleware` slot is what enforces from __future__ import annotations +import time as _perf_time from typing import Any from langchain_core.language_models import BaseChatModel -from app.agents.multi_agent_chat.middleware.shared.anthropic_cache import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.anthropic_cache import ( build_anthropic_cache_mw, ) -from app.agents.multi_agent_chat.middleware.shared.compaction import ( +from app.agents.chat.multi_agent_chat.shared.middleware.compaction import ( build_compaction_mw, ) -from app.agents.multi_agent_chat.middleware.shared.filesystem import ( +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem import ( build_filesystem_mw, ) -from app.agents.multi_agent_chat.middleware.shared.kb_context_projection import ( +from app.agents.chat.multi_agent_chat.shared.middleware.kb_context_projection import ( build_kb_context_projection_mw, ) -from app.agents.multi_agent_chat.middleware.shared.patch_tool_calls import ( +from app.agents.chat.multi_agent_chat.shared.middleware.patch_tool_calls import ( build_patch_tool_calls_mw, ) -from app.agents.multi_agent_chat.middleware.shared.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions import ( + Ruleset, build_permission_mw, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.permissions import Ruleset +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() def _kb_user_allowlist( @@ -93,25 +97,62 @@ def build_kb_middleware( user_allowlist = _kb_user_allowlist(dependencies, subagent_name) if user_allowlist is not None: rulesets.append(user_allowlist) + _t0 = _perf_time.perf_counter() permission_mw = build_permission_mw( flags=flags, subagent_rulesets=rulesets, trusted_tool_saver=dependencies.get("trusted_tool_saver"), ) + _t_perm = _perf_time.perf_counter() - _t0 + else: + _t_perm = 0.0 + + _t0 = _perf_time.perf_counter() + kb_ctx_mw = build_kb_context_projection_mw() + _t_ctx = _perf_time.perf_counter() - _t0 + + _t0 = _perf_time.perf_counter() + fs_mw = build_filesystem_mw( + backend_resolver=dependencies["backend_resolver"], + filesystem_mode=filesystem_mode, + search_space_id=dependencies["search_space_id"], + user_id=dependencies.get("user_id"), + thread_id=dependencies.get("thread_id"), + read_only=read_only, + ) + _t_fs = _perf_time.perf_counter() - _t0 + + _t0 = _perf_time.perf_counter() + compaction_mw = build_compaction_mw(llm) + _t_comp = _perf_time.perf_counter() - _t0 + + _t0 = _perf_time.perf_counter() + patch_mw = build_patch_tool_calls_mw() + _t_patch = _perf_time.perf_counter() - _t0 + + _t0 = _perf_time.perf_counter() + cache_mw = build_anthropic_cache_mw() + _t_cache = _perf_time.perf_counter() - _t0 + + _perf_log.info( + "[kb_middleware] name=%s ro=%s ctx=%.3fs filesystem=%.3fs " + "compaction=%.3fs patch=%.3fs anthropic_cache=%.3fs permission=%.3fs", + subagent_name, + read_only, + _t_ctx, + _t_fs, + _t_comp, + _t_patch, + _t_cache, + _t_perm, + ) return [ mws["todos"], - build_kb_context_projection_mw(), - build_filesystem_mw( - backend_resolver=dependencies["backend_resolver"], - filesystem_mode=filesystem_mode, - search_space_id=dependencies["search_space_id"], - user_id=dependencies.get("user_id"), - thread_id=dependencies.get("thread_id"), - read_only=read_only, - ), - build_compaction_mw(llm), - build_patch_tool_calls_mw(), + kb_ctx_mw, + fs_mw, + compaction_mw, + patch_mw, *([permission_mw] if permission_mw is not None else []), *resilience_mws, - build_anthropic_cache_mw(), + cache_mw, ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py similarity index 83% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py index 617bb2a85..ea9ae4706 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/prompts.py @@ -2,8 +2,10 @@ from __future__ import annotations -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) def load_system_prompt(filesystem_mode: FilesystemMode) -> str: diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_cloud.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_cloud.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_cloud.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_cloud.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_desktop.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_desktop.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_desktop.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_desktop.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_cloud.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_cloud.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_cloud.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_cloud.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_desktop.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_desktop.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_desktop.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/system_prompt_readonly_desktop.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/index.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/knowledge_base/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/tools/index.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/agent.py similarity index 78% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/agent.py index 84ab0c2fb..4038b13de 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/agent.py @@ -7,9 +7,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/index.py index b6e06dcdd..0afce9dec 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/index.py @@ -6,7 +6,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from app.db import ChatVisibility from .update_memory import create_update_memory_tool, create_update_team_memory_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/update_memory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/update_memory.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/memory/tools/update_memory.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/memory/tools/update_memory.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py similarity index 78% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py index 37026bebd..9a694872b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/agent.py @@ -7,9 +7,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py index d8abce46c..1e823fafa 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py @@ -6,7 +6,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .scrape_webpage import create_scrape_webpage_tool from .web_search import create_web_search_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py similarity index 87% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py index bb7c8e5a3..f367d7b57 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py @@ -2,18 +2,19 @@ import hashlib import logging +import time from typing import Any from urllib.parse import urlparse -import aiohttp from fake_useragent import UserAgent from langchain_core.tools import tool from requests import Session +from scrapling.fetchers import AsyncFetcher from youtube_transcript_api import YouTubeTranscriptApi from app.connectors.webcrawler_connector import WebCrawlerConnector from app.tasks.document_processors.youtube_processor import get_youtube_video_id -from app.utils.proxy_config import get_requests_proxies +from app.utils.proxy import get_proxy_url, get_requests_proxies logger = logging.getLogger(__name__) @@ -23,7 +24,6 @@ def extract_domain(url: str) -> str: try: parsed = urlparse(url) domain = parsed.netloc - # Remove 'www.' prefix if present if domain.startswith("www."): domain = domain[4:] return domain @@ -47,14 +47,13 @@ def truncate_content(content: str, max_length: int = 50000) -> tuple[str, bool]: if len(content) <= max_length: return content, False - # Try to truncate at a sentence boundary + # Prefer truncating at a sentence/paragraph boundary. truncated = content[:max_length] last_period = truncated.rfind(".") last_newline = truncated.rfind("\n\n") - # Use the later of the two boundaries, or just truncate boundary = max(last_period, last_newline) - if boundary > max_length * 0.8: # Only use boundary if it's not too far back + if boundary > max_length * 0.8: # only if the boundary isn't too far back truncated = content[: boundary + 1] return truncated + "\n\n[Content truncated...]", True @@ -81,15 +80,20 @@ async def _scrape_youtube_video( oembed_url = "https://www.youtube.com/oembed" try: - async with ( - aiohttp.ClientSession() as http_session, - http_session.get( - oembed_url, - params=params, - proxy=residential_proxies["http"] if residential_proxies else None, - ) as response, - ): - video_data = await response.json() + oembed_fetch_start = time.perf_counter() + oembed_page = await AsyncFetcher.get( + oembed_url, + params=params, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + logger.info( + "[scrape_webpage][perf] source=oembed video=%s status=%s fetch_ms=%.1f", + video_id, + getattr(oembed_page, "status", None), + (time.perf_counter() - oembed_fetch_start) * 1000, + ) + video_data = oembed_page.json() except Exception: video_data = {} @@ -98,6 +102,7 @@ async def _scrape_youtube_video( # --- Transcript via YouTubeTranscriptApi --- try: + transcript_fetch_start = time.perf_counter() ua = UserAgent() http_client = Session() http_client.headers.update({"User-Agent": ua.random}) @@ -105,12 +110,17 @@ async def _scrape_youtube_video( http_client.proxies.update(residential_proxies) ytt_api = YouTubeTranscriptApi(http_client=http_client) - # List all available transcripts and pick the first one - # (the video's primary language) instead of defaulting to English + # Pick the first transcript (video's primary language) rather than + # defaulting to English. transcript_list = ytt_api.list(video_id) transcript = next(iter(transcript_list)) captions = transcript.fetch() + logger.info( + "[scrape_webpage][perf] source=transcript video=%s fetch_ms=%.1f", + video_id, + (time.perf_counter() - transcript_fetch_start) * 1000, + ) logger.info( f"[scrape_webpage] Fetched transcript for {video_id} " f"in {transcript.language} ({transcript.language_code})" @@ -128,10 +138,8 @@ async def _scrape_youtube_video( logger.warning(f"[scrape_webpage] No transcript for video {video_id}: {e}") transcript_text = f"No captions available for this video. Error: {e!s}" - # Build combined content content = f"# {title}\n\n**Author:** {author}\n**Video ID:** {video_id}\n\n## Transcript\n\n{transcript_text}" - # Truncate if needed content, was_truncated = truncate_content(content, max_length) word_count = len(content.split()) @@ -206,20 +214,16 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): scrape_id = generate_scrape_id(url) domain = extract_domain(url) - # Validate and normalize URL if not url.startswith(("http://", "https://")): url = f"https://{url}" try: - # Check if this is a YouTube URL and use transcript API instead + # YouTube URLs use the transcript API instead of crawling. video_id = get_youtube_video_id(url) if video_id: return await _scrape_youtube_video(url, video_id, max_length) - # Create webcrawler connector connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key) - - # Crawl the URL result, error = await connector.crawl_url(url, formats=["markdown"]) if error: @@ -244,28 +248,21 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): "error": "No content returned from crawler", } - # Extract content and metadata content = result.get("content", "") metadata = result.get("metadata", {}) - # Get title from metadata title = metadata.get("title", "") if not title: title = domain or url.split("/")[-1] or "Webpage" - # Get description from metadata description = metadata.get("description", "") if not description and content: - # Use first paragraph as description first_para = content.split("\n\n")[0] if content else "" description = ( first_para[:300] + "..." if len(first_para) > 300 else first_para ) - # Truncate content if needed content, was_truncated = truncate_content(content, max_length) - - # Calculate word count word_count = len(content.split()) return { diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/web_search.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/web_search.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/research/tools/web_search.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/web_search.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py index d7648d407..87391371a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py index 9eebd2395..52cc8be2d 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/airtable/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/airtable/tools/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset NAME = "airtable" diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py index 7ef706c3d..b9b7b553a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py new file mode 100644 index 000000000..717199fef --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py @@ -0,0 +1,11 @@ +from .create_event import create_create_calendar_event_tool +from .delete_event import create_delete_calendar_event_tool +from .search_events import create_search_calendar_events_tool +from .update_event import create_update_calendar_event_tool + +__all__ = [ + "create_create_calendar_event_tool", + "create_delete_calendar_event_tool", + "create_search_calendar_events_tool", + "create_update_calendar_event_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py index e5262bd43..91a50b3cc 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/create_event.py @@ -8,7 +8,7 @@ from googleapiclient.discovery import build from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.google_calendar import GoogleCalendarToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py index 2f907e746..7682dae33 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/delete_event.py @@ -8,7 +8,7 @@ from googleapiclient.discovery import build from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.google_calendar import GoogleCalendarToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/index.py index 2570a51b2..b087105d4 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/index.py @@ -10,7 +10,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_event import create_create_calendar_event_tool from .delete_event import create_delete_calendar_event_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py index 6772d5a1e..cf9a015cf 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/search_events.py @@ -5,7 +5,9 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.new_chat.tools.gmail.search_emails import _build_credentials +from app.agents.chat.multi_agent_chat.subagents.connectors.google_auth import ( + build_credentials as _build_credentials, +) from app.db import SearchSourceConnector, SearchSourceConnectorType logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py index e6f9f098e..78d3b147b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/calendar/tools/update_event.py @@ -8,7 +8,7 @@ from googleapiclient.discovery import build from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.google_calendar import GoogleCalendarToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py index e1308a100..dd6ea6503 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py index b2c523080..c64da647a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/clickup/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/clickup/tools/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset NAME = "clickup" diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py index 5e95c876d..8322d901b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py index f33dc8e23..17497eee2 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/create_page.py @@ -5,7 +5,7 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.confluence_history import ConfluenceHistoryConnector diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py index 7a3a4f2c7..5e2bd9868 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/delete_page.py @@ -5,7 +5,7 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.confluence_history import ConfluenceHistoryConnector diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py index b38503c5c..73350974e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_page import create_create_confluence_page_tool from .delete_page import create_delete_confluence_page_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py index 7a8207a00..7db9a24dc 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/confluence/tools/update_page.py @@ -5,7 +5,7 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.confluence_history import ConfluenceHistoryConnector diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py index 567e72973..fe8f0df1e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py new file mode 100644 index 000000000..e6733a098 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/__init__.py @@ -0,0 +1,9 @@ +from .list_channels import create_list_discord_channels_tool +from .read_messages import create_read_discord_messages_tool +from .send_message import create_send_discord_message_tool + +__all__ = [ + "create_list_discord_channels_tool", + "create_read_discord_messages_tool", + "create_send_discord_message_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/_auth.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/_auth.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/_auth.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py index c69ef3e5c..fcef3401a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .list_channels import create_list_discord_channels_tool from .read_messages import create_read_discord_messages_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/list_channels.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/read_messages.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/send_message.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py similarity index 97% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/send_message.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py index 95890ed10..59ea1de30 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/send_message.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/discord/tools/send_message.py @@ -5,7 +5,7 @@ import httpx from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/agent.py index d3ae6dc83..841bcba6e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py new file mode 100644 index 000000000..f2b8303a5 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py @@ -0,0 +1,7 @@ +from .create_file import create_create_dropbox_file_tool +from .trash_file import create_delete_dropbox_file_tool + +__all__ = [ + "create_create_dropbox_file_tool", + "create_delete_dropbox_file_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py index 2de7c301f..7732c35e5 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/create_file.py @@ -8,7 +8,7 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.dropbox.client import DropboxClient diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/index.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/index.py index 68e02866a..440b4583c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_file import create_create_dropbox_file_tool from .trash_file import create_delete_dropbox_file_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py index 7cb652d5d..c713bdd00 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/dropbox/tools/trash_file.py @@ -6,7 +6,7 @@ from sqlalchemy import String, and_, cast, func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.dropbox.client import DropboxClient diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py index 082400eb9..be8adc17c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py new file mode 100644 index 000000000..1f0839c44 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py @@ -0,0 +1,15 @@ +from .create_draft import create_create_gmail_draft_tool +from .read_email import create_read_gmail_email_tool +from .search_emails import create_search_gmail_tool +from .send_email import create_send_gmail_email_tool +from .trash_email import create_trash_gmail_email_tool +from .update_draft import create_update_gmail_draft_tool + +__all__ = [ + "create_create_gmail_draft_tool", + "create_read_gmail_email_tool", + "create_search_gmail_tool", + "create_send_gmail_email_tool", + "create_trash_gmail_email_tool", + "create_update_gmail_draft_tool", +] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/_helpers.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/_helpers.py new file mode 100644 index 000000000..12d984352 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/_helpers.py @@ -0,0 +1,46 @@ +"""Gmail-specific helpers for the Gmail connector tools. + +Google OAuth credential construction lives in +``app.agents.chat.multi_agent_chat.subagents.connectors.google_auth`` (shared +with the Calendar connector). It is re-exported here under the legacy private +names so the existing Gmail tools keep importing it from this module. +""" + +from __future__ import annotations + +from typing import Any + +from app.agents.chat.multi_agent_chat.subagents.connectors.google_auth import ( + build_credentials as _build_credentials, + get_token_encryption as _get_token_encryption, +) + +__all__ = [ + "_build_credentials", + "_format_gmail_summary", + "_get_token_encryption", + "_gmail_headers", +] + + +def _gmail_headers(message: dict[str, Any]) -> dict[str, str]: + headers = message.get("payload", {}).get("headers", []) + return { + header.get("name", "").lower(): header.get("value", "") + for header in headers + if isinstance(header, dict) + } + + +def _format_gmail_summary(message: dict[str, Any]) -> dict[str, Any]: + headers = _gmail_headers(message) + return { + "message_id": message.get("id") or message.get("messageId"), + "thread_id": message.get("threadId"), + "subject": message.get("subject") or headers.get("subject", "No Subject"), + "from": message.get("sender") or headers.get("from", "Unknown"), + "to": message.get("to") or headers.get("to", ""), + "date": message.get("messageTimestamp") or headers.get("date", ""), + "snippet": message.get("snippet") or message.get("messageText", "")[:300], + "labels": message.get("labelIds", []), + } diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py index fb1461d7c..3f25305c5 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/create_draft.py @@ -8,7 +8,7 @@ from typing import Any from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.gmail import GmailToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/index.py index 020089ebb..60405dcf7 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_draft import create_create_gmail_draft_tool from .read_email import create_read_gmail_email_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py similarity index 95% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py index 39526f25e..10c64c6c5 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/read_email.py @@ -61,11 +61,10 @@ def create_read_gmail_email_tool( "message": "Composio connected account ID not found for this Gmail connector.", } - from app.agents.new_chat.tools.gmail.search_emails import ( - _format_gmail_summary, - ) from app.services.composio_service import ComposioService + from ._helpers import _format_gmail_summary + detail, error = await ComposioService().get_gmail_message_detail( connected_account_id=cca_id, entity_id=f"surfsense_{user_id}", @@ -97,9 +96,7 @@ def create_read_gmail_email_tool( "content": content, } - from app.agents.new_chat.tools.gmail.search_emails import ( - _build_credentials, - ) + from ._helpers import _build_credentials creds = _build_credentials(connector) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py similarity index 95% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py index a9d7cdedf..2c633d629 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/search_emails.py @@ -69,11 +69,10 @@ def create_search_gmail_tool( "message": "Composio connected account ID not found for this Gmail connector.", } - from app.agents.new_chat.tools.gmail.search_emails import ( - _format_gmail_summary, - ) from app.services.composio_service import ComposioService + from ._helpers import _format_gmail_summary + ( messages, _next, @@ -98,9 +97,7 @@ def create_search_gmail_tool( } return {"status": "success", "emails": emails, "total": len(emails)} - from app.agents.new_chat.tools.gmail.search_emails import ( - _build_credentials, - ) + from ._helpers import _build_credentials creds = _build_credentials(connector) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py index 0680e51cb..3431a2bc3 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/send_email.py @@ -10,11 +10,11 @@ from langchain_core.tools import tool from langgraph.types import Command from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt from app.services.gmail import GmailToolMetadataService logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py index b24e9ebe4..ef5882074 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/trash_email.py @@ -6,7 +6,7 @@ from typing import Any from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.gmail import GmailToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py index 1ab9d30cf..ef7839a1a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/gmail/tools/update_draft.py @@ -8,7 +8,7 @@ from typing import Any from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.services.gmail import GmailToolMetadataService diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_auth.py new file mode 100644 index 000000000..6eb60ef2a --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_auth.py @@ -0,0 +1,59 @@ +"""Google OAuth credential construction shared across Google connectors. + +Both the Gmail and Calendar connector tools are Google OAuth backed and build +``google.oauth2.credentials.Credentials`` from a stored ``SearchSourceConnector`` +the same way. This module is the single owner of that logic so neither connector +has to import the other. +""" + +from __future__ import annotations + +from datetime import datetime + +from app.db import SearchSourceConnector + +_token_encryption_cache: object | None = None + + +def get_token_encryption(): + global _token_encryption_cache + if _token_encryption_cache is None: + from app.config import config + from app.utils.oauth_security import TokenEncryption + + if not config.SECRET_KEY: + raise RuntimeError("SECRET_KEY not configured for token decryption.") + _token_encryption_cache = TokenEncryption(config.SECRET_KEY) + return _token_encryption_cache + + +def build_credentials(connector: SearchSourceConnector): + """Build Google OAuth Credentials from a connector's stored config. + + Handles both native OAuth connectors (with encrypted tokens) and + Composio-backed connectors. Shared by Gmail and Calendar tools. + """ + from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES + + if connector.connector_type in COMPOSIO_GOOGLE_CONNECTOR_TYPES: + raise ValueError("Composio connectors must use Composio tool execution.") + + from google.oauth2.credentials import Credentials + + cfg = dict(connector.config) + if cfg.get("_token_encrypted"): + enc = get_token_encryption() + for key in ("token", "refresh_token", "client_secret"): + if cfg.get(key): + cfg[key] = enc.decrypt_token(cfg[key]) + + exp = (cfg.get("expiry") or "").replace("Z", "") + return Credentials( + token=cfg.get("token"), + refresh_token=cfg.get("refresh_token"), + token_uri=cfg.get("token_uri"), + client_id=cfg.get("client_id"), + client_secret=cfg.get("client_secret"), + scopes=cfg.get("scopes", []), + expiry=datetime.fromisoformat(exp) if exp else None, + ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/agent.py index fb4a24ddd..1597d025e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py new file mode 100644 index 000000000..403140a5d --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py @@ -0,0 +1,7 @@ +from .create_file import create_create_google_drive_file_tool +from .trash_file import create_delete_google_drive_file_tool + +__all__ = [ + "create_create_google_drive_file_tool", + "create_delete_google_drive_file_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py index 70f5eea74..9de4e0a4b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/create_file.py @@ -5,7 +5,7 @@ from googleapiclient.errors import HttpError from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.google_drive.client import GoogleDriveClient diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/index.py index dd05374a1..caf06d6ba 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_file import create_create_google_drive_file_tool from .trash_file import create_delete_google_drive_file_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py index 7fbcd74a3..c89b54c8e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/google_drive/tools/trash_file.py @@ -5,7 +5,7 @@ from googleapiclient.errors import HttpError from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.google_drive.client import GoogleDriveClient diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py index ff71d4cf7..693d5980a 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py similarity index 93% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py index 24f1bdc01..20c67671b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/jira/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/jira/tools/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset NAME = "jira" diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py index d9b282f2b..d88ec03f1 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py similarity index 94% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py index 4a71a31b8..a06b33359 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/linear/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/linear/tools/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset NAME = "linear" diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py index d84efaed8..49973d08c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py new file mode 100644 index 000000000..c089eab4b --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/__init__.py @@ -0,0 +1,9 @@ +from .create_event import create_create_luma_event_tool +from .list_events import create_list_luma_events_tool +from .read_event import create_read_luma_event_tool + +__all__ = [ + "create_create_luma_event_tool", + "create_list_luma_events_tool", + "create_read_luma_event_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/_auth.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/_auth.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/_auth.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/create_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/create_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py index e3e1126fd..0dffb2d2c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/create_event.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/create_event.py @@ -5,7 +5,7 @@ import httpx from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py index dbde01061..a479331bb 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_event import create_create_luma_event_tool from .list_events import create_list_luma_events_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/list_events.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/list_events.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/list_events.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/list_events.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/read_event.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/read_event.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/read_event.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/luma/tools/read_event.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py index 8de86b2d8..a4b2d61cf 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/create_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/create_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py index 20862eb56..49ee0f3aa 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/create_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/create_page.py @@ -4,7 +4,7 @@ from typing import Any from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py index c98b25811..a187b2cbc 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/delete_page.py @@ -6,11 +6,11 @@ from langchain_core.tools import tool from langgraph.types import Command from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.shared.receipts.command import with_receipt +from app.agents.chat.multi_agent_chat.shared.receipts.receipt import make_receipt +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) -from app.agents.shared.receipt import make_receipt -from app.agents.shared.receipt_command import with_receipt from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector from app.services.notion.tool_metadata_service import NotionToolMetadataService diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py index 0475e9dd0..b8f662b03 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_page import create_create_notion_page_tool from .delete_page import create_delete_notion_page_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/update_page.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/update_page.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py index 2b9ce3a6c..6950f0abd 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/notion/tools/update_page.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/notion/tools/update_page.py @@ -4,7 +4,7 @@ from typing import Any from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/agent.py index f7634d8ef..e2fcdac90 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py new file mode 100644 index 000000000..406b9b6d2 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py @@ -0,0 +1,7 @@ +from .create_file import create_create_onedrive_file_tool +from .trash_file import create_delete_onedrive_file_tool + +__all__ = [ + "create_create_onedrive_file_tool", + "create_delete_onedrive_file_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py index 41fa65787..11160650d 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/create_file.py @@ -8,7 +8,7 @@ from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.onedrive.client import OneDriveClient diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/index.py similarity index 91% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/index.py index e09b43200..4f0a2a7d6 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .create_file import create_create_onedrive_file_tool from .trash_file import create_delete_onedrive_file_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py similarity index 99% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py index 1f7c51ac5..7b4e0b98c 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/onedrive/tools/trash_file.py @@ -6,7 +6,7 @@ from sqlalchemy import String, and_, cast, func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) from app.connectors.onedrive.client import OneDriveClient diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py similarity index 80% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py index e16956b25..9951a63f0 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/system_prompt.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py similarity index 89% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py index 44b96661c..a26b537a6 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/slack/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/slack/tools/index.py @@ -2,7 +2,7 @@ from __future__ import annotations -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset NAME = "slack" diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py similarity index 81% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/agent.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py index ab808b745..ab927654b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/agent.py @@ -12,9 +12,13 @@ from typing import Any from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import read_md_file -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import pack_subagent +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) from .tools.index import NAME, RULESET, load_tools diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/description.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/description.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/description.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/description.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/system_prompt.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/system_prompt.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/system_prompt.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/system_prompt.md diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py new file mode 100644 index 000000000..dbf966307 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/__init__.py @@ -0,0 +1,9 @@ +from .list_channels import create_list_teams_channels_tool +from .read_messages import create_read_teams_messages_tool +from .send_message import create_send_teams_message_tool + +__all__ = [ + "create_list_teams_channels_tool", + "create_read_teams_messages_tool", + "create_send_teams_message_tool", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/_auth.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/_auth.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/_auth.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/_auth.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py similarity index 92% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py index 41661651f..d144eee82 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/index.py @@ -9,7 +9,7 @@ from typing import Any from langchain_core.tools import BaseTool -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset from .list_channels import create_list_teams_channels_tool from .read_messages import create_read_teams_messages_tool diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/list_channels.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/read_messages.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/send_message.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py similarity index 97% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/send_message.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py index f1469e3e1..c4491e82e 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/send_message.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/connectors/teams/tools/send_message.py @@ -5,7 +5,7 @@ import httpx from langchain_core.tools import tool from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/mcp_tools/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/mcp_tools/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/mcp_tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py similarity index 96% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/mcp_tools/index.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py index 16dc09ac5..436b13aea 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/mcp_tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/mcp_tools/index.py @@ -18,10 +18,10 @@ from sqlalchemy import cast, select from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.constants import ( +from app.agents.chat.multi_agent_chat.constants import ( CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS, ) -from app.agents.new_chat.tools.mcp_tool import load_mcp_tools +from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import load_mcp_tools from app.db import SearchSourceConnector logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/subagent/middleware_stack.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py similarity index 82% rename from surfsense_backend/app/agents/multi_agent_chat/middleware/subagent/middleware_stack.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py index aa6211fcc..124ccf704 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/subagent/middleware_stack.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/middleware_stack.py @@ -14,11 +14,14 @@ from __future__ import annotations from typing import Any -from app.agents.new_chat.feature_flags import AgentFeatureFlags - -from ..shared.permissions import build_permission_mw -from ..shared.resilience import ResilienceMiddlewares -from ..shared.todos import build_todos_mw +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.middleware.resilience import ( + ResilienceMiddlewares, +) +from app.agents.chat.multi_agent_chat.shared.middleware.todos import build_todos_mw +from app.agents.chat.multi_agent_chat.shared.permissions import ( + build_permission_mw, +) def build_subagent_middleware_stack( diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py similarity index 74% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/registry.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py index 27c147672..c48b7f7ac 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/registry.py @@ -2,76 +2,80 @@ from __future__ import annotations +import time as _perf_time from typing import Any, Protocol from deepagents import SubAgent from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.constants import ( +from app.agents.chat.multi_agent_chat.constants import ( SUBAGENT_TO_REQUIRED_CONNECTOR_MAP, ) -from app.agents.multi_agent_chat.subagents.builtins.deliverables.agent import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.agent import ( build_subagent as build_deliverables_subagent, ) -from app.agents.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( build_subagent as build_knowledge_base_subagent, ) -from app.agents.multi_agent_chat.subagents.builtins.memory.agent import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.memory.agent import ( build_subagent as build_memory_subagent, ) -from app.agents.multi_agent_chat.subagents.builtins.research.agent import ( +from app.agents.chat.multi_agent_chat.subagents.builtins.research.agent import ( build_subagent as build_research_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.airtable.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.airtable.agent import ( build_subagent as build_airtable_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.calendar.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.calendar.agent import ( build_subagent as build_calendar_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.clickup.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.clickup.agent import ( build_subagent as build_clickup_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.confluence.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.confluence.agent import ( build_subagent as build_confluence_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.discord.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.discord.agent import ( build_subagent as build_discord_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.dropbox.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.dropbox.agent import ( build_subagent as build_dropbox_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.gmail.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.gmail.agent import ( build_subagent as build_gmail_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.google_drive.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.google_drive.agent import ( build_subagent as build_google_drive_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.jira.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.jira.agent import ( build_subagent as build_jira_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.linear.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.linear.agent import ( build_subagent as build_linear_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.luma.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.luma.agent import ( build_subagent as build_luma_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.notion.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.notion.agent import ( build_subagent as build_notion_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.onedrive.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.onedrive.agent import ( build_subagent as build_onedrive_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.slack.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.slack.agent import ( build_subagent as build_slack_subagent, ) -from app.agents.multi_agent_chat.subagents.connectors.teams.agent import ( +from app.agents.chat.multi_agent_chat.subagents.connectors.teams.agent import ( build_subagent as build_teams_subagent, ) -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import ( +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( read_md_file, ) -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.utils.perf import get_perf_logger + +_perf_log = get_perf_logger() class SubagentBuilder(Protocol): @@ -192,19 +196,25 @@ def build_subagents( if exclude: excluded.extend(exclude) disabled_names = frozenset(disabled_tools or ()) + _timings: list[tuple[str, float]] = [] for name in sorted(SUBAGENT_BUILDERS_BY_NAME): if name in excluded: continue builder = SUBAGENT_BUILDERS_BY_NAME[name] + _t0 = _perf_time.perf_counter() result = builder( dependencies=dependencies, model=model, middleware_stack=middleware_stack, mcp_tools=mcp.get(name), ) + _timings.append((name, _perf_time.perf_counter() - _t0)) spec = result.spec _filter_disabled_tools_in_place(spec, disabled_names) if ask_kb_tool is not None: _inject_ask_kb_tool_in_place(spec, ask_kb_tool) specs.append(spec) + if _timings: + _detail = " ".join(f"{n}={dt:.3f}s" for n, dt in _timings) + _perf_log.info("[build_subagents.detail] %s", _detail) return specs diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/__init__.py new file mode 100644 index 000000000..4ed3a5d8e --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/__init__.py @@ -0,0 +1,17 @@ +"""Cross-slice helpers for route subagents.""" + +from __future__ import annotations + +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, +) +from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + pack_subagent, +) + +__all__ = [ + "SurfSenseSubagentSpec", + "pack_subagent", + "read_md_file", +] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/auto_approved.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/auto_approved.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/auto_approved.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/auto_approved.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py similarity index 98% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py index 2f7e3cd35..8771b1506 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/request.py @@ -19,7 +19,7 @@ from typing import Any from langgraph.types import interrupt -from app.agents.multi_agent_chat.subagents.shared.hitl.wire import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.wire import ( LC_DECISION_APPROVE, LC_DECISION_EDIT, LC_DECISION_REJECT, diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/result.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/result.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/result.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/result.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/decision.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/decision.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/decision.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/decision.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/payload.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/payload.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/hitl/wire/payload.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/hitl/wire/payload.py diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/invocation.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/invocation.py new file mode 100644 index 000000000..63a63cbc3 --- /dev/null +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/invocation.py @@ -0,0 +1,69 @@ +"""Subagent-invocation contract shared by the orchestrator and nested subagents. + +Both the main-agent ``task`` middleware (``checkpointed_subagent_middleware``) +and subagents that themselves invoke another subagent (e.g. +``ask_knowledge_base``) need the same two things when spawning a child run: + +- a ``RunnableConfig`` that raises the recursion limit and isolates the child's + ``thread_id`` so each invocation lands in its own checkpoint slot + (``subagent_invoke_config``), and +- the set of parent state keys that must *not* be forwarded into / merged back + from the child (``EXCLUDED_STATE_KEYS``). + +Keeping this here (rather than inside the main-agent middleware) lets subagents +reuse the contract without importing main-agent internals. +""" + +from __future__ import annotations + +from typing import Any + +from langchain.tools import ToolRuntime + +# Mirror of deepagents.middleware.subagents._EXCLUDED_STATE_KEYS. +EXCLUDED_STATE_KEYS = frozenset( + { + "messages", + "todos", + "structured_response", + "skills_metadata", + "memory_contents", + } +) + +# Match the parent graph's budget; the LangGraph default of 25 trips on +# multi-step subagent runs. +DEFAULT_SUBAGENT_RECURSION_LIMIT = 10_000 + + +def subagent_invoke_config(runtime: ToolRuntime) -> dict[str, Any]: + """RunnableConfig for the nested invoke; raises ``recursion_limit`` and isolates ``thread_id``. + + Each parallel subagent invocation lands in its own checkpoint slot keyed + by an extended ``thread_id`` of the form ``{parent_thread}::task:{tool_call_id}``. + The same call across the resume cycle keeps reading from the same snapshot + (``tool_call_id`` is stable per LLM-emitted call). + + We namespace via ``thread_id`` rather than ``checkpoint_ns`` because + langgraph's ``aget_state`` interprets a non-empty ``checkpoint_ns`` as a + subgraph path and raises ``ValueError("Subgraph X not found")``. + """ + merged: dict[str, Any] = dict(runtime.config) if runtime.config else {} + current_limit = merged.get("recursion_limit") + try: + current_int = int(current_limit) if current_limit is not None else 0 + except (TypeError, ValueError): + current_int = 0 + if current_int < DEFAULT_SUBAGENT_RECURSION_LIMIT: + merged["recursion_limit"] = DEFAULT_SUBAGENT_RECURSION_LIMIT + + configurable: dict[str, Any] = dict(merged.get("configurable") or {}) + parent_thread_id = configurable.get("thread_id") + per_call_suffix = f"task:{runtime.tool_call_id}" + configurable["thread_id"] = ( + f"{parent_thread_id}::{per_call_suffix}" + if parent_thread_id + else per_call_suffix + ) + merged["configurable"] = configurable + return merged diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/md_file_reader.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/md_file_reader.py similarity index 90% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/md_file_reader.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/md_file_reader.py index 5694e4326..786086f60 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/md_file_reader.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/md_file_reader.py @@ -5,7 +5,7 @@ from __future__ import annotations from functools import lru_cache from importlib import resources -_SHARED_SNIPPETS_PACKAGE = "app.agents.multi_agent_chat.subagents.shared.snippets" +_SHARED_SNIPPETS_PACKAGE = "app.agents.chat.multi_agent_chat.subagents.shared.snippets" def read_md_file(package: str, stem: str) -> str: diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/__init__.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/__init__.py similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/__init__.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/__init__.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/output_contract_base.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/output_contract_base.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/output_contract_base.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/verifiable_handle.md b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/verifiable_handle.md similarity index 100% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/snippets/verifiable_handle.md rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/snippets/verifiable_handle.md diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/spec.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/spec.py similarity index 77% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/spec.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/spec.py index f891f94d2..6c68b96db 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/spec.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/spec.py @@ -8,7 +8,7 @@ from typing import Any from deepagents import SubAgent -from app.agents.new_chat.permissions import Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset # A context-hint provider receives the parent-agent ``runtime.state`` mapping # and the ``description`` the orchestrator wrote, and returns a short string @@ -26,6 +26,16 @@ ContextHintProvider = Callable[[Mapping[str, Any], str], str | None] # The prefix avoids any collision with future deepagents fields. SURF_CONTEXT_HINT_PROVIDER_KEY = "surf_context_hint_provider" +# Custom key carrying a zero-arg callable that builds the full deepagents +# ``SubAgent`` spec dict on demand. A descriptor dict carrying only +# ``name`` / ``description`` / this key lets the checkpointed subagent +# middleware register a subagent's catalog entry cheaply while deferring the +# expensive spec construction (e.g. the knowledge_base filesystem middleware, +# which builds ~13 tool schemas at ~150ms each) until the first +# ``task(name)`` call. Most turns never invoke a subagent, so this keeps the +# cost off the cold agent-build / time-to-first-token path. +SURF_LAZY_SPEC_FACTORY_KEY = "surf_lazy_spec_factory" + @dataclass(frozen=True, slots=True) class SurfSenseSubagentSpec: @@ -54,6 +64,7 @@ class SurfSenseSubagentSpec: __all__ = [ "SURF_CONTEXT_HINT_PROVIDER_KEY", + "SURF_LAZY_SPEC_FACTORY_KEY", "ContextHintProvider", "SurfSenseSubagentSpec", ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/subagent_builder.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py similarity index 88% rename from surfsense_backend/app/agents/multi_agent_chat/subagents/shared/subagent_builder.py rename to surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py index 5025b32e7..b8182ef24 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/subagent_builder.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging import re +import time as _perf_time from typing import Any, cast from deepagents import SubAgent @@ -11,20 +12,22 @@ from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool -from app.agents.multi_agent_chat.middleware.shared.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions import ( + Ruleset, build_permission_mw, ) -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import ( +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( read_shared_snippet, ) -from app.agents.multi_agent_chat.subagents.shared.spec import ( +from app.agents.chat.multi_agent_chat.subagents.shared.spec import ( SURF_CONTEXT_HINT_PROVIDER_KEY, ContextHintProvider, SurfSenseSubagentSpec, ) -from app.agents.new_chat.permissions import Ruleset +from app.utils.perf import get_perf_logger logger = logging.getLogger(__name__) +_perf_log = get_perf_logger() # ```` directive. Matches an XML-style self-closing # tag whose ``snippet`` attribute names a file in ``shared/snippets/``. @@ -110,19 +113,31 @@ def pack_subagent( msg = f"Subagent {name!r}: system_prompt is empty" raise ValueError(msg) + _t0 = _perf_time.perf_counter() system_prompt = _resolve_includes(system_prompt, subagent_name=name) + _t_resolve = _perf_time.perf_counter() - _t0 flags = dependencies["flags"] user_allowlist = _user_allowlist_for(dependencies, name) subagent_rulesets: list[Ruleset] = [ruleset] if user_allowlist is not None: subagent_rulesets.append(user_allowlist) + _t0 = _perf_time.perf_counter() per_subagent_perm = build_permission_mw( flags=flags, subagent_rulesets=subagent_rulesets, tools=tools, trusted_tool_saver=dependencies.get("trusted_tool_saver"), ) + _t_perm = _perf_time.perf_counter() - _t0 + _perf_log.info( + "[pack_subagent] name=%s tools=%d resolve_includes=%.3fs " + "build_permission_mw=%.3fs", + name, + len(tools), + _t_resolve, + _t_perm, + ) prepended: list[Any] = [] for slot, mw in (middleware_stack or {}).items(): diff --git a/surfsense_backend/app/agents/chat/runtime/__init__.py b/surfsense_backend/app/agents/chat/runtime/__init__.py new file mode 100644 index 000000000..9cc63f289 --- /dev/null +++ b/surfsense_backend/app/agents/chat/runtime/__init__.py @@ -0,0 +1,16 @@ +"""Lower-level runtime infrastructure for the chat agents. + +Modules here are the foundation layer used to *run* chat agents: wired by the +boundary (routes/tasks) and/or imported by the agent factory + shared +middleware, but never part of any single agent's domain logic. Because they sit +below the agent packages, both the boundary and the agents may depend on them +(forward dependency), while they never import agent code. + +Contents: +- ``checkpointer`` LangGraph Postgres checkpoint saver (boundary lifespan) +- ``llm_config`` LLM provider/model configuration resolution +- ``prompt_caching`` LiteLLM prompt-caching configuration +- ``errors`` agent-runtime error contracts (raised by MW, caught at boundary) +- ``path_resolver`` filesystem path resolution helpers +- ``mention_resolver`` @-mention resolution helpers +""" diff --git a/surfsense_backend/app/agents/new_chat/checkpointer.py b/surfsense_backend/app/agents/chat/runtime/checkpointer.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/checkpointer.py rename to surfsense_backend/app/agents/chat/runtime/checkpointer.py diff --git a/surfsense_backend/app/agents/new_chat/errors.py b/surfsense_backend/app/agents/chat/runtime/errors.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/errors.py rename to surfsense_backend/app/agents/chat/runtime/errors.py diff --git a/surfsense_backend/app/agents/new_chat/llm_config.py b/surfsense_backend/app/agents/chat/runtime/llm_config.py similarity index 64% rename from surfsense_backend/app/agents/new_chat/llm_config.py rename to surfsense_backend/app/agents/chat/runtime/llm_config.py index bc37bf1c4..aad432edb 100644 --- a/surfsense_backend/app/agents/new_chat/llm_config.py +++ b/surfsense_backend/app/agents/chat/runtime/llm_config.py @@ -27,7 +27,9 @@ from litellm import get_model_info from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.prompt_caching import apply_litellm_prompt_caching +from app.agents.chat.runtime.prompt_caching import ( + apply_litellm_prompt_caching, +) from app.services.llm_router_service import ( AUTO_MODE_ID, ChatLiteLLMRouter, @@ -90,15 +92,9 @@ class SanitizedChatLiteLLM(ChatLiteLLM): yield chunk -# Provider mapping for LiteLLM model string construction. -# -# Single source of truth lives in -# :mod:`app.services.provider_capabilities` so the YAML loader (which -# runs during ``app.config`` class-body init) can resolve provider -# prefixes without dragging the agent / tools tree into module load -# order. Re-exported here under the historical ``PROVIDER_MAP`` name -# so existing callers (``llm_router_service``, ``image_gen_router_service``, -# tests) keep working unchanged. +# Re-exported under the historical name ``PROVIDER_MAP``. Source of truth lives +# in provider_capabilities so the YAML loader can resolve prefixes during +# app.config init without importing the agent/tools tree. from app.services.provider_capabilities import ( # noqa: E402 _PROVIDER_PREFIX_MAP as PROVIDER_MAP, ) @@ -155,25 +151,14 @@ class AgentConfig: anonymous_enabled: bool = False quota_reserve_tokens: int | None = None - # Capability flag: best-effort True for the chat selector / catalog. - # Resolved via :func:`provider_capabilities.derive_supports_image_input` - # which prefers OpenRouter's ``architecture.input_modalities`` and - # otherwise consults LiteLLM's authoritative model map. Default True - # is the conservative-allow stance — the streaming-task safety net - # (``is_known_text_only_chat_model``) is the *only* place a False - # actually blocks a request. Setting this to False here without an - # authoritative source would silently hide vision-capable models - # (the regression we're fixing). + # Default-allow: only the streaming safety net (is_known_text_only_chat_model) + # actually blocks on False, so defaulting False would silently hide + # vision-capable models. Resolved via derive_supports_image_input. supports_image_input: bool = True @classmethod def from_auto_mode(cls) -> "AgentConfig": - """ - Create an AgentConfig for Auto mode (LiteLLM Router load balancing). - - Returns: - AgentConfig instance configured for Auto mode - """ + """Build an AgentConfig for Auto mode (LiteLLM Router load balancing).""" return cls( provider="AUTO", model_name="auto", @@ -191,27 +176,15 @@ class AgentConfig: is_premium=False, anonymous_enabled=False, quota_reserve_tokens=None, - # Auto routes across the configured pool, which usually - # contains at least one vision-capable deployment; the router - # will surface a 404 from a non-vision deployment as a normal - # ``allowed_fails`` event and fail over rather than blocking - # the request outright. + # Auto fails over across the pool, so a non-vision deployment's 404 + # is just an allowed_fails event rather than a hard block. supports_image_input=True, ) @classmethod def from_new_llm_config(cls, config) -> "AgentConfig": - """ - Create an AgentConfig from a NewLLMConfig database model. - - Args: - config: NewLLMConfig database model instance - - Returns: - AgentConfig instance - """ - # Lazy import to avoid pulling provider_capabilities (and its - # transitive litellm import) into module-init order. + """Build an AgentConfig from a NewLLMConfig database model.""" + # Lazy import: keeps provider_capabilities (and litellm) out of init order. from app.services.provider_capabilities import derive_supports_image_input provider_value = ( @@ -243,10 +216,8 @@ class AgentConfig: is_premium=False, anonymous_enabled=False, quota_reserve_tokens=None, - # BYOK rows have no operator-curated capability flag, so we - # ask LiteLLM (default-allow on unknown). The streaming - # safety net still blocks if the model is *explicitly* - # marked text-only. + # BYOK rows have no curated flag; ask LiteLLM (default-allow on + # unknown). The streaming safety net still blocks explicit text-only. supports_image_input=derive_supports_image_input( provider=provider_value, model_name=config.model_name, @@ -257,25 +228,14 @@ class AgentConfig: @classmethod def from_yaml_config(cls, yaml_config: dict) -> "AgentConfig": + """Build an AgentConfig from a YAML configuration dictionary. + + Supports the same prompt fields as NewLLMConfig (system_instructions, + use_default_system_instructions, citations_enabled). """ - Create an AgentConfig from a YAML configuration dictionary. - - YAML configs now support the same prompt configuration fields as NewLLMConfig: - - system_instructions: Custom system instructions (empty string uses defaults) - - use_default_system_instructions: Whether to use default instructions - - citations_enabled: Whether citations are enabled - - Args: - yaml_config: Configuration dictionary from YAML file - - Returns: - AgentConfig instance - """ - # Lazy import to avoid pulling provider_capabilities (and its - # transitive litellm import) into module-init order. + # Lazy import: keeps provider_capabilities (and litellm) out of init order. from app.services.provider_capabilities import derive_supports_image_input - # Get system instructions from YAML, default to empty string system_instructions = yaml_config.get("system_instructions", "") provider = yaml_config.get("provider", "").upper() @@ -288,13 +248,8 @@ class AgentConfig: else None ) - # Explicit YAML override wins; otherwise derive from LiteLLM / - # OpenRouter modalities. The YAML loader already populates this - # field, but this method is also called from - # ``load_global_llm_config_by_id``'s file fallback (hot reload), - # so we re-derive here for safety. The bool() coercion preserves - # the loader's behaviour for explicit ``true`` / ``false`` - # strings that PyYAML may surface. + # Explicit YAML override wins; otherwise re-derive (the hot-reload file + # fallback reaches this method without the loader having populated it). if "supports_image_input" in yaml_config: supports_image_input = bool(yaml_config.get("supports_image_input")) else: @@ -312,7 +267,6 @@ class AgentConfig: api_base=yaml_config.get("api_base"), custom_provider=custom_provider, litellm_params=yaml_config.get("litellm_params"), - # Prompt configuration from YAML (with defaults for backwards compatibility) system_instructions=system_instructions if system_instructions else None, use_default_system_instructions=yaml_config.get( "use_default_system_instructions", True @@ -330,20 +284,10 @@ class AgentConfig: def load_llm_config_from_yaml(llm_config_id: int = -1) -> dict | None: - """ - Load a specific LLM config from global_llm_config.yaml. - - Args: - llm_config_id: The id of the config to load (default: -1) - - Returns: - LLM config dict or None if not found - """ - # Get the config file path + """Load a specific LLM config from global_llm_config.yaml.""" base_dir = Path(__file__).resolve().parent.parent.parent.parent config_file = base_dir / "app" / "config" / "global_llm_config.yaml" - # Fallback to example file if main config doesn't exist if not config_file.exists(): config_file = base_dir / "app" / "config" / "global_llm_config.example.yaml" if not config_file.exists(): @@ -366,24 +310,17 @@ def load_llm_config_from_yaml(llm_config_id: int = -1) -> dict | None: def load_global_llm_config_by_id(llm_config_id: int) -> dict | None: - """ - Load a global LLM config by ID, checking in-memory configs first. + """Load a global LLM config by ID, checking in-memory configs first. - This handles both static YAML configs and dynamically injected configs - (e.g. OpenRouter integration models that only exist in memory). - - Args: - llm_config_id: The negative ID of the global config to load - - Returns: - LLM config dict or None if not found + In-memory covers both static YAML and dynamically injected configs (e.g. + OpenRouter integration models that only exist in memory). """ from app.config import config as app_config for cfg in app_config.GLOBAL_LLM_CONFIGS: if cfg.get("id") == llm_config_id: return cfg - # Fallback to YAML file read (covers edge cases like hot-reload) + # Fallback to YAML file read (covers hot-reload edge cases). return load_llm_config_from_yaml(llm_config_id) @@ -391,17 +328,7 @@ async def load_new_llm_config_from_db( session: AsyncSession, config_id: int, ) -> "AgentConfig | None": - """ - Load a NewLLMConfig from the database by ID. - - Args: - session: AsyncSession for database access - config_id: The ID of the NewLLMConfig to load - - Returns: - AgentConfig instance or None if not found - """ - # Import here to avoid circular imports + """Load a NewLLMConfig from the database by ID.""" from app.db import NewLLMConfig try: @@ -424,26 +351,13 @@ async def load_agent_llm_config_for_search_space( session: AsyncSession, search_space_id: int, ) -> "AgentConfig | None": + """Load the agent LLM config for a search space via its agent_llm_id. + + Positive id -> DB; negative -> YAML; None -> first global config (-1). """ - Load the agent LLM configuration for a search space. - - This loads the LLM config based on the search space's agent_llm_id setting: - - Positive ID: Load from NewLLMConfig database table - - Negative ID: Load from YAML global configs - - None: Falls back to first global config (id=-1) - - Args: - session: AsyncSession for database access - search_space_id: The search space ID - - Returns: - AgentConfig instance or None if not found - """ - # Import here to avoid circular imports from app.db import SearchSpace try: - # Get the search space to check its agent_llm_id preference result = await session.execute( select(SearchSpace).filter(SearchSpace.id == search_space_id) ) @@ -453,12 +367,9 @@ async def load_agent_llm_config_for_search_space( print(f"Error: SearchSpace with id {search_space_id} not found") return None - # Use agent_llm_id from search space, fallback to -1 (first global config) config_id = ( search_space.agent_llm_id if search_space.agent_llm_id is not None else -1 ) - - # Load the config using the unified loader return await load_agent_config(session, config_id, search_space_id) except Exception as e: print(f"Error loading agent LLM config for search space {search_space_id}: {e}") @@ -470,23 +381,7 @@ async def load_agent_config( config_id: int, search_space_id: int | None = None, ) -> "AgentConfig | None": - """ - Load an agent configuration, supporting Auto mode, YAML, and database configs. - - This is the main entry point for loading configurations: - - ID 0: Auto mode (uses LiteLLM Router for load balancing) - - Negative IDs: Load from YAML file (global configs) - - Positive IDs: Load from NewLLMConfig database table - - Args: - session: AsyncSession for database access - config_id: The config ID (0 for Auto, negative for YAML, positive for database) - search_space_id: Optional search space ID for context - - Returns: - AgentConfig instance or None if not found - """ - # Auto mode (ID 0) - use LiteLLM Router + """Main config loader: id 0 -> Auto mode; negative -> YAML; positive -> DB.""" if is_auto_mode(config_id): if not LLMRouterService.is_initialized(): print("Error: Auto mode requested but LLM Router not initialized") @@ -494,33 +389,22 @@ async def load_agent_config( return AgentConfig.from_auto_mode() if config_id < 0: - # Check in-memory configs first (includes static YAML + dynamic OpenRouter) + # In-memory covers static YAML + dynamic OpenRouter configs. from app.config import config as app_config for cfg in app_config.GLOBAL_LLM_CONFIGS: if cfg.get("id") == config_id: return AgentConfig.from_yaml_config(cfg) - # Fallback to YAML file read for safety yaml_config = load_llm_config_from_yaml(config_id) if yaml_config: return AgentConfig.from_yaml_config(yaml_config) return None else: - # Load from database (NewLLMConfig) return await load_new_llm_config_from_db(session, config_id) def create_chat_litellm_from_config(llm_config: dict) -> ChatLiteLLM | None: - """ - Create a ChatLiteLLM instance from a global LLM config dictionary. - - Args: - llm_config: LLM configuration dictionary from YAML - - Returns: - ChatLiteLLM instance or None on error - """ - # Build the model string + """Create a ChatLiteLLM instance from a global LLM config dictionary.""" if llm_config.get("custom_provider"): model_string = f"{llm_config['custom_provider']}/{llm_config['model_name']}" else: @@ -528,27 +412,20 @@ def create_chat_litellm_from_config(llm_config: dict) -> ChatLiteLLM | None: provider_prefix = PROVIDER_MAP.get(provider, provider.lower()) model_string = f"{provider_prefix}/{llm_config['model_name']}" - # Create ChatLiteLLM instance with streaming enabled litellm_kwargs = { "model": model_string, "api_key": llm_config.get("api_key"), - "streaming": True, # Enable streaming for real-time token streaming + "streaming": True, } - - # Add optional parameters if llm_config.get("api_base"): litellm_kwargs["api_base"] = llm_config["api_base"] - - # Add any additional litellm parameters if llm_config.get("litellm_params"): litellm_kwargs.update(llm_config["litellm_params"]) llm = SanitizedChatLiteLLM(**litellm_kwargs) _attach_model_profile(llm, model_string) - # Configure LiteLLM-native prompt caching (cache_control_injection_points - # for Anthropic/Bedrock/Vertex/Gemini/Azure-AI/OpenRouter/Databricks/etc.). - # ``agent_config=None`` here — the YAML path doesn't have provider intent - # in a structured form, so we set only the universal injection points. + # agent_config=None: the YAML path lacks structured provider intent, so set + # only the universal cache_control_injection_points. apply_litellm_prompt_caching(llm) return llm @@ -556,19 +433,7 @@ def create_chat_litellm_from_config(llm_config: dict) -> ChatLiteLLM | None: def create_chat_litellm_from_agent_config( agent_config: AgentConfig, ) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """ - Create a ChatLiteLLM or ChatLiteLLMRouter instance from an AgentConfig. - - For Auto mode configs, returns a ChatLiteLLMRouter that uses LiteLLM Router - for automatic load balancing across available providers. - - Args: - agent_config: AgentConfig instance - - Returns: - ChatLiteLLM or ChatLiteLLMRouter instance, or None on error - """ - # Handle Auto mode - return ChatLiteLLMRouter + """Create a ChatLiteLLM (or, for Auto mode, a load-balancing router) from config.""" if agent_config.is_auto_mode: if not LLMRouterService.is_initialized(): print("Error: Auto mode requested but LLM Router not initialized") @@ -576,19 +441,14 @@ def create_chat_litellm_from_agent_config( try: router_llm = get_auto_mode_llm() if router_llm is not None: - # Universal cache_control_injection_points only — auto-mode - # fans out across providers, so OpenAI-only kwargs (e.g. - # ``prompt_cache_key``) are left off here. ``drop_params`` - # would strip them at the provider boundary anyway, but - # there's no point setting them when we don't know the - # destination. + # Universal injection points only: auto-mode fans out across + # providers, so provider-specific kwargs have no known target. apply_litellm_prompt_caching(router_llm, agent_config=agent_config) return router_llm except Exception as e: print(f"Error creating ChatLiteLLMRouter: {e}") return None - # Build the model string if agent_config.custom_provider: model_string = f"{agent_config.custom_provider}/{agent_config.model_name}" else: @@ -597,26 +457,19 @@ def create_chat_litellm_from_agent_config( ) model_string = f"{provider_prefix}/{agent_config.model_name}" - # Create ChatLiteLLM instance with streaming enabled litellm_kwargs = { "model": model_string, "api_key": agent_config.api_key, - "streaming": True, # Enable streaming for real-time token streaming + "streaming": True, } - - # Add optional parameters if agent_config.api_base: litellm_kwargs["api_base"] = agent_config.api_base - - # Add any additional litellm parameters if agent_config.litellm_params: litellm_kwargs.update(agent_config.litellm_params) llm = SanitizedChatLiteLLM(**litellm_kwargs) _attach_model_profile(llm, model_string) - # Build-time prompt caching: sets ``cache_control_injection_points`` for - # all providers and (for OpenAI/DeepSeek/xAI) ``prompt_cache_retention``. - # Per-thread ``prompt_cache_key`` is layered on later in - # ``create_surfsense_deep_agent`` once ``thread_id`` is known. + # Build-time caching only; the per-thread prompt_cache_key is layered on + # later in create_surfsense_deep_agent once thread_id is known. apply_litellm_prompt_caching(llm, agent_config=agent_config) return llm diff --git a/surfsense_backend/app/agents/new_chat/mention_resolver.py b/surfsense_backend/app/agents/chat/runtime/mention_resolver.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/mention_resolver.py rename to surfsense_backend/app/agents/chat/runtime/mention_resolver.py index f13dbc6ae..a47ed8f36 100644 --- a/surfsense_backend/app/agents/new_chat/mention_resolver.py +++ b/surfsense_backend/app/agents/chat/runtime/mention_resolver.py @@ -36,7 +36,7 @@ from dataclasses import dataclass, field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, build_path_index, doc_to_virtual_path, diff --git a/surfsense_backend/app/agents/new_chat/path_resolver.py b/surfsense_backend/app/agents/chat/runtime/path_resolver.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/path_resolver.py rename to surfsense_backend/app/agents/chat/runtime/path_resolver.py diff --git a/surfsense_backend/app/agents/chat/runtime/prompt_caching.py b/surfsense_backend/app/agents/chat/runtime/prompt_caching.py new file mode 100644 index 000000000..5a5fd7418 --- /dev/null +++ b/surfsense_backend/app/agents/chat/runtime/prompt_caching.py @@ -0,0 +1,162 @@ +r"""LiteLLM-native prompt caching for SurfSense agents. + +Replaces the legacy ``AnthropicPromptCachingMiddleware`` (its +``isinstance(model, ChatAnthropic)`` gate never matched our LiteLLM stack) +with LiteLLM's universal ``cache_control_injection_points`` mechanism, which +covers the Anthropic/Bedrock/Vertex/Gemini/OpenRouter/etc. marker-based +providers and the auto-caching OpenAI family. + +Two breakpoints per request: + +- ``index: 0`` pins the head-of-request system prompt. We use ``index: 0``, + NOT ``role: system``: ``before_agent`` injectors accumulate many + SystemMessages, and tagging all of them overflows Anthropic's 4-block cap + (upstream 400 via OpenRouter). +- ``index: -1`` pins the latest message so longest-prefix lookup compounds + multi-turn savings. + +OpenAI-family configs also get ``prompt_cache_key`` (per-thread routing hint) +and ``prompt_cache_retention="24h"``. Azure is excluded from the latter +because LiteLLM's Azure transformer drops it (see +``_PROMPT_CACHE_RETENTION_PROVIDERS``). + +Safety net: ``litellm.drop_params=True`` (set in ``app.services.llm_service``) +strips any kwarg the destination provider rejects, so an auto-mode fallback +can't 400 on these extras. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from langchain_core.language_models import BaseChatModel + +if TYPE_CHECKING: + from app.agents.chat.runtime.llm_config import AgentConfig + +logger = logging.getLogger(__name__) + + +# Head-of-request + latest message (see module docstring for the index:0 vs +# role:system rationale and Anthropic's 4-block cap). +_DEFAULT_INJECTION_POINTS: tuple[dict[str, Any], ...] = ( + {"location": "message", "index": 0}, + {"location": "message", "index": -1}, +) + +# Providers that accept the OpenAI ``prompt_cache_key`` routing hint. Strict +# whitelist: many providers route through litellm's ``openai`` prefix without +# the prompt-cache surface, so the prefix alone isn't enough to infer family. +_PROMPT_CACHE_KEY_PROVIDERS: frozenset[str] = frozenset( + {"OPENAI", "DEEPSEEK", "XAI", "AZURE", "AZURE_OPENAI"} +) + +# Subset that also accepts ``prompt_cache_retention="24h"``. Azure is excluded +# because LiteLLM's Azure transformer omits the param (drop_params strips it). +_PROMPT_CACHE_RETENTION_PROVIDERS: frozenset[str] = frozenset( + {"OPENAI", "DEEPSEEK", "XAI"} +) + + +def _is_router_llm(llm: BaseChatModel) -> bool: + """Detect ``ChatLiteLLMRouter`` by class name to avoid an import cycle.""" + return type(llm).__name__ == "ChatLiteLLMRouter" + + +def _provider_supports_prompt_cache_key(agent_config: AgentConfig | None) -> bool: + """Whether the config targets a provider that accepts ``prompt_cache_key``. + + Strict — only returns True for explicitly chosen OPENAI, DEEPSEEK, + XAI, AZURE, or AZURE_OPENAI providers. Auto-mode and custom + providers return False because we can't statically know the + destination and the router fans out across mixed providers. + """ + if agent_config is None or not agent_config.provider: + return False + if agent_config.is_auto_mode: + return False + if agent_config.custom_provider: + return False + return agent_config.provider.upper() in _PROMPT_CACHE_KEY_PROVIDERS + + +def _provider_supports_prompt_cache_retention( + agent_config: AgentConfig | None, +) -> bool: + """Whether the config targets a provider that accepts ``prompt_cache_retention``. + + Tighter than :func:`_provider_supports_prompt_cache_key` — Azure + deployments are excluded until LiteLLM ships the param in its Azure + transformer (see module docstring). + """ + if agent_config is None or not agent_config.provider: + return False + if agent_config.is_auto_mode: + return False + if agent_config.custom_provider: + return False + return agent_config.provider.upper() in _PROMPT_CACHE_RETENTION_PROVIDERS + + +def _get_or_init_model_kwargs(llm: BaseChatModel) -> dict[str, Any] | None: + """Return ``llm.model_kwargs`` as a writable dict, or ``None`` to bail. + + Initialises the field to ``{}`` when present-but-None on a Pydantic v2 + model. Returns ``None`` if the LLM type doesn't expose a writable + ``model_kwargs`` attribute (caller should treat as no-op). + """ + model_kwargs = getattr(llm, "model_kwargs", None) + if isinstance(model_kwargs, dict): + return model_kwargs + try: + llm.model_kwargs = {} # type: ignore[attr-defined] + except Exception: + return None + refreshed = getattr(llm, "model_kwargs", None) + return refreshed if isinstance(refreshed, dict) else None + + +def apply_litellm_prompt_caching( + llm: BaseChatModel, + *, + agent_config: AgentConfig | None = None, + thread_id: int | None = None, +) -> None: + """Configure LiteLLM prompt caching on a ChatLiteLLM/ChatLiteLLMRouter. + + Idempotent (existing ``model_kwargs`` values are preserved) and mutates + ``llm.model_kwargs`` in place. Without ``agent_config`` (or in auto-mode) + only the universal injection points are set; ``thread_id`` adds a per-thread + ``prompt_cache_key`` for OpenAI-family providers to improve routing affinity. + """ + model_kwargs = _get_or_init_model_kwargs(llm) + if model_kwargs is None: + logger.debug( + "apply_litellm_prompt_caching: %s exposes no writable model_kwargs; skipping", + type(llm).__name__, + ) + return + + if "cache_control_injection_points" not in model_kwargs: + model_kwargs["cache_control_injection_points"] = [ + dict(point) for point in _DEFAULT_INJECTION_POINTS + ] + + # OpenAI-style extras only when the destination is statically known. The + # auto-mode router fans out across mixed providers, so skip them there. + if _is_router_llm(llm): + return + + if ( + thread_id is not None + and "prompt_cache_key" not in model_kwargs + and _provider_supports_prompt_cache_key(agent_config) + ): + model_kwargs["prompt_cache_key"] = f"surfsense-thread-{thread_id}" + + if ( + "prompt_cache_retention" not in model_kwargs + and _provider_supports_prompt_cache_retention(agent_config) + ): + model_kwargs["prompt_cache_retention"] = "24h" diff --git a/surfsense_backend/app/agents/shared/__init__.py b/surfsense_backend/app/agents/chat/shared/__init__.py similarity index 79% rename from surfsense_backend/app/agents/shared/__init__.py rename to surfsense_backend/app/agents/chat/shared/__init__.py index 7c46c65ff..e84bc7543 100644 --- a/surfsense_backend/app/agents/shared/__init__.py +++ b/surfsense_backend/app/agents/chat/shared/__init__.py @@ -2,7 +2,7 @@ Symbols here are intentionally framework-light (no LangGraph / deepagents internals) so they can be imported from both ``app.agents.new_chat`` and -``app.agents.multi_agent_chat`` without creating a circular dependency +``app.agents.chat.multi_agent_chat`` without creating a circular dependency between the two packages. See ``receipt.py`` for the rationale. """ diff --git a/surfsense_backend/app/agents/new_chat/context.py b/surfsense_backend/app/agents/chat/shared/context.py similarity index 97% rename from surfsense_backend/app/agents/new_chat/context.py rename to surfsense_backend/app/agents/chat/shared/context.py index 1b3ea3d20..50b761f5b 100644 --- a/surfsense_backend/app/agents/new_chat/context.py +++ b/surfsense_backend/app/agents/chat/shared/context.py @@ -50,8 +50,8 @@ class SurfSenseContextSchema: (cloud filesystem mode). Surfaced as ``[USER-MENTIONED]`` entries in ```` so the agent prioritises walking those folders with ``ls`` / ``find_documents``. - file_operation_contract: One-shot file operation contract emitted - by ``FileIntentMiddleware`` for the upcoming turn. + file_operation_contract: One-shot file operation contract for the + upcoming turn (reserved; not currently populated). turn_id / request_id: Correlation IDs surfaced by the streaming task; populated for telemetry. diff --git a/surfsense_backend/app/agents/chat/shared/middleware/__init__.py b/surfsense_backend/app/agents/chat/shared/middleware/__init__.py new file mode 100644 index 000000000..90339137b --- /dev/null +++ b/surfsense_backend/app/agents/chat/shared/middleware/__init__.py @@ -0,0 +1,13 @@ +"""Shared middleware components for the SurfSense chat agents.""" + +from app.agents.chat.shared.middleware.compaction import ( + SurfSenseCompactionMiddleware, + create_surfsense_compaction_middleware, +) +from app.agents.chat.shared.middleware.retry_after import RetryAfterMiddleware + +__all__ = [ + "RetryAfterMiddleware", + "SurfSenseCompactionMiddleware", + "create_surfsense_compaction_middleware", +] diff --git a/surfsense_backend/app/agents/new_chat/middleware/compaction.py b/surfsense_backend/app/agents/chat/shared/middleware/compaction.py similarity index 70% rename from surfsense_backend/app/agents/new_chat/middleware/compaction.py rename to surfsense_backend/app/agents/chat/shared/middleware/compaction.py index f8d340e5d..f91af6a70 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/compaction.py +++ b/surfsense_backend/app/agents/chat/shared/middleware/compaction.py @@ -1,26 +1,13 @@ -""" -SurfSense compaction middleware. +"""SurfSense compaction middleware. -Subclasses :class:`deepagents.middleware.summarization.SummarizationMiddleware` -to add SurfSense-specific behavior: +Extends ``SummarizationMiddleware`` with three SurfSense behaviors: -1. **Structured summary template** (OpenCode-style ``## Goal / Constraints / - Progress / Key Decisions / Next Steps / Critical Context / Relevant Files``) - — see :data:`SURFSENSE_SUMMARY_PROMPT` below. The base - ``SummarizationMiddleware`` only ships a freeform "summarize this" - prompt; the structured template is ported from OpenCode's - ``compaction.ts``. -2. **Protect SurfSense-specific SystemMessages** so injected hints - (````, ````, ````, - ````, ````, ````, ````) - are *not* summarized away and are kept verbatim in the post-summary - message list. Mirrors OpenCode's ``PRUNE_PROTECTED_TOOLS`` philosophy - (some message types are part of the agent's contract and must survive - compaction unchanged). -3. **Sanitize ``content=None``** when feeding messages into ``get_buffer_string`` - (Azure OpenAI / LiteLLM defense — when a provider streams an AIMessage - containing only tool_calls and no text, ``content`` can be ``None`` and - ``get_buffer_string`` crashes iterating over ``None``). SurfSense-specific. +1. A structured summary template (:data:`SURFSENSE_SUMMARY_PROMPT`) instead of + the base freeform prompt. +2. Protected SystemMessages (injected hints like ````) are + kept verbatim instead of being summarized away. +3. ``content=None`` is sanitized before ``get_buffer_string`` (some providers + stream tool-only AIMessages with ``None`` content, which would crash it). """ from __future__ import annotations @@ -43,9 +30,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Structured summary template ported from OpenCode's -# ``opencode/packages/opencode/src/session/compaction.ts:40-75``. Kept as a -# module-level constant so unit tests can assert on its sections. +# Module-level constant so unit tests can assert on its sections. SURFSENSE_SUMMARY_PROMPT = """ SurfSense Conversation Compaction Assistant @@ -94,7 +79,7 @@ Respond ONLY with the structured summary. Do not include any text before or afte PROTECTED_SYSTEM_PREFIXES: tuple[str, ...] = ( "", # KnowledgePriorityMiddleware "", # KnowledgeTreeMiddleware - "", # FileIntentMiddleware + "", # reserved file-operation contract prefix "", # MemoryInjectionMiddleware "", # MemoryInjectionMiddleware "", # MemoryInjectionMiddleware @@ -114,13 +99,10 @@ def _is_protected_system_message(msg: AnyMessage) -> bool: def _sanitize_message_content(msg: AnyMessage) -> AnyMessage: - """Return ``msg`` with ``content=None`` coerced to ``""``. + """Return a copy of ``msg`` with ``content=None`` coerced to ``""``. - Folds in the historical defense from ``safe_summarization.py`` — - ``get_buffer_string`` reads ``m.text`` which iterates ``self.content``, - so a ``None`` content (Azure OpenAI / LiteLLM streaming a tool-only - AIMessage) explodes. We return a copy with empty string content so - downstream consumers see an empty body without mutating the original. + ``get_buffer_string`` reads ``m.text`` (iterating ``content``), so a + tool-only AIMessage with ``None`` content would crash it. """ if getattr(msg, "content", "not-missing") is not None: return msg @@ -159,20 +141,11 @@ class SurfSenseCompactionMiddleware(SummarizationMiddleware): conversation_messages: list[AnyMessage], cutoff_index: int, ) -> tuple[list[AnyMessage], list[AnyMessage]]: - """Split messages but always preserve SurfSense protected SystemMessages. + """Split messages, always preserving protected SystemMessages. - Mirrors OpenCode's ``PRUNE_PROTECTED_TOOLS`` philosophy - (``opencode/packages/opencode/src/session/compaction.ts``): some - message types are always kept verbatim because they are part of the - agent's working contract, not transient output. - - Also opens a ``compaction.run`` OTel span (no-op when OTel is off) - so dashboards can count compaction events and message-volume - without having to instrument upstream callers. + Also opens a ``compaction.run`` OTel span (no-op when OTel is off) here, + since partitioning is the first call once summarization is decided. """ - # Opening a span here is appropriate because partitioning is the - # first call SummarizationMiddleware makes when it has decided to - # summarize; we record the volume and then close as a normal span. with ot.compaction_span( reason="auto", messages_in=len(conversation_messages), @@ -191,20 +164,15 @@ class SurfSenseCompactionMiddleware(SummarizationMiddleware): else: kept_for_summary.append(msg) - # Place protected blocks at the *front* of preserved_messages so - # they keep their original ordering relative to the summary - # HumanMessage that precedes the rest of the preserved tail. + # Protected blocks go at the front of preserved_messages to keep + # ordering relative to the summary HumanMessage. return kept_for_summary, [*protected, *preserved_messages] def _filter_summary_messages( # type: ignore[override] self, messages: list[AnyMessage] ) -> list[AnyMessage]: - """Filter previous summaries AND sanitize ``content=None``. - - Folds the ``safe_summarization.py`` defense in: when the buffer - builder iterates ``m.text`` over ``None`` it explodes; sanitizing - here covers both the sync and async offload paths. - """ + """Filter previous summaries and sanitize ``content=None`` (covers the + sync and async offload paths).""" filtered = super()._filter_summary_messages(messages) return [_sanitize_message_content(m) for m in filtered] diff --git a/surfsense_backend/app/agents/new_chat/middleware/retry_after.py b/surfsense_backend/app/agents/chat/shared/middleware/retry_after.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/middleware/retry_after.py rename to surfsense_backend/app/agents/chat/shared/middleware/retry_after.py diff --git a/surfsense_backend/app/agents/chat/shared/tools/__init__.py b/surfsense_backend/app/agents/chat/shared/tools/__init__.py new file mode 100644 index 000000000..342fe9169 --- /dev/null +++ b/surfsense_backend/app/agents/chat/shared/tools/__init__.py @@ -0,0 +1,5 @@ +"""Cross-agent shared tools. + +Only genuinely cross-agent tool code lives here (currently web_search, imported +directly from its module). +""" diff --git a/surfsense_backend/app/agents/new_chat/tools/web_search.py b/surfsense_backend/app/agents/chat/shared/tools/web_search.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/tools/web_search.py rename to surfsense_backend/app/agents/chat/shared/tools/web_search.py diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/provider_hints.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/provider_hints.py deleted file mode 100644 index 78d7b08ec..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/provider_hints.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Provider-specific style hints from ``prompts/providers/`` (main agent only).""" - -from __future__ import annotations - -import re - -from .load_md import read_prompt_md - -ProviderVariant = str - -_OPENAI_CODEX_RE = re.compile( - r"\b(gpt-codex|codex-mini|gpt-[\d.]+-codex)\b", re.IGNORECASE -) -_OPENAI_REASONING_RE = re.compile(r"\b(gpt-5|o\d|o-)", re.IGNORECASE) -_OPENAI_CLASSIC_RE = re.compile(r"\bgpt-4", re.IGNORECASE) -_ANTHROPIC_RE = re.compile(r"\bclaude\b", re.IGNORECASE) -_GOOGLE_RE = re.compile(r"\bgemini\b", re.IGNORECASE) -_KIMI_RE = re.compile(r"\b(kimi[-\d.]*|moonshot)\b", re.IGNORECASE) -_GROK_RE = re.compile(r"\bgrok\b", re.IGNORECASE) -_DEEPSEEK_RE = re.compile(r"\bdeepseek\b", re.IGNORECASE) - - -def detect_provider_variant(model_name: str | None) -> ProviderVariant: - if not model_name: - return "default" - name = model_name.strip() - if _OPENAI_CODEX_RE.search(name): - return "openai_codex" - if _OPENAI_REASONING_RE.search(name): - return "openai_reasoning" - if _OPENAI_CLASSIC_RE.search(name): - return "openai_classic" - if _ANTHROPIC_RE.search(name): - return "anthropic" - if _GOOGLE_RE.search(name): - return "google" - if _KIMI_RE.search(name): - return "kimi" - if _GROK_RE.search(name): - return "grok" - if _DEEPSEEK_RE.search(name): - return "deepseek" - return "default" - - -def build_provider_hint_block(provider_variant: ProviderVariant) -> str: - if not provider_variant or provider_variant == "default": - return "" - text = read_prompt_md(f"providers/{provider_variant}.md") - return f"\n{text}\n" if text else "" diff --git a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/provider.py b/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/provider.py deleted file mode 100644 index 7de722080..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/main_agent/system_prompt/builder/sections/provider.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Provider-specific style hints.""" - -from __future__ import annotations - -from ..provider_hints import build_provider_hint_block, detect_provider_variant - - -def build_provider_section(*, model_name: str | None) -> str: - return build_provider_hint_block(detect_provider_variant(model_name)) diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/middleware.py b/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/middleware.py deleted file mode 100644 index 6cc71f252..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/checkpointed_subagent_middleware/middleware.py +++ /dev/null @@ -1,152 +0,0 @@ -"""SubAgent middleware that compiles each subagent against the parent checkpointer.""" - -from __future__ import annotations - -import time -from typing import Any, cast - -from deepagents.backends.protocol import BackendFactory, BackendProtocol -from deepagents.middleware.subagents import ( - TASK_SYSTEM_PROMPT, - CompiledSubAgent, - SubAgent, - SubAgentMiddleware, -) -from langchain.agents import create_agent -from langchain.chat_models import init_chat_model -from langgraph.types import Checkpointer - -from app.agents.multi_agent_chat.subagents.shared.spec import ( - SURF_CONTEXT_HINT_PROVIDER_KEY, -) -from app.utils.perf import get_perf_logger - -from .task_tool import build_task_tool_with_parent_config - -_perf_log = get_perf_logger() - - -class SurfSenseCheckpointedSubAgentMiddleware(SubAgentMiddleware): - """``SubAgentMiddleware`` variant that compiles each subagent against the parent checkpointer.""" - - def __init__( - self, - *, - checkpointer: Checkpointer, - backend: BackendProtocol | BackendFactory, - subagents: list[SubAgent | CompiledSubAgent], - system_prompt: str | None = TASK_SYSTEM_PROMPT, - task_description: str | None = None, - search_space_id: int | None = None, - ) -> None: - self._surf_checkpointer = checkpointer - super(SubAgentMiddleware, self).__init__() - if not subagents: - raise ValueError( - "At least one subagent must be specified when using the new API" - ) - self._backend = backend - self._subagents = subagents - # Search-space id is captured at build time (the orchestrator runs in - # exactly one search space for its lifetime). The spawn-paused kill - # switch keys on it so an operator can quarantine one workspace - # without affecting the rest of the deployment. - self._search_space_id = search_space_id - subagent_specs = self._surf_compile_subagent_graphs() - task_tool = build_task_tool_with_parent_config( - subagent_specs, - task_description, - search_space_id=search_space_id, - ) - if system_prompt and subagent_specs: - agents_desc = "\n".join( - f"- {s['name']}: {s['description']}" for s in subagent_specs - ) - self.system_prompt = ( - system_prompt + "\n\nAvailable subagent types:\n" + agents_desc - ) - else: - self.system_prompt = system_prompt - self.tools = [task_tool] - - def _surf_compile_subagent_graphs(self) -> list[dict[str, Any]]: - """Mirror of ``SubAgentMiddleware._get_subagents`` that threads the parent checkpointer.""" - specs: list[dict[str, Any]] = [] - loop_start = time.perf_counter() - timings: list[tuple[str, float, str]] = [] # (name, elapsed, source) - - for spec in self._subagents: - spec_start = time.perf_counter() - # Provider may be ``None`` (no hint), in which case task_tool - # skips the prepend step. We forward the key unconditionally so - # the registry shape is uniform. - hint_provider = cast(dict, spec).get(SURF_CONTEXT_HINT_PROVIDER_KEY) - if "runnable" in spec: - compiled = cast(CompiledSubAgent, spec) - specs.append( - { - "name": compiled["name"], - "description": compiled["description"], - "runnable": compiled["runnable"], - SURF_CONTEXT_HINT_PROVIDER_KEY: hint_provider, - } - ) - timings.append( - (compiled["name"], time.perf_counter() - spec_start, "precompiled") - ) - continue - - if "model" not in spec: - msg = f"SubAgent '{spec['name']}' must specify 'model'" - raise ValueError(msg) - if "tools" not in spec: - msg = f"SubAgent '{spec['name']}' must specify 'tools'" - raise ValueError(msg) - - model = spec["model"] - if isinstance(model, str): - model = init_chat_model(model) - - middleware: list[Any] = list(spec.get("middleware", [])) - tools_count = len(spec.get("tools") or []) - mw_count = len(middleware) - - compile_start = time.perf_counter() - runnable = create_agent( - model, - system_prompt=spec["system_prompt"], - tools=spec["tools"], - middleware=middleware, - name=spec["name"], - checkpointer=self._surf_checkpointer, - ) - compile_elapsed = time.perf_counter() - compile_start - specs.append( - { - "name": spec["name"], - "description": spec["description"], - "runnable": runnable, - SURF_CONTEXT_HINT_PROVIDER_KEY: hint_provider, - } - ) - timings.append( - ( - spec["name"], - compile_elapsed, - f"compiled tools={tools_count} mw={mw_count}", - ) - ) - - total_elapsed = time.perf_counter() - loop_start - per_subagent = ", ".join( - f"{name}={elapsed * 1000:.0f}ms[{source}]" - for name, elapsed, source in timings - ) - _perf_log.info( - "[subagent_compile] total=%.3fs count=%d details=[%s]", - total_elapsed, - len(timings), - per_subagent, - ) - - return specs diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/dedup_hitl.py b/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/dedup_hitl.py deleted file mode 100644 index 66cae300b..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/main_agent/dedup_hitl.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Drop duplicate HITL tool calls before execution.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from langchain_core.tools import BaseTool - -from app.agents.new_chat.middleware import DedupHITLToolCallsMiddleware - - -def build_dedup_hitl_mw(tools: Sequence[BaseTool]) -> DedupHITLToolCallsMiddleware: - return DedupHITLToolCallsMiddleware(agent_tools=list(tools)) diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/__init__.py deleted file mode 100644 index c25c2b281..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/shared/permissions/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Pattern-based allow/deny/ask middleware with HITL fallback (vertical slice). - -Public surface (one entry point only — every other symbol is an internal of -the rule engine and stays inside ``middleware/``, ``ask/``, or ``deny.py``): - -- :func:`build_permission_mw` — construction recipe shared by every stack. -""" - -from .middleware.factory import build_permission_mw - -__all__ = ["build_permission_mw"] diff --git a/surfsense_backend/app/agents/multi_agent_chat/middleware/stack.py b/surfsense_backend/app/agents/multi_agent_chat/middleware/stack.py deleted file mode 100644 index 3b20d8915..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/middleware/stack.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Main-agent middleware list assembly: one line per slot. - -The main agent is a pure router — filesystem reads/writes are owned by the -``knowledge_base`` subagent and delegated via the ``task`` tool. The stack -here only renders KB context (workspace tree + priority docs), projects it -into system messages, and commits any subagent-side staged writes at end of -turn (cloud mode). -""" - -from __future__ import annotations - -import logging -from collections.abc import Sequence -from typing import Any - -from deepagents import SubAgent -from deepagents.backends import StateBackend -from langchain.agents import create_agent -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool -from langgraph.types import Checkpointer - -from app.agents.multi_agent_chat.subagents import ( - build_subagents, - get_subagents_to_exclude, -) -from app.agents.multi_agent_chat.subagents.builtins.knowledge_base.agent import ( - READONLY_NAME as KB_READONLY_NAME, - build_readonly_subagent as build_kb_readonly_subagent, -) -from app.agents.multi_agent_chat.subagents.builtins.knowledge_base.ask_knowledge_base_tool import ( - build_ask_knowledge_base_tool, -) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.db import ChatVisibility - -from .main_agent.action_log import build_action_log_mw -from .main_agent.anonymous_doc import build_anonymous_doc_mw -from .main_agent.busy_mutex import build_busy_mutex_mw -from .main_agent.checkpointed_subagent_middleware import ( - SurfSenseCheckpointedSubAgentMiddleware, -) -from .main_agent.checkpointed_subagent_middleware.task_description import ( - TASK_TOOL_DESCRIPTION, -) -from .main_agent.context_editing import build_context_editing_mw -from .main_agent.dedup_hitl import build_dedup_hitl_mw -from .main_agent.doom_loop import build_doom_loop_mw -from .main_agent.kb_persistence import build_kb_persistence_mw -from .main_agent.knowledge_priority import build_knowledge_priority_mw -from .main_agent.knowledge_tree import build_knowledge_tree_mw -from .main_agent.noop_injection import build_noop_injection_mw -from .main_agent.otel import build_otel_mw -from .main_agent.plugins import build_plugin_middlewares -from .main_agent.repair import build_repair_mw -from .main_agent.skills import build_skills_mw -from .shared.anthropic_cache import build_anthropic_cache_mw -from .shared.compaction import build_compaction_mw -from .shared.kb_context_projection import build_kb_context_projection_mw -from .shared.memory import build_memory_mw -from .shared.patch_tool_calls import build_patch_tool_calls_mw -from .shared.permissions import build_permission_mw -from .shared.resilience import build_resilience_middlewares -from .shared.todos import build_todos_mw -from .subagent.middleware_stack import build_subagent_middleware_stack - - -def build_main_agent_deepagent_middleware( - *, - llm: BaseChatModel, - tools: Sequence[BaseTool], - backend_resolver: Any, - filesystem_mode: FilesystemMode, - search_space_id: int, - user_id: str | None, - thread_id: int | None, - visibility: ChatVisibility, - anon_session_id: str | None, - available_connectors: list[str] | None, - available_document_types: list[str] | None, - mentioned_document_ids: list[int] | None, - max_input_tokens: int | None, - flags: AgentFeatureFlags, - subagent_dependencies: dict[str, Any], - checkpointer: Checkpointer, - mcp_tools_by_agent: dict[str, list[BaseTool]] | None = None, - disabled_tools: list[str] | None = None, -) -> list[Any]: - """Ordered middleware for ``create_agent`` (None entries already stripped).""" - resilience = build_resilience_middlewares(flags) - - memory_mw = build_memory_mw( - user_id=user_id, - search_space_id=search_space_id, - visibility=visibility, - ) - - subagent_dependencies = { - **subagent_dependencies, - "backend_resolver": backend_resolver, - "filesystem_mode": filesystem_mode, - "flags": flags, - } - shared_subagent_middleware = build_subagent_middleware_stack( - resilience=resilience, - flags=flags, - ) - - kb_readonly = build_kb_readonly_subagent( - dependencies=subagent_dependencies, - model=llm, - middleware_stack=shared_subagent_middleware, - ) - kb_readonly_spec = kb_readonly.spec - kb_readonly_runnable = create_agent( - llm, - system_prompt=kb_readonly_spec["system_prompt"], - tools=kb_readonly_spec["tools"], - middleware=kb_readonly_spec["middleware"], - name=KB_READONLY_NAME, - checkpointer=checkpointer, - ) - ask_kb_tool = build_ask_knowledge_base_tool(kb_readonly_runnable) - - subagents: list[SubAgent] = build_subagents( - dependencies=subagent_dependencies, - model=llm, - middleware_stack=shared_subagent_middleware, - mcp_tools_by_agent=mcp_tools_by_agent or {}, - exclude=get_subagents_to_exclude(available_connectors), - disabled_tools=disabled_tools, - ask_kb_tool=ask_kb_tool, - ) - logging.debug("Subagents registry: %s", [s["name"] for s in subagents]) - - stack: list[Any] = [ - build_busy_mutex_mw(flags), - build_otel_mw(flags), - build_todos_mw(system_prompt=""), - memory_mw, - build_anonymous_doc_mw( - filesystem_mode=filesystem_mode, anon_session_id=anon_session_id - ), - build_knowledge_tree_mw( - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - llm=llm, - ), - build_knowledge_priority_mw( - llm=llm, - search_space_id=search_space_id, - filesystem_mode=filesystem_mode, - available_connectors=available_connectors, - available_document_types=available_document_types, - mentioned_document_ids=mentioned_document_ids, - ), - build_kb_context_projection_mw(), - build_kb_persistence_mw( - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - user_id=user_id, - thread_id=thread_id, - ), - build_skills_mw( - flags=flags, - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - ), - SurfSenseCheckpointedSubAgentMiddleware( - checkpointer=checkpointer, - backend=StateBackend, - subagents=subagents, - system_prompt=None, - task_description=TASK_TOOL_DESCRIPTION, - search_space_id=search_space_id, - ), - resilience.model_call_limit, - resilience.tool_call_limit, - build_context_editing_mw( - flags=flags, - max_input_tokens=max_input_tokens, - tools=tools, - backend_resolver=backend_resolver, - ), - build_compaction_mw(llm), - build_noop_injection_mw(flags), - resilience.retry, - resilience.fallback, - build_repair_mw(flags=flags, tools=tools), - build_permission_mw(flags=flags), - build_doom_loop_mw(flags), - build_action_log_mw( - flags=flags, - thread_id=thread_id, - search_space_id=search_space_id, - user_id=user_id, - ), - build_patch_tool_calls_mw(), - build_dedup_hitl_mw(tools), - *build_plugin_middlewares( - flags=flags, - search_space_id=search_space_id, - user_id=user_id, - visibility=visibility, - llm=llm, - ), - build_anthropic_cache_mw(), - ] - return [m for m in stack if m is not None] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py deleted file mode 100644 index 13d4c06cb..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/calendar/tools/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from app.agents.new_chat.tools.google_calendar.create_event import ( - create_create_calendar_event_tool, -) -from app.agents.new_chat.tools.google_calendar.delete_event import ( - create_delete_calendar_event_tool, -) -from app.agents.new_chat.tools.google_calendar.search_events import ( - create_search_calendar_events_tool, -) -from app.agents.new_chat.tools.google_calendar.update_event import ( - create_update_calendar_event_tool, -) - -__all__ = [ - "create_create_calendar_event_tool", - "create_delete_calendar_event_tool", - "create_search_calendar_events_tool", - "create_update_calendar_event_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/__init__.py deleted file mode 100644 index b4eaec1f0..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/discord/tools/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.discord.list_channels import ( - create_list_discord_channels_tool, -) -from app.agents.new_chat.tools.discord.read_messages import ( - create_read_discord_messages_tool, -) -from app.agents.new_chat.tools.discord.send_message import ( - create_send_discord_message_tool, -) - -__all__ = [ - "create_list_discord_channels_tool", - "create_read_discord_messages_tool", - "create_send_discord_message_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py deleted file mode 100644 index 836b9ee41..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/dropbox/tools/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.dropbox.create_file import ( - create_create_dropbox_file_tool, -) -from app.agents.new_chat.tools.dropbox.trash_file import ( - create_delete_dropbox_file_tool, -) - -__all__ = [ - "create_create_dropbox_file_tool", - "create_delete_dropbox_file_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py deleted file mode 100644 index 294840122..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/gmail/tools/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from app.agents.new_chat.tools.gmail.create_draft import ( - create_create_gmail_draft_tool, -) -from app.agents.new_chat.tools.gmail.read_email import ( - create_read_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.search_emails import ( - create_search_gmail_tool, -) -from app.agents.new_chat.tools.gmail.send_email import ( - create_send_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.trash_email import ( - create_trash_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.update_draft import ( - create_update_gmail_draft_tool, -) - -__all__ = [ - "create_create_gmail_draft_tool", - "create_read_gmail_email_tool", - "create_search_gmail_tool", - "create_send_gmail_email_tool", - "create_trash_gmail_email_tool", - "create_update_gmail_draft_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py deleted file mode 100644 index 9c63bceb1..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/google_drive/tools/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.google_drive.create_file import ( - create_create_google_drive_file_tool, -) -from app.agents.new_chat.tools.google_drive.trash_file import ( - create_delete_google_drive_file_tool, -) - -__all__ = [ - "create_create_google_drive_file_tool", - "create_delete_google_drive_file_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/__init__.py deleted file mode 100644 index 255119bee..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/luma/tools/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.luma.create_event import ( - create_create_luma_event_tool, -) -from app.agents.new_chat.tools.luma.list_events import ( - create_list_luma_events_tool, -) -from app.agents.new_chat.tools.luma.read_event import ( - create_read_luma_event_tool, -) - -__all__ = [ - "create_create_luma_event_tool", - "create_list_luma_events_tool", - "create_read_luma_event_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py deleted file mode 100644 index 8edb4857e..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/onedrive/tools/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.onedrive.create_file import ( - create_create_onedrive_file_tool, -) -from app.agents.new_chat.tools.onedrive.trash_file import ( - create_delete_onedrive_file_tool, -) - -__all__ = [ - "create_create_onedrive_file_tool", - "create_delete_onedrive_file_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/__init__.py deleted file mode 100644 index 60e2add49..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/connectors/teams/tools/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.teams.list_channels import ( - create_list_teams_channels_tool, -) -from app.agents.new_chat.tools.teams.read_messages import ( - create_read_teams_messages_tool, -) -from app.agents.new_chat.tools.teams.send_message import ( - create_send_teams_message_tool, -) - -__all__ = [ - "create_list_teams_channels_tool", - "create_read_teams_messages_tool", - "create_send_teams_message_tool", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/__init__.py deleted file mode 100644 index 70d3dfe39..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/shared/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Cross-slice helpers for route subagents.""" - -from __future__ import annotations - -from app.agents.multi_agent_chat.subagents.shared.md_file_reader import ( - read_md_file, -) -from app.agents.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import ( - pack_subagent, -) - -__all__ = [ - "SurfSenseSubagentSpec", - "pack_subagent", - "read_md_file", -] diff --git a/surfsense_backend/app/agents/new_chat/__init__.py b/surfsense_backend/app/agents/new_chat/__init__.py deleted file mode 100644 index 4b2eb89eb..000000000 --- a/surfsense_backend/app/agents/new_chat/__init__.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -SurfSense New Chat Agent Module. - -This module provides the SurfSense deep agent with configurable tools, -middleware, and preloaded knowledge-base filesystem behavior. - -Directory Structure: -- tools/: All agent tools (podcast, generate_image, web, memory, etc.) -- middleware/: Custom middleware (knowledge search, filesystem, dedup, etc.) -- chat_deepagent.py: Main agent factory -- system_prompt.py: System prompts and instructions -- context.py: Context schema for the agent -- checkpointer.py: LangGraph checkpointer setup -- llm_config.py: LLM configuration utilities -- utils.py: Shared utilities -""" - -# Agent factory -from .chat_deepagent import create_surfsense_deep_agent - -# Context -from .context import SurfSenseContextSchema - -# LLM config -from .llm_config import ( - create_chat_litellm_from_config, - load_global_llm_config_by_id, - load_llm_config_from_yaml, -) - -# Middleware -from .middleware import ( - DedupHITLToolCallsMiddleware, - KnowledgeBaseSearchMiddleware, - SurfSenseFilesystemMiddleware, -) - -# System prompt -from .system_prompt import ( - SURFSENSE_CITATION_INSTRUCTIONS, - SURFSENSE_SYSTEM_PROMPT, - build_surfsense_system_prompt, -) - -# Tools - registry exports -# Tools - factory exports (for direct use) -# Tools - knowledge base utilities -from .tools import ( - BUILTIN_TOOLS, - ToolDefinition, - build_tools, - create_generate_podcast_tool, - create_scrape_webpage_tool, - format_documents_for_context, - get_all_tool_names, - get_default_enabled_tools, - get_tool_by_name, - search_knowledge_base_async, -) - -__all__ = [ - # Tools registry - "BUILTIN_TOOLS", - # System prompt - "SURFSENSE_CITATION_INSTRUCTIONS", - "SURFSENSE_SYSTEM_PROMPT", - # Middleware - "DedupHITLToolCallsMiddleware", - "KnowledgeBaseSearchMiddleware", - # Context - "SurfSenseContextSchema", - "SurfSenseFilesystemMiddleware", - "ToolDefinition", - "build_surfsense_system_prompt", - "build_tools", - # LLM config - "create_chat_litellm_from_config", - # Tool factories - "create_generate_podcast_tool", - "create_scrape_webpage_tool", - # Agent factory - "create_surfsense_deep_agent", - # Knowledge base utilities - "format_documents_for_context", - "get_all_tool_names", - "get_default_enabled_tools", - "get_tool_by_name", - "load_global_llm_config_by_id", - "load_llm_config_from_yaml", - "search_knowledge_base_async", -] diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py deleted file mode 100644 index f8db333ba..000000000 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ /dev/null @@ -1,1166 +0,0 @@ -""" -SurfSense deep agent implementation. - -This module provides the factory function for creating SurfSense deep agents -with configurable tools via the tools registry and configurable prompts -via NewLLMConfig. - -We use ``create_agent`` (from langchain) rather than ``create_deep_agent`` -(from deepagents) so that the middleware stack is fully under our control. -This lets us swap in ``SurfSenseFilesystemMiddleware`` — a customisable -subclass of the default ``FilesystemMiddleware`` — while preserving every -other behaviour that ``create_deep_agent`` provides (todo-list, subagents, -summarisation, etc.). Prompt caching is configured at LLM-build time via -``apply_litellm_prompt_caching`` (LiteLLM-native, multi-provider) rather -than as a middleware. -""" - -import asyncio -import logging -import time -from collections.abc import Sequence -from typing import Any - -from deepagents import SubAgent, SubAgentMiddleware, __version__ as deepagents_version -from deepagents.backends import StateBackend -from deepagents.graph import BASE_AGENT_PROMPT -from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware -from deepagents.middleware.skills import SkillsMiddleware -from deepagents.middleware.subagents import GENERAL_PURPOSE_SUBAGENT -from langchain.agents import create_agent -from langchain.agents.middleware import ( - LLMToolSelectorMiddleware, - ModelCallLimitMiddleware, - TodoListMiddleware, - ToolCallLimitMiddleware, -) -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool -from langgraph.types import Checkpointer -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.agent_cache import ( - flags_signature, - get_cache, - stable_hash, - system_prompt_hash, - tools_signature, -) -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.feature_flags import AgentFeatureFlags, get_flags -from app.agents.new_chat.filesystem_backends import build_backend_resolver -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.llm_config import AgentConfig -from app.agents.new_chat.middleware import ( - ActionLogMiddleware, - AnonymousDocumentMiddleware, - BusyMutexMiddleware, - ClearToolUsesEdit, - DedupHITLToolCallsMiddleware, - DoomLoopMiddleware, - FileIntentMiddleware, - FlattenSystemMessageMiddleware, - KnowledgeBasePersistenceMiddleware, - KnowledgePriorityMiddleware, - KnowledgeTreeMiddleware, - MemoryInjectionMiddleware, - NoopInjectionMiddleware, - OtelSpanMiddleware, - PermissionMiddleware, - RetryAfterMiddleware, - SpillingContextEditingMiddleware, - SpillToBackendEdit, - SurfSenseFilesystemMiddleware, - ToolCallNameRepairMiddleware, - build_skills_backend_factory, - create_surfsense_compaction_middleware, - default_skills_sources, -) -from app.agents.new_chat.middleware.scoped_model_fallback import ( - ScopedModelFallbackMiddleware, -) -from app.agents.new_chat.permissions import Rule, Ruleset -from app.agents.new_chat.plugin_loader import ( - PluginContext, - load_allowed_plugin_names_from_env, - load_plugin_middlewares, -) -from app.agents.new_chat.prompt_caching import apply_litellm_prompt_caching -from app.agents.new_chat.subagents import build_specialized_subagents -from app.agents.new_chat.system_prompt import ( - build_configurable_system_prompt, - build_surfsense_system_prompt, -) -from app.agents.new_chat.tools.invalid_tool import ( - INVALID_TOOL_NAME, - invalid_tool, -) -from app.agents.new_chat.tools.registry import ( - BUILTIN_TOOLS, - build_tools_async, - get_connector_gated_tools, -) -from app.db import ChatVisibility -from app.services.connector_service import ConnectorService -from app.services.llm_service import get_planner_llm -from app.utils.perf import get_perf_logger - -_perf_log = get_perf_logger() - - -def _resolve_prompt_model_name( - agent_config: AgentConfig | None, - llm: BaseChatModel, -) -> str | None: - """Resolve the model id to feed to provider-variant detection. - - Preference order (matches the established idiom in - ``llm_router_service.py`` — see ``params.get("base_model") or - params.get("model", "")`` usages there): - - 1. ``agent_config.litellm_params["base_model"]`` — required for Azure - deployments where ``model_name`` is the deployment slug, not the - underlying family. Without this, a deployment named e.g. - ``"prod-chat-001"`` would silently miss every provider regex. - 2. ``agent_config.model_name`` — the user's configured model id. - 3. ``getattr(llm, "model", None)`` — fallback for direct callers that - don't supply an ``AgentConfig`` (currently a defensive path; all - production callers pass ``agent_config``). - - Returns ``None`` when nothing is available; ``compose_system_prompt`` - treats that as the ``"default"`` variant (no provider block emitted). - """ - if agent_config is not None: - params = agent_config.litellm_params or {} - base_model = params.get("base_model") - if isinstance(base_model, str) and base_model.strip(): - return base_model - if agent_config.model_name: - return agent_config.model_name - return getattr(llm, "model", None) - - -# ============================================================================= -# Connector Type Mapping -# ============================================================================= - -# Maps SearchSourceConnectorType enum values to the searchable document/connector types -# used by pre-search middleware and web_search. -# Live search connectors (TAVILY_API, LINKUP_API, BAIDU_SEARCH_API) are routed to -# the web_search tool; all others are considered local/indexed data. -_CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = { - # Live search connectors (handled by web_search tool) - "TAVILY_API": "TAVILY_API", - "LINKUP_API": "LINKUP_API", - "BAIDU_SEARCH_API": "BAIDU_SEARCH_API", - # Local/indexed connectors (handled by KB pre-search middleware) - "SLACK_CONNECTOR": "SLACK_CONNECTOR", - "TEAMS_CONNECTOR": "TEAMS_CONNECTOR", - "NOTION_CONNECTOR": "NOTION_CONNECTOR", - "GITHUB_CONNECTOR": "GITHUB_CONNECTOR", - "LINEAR_CONNECTOR": "LINEAR_CONNECTOR", - "DISCORD_CONNECTOR": "DISCORD_CONNECTOR", - "JIRA_CONNECTOR": "JIRA_CONNECTOR", - "CONFLUENCE_CONNECTOR": "CONFLUENCE_CONNECTOR", - "CLICKUP_CONNECTOR": "CLICKUP_CONNECTOR", - "GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", - "GOOGLE_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", - "GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", # Connector type differs from document type - "AIRTABLE_CONNECTOR": "AIRTABLE_CONNECTOR", - "LUMA_CONNECTOR": "LUMA_CONNECTOR", - "ELASTICSEARCH_CONNECTOR": "ELASTICSEARCH_CONNECTOR", - "WEBCRAWLER_CONNECTOR": "CRAWLED_URL", # Maps to document type - "BOOKSTACK_CONNECTOR": "BOOKSTACK_CONNECTOR", - "CIRCLEBACK_CONNECTOR": "CIRCLEBACK", # Connector type differs from document type - "OBSIDIAN_CONNECTOR": "OBSIDIAN_CONNECTOR", - "DROPBOX_CONNECTOR": "DROPBOX_FILE", # Connector type differs from document type - "ONEDRIVE_CONNECTOR": "ONEDRIVE_FILE", # Connector type differs from document type - # Composio connectors (unified to native document types). - # Reverse of NATIVE_TO_LEGACY_DOCTYPE in app.db. - "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", - "COMPOSIO_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", - "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", -} - -# Document types that don't come from SearchSourceConnector but should always be searchable -_ALWAYS_AVAILABLE_DOC_TYPES: list[str] = [ - "EXTENSION", # Browser extension data - "FILE", # Uploaded files - "NOTE", # User notes - "YOUTUBE_VIDEO", # YouTube videos -] - - -def _map_connectors_to_searchable_types( - connector_types: list[Any], -) -> list[str]: - """ - Map SearchSourceConnectorType enums to searchable document/connector types. - - This function: - 1. Converts connector type enums to their searchable counterparts - 2. Includes always-available document types (EXTENSION, FILE, NOTE, YOUTUBE_VIDEO) - 3. Deduplicates while preserving order - - Args: - connector_types: List of SearchSourceConnectorType enum values - - Returns: - List of searchable connector/document type strings - """ - result_set: set[str] = set() - result_list: list[str] = [] - - # Add always-available document types first - for doc_type in _ALWAYS_AVAILABLE_DOC_TYPES: - if doc_type not in result_set: - result_set.add(doc_type) - result_list.append(doc_type) - - # Map each connector type to its searchable equivalent - for ct in connector_types: - # Handle both enum and string types - ct_str = ct.value if hasattr(ct, "value") else str(ct) - searchable = _CONNECTOR_TYPE_TO_SEARCHABLE.get(ct_str) - if searchable and searchable not in result_set: - result_set.add(searchable) - result_list.append(searchable) - - return result_list - - -# ============================================================================= -# Deep Agent Factory -# ============================================================================= - - -async def create_surfsense_deep_agent( - llm: BaseChatModel, - search_space_id: int, - db_session: AsyncSession, - connector_service: ConnectorService, - checkpointer: Checkpointer, - user_id: str | None = None, - thread_id: int | None = None, - agent_config: AgentConfig | None = None, - enabled_tools: list[str] | None = None, - disabled_tools: list[str] | None = None, - additional_tools: Sequence[BaseTool] | None = None, - firecrawl_api_key: str | None = None, - thread_visibility: ChatVisibility | None = None, - mentioned_document_ids: list[int] | None = None, - anon_session_id: str | None = None, - filesystem_selection: FilesystemSelection | None = None, -): - """ - Create a SurfSense deep agent with configurable tools and prompts. - - The agent comes with built-in tools that can be configured: - - generate_podcast: Generate audio podcasts from content - - generate_image: Generate images from text descriptions using AI models - - scrape_webpage: Extract content from webpages - - update_memory: Update the user's personal or team memory document - - The agent also includes TodoListMiddleware by default (via create_deep_agent) which provides: - - write_todos: Create and update planning/todo lists for complex tasks - - The system prompt can be configured via agent_config: - - Custom system instructions (or use defaults) - - Citation toggle (enable/disable citation requirements) - - Args: - llm: ChatLiteLLM instance for the agent's language model - search_space_id: The user's search space ID - db_session: Database session for tools that need DB access - connector_service: Initialized connector service for knowledge base search - checkpointer: LangGraph checkpointer for conversation state persistence. - Use AsyncPostgresSaver for production or MemorySaver for testing. - user_id: The current user's UUID string (required for memory tools) - agent_config: Optional AgentConfig from NewLLMConfig for prompt configuration. - If None, uses default system prompt with citations enabled. - enabled_tools: Explicit list of tool names to enable. If None, all default tools - are enabled. Use this to limit which tools are available. - disabled_tools: List of tool names to disable. Applied after enabled_tools. - Use this to exclude specific tools from the defaults. - additional_tools: Extra custom tools to add beyond the built-in ones. - These are always added regardless of enabled/disabled settings. - firecrawl_api_key: Optional Firecrawl API key for premium web scraping. - Falls back to Chromium/Trafilatura if not provided. - - Returns: - CompiledStateGraph: The configured deep agent - - Examples: - # Create agent with all default tools and default prompt - agent = create_surfsense_deep_agent(llm, search_space_id, db_session, ...) - - # Create agent with custom prompt configuration - agent = create_surfsense_deep_agent( - llm, search_space_id, db_session, ..., - agent_config=AgentConfig( - provider="OPENAI", - model_name="gpt-4", - api_key="...", - system_instructions="Custom instructions...", - citations_enabled=False, - ) - ) - - # Create agent with only specific tools - agent = create_surfsense_deep_agent( - llm, search_space_id, db_session, ..., - enabled_tools=["scrape_webpage"] - ) - - # Create agent without podcast generation - agent = create_surfsense_deep_agent( - llm, search_space_id, db_session, ..., - disabled_tools=["generate_podcast"] - ) - - # Add custom tools - agent = create_surfsense_deep_agent( - llm, search_space_id, db_session, ..., - additional_tools=[my_custom_tool] - ) - """ - _t_agent_total = time.perf_counter() - - # Layer thread-aware prompt caching onto the LLM. Idempotent with the - # build-time call in ``llm_config.py``; this run merely adds - # ``prompt_cache_key=f"surfsense-thread-{thread_id}"`` for OpenAI-family - # configs now that ``thread_id`` is known. No-op when ``thread_id`` is - # None or the provider is non-OpenAI-family. - apply_litellm_prompt_caching(llm, agent_config=agent_config, thread_id=thread_id) - - filesystem_selection = filesystem_selection or FilesystemSelection() - backend_resolver = build_backend_resolver( - filesystem_selection, - search_space_id=search_space_id - if filesystem_selection.mode == FilesystemMode.CLOUD - else None, - ) - - # Discover available connectors and document types for this search space. - # - # NOTE: These two calls cannot be parallelized via ``asyncio.gather``. - # ``ConnectorService`` shares a single ``AsyncSession`` (``self.session``); - # SQLAlchemy explicitly forbids concurrent operations on the same session - # ("This session is provisioning a new connection; concurrent operations - # are not permitted on the same session"). The Phase 1.4 in-process TTL - # cache in ``connector_service`` already collapses the warm path to a - # near-zero pair of dict lookups, so sequential awaits cost nothing in - # the common case while remaining correct on cold cache misses. - available_connectors: list[str] | None = None - available_document_types: list[str] | None = None - - _t0 = time.perf_counter() - try: - try: - connector_types_result = await connector_service.get_available_connectors( - search_space_id - ) - if connector_types_result: - available_connectors = _map_connectors_to_searchable_types( - connector_types_result - ) - except Exception as e: - logging.warning("Failed to discover available connectors: %s", e) - - try: - available_document_types = ( - await connector_service.get_available_document_types(search_space_id) - ) - except Exception as e: - logging.warning("Failed to discover available document types: %s", e) - except Exception as e: # pragma: no cover - defensive outer guard - logging.warning(f"Failed to discover available connectors/document types: {e}") - _perf_log.info( - "[create_agent] Connector/doc-type discovery in %.3fs", - time.perf_counter() - _t0, - ) - - # Build dependencies dict for the tools registry - visibility = thread_visibility or ChatVisibility.PRIVATE - - # Extract the model's context window so tools can size their output. - _model_profile = getattr(llm, "profile", None) - _max_input_tokens: int | None = ( - _model_profile.get("max_input_tokens") - if isinstance(_model_profile, dict) - else None - ) - - dependencies = { - "search_space_id": search_space_id, - "db_session": db_session, - "connector_service": connector_service, - "firecrawl_api_key": firecrawl_api_key, - "user_id": user_id, - "thread_id": thread_id, - "thread_visibility": visibility, - "available_connectors": available_connectors, - "available_document_types": available_document_types, - "max_input_tokens": _max_input_tokens, - "llm": llm, - } - - modified_disabled_tools = list(disabled_tools) if disabled_tools else [] - modified_disabled_tools.extend(get_connector_gated_tools(available_connectors)) - - # Remove direct KB search tool; KnowledgePriorityMiddleware now runs hybrid - # search per turn and surfaces hits as a hint plus - # `` markers inside lazy-loaded XML. - if "search_knowledge_base" not in modified_disabled_tools: - modified_disabled_tools.append("search_knowledge_base") - - # Build tools using the async registry (includes MCP tools) - _t0 = time.perf_counter() - tools = await build_tools_async( - dependencies=dependencies, - enabled_tools=enabled_tools, - disabled_tools=modified_disabled_tools, - additional_tools=list(additional_tools) if additional_tools else None, - ) - - # Register the ``invalid`` tool only when tool-call repair is on. It - # is dispatched only when :class:`ToolCallNameRepairMiddleware` - # rewrites a malformed call. We intentionally append it AFTER - # ``build_tools_async`` so it never appears in the system-prompt - # tool list (which is built from the registry, not the bound tool - # list). - _flags: AgentFeatureFlags = get_flags() - if _flags.enable_tool_call_repair and INVALID_TOOL_NAME not in { - t.name for t in tools - }: - tools = [*list(tools), invalid_tool] - _perf_log.info( - "[create_agent] build_tools_async in %.3fs (%d tools)", - time.perf_counter() - _t0, - len(tools), - ) - - # Build system prompt based on agent_config, scoped to the tools actually enabled - _t0 = time.perf_counter() - _enabled_tool_names = {t.name for t in tools} - _user_disabled_tool_names = set(disabled_tools) if disabled_tools else set() - - # Collect generic MCP connector info so the system prompt can route queries - # to their tools instead of falling back to "not in knowledge base". - _mcp_connector_tools: dict[str, list[str]] = {} - for t in tools: - meta = getattr(t, "metadata", None) or {} - if meta.get("mcp_is_generic") and meta.get("mcp_connector_name"): - _mcp_connector_tools.setdefault( - meta["mcp_connector_name"], - [], - ).append(t.name) - - if _mcp_connector_tools: - _perf_log.info("MCP connector tool routing: %s", _mcp_connector_tools) - - if agent_config is not None: - system_prompt = build_configurable_system_prompt( - custom_system_instructions=agent_config.system_instructions, - use_default_system_instructions=agent_config.use_default_system_instructions, - citations_enabled=agent_config.citations_enabled, - thread_visibility=thread_visibility, - enabled_tool_names=_enabled_tool_names, - disabled_tool_names=_user_disabled_tool_names, - mcp_connector_tools=_mcp_connector_tools, - model_name=_resolve_prompt_model_name(agent_config, llm), - ) - else: - system_prompt = build_surfsense_system_prompt( - thread_visibility=thread_visibility, - enabled_tool_names=_enabled_tool_names, - disabled_tool_names=_user_disabled_tool_names, - mcp_connector_tools=_mcp_connector_tools, - model_name=_resolve_prompt_model_name(agent_config, llm), - ) - _perf_log.info( - "[create_agent] System prompt built in %.3fs", time.perf_counter() - _t0 - ) - - # Combine system_prompt with BASE_AGENT_PROMPT (same as create_deep_agent) - final_system_prompt = system_prompt + "\n\n" + BASE_AGENT_PROMPT - - # The middleware stack — and especially ``SubAgentMiddleware`` — is *not* - # cheap to build. ``SubAgentMiddleware.__init__`` calls ``create_agent`` - # synchronously to compile the general-purpose subagent's full state graph - # (every tool + every middleware → pydantic schemas + langgraph compile). - # On gpt-5.x agents that's roughly 1.5-2s of pure CPU work. If we run it - # directly here it blocks the asyncio event loop for the whole streaming - # task (and any other coroutine sharing this loop), which is why - # "agent creation" wall-clock time used to stretch to ~3-4s. Move the - # entire middleware build + main-graph compile into a single - # ``asyncio.to_thread`` so the heavy CPU work runs off-loop and the - # event loop stays responsive. - # - # PHASE 1: cache the resulting compiled graph. ``agent_cache`` is keyed - # on every per-request value that any middleware in the stack closes - # over in ``__init__`` — drop one and you risk leaking state across - # threads. Hits collapse this whole block to a microsecond lookup; - # misses pay the original CPU cost AND populate the cache. - config_id = agent_config.config_id if agent_config is not None else None - - async def _build_agent() -> Any: - return await asyncio.to_thread( - _build_compiled_agent_blocking, - llm=llm, - tools=tools, - final_system_prompt=final_system_prompt, - backend_resolver=backend_resolver, - filesystem_mode=filesystem_selection.mode, - search_space_id=search_space_id, - user_id=user_id, - thread_id=thread_id, - visibility=visibility, - anon_session_id=anon_session_id, - available_connectors=available_connectors, - available_document_types=available_document_types, - # ``mentioned_document_ids`` is consumed by - # ``KnowledgePriorityMiddleware`` per turn via - # ``runtime.context`` (Phase 1.5). We still pass the - # caller-provided list here for the legacy fallback path - # (cache disabled / context not propagated) — the middleware - # drains its own copy after the first read so a cached graph - # never replays stale mentions. - mentioned_document_ids=mentioned_document_ids, - max_input_tokens=_max_input_tokens, - flags=_flags, - checkpointer=checkpointer, - ) - - _t0 = time.perf_counter() - if _flags.enable_agent_cache and not _flags.disable_new_agent_stack: - # Cache key components — order matters only for human readability; - # the resulting hash is what's stored. Every component must - # rotate on a real shape change AND stay stable across identical - # invocations. - cache_key = stable_hash( - "v1", # schema version of the key — bump if components change - config_id, - thread_id, - user_id, - search_space_id, - visibility, - filesystem_selection.mode, - anon_session_id, - tools_signature( - tools, - available_connectors=available_connectors, - available_document_types=available_document_types, - ), - flags_signature(_flags), - system_prompt_hash(final_system_prompt), - _max_input_tokens, - # ``mentioned_document_ids`` deliberately omitted — middleware - # reads it from ``runtime.context`` (Phase 1.5). - ) - agent = await get_cache().get_or_build(cache_key, builder=_build_agent) - else: - agent = await _build_agent() - _perf_log.info( - "[create_agent] Middleware stack + graph compiled in %.3fs (cache=%s)", - time.perf_counter() - _t0, - "on" - if _flags.enable_agent_cache and not _flags.disable_new_agent_stack - else "off", - ) - - _perf_log.info( - "[create_agent] Total agent creation in %.3fs", - time.perf_counter() - _t_agent_total, - ) - return agent - - -# Tools whose output is too costly / lossy to discard. Keep this -# conservative — anything listed here is *never* pruned by -# :class:`ContextEditingMiddleware`. The list is filtered against -# actually-bound tool names so disabled connectors don't show up here. -_PRUNE_PROTECTED_TOOL_NAMES: frozenset[str] = frozenset( - { - "generate_report", - "generate_resume", - "generate_podcast", - "generate_video_presentation", - "generate_image", - # Read-heavy connector reads — recomputing them is expensive - "read_email", - "search_emails", - # The fallback for malformed tool calls — keep its replies visible - "invalid", - } -) - - -def _safe_exclude_tools(tools: Sequence[BaseTool]) -> tuple[str, ...]: - """Return ``exclude_tools`` derived from the actually-bound tool list. - - Filters :data:`_PRUNE_PROTECTED_TOOL_NAMES` against the bound tools - so we never list tools that don't exist (would be a silent no-op). - """ - enabled = {t.name for t in tools} - return tuple(name for name in _PRUNE_PROTECTED_TOOL_NAMES if name in enabled) - - -# Connector gating: any tool whose ``ToolDefinition.required_connector`` -# isn't actually wired up gets a synthesized permission deny rule so -# execution attempts short-circuit with ``permission_denied`` instead of -# bubbling up provider-specific 401/404 errors. Mirrors OpenCode's -# ``Permission.disabled`` (declarative, per-tool gating) — replaces the -# legacy binary ``_CONNECTOR_TYPE_TO_SEARCHABLE`` substring-heuristic. -def _synthesize_connector_deny_rules( - *, - available_connectors: list[str] | None, - enabled_tool_names: set[str], -) -> list[Rule]: - """Build deny rules for tools whose required connector is not enabled. - - Source of truth is ``ToolDefinition.required_connector`` in - :data:`BUILTIN_TOOLS`. A tool only gets a deny rule when: - - 1. It is currently bound (``enabled_tool_names``). - 2. It declares a ``required_connector``. - 3. That connector is *not* in ``available_connectors``. - """ - available = set(available_connectors or []) - deny: list[Rule] = [] - for tool_def in BUILTIN_TOOLS: - if tool_def.name not in enabled_tool_names: - continue - rc = tool_def.required_connector - if rc and rc not in available: - deny.append(Rule(permission=tool_def.name, pattern="*", action="deny")) - return deny - - -def _build_compiled_agent_blocking( - *, - llm: BaseChatModel, - tools: Sequence[BaseTool], - final_system_prompt: str, - backend_resolver: Any, - filesystem_mode: FilesystemMode, - search_space_id: int, - user_id: str | None, - thread_id: int | None, - visibility: ChatVisibility, - anon_session_id: str | None, - available_connectors: list[str] | None, - available_document_types: list[str] | None, - mentioned_document_ids: list[int] | None, - max_input_tokens: int | None, - flags: AgentFeatureFlags, - checkpointer: Checkpointer, -): - """Build the middleware stack and compile the agent graph synchronously. - - Runs in a worker thread (see ``asyncio.to_thread`` call site) so the heavy - CPU work — most notably ``SubAgentMiddleware.__init__`` eagerly calling - ``create_agent`` to compile the general-purpose subagent — does not block - the event loop. - """ - _memory_middleware = MemoryInjectionMiddleware( - user_id=user_id, - search_space_id=search_space_id, - thread_visibility=visibility, - ) - - # General-purpose subagent middleware - # Subagent omits AnonymousDocumentMiddleware, KnowledgeTreeMiddleware, - # KnowledgePriorityMiddleware, and KnowledgeBasePersistenceMiddleware - it - # inherits state and tools from the parent, but should not (a) re-load - # anon docs / re-render the tree / re-run hybrid search, or (b) commit at - # its own completion (only the top-level agent's aafter_agent commits). - gp_middleware = [ - TodoListMiddleware(), - _memory_middleware, - FileIntentMiddleware(llm=llm), - SurfSenseFilesystemMiddleware( - backend=backend_resolver, - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - created_by_id=user_id, - thread_id=thread_id, - ), - create_surfsense_compaction_middleware(llm, StateBackend), - PatchToolCallsMiddleware(), - ] - - general_purpose_spec: SubAgent = { # type: ignore[typeddict-unknown-key] - **GENERAL_PURPOSE_SUBAGENT, - "model": llm, - "tools": tools, - "middleware": gp_middleware, - } - - # Specialized user-facing subagents (explore, report_writer, - # connector_negotiator). Registered through SubAgentMiddleware alongside - # the general-purpose spec so the parent's `task` tool can address them - # by name. Off by default until the flag flips so existing deployments - # don't see new agent types in the task tool description. - specialized_subagents: list[SubAgent] = [] - if flags.enable_specialized_subagents and not flags.disable_new_agent_stack: - try: - # Specialized subagents share the parent's filesystem + - # todo view so their system prompts (which promise - # ``read_file``, ``ls``, ``grep``, ``glob``, ``write_todos``) - # actually match runtime behavior. Build *fresh* instances - # rather than aliasing the parent's GP middleware to avoid - # subtle state coupling across compiled graphs. - subagent_extra_middleware: list = [ - TodoListMiddleware(), - SurfSenseFilesystemMiddleware( - backend=backend_resolver, - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - created_by_id=user_id, - thread_id=thread_id, - ), - ] - specialized_subagents = build_specialized_subagents( - tools=tools, - model=llm, - extra_middleware=subagent_extra_middleware, - ) - logging.info( - "Specialized subagents registered for task tool: %s", - [s["name"] for s in specialized_subagents], - ) - except Exception as exc: # pragma: no cover - defensive - logging.warning( - "Specialized subagent build failed; running without them: %s", - exc, - ) - specialized_subagents = [] - - subagent_specs: list[SubAgent] = [general_purpose_spec, *specialized_subagents] - - # Main agent middleware - # Order: AnonDoc -> Tree -> Priority -> FileIntent -> Filesystem -> Persistence -> ... - # before_agent hooks run in declared order; later injections sit closer to - # the latest human turn. Tree (large + cacheable) is injected earliest so - # provider-side prefix caching has more material to hit; FileIntent (most - # actionable per-turn contract) is injected closest to the user message. - # - # ``wrap_model_call`` ordering: the FIRST middleware in the list is the - # OUTERMOST wrapper. To ensure prune executes before summarization, - # place ``SpillingContextEditingMiddleware`` before - # ``SurfSenseCompactionMiddleware``. Compaction is the canonical - # token-budget defense; the Bedrock buffer-empty defense is folded - # into ``SurfSenseCompactionMiddleware``. - summarization_mw = create_surfsense_compaction_middleware(llm, StateBackend) - _ = flags.enable_compaction_v2 # historical flag; retained for telemetry parity - - # ContextEditing prune. Trigger at 55% of ``max_input_tokens``, - # earlier than summarization (~85%). When disabled, no edit runs. - context_edit_mw = None - if ( - flags.enable_context_editing - and not flags.disable_new_agent_stack - and max_input_tokens - ): - spill_edit = SpillToBackendEdit( - trigger=int(max_input_tokens * 0.55), - clear_at_least=int(max_input_tokens * 0.15), - keep=5, - exclude_tools=_safe_exclude_tools(tools), - clear_tool_inputs=True, - ) - clear_edit = ClearToolUsesEdit( - trigger=int(max_input_tokens * 0.55), - clear_at_least=int(max_input_tokens * 0.15), - keep=5, - exclude_tools=_safe_exclude_tools(tools), - clear_tool_inputs=True, - placeholder="[cleared - older tool output trimmed for context]", - ) - context_edit_mw = SpillingContextEditingMiddleware( - edits=[spill_edit, clear_edit], - backend_resolver=backend_resolver, - ) - - # Resilience knobs: header-aware retry, model fallback, and - # per-thread / per-run call-count limits. The fallback / limit - # middlewares are vanilla LangChain primitives; ``RetryAfter`` is - # SurfSense's header-aware variant (see its module docstring). - retry_mw = ( - RetryAfterMiddleware(max_retries=3) - if flags.enable_retry_after and not flags.disable_new_agent_stack - else None - ) - # Fallback chain — primary is the agent's own model; we add cheap - # alternatives. Off by default; only the first call site that - # configures the chain via env should enable it. - fallback_mw: ScopedModelFallbackMiddleware | None = None - if flags.enable_model_fallback and not flags.disable_new_agent_stack: - try: - fallback_mw = ScopedModelFallbackMiddleware( - "openai:gpt-4o-mini", - "anthropic:claude-3-5-haiku-20241022", - ) - except Exception: - logging.warning("ScopedModelFallbackMiddleware init failed; skipping.") - fallback_mw = None - model_call_limit_mw = ( - ModelCallLimitMiddleware( - thread_limit=120, - run_limit=80, - exit_behavior="end", - ) - if flags.enable_model_call_limit and not flags.disable_new_agent_stack - else None - ) - tool_call_limit_mw = ( - ToolCallLimitMiddleware( - thread_limit=300, run_limit=80, exit_behavior="continue" - ) - if flags.enable_tool_call_limit and not flags.disable_new_agent_stack - else None - ) - - # Provider-compat ``_noop`` injection (mirrors OpenCode's - # ``llm.ts`` workaround for providers that reject empty assistant - # turns or alternating-role constraints). - noop_mw = ( - NoopInjectionMiddleware() - if flags.enable_compaction_v2 and not flags.disable_new_agent_stack - else None - ) - - # Tool-call name repair (lowercase + ``invalid`` fallback). - # - # ``registered_tool_names`` MUST cover every tool the model can legitimately - # call. That includes the bound ``tools`` list AND every tool provided by - # middleware in the stack — ``FilesystemMiddleware`` (read_file, ls, grep, - # glob, edit_file, write_file, execute), ``TodoListMiddleware`` - # (write_todos), ``SubAgentMiddleware`` (task), ``SkillsMiddleware`` (skill - # loaders), etc. If we only inspect ``tools`` here, every call to - # ``read_file`` / ``ls`` / ``grep`` from the model will be rewritten to - # ``invalid`` because the repair middleware doesn't recognize them. The - # built-in deepagents middleware aren't in scope yet at this point of the - # function but they're added unconditionally below, so we hard-code their - # canonical names alongside the dynamic ``tools`` set. - repair_mw = None - if flags.enable_tool_call_repair and not flags.disable_new_agent_stack: - registered_names: set[str] = {t.name for t in tools} - # Tools owned by the standard deepagents middleware stack and the - # SurfSense filesystem extension. - registered_names |= { - "write_todos", - "ls", - "read_file", - "write_file", - "edit_file", - "glob", - "grep", - "execute", - "task", - "mkdir", - "cd", - "pwd", - "move_file", - "rm", - "rmdir", - "list_tree", - "execute_code", - } - repair_mw = ToolCallNameRepairMiddleware( - registered_tool_names=registered_names, - # Disable fuzzy matching to avoid silent rewrites; the - # lowercase + ``invalid`` fallback alone covers >95% of - # observed model errors. - fuzzy_match_threshold=None, - ) - - # Doom-loop detector. Off by default until the frontend handles - # ``permission == "doom_loop"`` interrupts. - doom_loop_mw = ( - DoomLoopMiddleware(threshold=3) - if flags.enable_doom_loop and not flags.disable_new_agent_stack - else None - ) - - # PermissionMiddleware. Layers, earliest -> latest (last match wins, - # same evaluation order as OpenCode's ``permission/index.ts``): - # - # 1. ``surfsense_defaults`` — single ``allow */*`` rule. SurfSense - # already runs per-tool HITL (see ``tools/hitl.py``) for mutating - # connector tools, so we only want PermissionMiddleware to *deny* - # things the user has gated off; the default fallback in - # ``permissions.evaluate`` is ``ask``, which would double-prompt - # on every safe read-only call (``ls``, ``read_file``, ``grep``, - # ``glob``, ``web_search`` …) and, on resume, replay the previous - # reject decision into innocent calls. - # 2. ``desktop_safety`` — ``ask`` for destructive filesystem ops when - # the agent is operating against the user's real disk. Cloud mode - # has full revision-based revert via ``revert_service``, but - # desktop mode hits disk immediately with no undo, so an - # accidental ``rm`` / ``rmdir`` / ``move_file`` / ``edit_file`` / - # ``write_file`` is unrecoverable. This layer is forced on in - # desktop mode regardless of ``enable_permission`` because the - # safety net is non-negotiable. - # 3. ``connector_synthesized`` — deny rules for tools whose required - # connector is not connected to this space. Overrides #1/#2. - # 4. (future) user-defined rules from ``agent_permission_rules`` table - # via the Agent Permissions UI. Loaded last so they override all. - permission_mw: PermissionMiddleware | None = None - is_desktop_fs = filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER - permission_enabled = flags.enable_permission and not flags.disable_new_agent_stack - # Build the middleware whenever it has work to do: either the user - # opted into the rule engine, OR we're in desktop mode and need the - # safety rules unconditionally. - if permission_enabled or is_desktop_fs: - rulesets: list[Ruleset] = [ - Ruleset( - rules=[Rule(permission="*", pattern="*", action="allow")], - origin="surfsense_defaults", - ), - ] - if is_desktop_fs: - rulesets.append( - Ruleset( - rules=[ - Rule(permission="rm", pattern="*", action="ask"), - Rule(permission="rmdir", pattern="*", action="ask"), - Rule(permission="move_file", pattern="*", action="ask"), - Rule(permission="edit_file", pattern="*", action="ask"), - Rule(permission="write_file", pattern="*", action="ask"), - ], - origin="desktop_safety", - ) - ) - if permission_enabled: - synthesized = _synthesize_connector_deny_rules( - available_connectors=available_connectors, - enabled_tool_names={t.name for t in tools}, - ) - rulesets.append(Ruleset(rules=synthesized, origin="connector_synthesized")) - permission_mw = PermissionMiddleware(rulesets=rulesets) - - # ActionLogMiddleware. Off by default until the ``agent_action_log`` - # table is migrated. When enabled, persists one row per tool call - # with optional reverse_descriptor for - # ``POST /api/threads/{thread_id}/revert/{action_id}``. Sits inside - # ``permission`` so denied calls aren't logged as completions. - action_log_mw: ActionLogMiddleware | None = None - if ( - flags.enable_action_log - and not flags.disable_new_agent_stack - and thread_id is not None - ): - try: - tool_defs_by_name = {td.name: td for td in BUILTIN_TOOLS} - action_log_mw = ActionLogMiddleware( - thread_id=thread_id, - search_space_id=search_space_id, - user_id=user_id, - tool_definitions=tool_defs_by_name, - ) - except Exception: # pragma: no cover - defensive - logging.warning( - "ActionLogMiddleware init failed; running without it.", - exc_info=True, - ) - action_log_mw = None - - # Per-thread busy mutex (refuse a second concurrent turn on the same - # thread; see :class:`BusyMutexMiddleware` docstring). - busy_mutex_mw: BusyMutexMiddleware | None = ( - BusyMutexMiddleware() - if flags.enable_busy_mutex and not flags.disable_new_agent_stack - else None - ) - - # OpenTelemetry spans (model.call + tool.call). Lives just inside - # BusyMutex so it spans every retry/fallback attempt of the current - # turn but never wraps a queued/blocked turn. - otel_mw: OtelSpanMiddleware | None = ( - OtelSpanMiddleware() - if flags.enable_otel and not flags.disable_new_agent_stack - else None - ) - - # Plugin entry-point loader. Off by default; opt-in via the - # ``SURFSENSE_ENABLE_PLUGIN_LOADER`` flag. The allowlist is read from - # the ``SURFSENSE_ALLOWED_PLUGINS`` env var (comma-separated). A future - # PR can wire it through ``global_llm_config.yaml``. - plugin_middlewares: list[Any] = [] - if flags.enable_plugin_loader and not flags.disable_new_agent_stack: - try: - allowed_names = load_allowed_plugin_names_from_env() - if allowed_names: - plugin_middlewares = load_plugin_middlewares( - PluginContext.build( - search_space_id=search_space_id, - user_id=user_id, - thread_visibility=visibility, - llm=llm, - ), - allowed_plugin_names=allowed_names, - ) - except Exception: # pragma: no cover - defensive - logging.warning( - "Plugin loader failed; continuing without plugins.", - exc_info=True, - ) - plugin_middlewares = [] - - # SkillsMiddleware (deepagents) loads built-in + space-authored - # skills via a CompositeBackend. Sources are layered: built-in first, - # space last, so a search-space-authored skill of the same name - # overrides the bundled one. - skills_mw: SkillsMiddleware | None = None - if flags.enable_skills and not flags.disable_new_agent_stack: - try: - skills_factory = build_skills_backend_factory( - search_space_id=search_space_id - if filesystem_mode == FilesystemMode.CLOUD - else None, - ) - skills_mw = SkillsMiddleware( - backend=skills_factory, - sources=default_skills_sources(), - ) - except Exception as exc: # pragma: no cover - defensive - logging.warning("SkillsMiddleware init failed; skipping: %s", exc) - skills_mw = None - - # LangChain's LLM-driven tool selection — only enabled for stacks - # large enough to need narrowing (>30 tools). - selector_mw: LLMToolSelectorMiddleware | None = None - if ( - flags.enable_llm_tool_selector - and not flags.disable_new_agent_stack - and len(tools) > 30 - ): - try: - selector_mw = LLMToolSelectorMiddleware( - model="openai:gpt-4o-mini", - max_tools=12, - always_include=[ - name - for name in ( - "update_memory", - "get_connected_accounts", - "scrape_webpage", - ) - if name in {t.name for t in tools} - ], - ) - except Exception: - logging.warning("LLMToolSelectorMiddleware init failed; skipping.") - selector_mw = None - - deepagent_middleware = [ - # BusyMutex is OUTERMOST: it must wrap the entire stream so no - # other turn can sneak in while this one is mid-flight. - busy_mutex_mw, - # OTel spans sit just inside BusyMutex so each retry attempt - # gets its own model.call / tool.call span. - otel_mw, - TodoListMiddleware(), - _memory_middleware, - AnonymousDocumentMiddleware( - anon_session_id=anon_session_id, - ) - if filesystem_mode == FilesystemMode.CLOUD - else None, - KnowledgeTreeMiddleware( - search_space_id=search_space_id, - filesystem_mode=filesystem_mode, - llm=llm, - ) - if filesystem_mode == FilesystemMode.CLOUD - else None, - KnowledgePriorityMiddleware( - llm=llm, - planner_llm=get_planner_llm(), - search_space_id=search_space_id, - filesystem_mode=filesystem_mode, - available_connectors=available_connectors, - available_document_types=available_document_types, - mentioned_document_ids=mentioned_document_ids, - ), - FileIntentMiddleware(llm=llm), - SurfSenseFilesystemMiddleware( - backend=backend_resolver, - filesystem_mode=filesystem_mode, - search_space_id=search_space_id, - created_by_id=user_id, - thread_id=thread_id, - ), - KnowledgeBasePersistenceMiddleware( - search_space_id=search_space_id, - created_by_id=user_id, - filesystem_mode=filesystem_mode, - thread_id=thread_id, - ) - if filesystem_mode == FilesystemMode.CLOUD - else None, - # Skill loader. Placed before SubAgentMiddleware so subagents - # inherit the same skill metadata (subagent specs reference the - # same source paths via ``default_skills_sources()``). - skills_mw, - SubAgentMiddleware(backend=StateBackend, subagents=subagent_specs), - # Tool selection (only when >30 tools and flag on). - selector_mw, - # Defensive caps, then prune, then summarize. - model_call_limit_mw, - tool_call_limit_mw, - context_edit_mw, - summarization_mw, - # Provider compatibility + retry chain — placed after prune/compact - # so retries happen on the already-trimmed payload. - noop_mw, - retry_mw, - fallback_mw, - # Coalesce a multi-text-block system message into one block - # immediately before the model call. Sits innermost on the - # system-message-mutation chain so it observes every appender - # (todo / filesystem / skills / subagents …) and prevents - # OpenRouter→Anthropic from redistributing ``cache_control`` - # across N blocks and tripping Anthropic's 4-breakpoint cap. - # See ``middleware/flatten_system.py`` for full rationale. - FlattenSystemMessageMiddleware(), - # Tool-call repair must run after model emits but before - # permission / dedup / doom-loop interpret the calls. - repair_mw, - # Permission deny/ask BEFORE the calls are forwarded to tool nodes. - permission_mw, - doom_loop_mw, - # Action log sits inside permission so denied calls don't appear - # as completions, and outside dedup so each unique tool invocation - # gets its own row. - action_log_mw, - PatchToolCallsMiddleware(), - DedupHITLToolCallsMiddleware(agent_tools=list(tools)), - # Plugin slot — sits at the tail so plugin-side transforms see the - # final tool result. Prompt caching is now applied at LLM build time - # via ``apply_litellm_prompt_caching`` (see prompt_caching.py), so no - # caching middleware is needed here. Multiple plugins run in declared - # order; loader filtered by the admin allowlist already. - *plugin_middlewares, - ] - deepagent_middleware = [m for m in deepagent_middleware if m is not None] - - agent = create_agent( - llm, - system_prompt=final_system_prompt, - tools=list(tools), - middleware=deepagent_middleware, - context_schema=SurfSenseContextSchema, - checkpointer=checkpointer, - ) - return agent.with_config( - { - "recursion_limit": 10_000, - "metadata": { - "ls_integration": "deepagents", - "versions": {"deepagents": deepagents_version}, - }, - } - ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/__init__.py b/surfsense_backend/app/agents/new_chat/middleware/__init__.py deleted file mode 100644 index 6742bd8de..000000000 --- a/surfsense_backend/app/agents/new_chat/middleware/__init__.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Middleware components for the SurfSense new chat agent.""" - -from app.agents.new_chat.middleware.action_log import ActionLogMiddleware -from app.agents.new_chat.middleware.anonymous_document import ( - AnonymousDocumentMiddleware, -) -from app.agents.new_chat.middleware.busy_mutex import BusyMutexMiddleware -from app.agents.new_chat.middleware.compaction import ( - SurfSenseCompactionMiddleware, - create_surfsense_compaction_middleware, -) -from app.agents.new_chat.middleware.context_editing import ( - ClearToolUsesEdit, - SpillingContextEditingMiddleware, - SpillToBackendEdit, -) -from app.agents.new_chat.middleware.dedup_tool_calls import ( - DedupHITLToolCallsMiddleware, -) -from app.agents.new_chat.middleware.doom_loop import DoomLoopMiddleware -from app.agents.new_chat.middleware.file_intent import ( - FileIntentMiddleware, -) -from app.agents.new_chat.middleware.filesystem import ( - SurfSenseFilesystemMiddleware, -) -from app.agents.new_chat.middleware.flatten_system import ( - FlattenSystemMessageMiddleware, -) -from app.agents.new_chat.middleware.kb_persistence import ( - KnowledgeBasePersistenceMiddleware, - commit_staged_filesystem_state, -) -from app.agents.new_chat.middleware.knowledge_search import ( - KnowledgeBaseSearchMiddleware, - KnowledgePriorityMiddleware, -) -from app.agents.new_chat.middleware.knowledge_tree import ( - KnowledgeTreeMiddleware, -) -from app.agents.new_chat.middleware.memory_injection import ( - MemoryInjectionMiddleware, -) -from app.agents.new_chat.middleware.noop_injection import NoopInjectionMiddleware -from app.agents.new_chat.middleware.otel_span import OtelSpanMiddleware -from app.agents.new_chat.middleware.permission import PermissionMiddleware -from app.agents.new_chat.middleware.retry_after import RetryAfterMiddleware -from app.agents.new_chat.middleware.skills_backends import ( - BuiltinSkillsBackend, - SearchSpaceSkillsBackend, - build_skills_backend_factory, - default_skills_sources, -) -from app.agents.new_chat.middleware.tool_call_repair import ( - ToolCallNameRepairMiddleware, -) - -__all__ = [ - "ActionLogMiddleware", - "AnonymousDocumentMiddleware", - "BuiltinSkillsBackend", - "BusyMutexMiddleware", - "ClearToolUsesEdit", - "DedupHITLToolCallsMiddleware", - "DoomLoopMiddleware", - "FileIntentMiddleware", - "FlattenSystemMessageMiddleware", - "KnowledgeBasePersistenceMiddleware", - "KnowledgeBaseSearchMiddleware", - "KnowledgePriorityMiddleware", - "KnowledgeTreeMiddleware", - "MemoryInjectionMiddleware", - "NoopInjectionMiddleware", - "OtelSpanMiddleware", - "PermissionMiddleware", - "RetryAfterMiddleware", - "SearchSpaceSkillsBackend", - "SpillToBackendEdit", - "SpillingContextEditingMiddleware", - "SurfSenseCompactionMiddleware", - "SurfSenseFilesystemMiddleware", - "ToolCallNameRepairMiddleware", - "build_skills_backend_factory", - "commit_staged_filesystem_state", - "create_surfsense_compaction_middleware", - "default_skills_sources", -] diff --git a/surfsense_backend/app/agents/new_chat/middleware/file_intent.py b/surfsense_backend/app/agents/new_chat/middleware/file_intent.py deleted file mode 100644 index 7897e13d6..000000000 --- a/surfsense_backend/app/agents/new_chat/middleware/file_intent.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Semantic file-intent routing middleware for new chat turns. - -This middleware classifies the latest human turn into a small intent set: -- chat_only -- file_write -- file_read - -For ``file_write`` turns it injects a strict system contract so the model -uses filesystem tools before claiming success, and provides a deterministic -fallback path when no filename is specified by the user. -""" - -from __future__ import annotations - -import json -import logging -import re -from datetime import UTC, datetime -from enum import StrEnum -from typing import Any - -from langchain.agents.middleware import AgentMiddleware, AgentState -from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage -from langgraph.runtime import Runtime -from pydantic import BaseModel, Field, ValidationError - -logger = logging.getLogger(__name__) - - -class FileOperationIntent(StrEnum): - CHAT_ONLY = "chat_only" - FILE_WRITE = "file_write" - FILE_READ = "file_read" - - -class FileIntentPlan(BaseModel): - intent: FileOperationIntent = Field( - description="Primary user intent for this turn." - ) - confidence: float = Field( - ge=0.0, - le=1.0, - default=0.5, - description="Model confidence in the selected intent.", - ) - suggested_filename: str | None = Field( - default=None, - description="Optional filename (e.g. notes.md) inferred from user request.", - ) - suggested_directory: str | None = Field( - default=None, - description=( - "Optional directory path (e.g. /reports/q2 or reports/q2) inferred from " - "user request." - ), - ) - suggested_path: str | None = Field( - default=None, - description=( - "Optional full file path (e.g. /reports/q2/summary.md). If present, this " - "takes precedence over suggested_directory + suggested_filename." - ), - ) - - -def _extract_text_from_message(message: BaseMessage) -> str: - content = getattr(message, "content", "") - if isinstance(content, str): - return content - if isinstance(content, list): - parts: list[str] = [] - for item in content: - if isinstance(item, str): - parts.append(item) - elif isinstance(item, dict) and item.get("type") == "text": - parts.append(str(item.get("text", ""))) - return "\n".join(part for part in parts if part) - return str(content) - - -def _extract_json_payload(text: str) -> str: - stripped = text.strip() - fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", stripped, re.DOTALL) - if fenced: - return fenced.group(1) - start = stripped.find("{") - end = stripped.rfind("}") - if start != -1 and end != -1 and end > start: - return stripped[start : end + 1] - return stripped - - -def _sanitize_filename(value: str) -> str: - name = re.sub(r"[\\/:*?\"<>|]+", "_", value).strip() - name = re.sub(r"\s+", "-", name) - name = name.strip("._-") - if not name: - name = "note" - if len(name) > 80: - name = name[:80].rstrip("-_.") - return name - - -def _sanitize_path_segment(value: str) -> str: - segment = re.sub(r"[\\/:*?\"<>|]+", "_", value).strip() - segment = re.sub(r"\s+", "_", segment) - segment = segment.strip("._-") - return segment - - -def _normalize_directory(value: str) -> str: - raw = value.strip().replace("\\", "/") - raw = raw.strip("/") - if not raw: - return "" - parts = [_sanitize_path_segment(part) for part in raw.split("/") if part.strip()] - parts = [part for part in parts if part] - return "/".join(parts) - - -def _normalize_file_path(value: str) -> str: - raw = value.strip().replace("\\", "/").strip() - if not raw: - return "" - had_trailing_slash = raw.endswith("/") - raw = raw.strip("/") - if not raw: - return "" - parts = [_sanitize_path_segment(part) for part in raw.split("/") if part.strip()] - parts = [part for part in parts if part] - if not parts: - return "" - if had_trailing_slash: - return f"/{'/'.join(parts)}/" - return f"/{'/'.join(parts)}" - - -def _infer_directory_from_user_text(user_text: str) -> str | None: - patterns = ( - r"\b(?:in|inside|under)\s+(?:the\s+)?([a-zA-Z0-9 _\-/]+?)\s+folder\b", - r"\b(?:in|inside|under)\s+([a-zA-Z0-9 _\-/]+?)\b", - ) - lowered = user_text.lower() - for pattern in patterns: - match = re.search(pattern, lowered, flags=re.IGNORECASE) - if not match: - continue - candidate = match.group(1).strip() - if candidate in {"the", "a", "an"}: - continue - normalized = _normalize_directory(candidate) - if normalized: - return normalized - return None - - -def _fallback_path( - suggested_filename: str | None, - *, - suggested_directory: str | None = None, - suggested_path: str | None = None, - user_text: str, -) -> str: - inferred_dir = _infer_directory_from_user_text(user_text) - - sanitized_filename = "" - if suggested_filename: - sanitized_filename = _sanitize_filename(suggested_filename) - if sanitized_filename.lower().endswith(".txt"): - sanitized_filename = f"{sanitized_filename[:-4]}.md" - if not sanitized_filename: - sanitized_filename = "notes.md" - elif "." not in sanitized_filename: - sanitized_filename = f"{sanitized_filename}.md" - - normalized_suggested_path = ( - _normalize_file_path(suggested_path) if suggested_path else "" - ) - if normalized_suggested_path: - if normalized_suggested_path.endswith("/"): - return f"{normalized_suggested_path.rstrip('/')}/{sanitized_filename}" - return normalized_suggested_path - - directory = _normalize_directory(suggested_directory or "") - if not directory and inferred_dir: - directory = inferred_dir - if directory: - return f"/{directory}/{sanitized_filename}" - - return f"/{sanitized_filename}" - - -def _build_classifier_prompt(*, recent_conversation: str, user_text: str) -> str: - return ( - "Classify the latest user request into a filesystem intent for an AI agent.\n" - "Return JSON only with this exact schema:\n" - '{"intent":"chat_only|file_write|file_read","confidence":0.0,"suggested_filename":"string or null","suggested_directory":"string or null","suggested_path":"string or null"}\n\n' - "Rules:\n" - "- Use semantic intent, not literal keywords.\n" - "- file_write: user asks to create/save/write/update/edit content as a file.\n" - "- file_read: user asks to open/read/list/search existing files.\n" - "- chat_only: conversational/analysis responses without required file operations.\n" - "- For file_write, choose a concise semantic suggested_filename and match the requested format.\n" - "- If the user mentions a folder/directory, populate suggested_directory.\n" - "- If user specifies an explicit full path, populate suggested_path.\n" - "- Use extensions that match user intent (e.g. .md, .json, .yaml, .csv, .py, .ts, .js, .html, .css, .sql).\n" - "- Do not use .txt; prefer .md for generic text notes.\n" - "- Do not include dates or timestamps in suggested_filename unless explicitly requested.\n" - "- Never include markdown or explanation.\n\n" - f"Recent conversation:\n{recent_conversation or '(none)'}\n\n" - f"Latest user message:\n{user_text}" - ) - - -def _build_recent_conversation( - messages: list[BaseMessage], *, max_messages: int = 6 -) -> str: - rows: list[str] = [] - filtered: list[tuple[str, BaseMessage]] = [] - for msg in messages: - role: str | None = None - if isinstance(msg, HumanMessage): - role = "user" - elif isinstance(msg, AIMessage): - if getattr(msg, "tool_calls", None): - continue - role = "assistant" - else: - continue - filtered.append((role, msg)) - for role, msg in filtered[-max_messages:]: - text = re.sub(r"\s+", " ", _extract_text_from_message(msg)).strip() - if text: - rows.append(f"{role}: {text[:280]}") - return "\n".join(rows) - - -class FileIntentMiddleware(AgentMiddleware): # type: ignore[type-arg] - """Classify file intent and inject a strict file-write contract.""" - - tools = () - - def __init__(self, *, llm: BaseChatModel | None = None) -> None: - self.llm = llm - - async def _classify_intent( - self, *, messages: list[BaseMessage], user_text: str - ) -> FileIntentPlan: - if self.llm is None: - return FileIntentPlan(intent=FileOperationIntent.CHAT_ONLY, confidence=0.0) - - prompt = _build_classifier_prompt( - recent_conversation=_build_recent_conversation(messages), - user_text=user_text, - ) - try: - response = await self.llm.ainvoke( - [HumanMessage(content=prompt)], - config={"tags": ["surfsense:internal"]}, - ) - payload = json.loads( - _extract_json_payload(_extract_text_from_message(response)) - ) - plan = FileIntentPlan.model_validate(payload) - return plan - except (json.JSONDecodeError, ValidationError, ValueError) as exc: - logger.warning("File intent classifier returned invalid output: %s", exc) - except Exception as exc: # pragma: no cover - defensive fallback - logger.warning("File intent classifier failed: %s", exc) - - return FileIntentPlan(intent=FileOperationIntent.CHAT_ONLY, confidence=0.0) - - async def abefore_agent( # type: ignore[override] - self, - state: AgentState, - runtime: Runtime[Any], - ) -> dict[str, Any] | None: - del runtime - messages = state.get("messages") or [] - if not messages: - return None - - last_human: HumanMessage | None = None - for msg in reversed(messages): - if isinstance(msg, HumanMessage): - last_human = msg - break - if last_human is None: - return None - - user_text = _extract_text_from_message(last_human).strip() - if not user_text: - return None - - plan = await self._classify_intent(messages=messages, user_text=user_text) - suggested_path = _fallback_path( - plan.suggested_filename, - suggested_directory=plan.suggested_directory, - suggested_path=plan.suggested_path, - user_text=user_text, - ) - contract = { - "intent": plan.intent.value, - "confidence": plan.confidence, - "suggested_path": suggested_path, - "timestamp": datetime.now(UTC).isoformat(), - "turn_id": state.get("turn_id", ""), - } - - if plan.intent != FileOperationIntent.FILE_WRITE: - return {"file_operation_contract": contract} - - contract_msg = SystemMessage( - content=( - "\n" - "This turn intent is file_write.\n" - f"Suggested default path: {suggested_path}\n" - "Rules:\n" - "- You MUST call write_file or edit_file before claiming success.\n" - "- If no path is provided by the user, use the suggested default path.\n" - "- Do not claim a file was created/updated unless tool output confirms it.\n" - "- If the write/edit fails, clearly report failure instead of success.\n" - "- Do not include timestamps or dates in generated file content unless the user explicitly asks for them.\n" - "- For open-ended requests (e.g., random note), generate useful concrete content, not placeholders.\n" - "" - ) - ) - - # Insert just before the latest human turn so it applies to this request. - new_messages = list(messages) - insert_at = max(len(new_messages) - 1, 0) - new_messages.insert(insert_at, contract_msg) - return {"messages": new_messages, "file_operation_contract": contract} diff --git a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py b/surfsense_backend/app/agents/new_chat/middleware/filesystem.py deleted file mode 100644 index c46eb98a5..000000000 --- a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py +++ /dev/null @@ -1,1998 +0,0 @@ -"""Custom filesystem middleware for the SurfSense agent. - -This middleware fully overrides every deepagents filesystem tool so that the -``Command(update=...)`` payload can carry SurfSense-specific state fields -(``cwd``, ``staged_dirs``, ``pending_moves``, ``doc_id_by_path``, -``dirty_paths``) atomically alongside the standard ``files`` update. - -In CLOUD mode the backend is :class:`KBPostgresBackend` (lazy DB reads, no DB -writes). End-of-turn persistence is handled by -:class:`KnowledgeBasePersistenceMiddleware`. In DESKTOP_LOCAL_FOLDER mode the -backend is :class:`MultiRootLocalFolderBackend` and writes go straight to disk. - -New tools introduced here: - -* ``mkdir`` — cloud-only stages folder paths to ``state['staged_dirs']``; - desktop creates real directories. -* ``cd`` / ``pwd`` — manage ``state['cwd']`` (per-thread). -* ``move_file`` — staged commit in cloud, real disk move in desktop. -* ``list_tree`` — works in both modes (cloud uses - :func:`KBPostgresBackend.alist_tree_listing`). - -The middleware no longer ships ``save_document``; persistence is inferred -from ``write_file`` / ``edit_file`` against ``/documents/*`` paths. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import posixpath -import re -import secrets -from typing import Annotated, Any - -from daytona.common.errors import DaytonaError -from deepagents import FilesystemMiddleware -from deepagents.backends.protocol import EditResult, WriteResult -from deepagents.backends.utils import ( - create_file_data, - format_read_response, - validate_path, -) -from langchain.tools import ToolRuntime -from langchain_core.messages import ToolMessage -from langchain_core.tools import BaseTool, StructuredTool -from langgraph.types import Command - -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.filesystem_state import SurfSenseFilesystemState -from app.agents.new_chat.middleware.kb_postgres_backend import ( - KBPostgresBackend, - paginate_listing, -) -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( - MultiRootLocalFolderBackend, -) -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT -from app.agents.new_chat.sandbox import ( - _evict_sandbox_cache, - delete_sandbox, - get_or_create_sandbox, - is_sandbox_enabled, -) -from app.agents.new_chat.state_reducers import _CLEAR - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# System Prompt (built per-session based on filesystem_mode) -# ============================================================================= -# -# Each chat session runs in exactly one filesystem mode. Including rules for -# the OTHER mode just wastes tokens and confuses the model, so we build the -# prompt + tool descriptions for the active mode only. - -_COMMON_PROMPT_HEADER = """## Following Conventions - -- Read files before editing — understand existing content before making changes. -- Mimic existing style, naming conventions, and patterns. -- Never claim a file was created/updated unless filesystem tool output confirms success. -- If a file write/edit fails, explicitly report the failure. -""" - -_CLOUD_SYSTEM_PROMPT = ( - _COMMON_PROMPT_HEADER - + """ -## Filesystem Tools - -All file paths must start with `/`. Relative paths resolve against the -current working directory (`cwd`, default `/documents`). - -- ls(path, offset=0, limit=200): list files and directories at the given path. -- read_file(path, offset, limit): read a file (paginated) from the filesystem. -- write_file(path, content): create a new text file in the workspace. -- edit_file(path, old, new): exact string-replacement edit (lazy-loads KB - documents on first edit). -- glob(pattern, path): find files matching a glob pattern. -- grep(pattern, path, glob): substring search across files. -- mkdir(path): create a folder under `/documents/` (committed at end of turn). -- cd(path): change the current working directory. -- pwd(): print the current working directory. -- move_file(source, dest): move/rename a file under `/documents/`. -- rm(path): delete a single file under `/documents/` (no `-r`). -- rmdir(path): delete an empty directory under `/documents/`. -- list_tree(path, max_depth, page_size): recursively list files/folders. - -## Persistence Rules - -- Files written under `/documents/<...>` are **persisted** at end of turn as - Documents in the user's knowledge base. -- Files whose **basename** starts with `temp_` (e.g. `temp_plan.md` or - `/documents/temp_scratch.md`) are **discarded** at end of turn — use this - prefix for any scratch/working content you do NOT want saved. -- All other paths (outside `/documents/` and not `temp_*`) are rejected. -- mkdir/move_file/rm/rmdir are staged this turn and committed at end of - turn alongside any new/edited documents. Snapshot/revert is enabled - for every destructive operation when action logging is on. - -## Reading Documents Efficiently - -Documents are formatted as XML. Each document contains: -- `` — title, type, URL, etc. -- `` — a table of every chunk with its **line range** and a - `matched="true"` flag for chunks that matched the search query. -- `` — the actual chunks in original document order. - -**Workflow**: when reading a large document, read the first ~20 lines to see -the ``, identify chunks marked `matched="true"`, then use -`read_file(path, offset=, limit=)` to jump directly to -those sections instead of reading the entire file sequentially. - -Use `` values as citation IDs in your answers. - -## Priority List - -You receive a `` system message each turn listing the -top-K paths most relevant to the user's query (by hybrid search). Read those -first — matched sections are flagged inside each document's ``. - -## Workspace Tree - -You receive a `` system message each turn with the current -folder/document layout. The tree may be truncated past a hard cap; in that -case, drill into specific folders with `ls(...)` or `list_tree(...)`. - -## grep Line Numbers - -`grep` searches across both your in-memory edits and the indexed chunks in -Postgres. State-cached files return real line numbers; database hits return -`line=0` because their position depends on per-document XML layout — call -`read_file(path)` to find the exact line. -""" -) - -_DESKTOP_SYSTEM_PROMPT = ( - _COMMON_PROMPT_HEADER - + """ -## Local Folder Mode - -This chat operates directly on the user's local folders. Writes and edits -hit disk immediately — there is no end-of-turn staging, no `/documents/` -namespace, and no `temp_` semantics. - -## Filesystem Tools - -All file paths must start with `/` and use mount-prefixed absolute paths -like `//file.ext`. Relative paths resolve against the current working -directory (`cwd`). - -- ls(path, offset=0, limit=200): list files and directories at the given path. -- read_file(path, offset, limit): read a file (paginated) from disk. -- write_file(path, content): write a file to disk. -- edit_file(path, old, new): exact string-replacement edit on disk. -- glob(pattern, path): find files matching a glob pattern. -- grep(pattern, path, glob): substring search across files. -- mkdir(path): create a directory on disk. -- cd(path): change the current working directory. -- pwd(): print the current working directory. -- move_file(source, dest): move/rename a file. -- rm(path): delete a single file from disk (no `-r`). NOT reversible. -- rmdir(path): delete an empty directory from disk. NOT reversible. -- list_tree(path, max_depth, page_size): recursively list files/folders. - -## Workflow Tips - -- If you are unsure which mounts are available, call `ls('/')` first. -- For large trees, prefer `list_tree` then `grep` then `read_file` over - brute-force directory traversal. -- Cross-mount moves are not supported. -- Desktop deletes hit disk immediately and cannot be undone via the - agent's revert flow — confirm before calling `rm`/`rmdir`. -""" -) - -_SANDBOX_PROMPT_ADDENDUM = ( - "\n- execute_code: run Python code in an isolated sandbox." - "\n\n## Code Execution" - "\n\nUse execute_code whenever a task benefits from running code." - " Never perform arithmetic manually." - "\n\nDocuments here are XML-wrapped markdown, not raw data files." - " To work with them programmatically, read the document first," - " extract the data, write it as a clean file (CSV, JSON, etc.)," - " and then run your code against it." -) - - -def _build_filesystem_system_prompt( - filesystem_mode: FilesystemMode, - *, - sandbox_available: bool, -) -> str: - """Build the filesystem system prompt for a given session mode. - - The prompt only describes rules and tools that actually apply in the - chosen mode — there is no cross-mode noise. - """ - base = ( - _CLOUD_SYSTEM_PROMPT - if filesystem_mode == FilesystemMode.CLOUD - else _DESKTOP_SYSTEM_PROMPT - ) - if sandbox_available: - base += _SANDBOX_PROMPT_ADDENDUM - return base - - -# Backwards-compatible alias retained for any external imports. -SURFSENSE_FILESYSTEM_SYSTEM_PROMPT = _CLOUD_SYSTEM_PROMPT - -# ============================================================================= -# Per-Tool Descriptions (shown to the LLM as the tool's docstring) -# ============================================================================= - -# ============================================================================= -# Per-Tool Descriptions (mode-specific; injected as the tool's docstring) -# ============================================================================= - -# --- mode-agnostic --------------------------------------------------------- - -SURFSENSE_READ_FILE_TOOL_DESCRIPTION = """Reads a file from the filesystem. - -Usage: -- By default, reads up to 100 lines from the beginning. -- Use `offset` and `limit` for pagination when files are large. -- Results include line numbers. -- Documents contain a `` near the top listing every chunk with - its line range and a `matched="true"` flag for search-relevant chunks. - Read the index first, then jump to matched chunks with - `read_file(path, offset=, limit=)`. -- Use chunk IDs (``) as citations in answers. -""" - -SURFSENSE_GLOB_TOOL_DESCRIPTION = """Find files matching a glob pattern. - -Supports standard glob patterns: `*`, `**`, `?`. -Returns absolute file paths. -""" - -SURFSENSE_CD_TOOL_DESCRIPTION = """Changes the current working directory (cwd). - -Args: -- path: absolute or relative directory path. Relative paths resolve against - the current cwd. - -The new cwd is used by other filesystem tools whenever a relative path is -given. Returns the resolved cwd. -""" - -SURFSENSE_PWD_TOOL_DESCRIPTION = """Prints the current working directory.""" - -SURFSENSE_EXECUTE_CODE_TOOL_DESCRIPTION = """Executes Python code in an isolated sandbox environment. - -Common data-science packages are pre-installed (pandas, numpy, matplotlib, -scipy, scikit-learn). - -Usage notes: -- No outbound network access. -- Returns combined stdout/stderr with exit code. -- Use print() to produce output. -- Use the optional timeout parameter to override the default timeout. -""" - -# --- cloud-only ------------------------------------------------------------ - -_CLOUD_LIST_FILES_TOOL_DESCRIPTION = """Lists files and directories at the given path. - -Usage: -- Provide an absolute path under `/documents` (relative paths resolve under - the current cwd, which defaults to `/documents`). -- For very large folders, use `offset` and `limit` to paginate the listing. -- Returns one entry per line; directories end with a trailing `/`. -""" - -_CLOUD_WRITE_FILE_TOOL_DESCRIPTION = """Writes a new text file to the workspace. - -Usage: -- Files written under `/documents/<...>` are persisted as Documents at end - of turn. -- Use a `temp_` filename prefix (e.g. `temp_plan.md` or `/documents/temp_x.md`) - for scratch/working files; they are automatically discarded at end of turn. -- Writes outside `/documents/` are rejected unless the basename starts with - `temp_`. -- Supported outputs include common LLM-friendly text formats like markdown, - json, yaml, csv, xml, html, css, sql, and code files. -- Avoid placeholders; produce concrete and useful text. -""" - -_CLOUD_EDIT_FILE_TOOL_DESCRIPTION = """Performs exact string replacements in files. - -IMPORTANT: -- Read the file before editing. -- Preserve exact indentation and formatting. -- Edits to documents under `/documents/` are persisted at end of turn. -- Edits to `temp_*` files are discarded at end of turn. -""" - -_CLOUD_MOVE_FILE_TOOL_DESCRIPTION = """Moves or renames a file or folder. - -Use absolute paths for both source and destination. - -Notes: -- `move_file` is staged this turn and committed at end of turn. -- The agent cannot overwrite an existing destination — pass a fresh dest - path or move the existing destination away first. -- The anonymous uploaded document is read-only and cannot be moved. -- Rename is a special case of move (same folder, different filename). -""" - -_CLOUD_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively in a single bounded call. - -Args: -- path: absolute path to start from. Defaults to `/documents`. -- max_depth: recursion depth limit (default 8). -- page_size: maximum number of entries returned (max 1000). -- include_files / include_dirs: filter returned entry types. - -Returns JSON with: -- entries: [{path, is_dir, size, modified_at, depth}] -- truncated: true when additional entries were omitted due to page_size. -""" - -_CLOUD_GREP_TOOL_DESCRIPTION = """Search for a literal text pattern across files. - -Searches both your in-memory edits and the indexed chunks in Postgres. -State-cached file matches include real line numbers; database hits return -`line=0` because their position depends on per-document XML layout — call -`read_file(path)` afterwards to find the exact line. -""" - -_CLOUD_MKDIR_TOOL_DESCRIPTION = """Creates a directory under `/documents/`. - -Stages the folder for end-of-turn commit; the Folder row is inserted only -after the agent's turn finishes successfully. - -Args: -- path: absolute path of the new directory (must start with - `/documents/`). - -Notes: -- Parent folders are created as needed. -""" - -_CLOUD_RM_TOOL_DESCRIPTION = """Deletes a single file under `/documents/`. - -Mirrors POSIX `rm path` (no `-r`, no glob expansion). Stages the deletion -for end-of-turn commit; the row is removed only after the agent's turn -finishes successfully. - -Args: -- path: absolute or relative file path. Cannot point at a directory — use - `rmdir` for empty folders. Cannot target the root or `/documents`. - -Notes: -- The action is reversible via the per-action revert flow when action - logging is enabled. -- The anonymous uploaded document is read-only and cannot be deleted. -""" - -_CLOUD_RMDIR_TOOL_DESCRIPTION = """Deletes an empty directory under `/documents/`. - -Mirrors POSIX `rmdir path`: refuses non-empty directories. Recursive -deletion (`rm -r`) is intentionally NOT supported — clear contents with -`rm` first. - -Args: -- path: absolute or relative directory path. Cannot target the root, - `/documents`, the current cwd, or any ancestor of cwd (use `cd` to - move out first). - -Notes: -- Emptiness is evaluated against the post-staged view, so a same-turn - `rm /a/x.md` followed by `rmdir /a` is fine. -- If the directory was added in this same turn via `mkdir` and never - committed, the staged mkdir is dropped instead of issuing a delete. -- The action is reversible via the per-action revert flow when action - logging is enabled. -""" - -# --- desktop-only ---------------------------------------------------------- - -_DESKTOP_LIST_FILES_TOOL_DESCRIPTION = """Lists files and directories at the given path. - -Usage: -- Provide an absolute path using a mount prefix (e.g. `//sub/dir`). - Use `ls('/')` to discover available mounts. -- For very large folders, use `offset` and `limit` to paginate the listing. -- Returns one entry per line; directories end with a trailing `/`. -""" - -_DESKTOP_WRITE_FILE_TOOL_DESCRIPTION = """Writes a text file to disk. - -Usage: -- Use mount-prefixed absolute paths like `//sub/file.ext`. -- Writes hit disk immediately. There is no end-of-turn staging. -- Supported outputs include common LLM-friendly text formats like markdown, - json, yaml, csv, xml, html, css, sql, and code files. -- Avoid placeholders; produce concrete and useful text. -""" - -_DESKTOP_EDIT_FILE_TOOL_DESCRIPTION = """Performs exact string replacements in files on disk. - -IMPORTANT: -- Read the file before editing. -- Preserve exact indentation and formatting. -- Edits hit disk immediately. -""" - -_DESKTOP_MOVE_FILE_TOOL_DESCRIPTION = """Moves or renames a file or folder on disk. - -Use mount-prefixed absolute paths for both source and destination -(e.g. `//old.txt` -> `//new.txt`). - -Notes: -- Cross-mount moves are not supported. -- Rename is a special case of move (same folder, different filename). -""" - -_DESKTOP_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively in a single bounded call. - -Args: -- path: absolute path to start from. Defaults to `/`. -- max_depth: recursion depth limit (default 8). -- page_size: maximum number of entries returned (max 1000). -- include_files / include_dirs: filter returned entry types. - -Returns JSON with: -- entries: [{path, is_dir, size, modified_at, depth}] -- truncated: true when additional entries were omitted due to page_size. -""" - -_DESKTOP_GREP_TOOL_DESCRIPTION = """Search for a literal text pattern across files. - -Searches files on disk and any in-memory edits. Returns real line numbers. -""" - -_DESKTOP_MKDIR_TOOL_DESCRIPTION = """Creates a directory on disk. - -Args: -- path: absolute mount-prefixed path of the new directory. - -Notes: -- Parent folders are created as needed. -""" - -_DESKTOP_RM_TOOL_DESCRIPTION = """Deletes a single file from disk. - -Mirrors POSIX `rm path` (no `-r`, no glob expansion). The deletion hits -disk immediately. Desktop deletes are NOT reversible via the agent's -revert flow. - -Args: -- path: absolute mount-prefixed file path. Cannot point at a directory — - use `rmdir` for empty folders. -""" - -_DESKTOP_RMDIR_TOOL_DESCRIPTION = """Deletes an empty directory from disk. - -Mirrors POSIX `rmdir path`: refuses non-empty directories. Recursive -deletion is NOT supported. The deletion hits disk immediately and is -NOT reversible via the agent's revert flow. - -Args: -- path: absolute mount-prefixed directory path. Cannot target the mount - root or any directory containing files/subfolders. -""" - - -def _build_tool_descriptions(filesystem_mode: FilesystemMode) -> dict[str, str]: - """Pick the active-mode description for every filesystem tool.""" - if filesystem_mode == FilesystemMode.CLOUD: - return { - "ls": _CLOUD_LIST_FILES_TOOL_DESCRIPTION, - "read_file": SURFSENSE_READ_FILE_TOOL_DESCRIPTION, - "write_file": _CLOUD_WRITE_FILE_TOOL_DESCRIPTION, - "edit_file": _CLOUD_EDIT_FILE_TOOL_DESCRIPTION, - "move_file": _CLOUD_MOVE_FILE_TOOL_DESCRIPTION, - "list_tree": _CLOUD_LIST_TREE_TOOL_DESCRIPTION, - "glob": SURFSENSE_GLOB_TOOL_DESCRIPTION, - "grep": _CLOUD_GREP_TOOL_DESCRIPTION, - "mkdir": _CLOUD_MKDIR_TOOL_DESCRIPTION, - "cd": SURFSENSE_CD_TOOL_DESCRIPTION, - "pwd": SURFSENSE_PWD_TOOL_DESCRIPTION, - "rm": _CLOUD_RM_TOOL_DESCRIPTION, - "rmdir": _CLOUD_RMDIR_TOOL_DESCRIPTION, - } - return { - "ls": _DESKTOP_LIST_FILES_TOOL_DESCRIPTION, - "read_file": SURFSENSE_READ_FILE_TOOL_DESCRIPTION, - "write_file": _DESKTOP_WRITE_FILE_TOOL_DESCRIPTION, - "edit_file": _DESKTOP_EDIT_FILE_TOOL_DESCRIPTION, - "move_file": _DESKTOP_MOVE_FILE_TOOL_DESCRIPTION, - "list_tree": _DESKTOP_LIST_TREE_TOOL_DESCRIPTION, - "glob": SURFSENSE_GLOB_TOOL_DESCRIPTION, - "grep": _DESKTOP_GREP_TOOL_DESCRIPTION, - "mkdir": _DESKTOP_MKDIR_TOOL_DESCRIPTION, - "cd": SURFSENSE_CD_TOOL_DESCRIPTION, - "pwd": SURFSENSE_PWD_TOOL_DESCRIPTION, - "rm": _DESKTOP_RM_TOOL_DESCRIPTION, - "rmdir": _DESKTOP_RMDIR_TOOL_DESCRIPTION, - } - - -# Backwards-compatible aliases retained for any external imports/tests that -# referenced the original CLOUD-flavoured constants. -SURFSENSE_LIST_FILES_TOOL_DESCRIPTION = _CLOUD_LIST_FILES_TOOL_DESCRIPTION -SURFSENSE_WRITE_FILE_TOOL_DESCRIPTION = _CLOUD_WRITE_FILE_TOOL_DESCRIPTION -SURFSENSE_EDIT_FILE_TOOL_DESCRIPTION = _CLOUD_EDIT_FILE_TOOL_DESCRIPTION -SURFSENSE_MOVE_FILE_TOOL_DESCRIPTION = _CLOUD_MOVE_FILE_TOOL_DESCRIPTION -SURFSENSE_LIST_TREE_TOOL_DESCRIPTION = _CLOUD_LIST_TREE_TOOL_DESCRIPTION -SURFSENSE_GREP_TOOL_DESCRIPTION = _CLOUD_GREP_TOOL_DESCRIPTION -SURFSENSE_MKDIR_TOOL_DESCRIPTION = _CLOUD_MKDIR_TOOL_DESCRIPTION - - -# ============================================================================= -# Helpers -# ============================================================================= - - -_TEMP_PREFIX = "temp_" - - -def _basename(path: str) -> str: - return path.rsplit("/", 1)[-1] - - -def _is_ancestor_of(candidate: str, target: str) -> bool: - """True iff ``candidate`` is a strict ancestor directory of ``target``. - - ``target`` itself is NOT considered an ancestor (use equality for that). - Both paths are assumed to be canonicalised, absolute, and free of - trailing slashes (except the root ``/``). - """ - if not candidate.startswith("/") or not target.startswith("/"): - return False - if candidate == target: - return False - prefix = candidate.rstrip("/") + "/" - return target.startswith(prefix) - - -class SurfSenseFilesystemMiddleware(FilesystemMiddleware): - """SurfSense-specific filesystem middleware (cloud + desktop).""" - - state_schema = SurfSenseFilesystemState - - _MAX_EXECUTE_TIMEOUT = 300 - - def __init__( - self, - *, - backend: Any = None, - filesystem_mode: FilesystemMode = FilesystemMode.CLOUD, - search_space_id: int | None = None, - created_by_id: str | None = None, - thread_id: int | str | None = None, - tool_token_limit_before_evict: int | None = 20000, - ) -> None: - self._filesystem_mode = filesystem_mode - self._search_space_id = search_space_id - self._created_by_id = created_by_id - self._thread_id = thread_id - self._sandbox_available = is_sandbox_enabled() and thread_id is not None - - # Build the prompt + tool descriptions for the active mode only — - # mixing both modes wastes tokens and confuses the model with rules - # it can't actually use this session. - system_prompt = _build_filesystem_system_prompt( - filesystem_mode, - sandbox_available=self._sandbox_available, - ) - - super().__init__( - backend=backend, - system_prompt=system_prompt, - custom_tool_descriptions=_build_tool_descriptions(filesystem_mode), - tool_token_limit_before_evict=tool_token_limit_before_evict, - max_execute_timeout=self._MAX_EXECUTE_TIMEOUT, - ) - self.tools = [t for t in self.tools if t.name != "execute"] - self.tools.append(self._create_mkdir_tool()) - self.tools.append(self._create_cd_tool()) - self.tools.append(self._create_pwd_tool()) - self.tools.append(self._create_move_file_tool()) - self.tools.append(self._create_rm_tool()) - self.tools.append(self._create_rmdir_tool()) - self.tools.append(self._create_list_tree_tool()) - if self._sandbox_available: - self.tools.append(self._create_execute_code_tool()) - - # ------------------------------------------------------------------ helpers - - def _is_cloud(self) -> bool: - return self._filesystem_mode == FilesystemMode.CLOUD - - @staticmethod - def _run_async_blocking(coro: Any) -> Any: - try: - loop = asyncio.get_running_loop() - if loop.is_running(): - return "Error: sync filesystem operation not supported inside an active event loop." - except RuntimeError: - pass - return asyncio.run(coro) - - @staticmethod - def _normalize_absolute_path(candidate: str) -> str: - normalized = re.sub(r"/+", "/", candidate.strip().replace("\\", "/")) - if not normalized: - return "/" - if normalized.startswith("/"): - return normalized - return f"/{normalized.lstrip('/')}" - - @staticmethod - def _extract_mount_from_path(path: str, mounts: tuple[str, ...]) -> str | None: - rel = path.lstrip("/") - if not rel: - return None - mount, _, _ = rel.partition("/") - if mount in mounts: - return mount - return None - - @staticmethod - def _local_parent_path(path: str) -> str: - rel = path.lstrip("/") - if "/" not in rel: - return "/" - parent = rel.rsplit("/", 1)[0].strip("/") - if not parent: - return "/" - return f"/{parent}" - - @staticmethod - def _path_exists_under_mount( - backend: MultiRootLocalFolderBackend, - mount: str, - local_path: str, - ) -> bool: - result = backend.list_tree( - f"/{mount}{local_path}", - max_depth=0, - page_size=1, - include_files=True, - include_dirs=True, - ) - return not bool(result.get("error")) - - def _normalize_local_mount_path( - self, - candidate: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - normalized = self._normalize_absolute_path(candidate) - backend = self._get_backend(runtime) - if not isinstance(backend, MultiRootLocalFolderBackend): - return normalized - - mounts = backend.list_mounts() - explicit_mount = self._extract_mount_from_path(normalized, mounts) - if explicit_mount: - return normalized - - if len(mounts) == 1: - return f"/{mounts[0]}{normalized}" - - suggested_mount: str | None = None - contract = runtime.state.get("file_operation_contract") or {} - suggested_path = contract.get("suggested_path") - if isinstance(suggested_path, str) and suggested_path.strip(): - normalized_suggested = self._normalize_absolute_path(suggested_path) - suggested_mount = self._extract_mount_from_path( - normalized_suggested, mounts - ) - - matching_mounts = [ - mount - for mount in mounts - if self._path_exists_under_mount(backend, mount, normalized) - ] - if len(matching_mounts) == 1: - return f"/{matching_mounts[0]}{normalized}" - - parent_path = self._local_parent_path(normalized) - if parent_path != "/": - parent_matching_mounts = [ - mount - for mount in mounts - if self._path_exists_under_mount(backend, mount, parent_path) - ] - if len(parent_matching_mounts) == 1: - return f"/{parent_matching_mounts[0]}{normalized}" - - if suggested_mount: - return f"/{suggested_mount}{normalized}" - - return f"/{backend.default_mount()}{normalized}" - - def _default_cwd(self) -> str: - return DOCUMENTS_ROOT if self._is_cloud() else "/" - - def _current_cwd(self, runtime: ToolRuntime[None, SurfSenseFilesystemState]) -> str: - cwd = runtime.state.get("cwd") if hasattr(runtime, "state") else None - if isinstance(cwd, str) and cwd.startswith("/"): - return cwd - return self._default_cwd() - - def _get_contract_suggested_path( - self, runtime: ToolRuntime[None, SurfSenseFilesystemState] - ) -> str: - contract = runtime.state.get("file_operation_contract") or {} - suggested = contract.get("suggested_path") - if isinstance(suggested, str) and suggested.strip(): - return self._normalize_absolute_path(suggested) - return self._default_cwd().rstrip("/") + "/notes.md" - - def _resolve_relative( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = path.strip() - if not candidate: - return self._current_cwd(runtime) - if candidate.startswith("/"): - return self._normalize_absolute_path(candidate) - cwd = self._current_cwd(runtime) - joined = posixpath.normpath(posixpath.join(cwd, candidate)) - return self._normalize_absolute_path(joined) - - def _resolve_write_target_path( - self, - file_path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = file_path.strip() - if not candidate: - return self._get_contract_suggested_path(runtime) - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - def _resolve_move_target_path( - self, - file_path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = file_path.strip() - if not candidate: - return "" - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - def _resolve_list_target_path( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = path.strip() or self._current_cwd(runtime) - if candidate == "/": - return "/" - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - # ------------------------------------------------------------------ namespace policy - - def _check_cloud_write_namespace( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str | None: - """Return an error string if cloud writes to ``path`` are not allowed. - - Order matters: - 1. Reject writes to the anonymous read-only doc. - 2. Allow ``/documents/*``. - 3. Allow ``temp_*`` basename anywhere. - 4. Reject everything else. - """ - if not self._is_cloud(): - return None - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict): - anon_path = str(anon.get("path") or "") - if anon_path and anon_path == path: - return "Error: the anonymous uploaded document is read-only." - if path.startswith(DOCUMENTS_ROOT + "/") or path == DOCUMENTS_ROOT: - return None - if _basename(path).startswith(_TEMP_PREFIX): - return None - return ( - "Error: cloud writes must target /documents/<...> or use a 'temp_' " - f"basename for scratch (got '{path}')." - ) - - # ------------------------------------------------------------------ tool: ls - - def _create_ls_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("ls") - or SURFSENSE_LIST_FILES_TOOL_DESCRIPTION - ) - - def sync_ls( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to the directory to list. Relative paths resolve against the current cwd.", - ] = "", - offset: Annotated[ - int, - "Number of entries to skip. Use for paginating large folders. Defaults to 0.", - ] = 0, - limit: Annotated[ - int, - "Maximum number of entries to return. Defaults to 200.", - ] = 200, - ) -> str: - return self._run_async_blocking( - async_ls(runtime, path=path, offset=offset, limit=limit) - ) - - async def async_ls( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to the directory to list. Relative paths resolve against the current cwd.", - ] = "", - offset: Annotated[ - int, - "Number of entries to skip. Use for paginating large folders. Defaults to 0.", - ] = 0, - limit: Annotated[ - int, - "Maximum number of entries to return. Defaults to 200.", - ] = 200, - ) -> str: - target = self._resolve_list_target_path(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - if offset < 0: - offset = 0 - if limit < 1: - limit = 1 - backend = self._get_backend(runtime) - infos = await backend.als_info(validated) - page = paginate_listing(infos, offset=offset, limit=limit) - paths = [ - f"{fi.get('path', '')}/" if fi.get("is_dir") else fi.get("path", "") - for fi in page - ] - total = len(infos) - shown = len(page) - header = ( - f"{validated} ({shown} of {total} entries" - f"{f', offset={offset}' if offset else ''})" - ) - if not paths: - return f"{header}\n(empty)" - body = "\n".join(paths) - if total > offset + shown: - body += ( - f"\n... {total - offset - shown} more — call ls(" - f"'{validated}', offset={offset + shown}, limit={limit})" - ) - return f"{header}\n{body}" - - return StructuredTool.from_function( - name="ls", - description=tool_description, - func=sync_ls, - coroutine=async_ls, - ) - - # ------------------------------------------------------------------ tool: read_file - - def _create_read_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("read_file") - or SURFSENSE_READ_FILE_TOOL_DESCRIPTION - ) - - async def async_read_file( - file_path: Annotated[ - str, - "Absolute path to the file to read. Relative paths resolve against the current cwd.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - offset: Annotated[ - int, - "Line number to start reading from (0-indexed).", - ] = 0, - limit: Annotated[ - int, - "Maximum number of lines to read.", - ] = 100, - ) -> Command | str: - target = self._resolve_relative(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - files = runtime.state.get("files") or {} - if validated in files: - return format_read_response(files[validated], offset, limit) - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: File '{validated}' not found" - file_data, doc_id = loaded - rendered = format_read_response(file_data, offset, limit) - update: dict[str, Any] = { - "files": {validated: file_data}, - "messages": [ - ToolMessage( - content=rendered, - tool_call_id=runtime.tool_call_id, - ) - ], - } - if doc_id is not None: - update["doc_id_by_path"] = {validated: doc_id} - return Command(update=update) - - try: - rendered = await backend.aread(validated, offset=offset, limit=limit) - except Exception as exc: # pragma: no cover - defensive - return f"Error: {exc}" - return rendered - - def sync_read_file( - file_path: Annotated[ - str, - "Absolute path to the file to read. Relative paths resolve against the current cwd.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - offset: Annotated[ - int, - "Line number to start reading from (0-indexed).", - ] = 0, - limit: Annotated[ - int, - "Maximum number of lines to read.", - ] = 100, - ) -> Command | str: - return self._run_async_blocking( - async_read_file(file_path, runtime, offset, limit) - ) - - return StructuredTool.from_function( - name="read_file", - description=tool_description, - func=sync_read_file, - coroutine=async_read_file, - ) - - # ------------------------------------------------------------------ tool: write_file - - def _create_write_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("write_file") - or SURFSENSE_WRITE_FILE_TOOL_DESCRIPTION - ) - - async def async_write_file( - file_path: Annotated[ - str, - "Absolute path where the file should be created. Relative paths resolve against the current cwd.", - ], - content: Annotated[str, "Text content to write to the file."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_write_target_path(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - namespace_error = self._check_cloud_write_namespace(validated, runtime) - if namespace_error: - return namespace_error - - backend = self._get_backend(runtime) - res: WriteResult = await backend.awrite(validated, content) - if res.error: - return res.error - - path = res.path or validated - files_update = res.files_update or {path: create_file_data(content)} - update: dict[str, Any] = { - "files": files_update, - "messages": [ - ToolMessage( - content=f"Updated file {path}", - tool_call_id=runtime.tool_call_id, - ) - ], - } - if self._is_cloud(): - update["dirty_paths"] = [path] - update["dirty_path_tool_calls"] = {path: runtime.tool_call_id} - return Command(update=update) - - def sync_write_file( - file_path: Annotated[ - str, - "Absolute path where the file should be created. Relative paths resolve against the current cwd.", - ], - content: Annotated[str, "Text content to write to the file."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking( - async_write_file(file_path, content, runtime) - ) - - return StructuredTool.from_function( - name="write_file", - description=tool_description, - func=sync_write_file, - coroutine=async_write_file, - ) - - # ------------------------------------------------------------------ tool: edit_file - - def _create_edit_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("edit_file") - or SURFSENSE_EDIT_FILE_TOOL_DESCRIPTION - ) - - async def async_edit_file( - file_path: Annotated[ - str, - "Absolute path to the file to edit. Relative paths resolve against the current cwd.", - ], - old_string: Annotated[ - str, - "Exact text to replace. Must be unique unless replace_all is True.", - ], - new_string: Annotated[ - str, - "Replacement text. Must differ from old_string.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - replace_all: Annotated[ - bool, - "If True, replace all occurrences of old_string. Defaults to False.", - ] = False, - ) -> Command | str: - target = self._resolve_relative(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - namespace_error = self._check_cloud_write_namespace(validated, runtime) - if namespace_error: - return namespace_error - - backend = self._get_backend(runtime) - files_state = runtime.state.get("files") or {} - doc_id_to_attach: int | None = None - - if ( - self._is_cloud() - and validated not in files_state - and isinstance(backend, KBPostgresBackend) - ): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: File '{validated}' not found" - _, doc_id_to_attach = loaded - - res: EditResult = await backend.aedit( - validated, old_string, new_string, replace_all=replace_all - ) - if res.error: - return res.error - - path = res.path or validated - files_update = res.files_update or {} - update: dict[str, Any] = { - "files": files_update, - "messages": [ - ToolMessage( - content=( - f"Successfully replaced {res.occurrences} instance(s) " - f"of the string in '{path}'" - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - if self._is_cloud(): - update["dirty_paths"] = [path] - update["dirty_path_tool_calls"] = {path: runtime.tool_call_id} - if doc_id_to_attach is not None: - update["doc_id_by_path"] = {path: doc_id_to_attach} - return Command(update=update) - - def sync_edit_file( - file_path: Annotated[ - str, - "Absolute path to the file to edit. Relative paths resolve against the current cwd.", - ], - old_string: Annotated[ - str, - "Exact text to replace. Must be unique unless replace_all is True.", - ], - new_string: Annotated[ - str, - "Replacement text. Must differ from old_string.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - replace_all: Annotated[ - bool, - "If True, replace all occurrences of old_string. Defaults to False.", - ] = False, - ) -> Command | str: - return self._run_async_blocking( - async_edit_file( - file_path, old_string, new_string, runtime, replace_all=replace_all - ) - ) - - return StructuredTool.from_function( - name="edit_file", - description=tool_description, - func=sync_edit_file, - coroutine=async_edit_file, - ) - - # ------------------------------------------------------------------ tool: mkdir - - def _create_mkdir_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("mkdir") - or SURFSENSE_MKDIR_TOOL_DESCRIPTION - ) - - async def async_mkdir( - path: Annotated[str, "Absolute or relative directory path to create."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if not ( - validated.startswith(DOCUMENTS_ROOT + "/") - or validated == DOCUMENTS_ROOT - ): - return ( - "Error: cloud mkdir must target a path under /documents/ " - f"(got '{validated}')." - ) - return Command( - update={ - "staged_dirs": [validated], - "staged_dir_tool_calls": { - validated: runtime.tool_call_id, - }, - "messages": [ - ToolMessage( - content=( - f"Staged directory '{validated}' (will be created " - "at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - backend = self._get_backend(runtime) - local_method = getattr(backend, "amkdir", None) or getattr( - backend, "mkdir", None - ) - if callable(local_method): - try: - res = local_method(validated, parents=True, exist_ok=True) - if asyncio.iscoroutine(res): - await res - except TypeError: - res = local_method(validated) - if asyncio.iscoroutine(res): - await res - except Exception as exc: # pragma: no cover - return f"Error: {exc}" - return f"Created directory {validated}" - - def sync_mkdir( - path: Annotated[str, "Absolute or relative directory path to create."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_mkdir(path, runtime)) - - return StructuredTool.from_function( - name="mkdir", - description=tool_description, - func=sync_mkdir, - coroutine=async_mkdir, - ) - - # ------------------------------------------------------------------ tool: cd - - def _create_cd_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("cd") or SURFSENSE_CD_TOOL_DESCRIPTION - ) - - async def async_cd( - path: Annotated[str, "Absolute or relative directory path to switch into."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - backend = self._get_backend(runtime) - try: - infos = await backend.als_info(validated) - except Exception as exc: # pragma: no cover - defensive - return f"Error: {exc}" - staged_dirs = list(runtime.state.get("staged_dirs") or []) - files = runtime.state.get("files") or {} - cwd_exists = ( - bool(infos) - or validated in staged_dirs - or any(p == validated for p in files) - or any( - isinstance(p, str) and p.startswith(validated.rstrip("/") + "/") - for p in files - ) - or validated == "/" - or validated == DOCUMENTS_ROOT - ) - if not cwd_exists: - return f"Error: directory '{validated}' not found." - return Command( - update={ - "cwd": validated, - "messages": [ - ToolMessage( - content=f"cwd changed to {validated}", - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - def sync_cd( - path: Annotated[str, "Absolute or relative directory path to switch into."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_cd(path, runtime)) - - return StructuredTool.from_function( - name="cd", - description=tool_description, - func=sync_cd, - coroutine=async_cd, - ) - - # ------------------------------------------------------------------ tool: pwd - - def _create_pwd_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("pwd") or SURFSENSE_PWD_TOOL_DESCRIPTION - ) - - def sync_pwd( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - return self._current_cwd(runtime) - - async def async_pwd( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - return self._current_cwd(runtime) - - return StructuredTool.from_function( - name="pwd", - description=tool_description, - func=sync_pwd, - coroutine=async_pwd, - ) - - # ------------------------------------------------------------------ tool: move_file - - def _create_move_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("move_file") - or SURFSENSE_MOVE_FILE_TOOL_DESCRIPTION - ) - - async def async_move_file( - source_path: Annotated[str, "Absolute or relative source path."], - destination_path: Annotated[str, "Absolute or relative destination path."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - overwrite: Annotated[ - bool, - "If True, replace existing destination. Cloud mode rejects True. Defaults to False.", - ] = False, - ) -> Command | str: - if not source_path.strip() or not destination_path.strip(): - return "Error: source_path and destination_path are required." - - source = self._resolve_move_target_path(source_path, runtime) - dest = self._resolve_move_target_path(destination_path, runtime) - try: - validated_source = validate_path(source) - validated_dest = validate_path(dest) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - return await self._cloud_move_file( - runtime, - validated_source, - validated_dest, - overwrite=overwrite, - ) - - backend = self._get_backend(runtime) - res: WriteResult = await backend.amove( - validated_source, validated_dest, overwrite=overwrite - ) - if res.error: - return res.error - update: dict[str, Any] = { - "messages": [ - ToolMessage( - content=f"Moved '{validated_source}' to '{res.path or validated_dest}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - if res.files_update is not None: - update["files"] = res.files_update - return Command(update=update) - - def sync_move_file( - source_path: Annotated[str, "Absolute or relative source path."], - destination_path: Annotated[str, "Absolute or relative destination path."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - overwrite: Annotated[ - bool, - "If True, replace existing destination. Cloud mode rejects True. Defaults to False.", - ] = False, - ) -> Command | str: - return self._run_async_blocking( - async_move_file( - source_path, destination_path, runtime, overwrite=overwrite - ) - ) - - return StructuredTool.from_function( - name="move_file", - description=tool_description, - func=sync_move_file, - coroutine=async_move_file, - ) - - async def _cloud_move_file( - self, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - source: str, - dest: str, - *, - overwrite: bool, - ) -> Command | str: - backend = self._get_backend(runtime) - if not isinstance(backend, KBPostgresBackend): - return "Error: cloud move requires KBPostgresBackend." - - if source == dest: - return f"Moved '{source}' to '{dest}' (no-op)" - if overwrite: - return ( - "Error: overwrite=True is not supported in cloud mode. Move/edit " - "the destination doc explicitly first." - ) - if not source.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud move_file source must be under /documents/ (got " - f"'{source}')." - ) - if not dest.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud move_file destination must be under /documents/ (got " - f"'{dest}')." - ) - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict): - anon_path = str(anon.get("path") or "") - if anon_path and (anon_path in (source, dest)): - return "Error: the anonymous uploaded document is read-only." - - files = runtime.state.get("files") or {} - doc_id_by_path = runtime.state.get("doc_id_by_path") or {} - pending_moves = list(runtime.state.get("pending_moves") or []) - - # Dest collision: occupied in state, in pending moves, or in DB. - if dest in files: - return f"Error: destination '{dest}' already exists." - if any(move.get("dest") == dest for move in pending_moves): - return f"Error: destination '{dest}' already exists." - if dest != source: - existing_dest = await backend._load_file_data(dest) - if existing_dest is not None: - return f"Error: destination '{dest}' already exists." - - # Source materialization: lazy load if not in state. - source_file_data = files.get(source) - source_doc_id = doc_id_by_path.get(source) - if source_file_data is None: - loaded = await backend._load_file_data(source) - if loaded is None: - return f"Error: source '{source}' not found." - source_file_data, loaded_doc_id = loaded - if source_doc_id is None: - source_doc_id = loaded_doc_id - - files_update: dict[str, Any] = {source: None, dest: source_file_data} - update: dict[str, Any] = { - "files": files_update, - "pending_moves": [ - { - "source": source, - "dest": dest, - "overwrite": False, - "tool_call_id": runtime.tool_call_id, - } - ], - "messages": [ - ToolMessage( - content=( - f"Moved '{source}' to '{dest}' (will commit at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - - doc_id_update: dict[str, int | None] = {source: None} - if source_doc_id is not None: - doc_id_update[dest] = source_doc_id - update["doc_id_by_path"] = doc_id_update - - dirty_paths = list(runtime.state.get("dirty_paths") or []) - if source in dirty_paths: - new_dirty: list[Any] = [_CLEAR] - for entry in dirty_paths: - new_dirty.append(dest if entry == source else entry) - update["dirty_paths"] = new_dirty - return Command(update=update) - - # ------------------------------------------------------------------ tool: rm - - def _create_rm_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("rm") or _CLOUD_RM_TOOL_DESCRIPTION - ) - - async def async_rm( - path: Annotated[ - str, - "Absolute or relative path to the file to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - if not path or not path.strip(): - return "Error: path is required." - - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if validated in ("/", DOCUMENTS_ROOT): - return f"Error: refusing to rm '{validated}'." - if not validated.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud rm must target a path under /documents/ " - f"(got '{validated}')." - ) - - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict) and str(anon.get("path") or "") == validated: - return "Error: the anonymous uploaded document is read-only." - - # Refuse if the path looks like a directory. - staged_dirs = list(runtime.state.get("staged_dirs") or []) - if validated in staged_dirs: - return ( - f"Error: '{validated}' is a directory. Use rmdir for " - "empty directories." - ) - pending_dir_deletes = list( - runtime.state.get("pending_dir_deletes") or [] - ) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_dir_deletes - ): - return f"Error: '{validated}' is already queued for rmdir." - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - # Detect "is a directory" via `ls`: if the path lists - # children we know it's a folder. Otherwise we still - # need to confirm it's a real file before staging. - children = await backend.als_info(validated) - if children: - return ( - f"Error: '{validated}' is a directory. Use rmdir for " - "empty directories." - ) - - # Already queued for delete this turn? - pending_deletes = list(runtime.state.get("pending_deletes") or []) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_deletes - ): - return f"'{validated}' is already queued for deletion." - - # Resolve doc_id (best-effort): file in state or DB. - files_state = runtime.state.get("files") or {} - doc_id_by_path = runtime.state.get("doc_id_by_path") or {} - resolved_doc_id: int | None = doc_id_by_path.get(validated) - if ( - validated not in files_state - and resolved_doc_id is None - and isinstance(backend, KBPostgresBackend) - ): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: file '{validated}' not found." - _, resolved_doc_id = loaded - - files_update: dict[str, Any] = {validated: None} - update: dict[str, Any] = { - "pending_deletes": [ - { - "path": validated, - "tool_call_id": runtime.tool_call_id, - } - ], - "files": files_update, - "doc_id_by_path": {validated: None}, - "messages": [ - ToolMessage( - content=( - f"Staged delete of '{validated}' (will commit at " - "end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - - # Drop the path from dirty_paths so a same-turn write+rm - # doesn't recreate the doc at commit time. - dirty_paths = list(runtime.state.get("dirty_paths") or []) - if validated in dirty_paths: - new_dirty: list[Any] = [_CLEAR] - for entry in dirty_paths: - if entry != validated: - new_dirty.append(entry) - update["dirty_paths"] = new_dirty - update["dirty_path_tool_calls"] = {validated: None} - - return Command(update=update) - - # Desktop mode — hit disk immediately. - backend = self._get_backend(runtime) - adelete = getattr(backend, "adelete_file", None) - if not callable(adelete): - return "Error: rm is not supported by the active backend." - res: WriteResult = await adelete(validated) - if res.error: - return res.error - update_desktop: dict[str, Any] = { - "files": {validated: None}, - "messages": [ - ToolMessage( - content=f"Deleted file '{res.path or validated}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - return Command(update=update_desktop) - - def sync_rm( - path: Annotated[ - str, - "Absolute or relative path to the file to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_rm(path, runtime)) - - return StructuredTool.from_function( - name="rm", - description=tool_description, - func=sync_rm, - coroutine=async_rm, - ) - - # ------------------------------------------------------------------ tool: rmdir - - def _create_rmdir_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("rmdir") or _CLOUD_RMDIR_TOOL_DESCRIPTION - ) - - async def async_rmdir( - path: Annotated[ - str, - "Absolute or relative path of the empty directory to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - if not path or not path.strip(): - return "Error: path is required." - - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if validated in ("/", DOCUMENTS_ROOT): - return f"Error: refusing to rmdir '{validated}'." - if not validated.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud rmdir must target a path under /documents/ " - f"(got '{validated}')." - ) - - cwd = self._current_cwd(runtime) - if validated == cwd or _is_ancestor_of(validated, cwd): - return ( - f"Error: cannot rmdir '{validated}' because the current " - "cwd is at or under it. cd out first." - ) - - staged_dirs = list(runtime.state.get("staged_dirs") or []) - pending_dir_deletes = list( - runtime.state.get("pending_dir_deletes") or [] - ) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_dir_deletes - ): - return f"'{validated}' is already queued for deletion." - - backend = self._get_backend(runtime) - - # The path must currently exist either in DB folder paths or - # in staged_dirs. We rely on KBPostgresBackend.als_info (which - # already accounts for pending deletes/moves) to evaluate - # both existence and emptiness against the post-staged view. - exists_in_staged = validated in staged_dirs - children: list[Any] = [] - if isinstance(backend, KBPostgresBackend): - children = list(await backend.als_info(validated)) - - # Detect "is a file" — if als_info returns no children but - # the path is actually a file, we should reject. We use - # _load_file_data to disambiguate file vs missing folder. - if ( - isinstance(backend, KBPostgresBackend) - and not children - and not exists_in_staged - ): - loaded = await backend._load_file_data(validated) - if loaded is not None: - return ( - f"Error: '{validated}' is a file. Use rm to delete files." - ) - # Confirm folder exists in DB by checking the parent listing. - parent = posixpath.dirname(validated) or "/" - parent_listing = await backend.als_info(parent) - parent_has_dir = any( - info.get("path") == validated and info.get("is_dir") - for info in parent_listing - ) - if not parent_has_dir: - return f"Error: directory '{validated}' not found." - - if children: - return ( - f"Error: directory '{validated}' is not empty. " - "Remove contents first." - ) - - # Same-turn mkdir un-stage: drop the staged_dirs entry - # entirely and skip queuing a DB delete (nothing was ever - # committed). - if exists_in_staged: - rest = [d for d in staged_dirs if d != validated] - return Command( - update={ - "staged_dirs": [_CLEAR, *rest], - "staged_dir_tool_calls": {validated: None}, - "messages": [ - ToolMessage( - content=(f"Un-staged directory '{validated}'."), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - return Command( - update={ - "pending_dir_deletes": [ - { - "path": validated, - "tool_call_id": runtime.tool_call_id, - } - ], - "messages": [ - ToolMessage( - content=( - f"Staged rmdir of '{validated}' (will commit " - "at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - # Desktop mode — hit disk immediately. - backend = self._get_backend(runtime) - armdir = getattr(backend, "armdir", None) - if not callable(armdir): - return "Error: rmdir is not supported by the active backend." - res: WriteResult = await armdir(validated) - if res.error: - return res.error - return Command( - update={ - "messages": [ - ToolMessage( - content=f"Deleted directory '{res.path or validated}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - def sync_rmdir( - path: Annotated[ - str, - "Absolute or relative path of the empty directory to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_rmdir(path, runtime)) - - return StructuredTool.from_function( - name="rmdir", - description=tool_description, - func=sync_rmdir, - coroutine=async_rmdir, - ) - - # ------------------------------------------------------------------ tool: list_tree - - def _create_list_tree_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("list_tree") - or SURFSENSE_LIST_TREE_TOOL_DESCRIPTION - ) - - async def async_list_tree( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to start from. Defaults to /documents in cloud mode.", - ] = "", - max_depth: Annotated[int, "Recursion depth limit. Default 8."] = 8, - page_size: Annotated[int, "Maximum entries returned. Max 1000."] = 500, - include_files: Annotated[bool, "Include file entries."] = True, - include_dirs: Annotated[bool, "Include directory entries."] = True, - ) -> str: - if max_depth < 0: - return "Error: max_depth must be >= 0." - if page_size < 1: - return "Error: page_size must be >= 1." - if not include_files and not include_dirs: - return "Error: include_files and include_dirs cannot both be false." - - target = self._resolve_list_target_path(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - result = await backend.alist_tree_listing( - validated, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - elif hasattr(backend, "alist_tree"): - result = await backend.alist_tree( - validated, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - else: - return "Error: list_tree is not supported by the active backend." - - if isinstance(result, dict) and isinstance(result.get("error"), str): - return result["error"] - return json.dumps(result, ensure_ascii=True) - - def sync_list_tree( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to start from. Defaults to /documents in cloud mode.", - ] = "", - max_depth: Annotated[int, "Recursion depth limit. Default 8."] = 8, - page_size: Annotated[int, "Maximum entries returned. Max 1000."] = 500, - include_files: Annotated[bool, "Include file entries."] = True, - include_dirs: Annotated[bool, "Include directory entries."] = True, - ) -> str: - return self._run_async_blocking( - async_list_tree( - runtime, - path=path, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - ) - - return StructuredTool.from_function( - name="list_tree", - description=tool_description, - func=sync_list_tree, - coroutine=async_list_tree, - ) - - # ------------------------------------------------------------------ tool: execute_code (sandbox) - - def _create_execute_code_tool(self) -> BaseTool: - def sync_execute_code( - command: Annotated[ - str, "Python code to execute. Use print() to see output." - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: Annotated[ - int | None, - "Optional timeout in seconds.", - ] = None, - ) -> str: - if timeout is not None: - if timeout < 0: - return f"Error: timeout must be non-negative, got {timeout}." - if timeout > self._MAX_EXECUTE_TIMEOUT: - return f"Error: timeout {timeout}s exceeds maximum ({self._MAX_EXECUTE_TIMEOUT}s)." - return self._run_async_blocking( - self._execute_in_sandbox(command, runtime, timeout) - ) - - async def async_execute_code( - command: Annotated[ - str, "Python code to execute. Use print() to see output." - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: Annotated[ - int | None, - "Optional timeout in seconds.", - ] = None, - ) -> str: - if timeout is not None: - if timeout < 0: - return f"Error: timeout must be non-negative, got {timeout}." - if timeout > self._MAX_EXECUTE_TIMEOUT: - return f"Error: timeout {timeout}s exceeds maximum ({self._MAX_EXECUTE_TIMEOUT}s)." - return await self._execute_in_sandbox(command, runtime, timeout) - - return StructuredTool.from_function( - name="execute_code", - description=SURFSENSE_EXECUTE_CODE_TOOL_DESCRIPTION, - func=sync_execute_code, - coroutine=async_execute_code, - ) - - @staticmethod - def _wrap_as_python(code: str) -> str: - sentinel = f"_PYEOF_{secrets.token_hex(8)}" - return f"python3 << '{sentinel}'\n{code}\n{sentinel}" - - async def _execute_in_sandbox( - self, - command: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: int | None, - ) -> str: - assert self._thread_id is not None - command = self._wrap_as_python(command) - try: - return await self._try_sandbox_execute(command, runtime, timeout) - except (DaytonaError, Exception) as first_err: - logger.warning( - "Sandbox execute failed for thread %s, retrying: %s", - self._thread_id, - first_err, - ) - try: - await delete_sandbox(self._thread_id) - except Exception: - _evict_sandbox_cache(self._thread_id) - try: - return await self._try_sandbox_execute(command, runtime, timeout) - except Exception: - logger.exception( - "Sandbox retry also failed for thread %s", self._thread_id - ) - return "Error: Code execution is temporarily unavailable. Please try again." - - async def _try_sandbox_execute( - self, - command: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: int | None, - ) -> str: - sandbox, _is_new = await get_or_create_sandbox(self._thread_id) - result = await sandbox.aexecute(command, timeout=timeout) - output = (result.output or "").strip() - if not output and result.exit_code == 0: - return ( - "[Code executed successfully but produced no output. " - "Use print() to display results, then try again.]" - ) - parts = [result.output] - if result.exit_code is not None: - status = "succeeded" if result.exit_code == 0 else "failed" - parts.append(f"\n[Command {status} with exit code {result.exit_code}]") - if result.truncated: - parts.append("\n[Output was truncated due to size limits]") - return "".join(parts) diff --git a/surfsense_backend/app/agents/new_chat/middleware/flatten_system.py b/surfsense_backend/app/agents/new_chat/middleware/flatten_system.py deleted file mode 100644 index 29cd57aa0..000000000 --- a/surfsense_backend/app/agents/new_chat/middleware/flatten_system.py +++ /dev/null @@ -1,233 +0,0 @@ -r"""Coalesce multi-block system messages into a single text block. - -Several middlewares in our deepagent stack each call -``append_to_system_message`` on the way down to the model -(``TodoListMiddleware``, ``SurfSenseFilesystemMiddleware``, -``SkillsMiddleware``, ``SubAgentMiddleware`` …). By the time the -request reaches the LLM, the system message has 5+ separate text blocks. - -Anthropic enforces a hard cap of **4 ``cache_control`` blocks per -request**, and we configure 2 injection points -(``index: 0`` + ``index: -1``). With ``index: 0`` always targeting -the prepended ``request.system_message``, this middleware is the -defensive partner: it guarantees that "the system block" is *one* -content block, so LiteLLM's ``AnthropicCacheControlHook`` and any -OpenRouter→Anthropic transformer can never multiply our budget into -several breakpoints by spreading ``cache_control`` across multiple -text blocks of a multi-block system content. - -Without flattening we used to see:: - - OpenrouterException - {"error":{"message":"Provider returned error", - "code":400,"metadata":{"raw":"...A maximum of 4 blocks with - cache_control may be provided. Found 5."}}} - -(Same error class documented in -https://github.com/BerriAI/litellm/issues/15696 and -https://github.com/BerriAI/litellm/issues/20485 — the litellm-side fix -in PR #15395 covers the litellm transformer but does not protect us -when the OpenRouter SaaS itself does the redistribution.) - -A separate fix in :mod:`app.agents.new_chat.prompt_caching` (switching -the first injection point from ``role: system`` to ``index: 0``) -neutralises the *primary* cause of the same 400 — multiple -``SystemMessage``\ s injected by ``before_agent`` middlewares -(priority/tree/memory/file-intent/anonymous-doc) accumulating across -turns, each tagged with ``cache_control`` by the ``role: system`` -matcher. This middleware remains useful as defence-in-depth against -the multi-block redistribution path. - -Placement: innermost on the system-message-mutation chain, after every -appender (``todo``/``filesystem``/``skills``/``subagents``) and after -summarization, but before ``noop``/``retry``/``fallback`` so each retry -attempt sees a flattened payload. See ``chat_deepagent.py``. - -Idempotent: a string-content system message is left untouched. A list -that contains anything other than plain text blocks (e.g. an image) is -also left untouched — those are rare on system messages and we'd lose -the non-text payload by joining. -""" - -from __future__ import annotations - -import logging -from collections.abc import Awaitable, Callable -from typing import Any - -from langchain.agents.middleware.types import ( - AgentMiddleware, - AgentState, - ContextT, - ModelRequest, - ModelResponse, - ResponseT, -) -from langchain_core.messages import SystemMessage - -logger = logging.getLogger(__name__) - - -def _flatten_text_blocks(content: list[Any]) -> str | None: - """Return joined text if every block is a plain ``{"type": "text"}``. - - Returns ``None`` when the list contains anything that isn't a text - block we can safely concatenate (image, audio, file, non-standard - blocks, dicts with extra non-cache_control fields). The caller - leaves the original content untouched in that case rather than - silently dropping payload. - - ``cache_control`` on individual blocks is intentionally discarded — - the whole point of flattening is to let LiteLLM's - ``cache_control_injection_points`` re-place a single breakpoint on - the resulting one-block system content. - """ - chunks: list[str] = [] - for block in content: - if isinstance(block, str): - chunks.append(block) - continue - if not isinstance(block, dict): - return None - if block.get("type") != "text": - return None - text = block.get("text") - if not isinstance(text, str): - return None - chunks.append(text) - return "\n\n".join(chunks) - - -def _flattened_request( - request: ModelRequest[ContextT], -) -> ModelRequest[ContextT] | None: - """Return a request with system_message flattened, or ``None`` for no-op.""" - sys_msg = request.system_message - if sys_msg is None: - return None - content = sys_msg.content - if not isinstance(content, list) or len(content) <= 1: - return None - - flattened = _flatten_text_blocks(content) - if flattened is None: - return None - - new_sys = SystemMessage( - content=flattened, - additional_kwargs=dict(sys_msg.additional_kwargs), - response_metadata=dict(sys_msg.response_metadata), - ) - if sys_msg.id is not None: - new_sys.id = sys_msg.id - return request.override(system_message=new_sys) - - -def _diagnostic_summary(request: ModelRequest[Any]) -> str: - """One-line dump of cache_control-relevant request shape. - - Temporary diagnostic to prove where the ``Found N`` cache_control - breakpoints are coming from when Anthropic 400s. Removed once the - root cause is confirmed and a fix is in place. - """ - sys_msg = request.system_message - if sys_msg is None: - sys_shape = "none" - elif isinstance(sys_msg.content, str): - sys_shape = f"str(len={len(sys_msg.content)})" - elif isinstance(sys_msg.content, list): - sys_shape = f"list(blocks={len(sys_msg.content)})" - else: - sys_shape = f"other({type(sys_msg.content).__name__})" - - role_hist: list[str] = [] - multi_block_msgs = 0 - msgs_with_cc = 0 - sys_msgs_in_history = 0 - for m in request.messages: - mtype = getattr(m, "type", type(m).__name__) - role_hist.append(mtype) - if isinstance(m, SystemMessage): - sys_msgs_in_history += 1 - c = getattr(m, "content", None) - if isinstance(c, list): - multi_block_msgs += 1 - for blk in c: - if isinstance(blk, dict) and "cache_control" in blk: - msgs_with_cc += 1 - break - if "cache_control" in getattr(m, "additional_kwargs", {}) or {}: - msgs_with_cc += 1 - - tools = request.tools or [] - tools_with_cc = 0 - for t in tools: - if isinstance(t, dict) and ( - "cache_control" in t or "cache_control" in t.get("function", {}) - ): - tools_with_cc += 1 - - return ( - f"sys={sys_shape} msgs={len(request.messages)} " - f"sys_msgs_in_history={sys_msgs_in_history} " - f"multi_block_msgs={multi_block_msgs} pre_existing_msg_cc={msgs_with_cc} " - f"tools={len(tools)} pre_existing_tool_cc={tools_with_cc} " - f"roles={role_hist[-8:]}" - ) - - -class FlattenSystemMessageMiddleware( - AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT] -): - """Collapse a multi-text-block system message to a single string. - - Sits innermost on the system-message-mutation chain so it observes - every middleware's contribution. Has no other side effect — the - body of every block is preserved, just joined with ``"\\n\\n"``. - """ - - def __init__(self) -> None: - super().__init__() - self.tools = [] - - def wrap_model_call( # type: ignore[override] - self, - request: ModelRequest[ContextT], - handler: Callable[[ModelRequest[ContextT]], ModelResponse[ResponseT]], - ) -> Any: - if logger.isEnabledFor(logging.DEBUG): - logger.debug("[flatten_system_diag] %s", _diagnostic_summary(request)) - flattened = _flattened_request(request) - if flattened is not None: - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "[flatten_system] collapsed %d system blocks to one", - len(request.system_message.content), # type: ignore[arg-type, union-attr] - ) - return handler(flattened) - return handler(request) - - async def awrap_model_call( # type: ignore[override] - self, - request: ModelRequest[ContextT], - handler: Callable[ - [ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]] - ], - ) -> Any: - if logger.isEnabledFor(logging.DEBUG): - logger.debug("[flatten_system_diag] %s", _diagnostic_summary(request)) - flattened = _flattened_request(request) - if flattened is not None: - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "[flatten_system] collapsed %d system blocks to one", - len(request.system_message.content), # type: ignore[arg-type, union-attr] - ) - return await handler(flattened) - return await handler(request) - - -__all__ = [ - "FlattenSystemMessageMiddleware", - "_flatten_text_blocks", - "_flattened_request", -] diff --git a/surfsense_backend/app/agents/new_chat/middleware/permission.py b/surfsense_backend/app/agents/new_chat/middleware/permission.py deleted file mode 100644 index 07549bedb..000000000 --- a/surfsense_backend/app/agents/new_chat/middleware/permission.py +++ /dev/null @@ -1,427 +0,0 @@ -""" -PermissionMiddleware — pattern-based allow/deny/ask with HITL fallback. - -LangChain's :class:`HumanInTheLoopMiddleware` only supports a static -"this tool always asks" decision per tool. There's no rule-based -allow/deny/ask layered ruleset, no glob patterns, no per-search-space or -per-thread overrides, and no auto-deny synthesis. - -This middleware ports OpenCode's ``packages/opencode/src/permission/index.ts`` -ruleset model on top of SurfSense's existing ``interrupt({type, action, -context})`` payload shape (see ``app/agents/new_chat/tools/hitl.py``) so -the frontend keeps working unchanged. - -Operation: -1. ``aafter_model`` inspects the latest ``AIMessage.tool_calls``. -2. For each call, the middleware builds a list of ``patterns`` (the - tool name plus any tool-specific patterns from the resolver). It - evaluates each pattern against the layered rulesets and aggregates - the results: ``deny`` > ``ask`` > ``allow``. -3. On ``deny``: replaces the call with a synthetic ``ToolMessage`` - containing a :class:`StreamingError`. -4. On ``ask``: raises a SurfSense-style ``interrupt(...)``. Both the legacy - SurfSense shape and LangChain HITL ``{"decisions": [{"type": ...}]}`` - replies are accepted via :func:`_normalize_permission_decision`. - - ``once``: proceed. - - ``approve_always``: also persist allow rules for ``request.always`` patterns. - - ``reject`` w/o feedback: raise :class:`RejectedError`. - - ``reject`` w/ feedback: raise :class:`CorrectedError`. -5. On ``allow``: proceed unchanged. - -The middleware also performs a *pre-model* tool-filter step (the -``before_model`` hook) so globally denied tools are stripped from the -exposed tool list before the model gets to see them. This mirrors -OpenCode's ``Permission.disabled`` and dramatically reduces the chance -the model emits a deny-only call. -""" - -from __future__ import annotations - -import logging -from collections.abc import Callable -from typing import Any - -from langchain.agents.middleware.types import ( - AgentMiddleware, - AgentState, - ContextT, -) -from langchain_core.messages import AIMessage, ToolMessage -from langgraph.runtime import Runtime -from langgraph.types import interrupt - -from app.agents.new_chat.errors import ( - CorrectedError, - RejectedError, - StreamingError, -) -from app.agents.new_chat.permissions import ( - Rule, - Ruleset, - aggregate_action, - evaluate_many, -) -from app.observability import metrics as ot_metrics, otel as ot - -logger = logging.getLogger(__name__) - - -# Mapping ``tool_name -> resolver`` that converts ``args`` to a list of -# patterns to evaluate. The first pattern is conventionally the bare -# tool name; later entries narrow down to specific resources. -PatternResolver = Callable[[dict[str, Any]], list[str]] - - -def _default_pattern_resolver(name: str) -> PatternResolver: - def _resolve(args: dict[str, Any]) -> list[str]: - # Bare name covers the default catch-all; primary-arg fallbacks - # are best added per-tool by callers. - del args - return [name] - - return _resolve - - -# Translation from the LangChain HITL envelope (what ``stream_resume_chat`` -# sends) to SurfSense's legacy ``decision_type`` shape. ``edit`` keeps the -# original tool args — tools needing argument edits should use -# ``request_approval`` from ``app/agents/new_chat/tools/hitl.py``. -_LC_TYPE_TO_PERMISSION_DECISION: dict[str, str] = { - "approve": "once", - "reject": "reject", - "edit": "once", - "approve_always": "approve_always", -} - - -def _normalize_permission_decision(decision: Any) -> dict[str, Any]: - """Coerce any accepted reply shape into ``{"decision_type": ..., "feedback"?}``. - - Falls back to ``reject`` (with a warning) on unrecognized payloads so the - middleware fails closed. - """ - if isinstance(decision, str): - return {"decision_type": decision} - if not isinstance(decision, dict): - logger.warning( - "Unrecognized permission resume value (%s); treating as reject", - type(decision).__name__, - ) - return {"decision_type": "reject"} - - if decision.get("decision_type"): - return decision - - payload: dict[str, Any] = decision - decisions = decision.get("decisions") - if isinstance(decisions, list) and decisions: - first = decisions[0] - if isinstance(first, dict): - payload = first - - raw_type = payload.get("type") or payload.get("decision_type") - if not raw_type: - logger.warning( - "Permission resume missing decision type (keys=%s); treating as reject", - list(payload.keys()), - ) - return {"decision_type": "reject"} - - raw_type = str(raw_type).lower() - mapped = _LC_TYPE_TO_PERMISSION_DECISION.get(raw_type) - if mapped is None: - # Tolerate legacy values arriving without ``decision_type`` wrapping. - if raw_type in {"once", "approve_always", "reject"}: - mapped = raw_type - else: - logger.warning( - "Unknown permission decision type %r; treating as reject", raw_type - ) - mapped = "reject" - - if raw_type == "edit": - logger.warning( - "Permission middleware received an 'edit' decision; original args " - "kept (edits not merged here)." - ) - - out: dict[str, Any] = {"decision_type": mapped} - feedback = payload.get("feedback") or payload.get("message") - if isinstance(feedback, str) and feedback.strip(): - out["feedback"] = feedback - return out - - -class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] - """Allow/deny/ask layer over the agent's tool calls. - - Args: - rulesets: Layered rulesets to evaluate. Earlier entries are - overridden by later ones (last-match-wins). Typical layering: - ``defaults < global < space < thread < runtime_approved``. - pattern_resolvers: Optional per-tool callables that return a list - of patterns to evaluate. When a tool isn't listed, the bare - tool name is used as the only pattern. - runtime_ruleset: Mutable :class:`Ruleset` that the middleware - extends in-place when the user replies ``"approve_always"`` to - an ask interrupt. Reused across all calls in the same agent - instance so newly-allowed rules apply to subsequent calls. - always_emit_interrupt_payload: If True, every ask uses the - SurfSense interrupt wire format (default). Set False to - disable interrupts and treat ``ask`` as ``deny`` for - non-interactive deployments. - """ - - tools = () - - def __init__( - self, - *, - rulesets: list[Ruleset] | None = None, - pattern_resolvers: dict[str, PatternResolver] | None = None, - runtime_ruleset: Ruleset | None = None, - always_emit_interrupt_payload: bool = True, - ) -> None: - super().__init__() - self._static_rulesets: list[Ruleset] = list(rulesets or []) - self._pattern_resolvers: dict[str, PatternResolver] = dict( - pattern_resolvers or {} - ) - self._runtime_ruleset: Ruleset = runtime_ruleset or Ruleset( - origin="runtime_approved" - ) - self._emit_interrupt = always_emit_interrupt_payload - - # ------------------------------------------------------------------ - # Tool-filter step (mirrors OpenCode's ``Permission.disabled``) - # ------------------------------------------------------------------ - - def _globally_denied(self, tool_name: str) -> bool: - """Return True if a deny rule with no narrowing pattern matches.""" - rules = evaluate_many(tool_name, ["*"], *self._all_rulesets()) - return aggregate_action(rules) == "deny" - - def _all_rulesets(self) -> list[Ruleset]: - return [*self._static_rulesets, self._runtime_ruleset] - - # NOTE: ``before_model`` filtering of the tools list is left to the - # agent factory. This middleware only blocks at execution time — and - # only via the rule-evaluator path, not by mutating ``request.tools``. - # Mutating ``request.tools`` per-call would invalidate provider - # prompt-cache prefixes (see Operational risks: prompt-cache regression). - - # ------------------------------------------------------------------ - # Tool-call evaluation - # ------------------------------------------------------------------ - - def _resolve_patterns(self, tool_name: str, args: dict[str, Any]) -> list[str]: - resolver = self._pattern_resolvers.get( - tool_name, _default_pattern_resolver(tool_name) - ) - try: - patterns = resolver(args or {}) - except Exception: - logger.exception( - "Pattern resolver for %s raised; using bare name", tool_name - ) - patterns = [tool_name] - if not patterns: - patterns = [tool_name] - return patterns - - def _evaluate( - self, tool_name: str, args: dict[str, Any] - ) -> tuple[str, list[str], list[Rule]]: - patterns = self._resolve_patterns(tool_name, args) - rules = evaluate_many(tool_name, patterns, *self._all_rulesets()) - action = aggregate_action(rules) - return action, patterns, rules - - # ------------------------------------------------------------------ - # HITL ask flow — SurfSense wire format - # ------------------------------------------------------------------ - - def _raise_interrupt( - self, - *, - tool_name: str, - args: dict[str, Any], - patterns: list[str], - rules: list[Rule], - ) -> dict[str, Any]: - """Block on user approval via SurfSense's ``interrupt`` shape.""" - if not self._emit_interrupt: - return {"decision_type": "reject"} - - # ``params`` (NOT ``args``) is what SurfSense's streaming - # normalizer forwards. Other fields move into ``context``. - payload = { - "type": "permission_ask", - "action": {"tool": tool_name, "params": args or {}}, - "context": { - "patterns": patterns, - "rules": [ - { - "permission": r.permission, - "pattern": r.pattern, - "action": r.action, - } - for r in rules - ], - # Rules of thumb for the frontend: surface the patterns - # the user can promote to "approve_always" with a single reply. - "always": patterns, - }, - } - # Open ``permission.asked`` + ``interrupt.raised`` OTel spans - # (no-op when OTel is disabled) so dashboards can correlate - # "we asked X" with "interrupt was actually delivered". - with ( - ot.permission_asked_span( - permission=tool_name, - pattern=patterns[0] if patterns else None, - extra={"permission.patterns": list(patterns)}, - ), - ot.interrupt_span(interrupt_type="permission_ask"), - ): - ot_metrics.record_permission_ask(permission=tool_name) - ot_metrics.record_interrupt(interrupt_type="permission_ask") - decision = interrupt(payload) - return _normalize_permission_decision(decision) - - def _persist_always(self, tool_name: str, patterns: list[str]) -> None: - """Promote ``approve_always`` reply into runtime allow rules. - - Persistence to ``agent_permission_rules`` is done by the - streaming layer (``stream_new_chat``) once it observes the - ``approve_always`` reply — the middleware just keeps an - in-memory copy so subsequent calls in the same stream see the rule. - """ - for pattern in patterns: - self._runtime_ruleset.rules.append( - Rule(permission=tool_name, pattern=pattern, action="allow") - ) - - # ------------------------------------------------------------------ - # Synthesizing deny -> ToolMessage - # ------------------------------------------------------------------ - - @staticmethod - def _deny_message( - tool_call: dict[str, Any], - rule: Rule, - ) -> ToolMessage: - err = StreamingError( - code="permission_denied", - retryable=False, - suggestion=( - f"rule permission={rule.permission!r} pattern={rule.pattern!r} " - f"blocked this call" - ), - ) - return ToolMessage( - content=( - f"Permission denied: rule {rule.permission}/{rule.pattern} " - f"blocked tool {tool_call.get('name')!r}." - ), - tool_call_id=tool_call.get("id") or "", - name=tool_call.get("name"), - status="error", - additional_kwargs={"error": err.model_dump()}, - ) - - # ------------------------------------------------------------------ - # The hook: aafter_model - # ------------------------------------------------------------------ - - def _process( - self, - state: AgentState, - runtime: Runtime[Any], - ) -> dict[str, Any] | None: - del runtime # unused - messages = state.get("messages") or [] - if not messages: - return None - last = messages[-1] - if not isinstance(last, AIMessage) or not last.tool_calls: - return None - - deny_messages: list[ToolMessage] = [] - kept_calls: list[dict[str, Any]] = [] - any_change = False - - for raw in last.tool_calls: - call = ( - dict(raw) - if isinstance(raw, dict) - else { - "name": getattr(raw, "name", None), - "args": getattr(raw, "args", {}), - "id": getattr(raw, "id", None), - "type": "tool_call", - } - ) - name = call.get("name") or "" - args = call.get("args") or {} - action, patterns, rules = self._evaluate(name, args) - - if action == "deny": - # Find the deny rule for the suggestion text - deny_rule = next((r for r in rules if r.action == "deny"), rules[0]) - deny_messages.append(self._deny_message(call, deny_rule)) - any_change = True - continue - - if action == "ask": - decision = self._raise_interrupt( - tool_name=name, args=args, patterns=patterns, rules=rules - ) - kind = str(decision.get("decision_type") or "reject").lower() - if kind == "once": - kept_calls.append(call) - elif kind == "approve_always": - self._persist_always(name, patterns) - kept_calls.append(call) - elif kind == "reject": - feedback = decision.get("feedback") - if isinstance(feedback, str) and feedback.strip(): - raise CorrectedError(feedback, tool=name) - raise RejectedError( - tool=name, pattern=patterns[0] if patterns else None - ) - else: - logger.warning( - "Unknown permission decision %r; treating as reject", kind - ) - raise RejectedError(tool=name) - continue - - # allow - kept_calls.append(call) - - if not any_change and len(kept_calls) == len(last.tool_calls): - return None - - updated = last.model_copy(update={"tool_calls": kept_calls}) - result_messages: list[Any] = [updated] - if deny_messages: - result_messages.extend(deny_messages) - return {"messages": result_messages} - - def after_model( # type: ignore[override] - self, state: AgentState, runtime: Runtime[ContextT] - ) -> dict[str, Any] | None: - return self._process(state, runtime) - - async def aafter_model( # type: ignore[override] - self, state: AgentState, runtime: Runtime[ContextT] - ) -> dict[str, Any] | None: - return self._process(state, runtime) - - -__all__ = [ - "PatternResolver", - "PermissionMiddleware", - "_normalize_permission_decision", -] diff --git a/surfsense_backend/app/agents/new_chat/prompt_caching.py b/surfsense_backend/app/agents/new_chat/prompt_caching.py deleted file mode 100644 index b58a48266..000000000 --- a/surfsense_backend/app/agents/new_chat/prompt_caching.py +++ /dev/null @@ -1,241 +0,0 @@ -r"""LiteLLM-native prompt caching configuration for SurfSense agents. - -Replaces the legacy ``AnthropicPromptCachingMiddleware`` (which never -activated for our LiteLLM-based stack — its ``isinstance(model, ChatAnthropic)`` -gate always failed) with LiteLLM's universal caching mechanism. - -Coverage: - -- Marker-based providers (need ``cache_control`` injection, which LiteLLM - performs automatically when ``cache_control_injection_points`` is set): - ``anthropic/``, ``bedrock/``, ``vertex_ai/``, ``gemini/``, ``azure_ai/``, - ``openrouter/`` (Claude/Gemini/MiniMax/GLM/z-ai routes), ``databricks/`` - (Claude), ``dashscope/`` (Qwen), ``minimax/``, ``zai/`` (GLM). -- Auto-cached (LiteLLM strips the marker silently): ``openai/``, - ``deepseek/``, ``xai/`` — these caches automatically for prompts ≥1024 - tokens and surface ``prompt_cache_key`` / ``prompt_cache_retention``. - -We inject **two** breakpoints per request: - -- ``index: 0`` — pins the SurfSense system prompt at the head of the - request (provider variant, citation rules, tool catalog, KB tree, - skills metadata). The langchain agent factory always prepends - ``request.system_message`` at index 0 (see ``factory.py`` - ``_execute_model_async``), so this targets exactly the main system - prompt regardless of how many other ``SystemMessage``\ s the - ``before_agent`` injectors (priority, tree, memory, file-intent, - anonymous-doc) have inserted into ``state["messages"]``. Using - ``role: system`` here would apply ``cache_control`` to **every** - system-role message and trip Anthropic's hard cap of 4 cache - breakpoints per request once the conversation accumulates enough - injected system messages — which surfaces as the upstream 400 - ``A maximum of 4 blocks with cache_control may be provided. Found N`` - via OpenRouter→Anthropic. -- ``index: -1`` — pins the latest message so multi-turn savings compound: - Anthropic-family providers use longest-matching-prefix lookup, so turn - N+1 still reads turn N's cache up to the shared prefix. - -For OpenAI-family configs we additionally pass: - -- ``prompt_cache_key=f"surfsense-thread-{thread_id}"`` — routing hint that - raises hit rate by sending requests with a shared prefix to the same - backend. Supported by ``openai/``, ``deepseek/``, ``xai/``, and - ``azure/`` (added to LiteLLM's Azure transformer in - https://github.com/BerriAI/litellm/pull/20989, Feb 2026; verified - against ``AzureOpenAIConfig.get_supported_openai_params`` in our - installed litellm 1.83.14 for ``azure/gpt-4o``, ``azure/gpt-4o-mini``, - ``azure/gpt-5.4``, ``azure/gpt-5.4-mini``). -- ``prompt_cache_retention="24h"`` — extends cache TTL beyond the default - 5-10 min in-memory cache. Set ONLY for OpenAI/DeepSeek/xAI: Azure's - server-side support landed in Microsoft's docs on 2026-05-13 but - LiteLLM 1.83.14's Azure transformer still omits it from its supported - params list, so it gets silently dropped by ``litellm.drop_params``. - Azure's default in-memory retention (5-10 min, max 1 h) already - bridges intra-conversation turns; revisit when LiteLLM bumps Azure. - -Safety net: ``litellm.drop_params=True`` is set globally in -``app.services.llm_service`` at module-load time. Any kwarg the destination -provider doesn't recognise is auto-stripped at the provider transformer -layer, so an OpenAI→Bedrock auto-mode fallback can't 400 on -``prompt_cache_key`` etc. -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any - -from langchain_core.language_models import BaseChatModel - -if TYPE_CHECKING: - from app.agents.new_chat.llm_config import AgentConfig - -logger = logging.getLogger(__name__) - - -# Two-breakpoint policy: head-of-request + latest message. See module -# docstring for rationale. Anthropic caps requests at 4 ``cache_control`` -# blocks; we use 2 here, leaving headroom for Phase-2 tool caching. -# -# IMPORTANT: ``index: 0`` (not ``role: system``). The deepagent stack's -# ``before_agent`` middlewares (priority, tree, memory, file-intent, -# anonymous-doc) insert ``SystemMessage`` instances into -# ``state["messages"]`` that accumulate across turns. With -# ``role: system`` the LiteLLM hook would tag *every* one of them with -# ``cache_control`` and overflow Anthropic's 4-block limit. ``index: 0`` -# always targets the langchain-prepended ``request.system_message`` -# (which our ``FlattenSystemMessageMiddleware`` reduces to a single text -# block), giving us exactly one stable cache breakpoint. -_DEFAULT_INJECTION_POINTS: tuple[dict[str, Any], ...] = ( - {"location": "message", "index": 0}, - {"location": "message", "index": -1}, -) - -# Providers (uppercase ``AgentConfig.provider`` values) that accept the -# OpenAI ``prompt_cache_key`` routing hint. Microsoft's Azure OpenAI docs -# (2026-05-13) confirm automatic prompt caching applies to every GPT-4o -# or newer Azure deployment at ≥1024 tokens with no configuration needed, -# and that ``prompt_cache_key`` is combined with the prefix hash to -# improve routing affinity and therefore cache hit rate. LiteLLM's Azure -# transformer ships ``prompt_cache_key`` in its supported params as of -# https://github.com/BerriAI/litellm/pull/20989. -# -# Strict whitelist — many other providers in ``PROVIDER_MAP`` route -# through litellm's ``openai`` prefix without implementing the OpenAI -# prompt-cache surface (e.g. MOONSHOT, ZHIPU, MINIMAX), so we can't infer -# family from the litellm prefix alone. -_PROMPT_CACHE_KEY_PROVIDERS: frozenset[str] = frozenset( - {"OPENAI", "DEEPSEEK", "XAI", "AZURE", "AZURE_OPENAI"} -) - -# Subset of ``_PROMPT_CACHE_KEY_PROVIDERS`` that also accept -# ``prompt_cache_retention="24h"``. Azure is excluded: see module -# docstring — LiteLLM 1.83.14's Azure transformer omits the param so -# ``drop_params`` silently strips it. Re-add Azure once a future LiteLLM -# release wires it into ``AzureOpenAIConfig.get_supported_openai_params``. -_PROMPT_CACHE_RETENTION_PROVIDERS: frozenset[str] = frozenset( - {"OPENAI", "DEEPSEEK", "XAI"} -) - - -def _is_router_llm(llm: BaseChatModel) -> bool: - """Detect ``ChatLiteLLMRouter`` (auto-mode) without an eager import. - - Importing ``app.services.llm_router_service`` at module-load time would - create a cycle via ``llm_config -> prompt_caching -> llm_router_service``. - Class-name comparison is sufficient since the class is defined in a - single place. - """ - return type(llm).__name__ == "ChatLiteLLMRouter" - - -def _provider_supports_prompt_cache_key(agent_config: AgentConfig | None) -> bool: - """Whether the config targets a provider that accepts ``prompt_cache_key``. - - Strict — only returns True for explicitly chosen OPENAI, DEEPSEEK, - XAI, AZURE, or AZURE_OPENAI providers. Auto-mode and custom - providers return False because we can't statically know the - destination and the router fans out across mixed providers. - """ - if agent_config is None or not agent_config.provider: - return False - if agent_config.is_auto_mode: - return False - if agent_config.custom_provider: - return False - return agent_config.provider.upper() in _PROMPT_CACHE_KEY_PROVIDERS - - -def _provider_supports_prompt_cache_retention( - agent_config: AgentConfig | None, -) -> bool: - """Whether the config targets a provider that accepts ``prompt_cache_retention``. - - Tighter than :func:`_provider_supports_prompt_cache_key` — Azure - deployments are excluded until LiteLLM ships the param in its Azure - transformer (see module docstring). - """ - if agent_config is None or not agent_config.provider: - return False - if agent_config.is_auto_mode: - return False - if agent_config.custom_provider: - return False - return agent_config.provider.upper() in _PROMPT_CACHE_RETENTION_PROVIDERS - - -def _get_or_init_model_kwargs(llm: BaseChatModel) -> dict[str, Any] | None: - """Return ``llm.model_kwargs`` as a writable dict, or ``None`` to bail. - - Initialises the field to ``{}`` when present-but-None on a Pydantic v2 - model. Returns ``None`` if the LLM type doesn't expose a writable - ``model_kwargs`` attribute (caller should treat as no-op). - """ - model_kwargs = getattr(llm, "model_kwargs", None) - if isinstance(model_kwargs, dict): - return model_kwargs - try: - llm.model_kwargs = {} # type: ignore[attr-defined] - except Exception: - return None - refreshed = getattr(llm, "model_kwargs", None) - return refreshed if isinstance(refreshed, dict) else None - - -def apply_litellm_prompt_caching( - llm: BaseChatModel, - *, - agent_config: AgentConfig | None = None, - thread_id: int | None = None, -) -> None: - """Configure LiteLLM prompt caching on a ChatLiteLLM/ChatLiteLLMRouter. - - Idempotent — values already present in ``llm.model_kwargs`` (e.g. from - ``agent_config.litellm_params`` overrides) are preserved. Mutates - ``llm.model_kwargs`` in place; the kwargs flow to ``litellm.completion`` - via ``ChatLiteLLM._default_params`` and via ``self.model_kwargs`` merge - in our custom ``ChatLiteLLMRouter``. - - Args: - llm: ChatLiteLLM, SanitizedChatLiteLLM, or ChatLiteLLMRouter instance. - agent_config: Optional ``AgentConfig`` driving provider-specific - behaviour. When omitted (or auto-mode), only the universal - ``cache_control_injection_points`` are set. - thread_id: Optional thread id used to construct a per-thread - ``prompt_cache_key`` for OpenAI-family providers. Caching still - works without it (server-side automatic), but the key improves - backend routing affinity and therefore hit rate. - """ - model_kwargs = _get_or_init_model_kwargs(llm) - if model_kwargs is None: - logger.debug( - "apply_litellm_prompt_caching: %s exposes no writable model_kwargs; skipping", - type(llm).__name__, - ) - return - - if "cache_control_injection_points" not in model_kwargs: - model_kwargs["cache_control_injection_points"] = [ - dict(point) for point in _DEFAULT_INJECTION_POINTS - ] - - # OpenAI-style extras only when we statically know the destination - # accepts them. Auto-mode router fans out across mixed providers so - # we can't safely set destination-specific kwargs there (drop_params - # would strip them but it's wasteful to set them in the first - # place). - if _is_router_llm(llm): - return - - if ( - thread_id is not None - and "prompt_cache_key" not in model_kwargs - and _provider_supports_prompt_cache_key(agent_config) - ): - model_kwargs["prompt_cache_key"] = f"surfsense-thread-{thread_id}" - - if ( - "prompt_cache_retention" not in model_kwargs - and _provider_supports_prompt_cache_retention(agent_config) - ): - model_kwargs["prompt_cache_retention"] = "24h" diff --git a/surfsense_backend/app/agents/new_chat/subagents/__init__.py b/surfsense_backend/app/agents/new_chat/subagents/__init__.py deleted file mode 100644 index bd1823b57..000000000 --- a/surfsense_backend/app/agents/new_chat/subagents/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Specialized user-facing subagents for the SurfSense agent. - -The :class:`deepagents.SubAgentMiddleware` already provides the -materialization machinery (each :class:`deepagents.SubAgent` typed-dict -spec is compiled into an ephemeral runnable invoked via the ``task`` -tool); what's specific to SurfSense is the *seeding* of those subagents -with declarative deny rules. - -Per-subagent permission rules are injected as a -:class:`PermissionMiddleware` entry inside the subagent's ``middleware`` -field. The auto-deny pattern (e.g. forbid ``task``/``todowrite`` -recursion, block write tools for read-only research roles) is borrowed -from OpenCode's ``packages/opencode/src/tool/task.ts``, which has -analogous logic for restricting child sessions. -""" - -from .config import ( - build_connector_negotiator_subagent, - build_explore_subagent, - build_report_writer_subagent, - build_specialized_subagents, -) -from .providers.linear import build_linear_specialist_subagent -from .providers.slack import build_slack_specialist_subagent - -__all__ = [ - "build_connector_negotiator_subagent", - "build_explore_subagent", - "build_linear_specialist_subagent", - "build_report_writer_subagent", - "build_slack_specialist_subagent", - "build_specialized_subagents", -] diff --git a/surfsense_backend/app/agents/new_chat/subagents/config.py b/surfsense_backend/app/agents/new_chat/subagents/config.py deleted file mode 100644 index 2cfd47441..000000000 --- a/surfsense_backend/app/agents/new_chat/subagents/config.py +++ /dev/null @@ -1,436 +0,0 @@ -"""Builders for specialized SurfSense subagents. - -Each subagent is built from three pieces: - -1. A name + description + system prompt (the user-facing contract for - when ``task`` should delegate to this role). -2. A filtered tool list (subset of the parent's bound tools). -3. A :class:`PermissionMiddleware` instance carrying a deny ruleset that - prevents the subagent from acting outside its scope (e.g. an - explore-only role cannot mutate state). - -Skill sources (``/skills/builtin/`` + ``/skills/space/``) are inherited -from the parent unconditionally — every subagent benefits from the same -authored guidance documents. -""" - -from __future__ import annotations - -import logging -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING, Any - -from app.agents.new_chat.middleware.skills_backends import default_skills_sources -from app.agents.new_chat.permissions import Rule, Ruleset -from app.agents.new_chat.subagents.providers.linear import ( - build_linear_specialist_subagent, -) -from app.agents.new_chat.subagents.providers.slack import ( - build_slack_specialist_subagent, -) - -if TYPE_CHECKING: - from deepagents import SubAgent - from langchain_core.language_models import BaseChatModel - from langchain_core.tools import BaseTool - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Tool name constants -# --------------------------------------------------------------------------- - -# Read-only tools that ``explore`` is permitted to use. Names match the -# tools provided by the deepagents ``FilesystemMiddleware`` (``ls``, ``read_file``, -# ``glob``, ``grep``) plus the SurfSense-side read tools. -EXPLORE_READ_TOOLS: frozenset[str] = frozenset( - { - "web_search", - "scrape_webpage", - "read_file", - "ls", - "glob", - "grep", - } -) - -# Tools ``report_writer`` may call. The set is intentionally narrow so the -# subagent doesn't drift into tangential research; if richer source-gathering -# is needed, the parent should hand off to ``explore`` first. -REPORT_WRITER_TOOLS: frozenset[str] = frozenset( - { - "read_file", - "generate_report", - } -) - -# Wildcard patterns that match write tools we deny by default in read-only -# subagents. Anchored at start AND end via :func:`Rule` semantics. We use -# substring-style ``*verb*`` patterns because connector tool names typically -# put the verb in the middle (``linear_create_issue``, ``slack_send_message``, -# ``notion_update_page``); strict suffix patterns (``*_create``) miss those. -# -# A handful of canonical exact-match names is appended so that bare verbs -# (``edit``, ``write``) are also blocked even when a connector dropped the -# usual prefix. -WRITE_TOOL_DENY_PATTERNS: tuple[str, ...] = ( - "*create*", - "*update*", - "*delete*", - "*send*", - "*write*", - "*edit*", - "*move*", - "*mkdir*", - "*upload*", - "edit_file", - "write_file", - "move_file", - "mkdir", - "rm", - "rmdir", - "update_memory", - "update_memory_team", - "update_memory_private", -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -# Tool names that are NOT in the registry's ``tools`` list because they -# are provided dynamically by middleware at compile time. We don't pass -# them through ``_filter_tools`` (the actual ``BaseTool`` instances live -# inside the middleware), but we do exempt them from the "missing" warning -# below — operators were seeing spurious noise like -# ``missing: ['glob', 'grep', 'ls', 'read_file']`` even though those -# tools are reachable via :class:`SurfSenseFilesystemMiddleware` once the -# subagent is compiled. -_MIDDLEWARE_PROVIDED_TOOL_NAMES: frozenset[str] = frozenset( - { - "ls", - "read_file", - "write_file", - "edit_file", - "glob", - "grep", - "execute", - "write_todos", - "task", - } -) - - -def _filter_tools( - tools: Sequence[BaseTool], - allowed_names: Iterable[str], -) -> list[BaseTool]: - """Return only tools whose ``name`` appears in ``allowed_names``. - - Tools are looked up by exact name. Names matching - :data:`_MIDDLEWARE_PROVIDED_TOOL_NAMES` are intentionally absent from - ``tools`` (they're injected by middleware at compile time) and are - silently excluded from the "missing" warning so operators don't see - false positives every build. - """ - allowed = set(allowed_names) - selected = [t for t in tools if t.name in allowed] - missing = sorted( - (allowed - {t.name for t in selected}) - _MIDDLEWARE_PROVIDED_TOOL_NAMES - ) - if missing: - logger.info( - "Subagent build: %d/%d registry tools available; missing: %s", - len(selected), - len(allowed - _MIDDLEWARE_PROVIDED_TOOL_NAMES), - missing, - ) - return selected - - -def _read_only_deny_rules() -> list[Rule]: - """Synthesize a list of deny rules covering common write-tool patterns.""" - return [ - Rule(permission=pattern, pattern="*", action="deny") - for pattern in WRITE_TOOL_DENY_PATTERNS - ] - - -def _build_permission_middleware(deny_rules: list[Rule], origin: str): - """Construct a :class:`PermissionMiddleware` seeded with ``deny_rules``. - - Imported lazily because the middleware module pulls in interrupt/HITL - machinery we don't want at import time of this config file. - """ - from app.agents.new_chat.middleware.permission import PermissionMiddleware - - return PermissionMiddleware( - rulesets=[Ruleset(rules=deny_rules, origin=origin)], - ) - - -def _wrap_with_subagent_essentials( - custom_middleware: list, - *, - agent_tools: Sequence[BaseTool], - extra_middleware: Sequence[Any] | None = None, -): - """Compose the final middleware list for a specialized subagent. - - Order, outer to inner: - - 1. ``extra_middleware`` — provided by the caller (typically the parent - agent's ``SurfSenseFilesystemMiddleware`` and ``TodoListMiddleware``) - so the subagent inherits the parent's filesystem/todo view. These - run **before** the subagent-local middleware so their tools are - wired up before permissioning kicks in. - 2. ``custom_middleware`` — subagent-local rules (e.g. permission deny - lists). - 3. :class:`PatchToolCallsMiddleware` — normalizes tool-call shapes. - 4. :class:`DedupHITLToolCallsMiddleware` — collapses duplicate HITL - calls using metadata declared at registry time. - - Without ``extra_middleware`` the subagent will only have the registry - tools listed in its ``tools`` field — meaning ``read_file``, ``ls``, - ``grep``, etc. won't exist. Always pass ``extra_middleware`` from the - parent unless you specifically want a sandboxed subagent. - """ - from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware - - from app.agents.new_chat.middleware import DedupHITLToolCallsMiddleware - - return [ - *(extra_middleware or []), - *custom_middleware, - PatchToolCallsMiddleware(), - DedupHITLToolCallsMiddleware(agent_tools=list(agent_tools)), - ] - - -# --------------------------------------------------------------------------- -# System prompts -# --------------------------------------------------------------------------- - -EXPLORE_SYSTEM_PROMPT = """You are the **explore** subagent for SurfSense. - -## Your job -Conduct read-only research across the user's knowledge base, the web, and any documents the parent agent has surfaced. Return a synthesized answer with explicit citations — never speculate beyond the sources you have actually inspected. - -## Tools available -- `web_search` — only when the user's KB clearly does not contain the answer. -- `scrape_webpage` — to read a URL the user or the search results provided. -- `read_file`, `ls`, `glob`, `grep` — to inspect specific documents or trees the parent has flagged. - -## Rules -- Read-only. You cannot create, edit, delete, send, or move anything. -- Cite every claim. Use `[citation:chunk_id]` exactly as the chunk tag specifies. -- If a sub-question has no support in the inspected sources, say so explicitly. Do not fabricate. -- Return the most useful synthesis in your single final message. The parent agent will not be able to follow up. -""" - - -REPORT_WRITER_SYSTEM_PROMPT = """You are the **report_writer** subagent for SurfSense. - -## Your job -Produce a single high-quality report deliverable using `generate_report`. The parent has already gathered (or knows where to gather) the underlying sources. - -## Workflow -1. **Outline first.** Before calling `generate_report`, write a one-paragraph outline of the sections you plan to produce. Confirm the outline reflects the parent's instructions. -2. **Source resolution.** Decide whether to call `read_file` for any final-checks, or whether the parent's earlier tool calls already cover the source set. -3. **One report.** Call `generate_report` exactly once with `source_strategy` chosen per the topic and chat history (see the `report-writing` skill). -4. **Confirm.** End with a one-sentence summary in your final message — never paste the report back into chat; the artifact card renders itself. -""" - - -CONNECTOR_NEGOTIATOR_SYSTEM_PROMPT = """You are the **connector_negotiator** subagent for SurfSense. - -## Your job -Coordinate cross-connector workflows: chains where the result of one service's tool feeds into another's. Common shapes include "find Linear issues mentioned in last week's Slack messages", "draft a Gmail reply citing a Notion doc", or "list Linear tickets opened by the same person who filed Jira FOO-123". - -## Workflow -1. **Plan.** Identify the connector hops needed and the order they should run in. Write a short plan in your first message. -2. **Verify access.** Use `get_connected_accounts` to confirm the relevant connectors are actually wired up before issuing tool calls. If a connector is missing, stop and report — do not fabricate. -3. **Execute.** Run each hop, citing IDs (issue keys, message ts, page IDs) in your scratch notes so the parent can audit. -4. **Hand back.** Return a structured summary with the final answer plus the chain of evidence (issue → message → page, etc.). - -## Caveats -- If a hop fails, do not retry blindly — return the partial result and explain. -- Mutating tools (create, update, delete, send) require parent permission; you are NOT cleared to call them on your own. -""" - - -# --------------------------------------------------------------------------- -# Subagent builders -# --------------------------------------------------------------------------- - - -def build_explore_subagent( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> SubAgent: - """Build the read-only ``explore`` subagent spec. - - Pass ``extra_middleware`` (typically the parent's filesystem + todo - middleware) so the subagent can actually use ``read_file``, ``ls``, - ``grep``, ``glob`` — which its system prompt promises but which only - exist when their middleware is mounted. - """ - from deepagents import SubAgent # noqa: F401 (TypedDict for type clarity) - - selected_tools = _filter_tools(tools, EXPLORE_READ_TOOLS) - deny_rules = _read_only_deny_rules() - permission_mw = _build_permission_middleware(deny_rules, origin="subagent_explore") - - spec: dict = { - "name": "explore", - "description": ( - "Read-only research across the user's knowledge base and the web. " - "Use when the parent needs deeply-cited synthesis without " - "modifying anything." - ), - "system_prompt": EXPLORE_SYSTEM_PROMPT, - "tools": selected_tools, - "middleware": _wrap_with_subagent_essentials( - [permission_mw], - agent_tools=selected_tools, - extra_middleware=extra_middleware, - ), - "skills": default_skills_sources(), - } - if model is not None: - spec["model"] = model - return spec # type: ignore[return-value] - - -def build_report_writer_subagent( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> SubAgent: - """Build the ``report_writer`` subagent spec. - - Read-only deny ruleset still applies — the subagent should call - ``generate_report`` and nothing else mutating. ``generate_report`` - creates a report artifact via a backend service and is intentionally - **not** denied. - - Pass ``extra_middleware`` (typically the parent's filesystem + todo - middleware) so the subagent can run ``read_file`` for source-checks - before calling ``generate_report``. - """ - selected_tools = _filter_tools(tools, REPORT_WRITER_TOOLS) - deny_rules = _read_only_deny_rules() - permission_mw = _build_permission_middleware( - deny_rules, origin="subagent_report_writer" - ) - - spec: dict = { - "name": "report_writer", - "description": ( - "Produce a single Markdown report artifact via generate_report, " - "using the outline-then-fill protocol. Use when the parent has " - "decided a deliverable is needed." - ), - "system_prompt": REPORT_WRITER_SYSTEM_PROMPT, - "tools": selected_tools, - "middleware": _wrap_with_subagent_essentials( - [permission_mw], - agent_tools=selected_tools, - extra_middleware=extra_middleware, - ), - "skills": default_skills_sources(), - } - if model is not None: - spec["model"] = model - return spec # type: ignore[return-value] - - -def build_connector_negotiator_subagent( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> SubAgent: - """Build the ``connector_negotiator`` subagent spec. - - Inherits all MCP / connector tools the parent has plus - ``get_connected_accounts``. Read-only by default; permission rules deny - write/mutation patterns. The parent agent re-asks for permission if a - connector mutation is genuinely needed. - - Pass ``extra_middleware`` (typically the parent's filesystem + todo - middleware) so this subagent shares the parent's filesystem view when - citing evidence across hops. - """ - parent_tool_names = {t.name for t in tools} - allowed: set[str] = set() - if "get_connected_accounts" in parent_tool_names: - allowed.add("get_connected_accounts") - # Inherit anything that smells connector- or MCP-related but is not a - # bulk-write API. Heuristic: keep all parent tools; rely on the deny - # ruleset to block mutation patterns. This mirrors the plan: "all - # MCP/connector tools the parent has". - for name in parent_tool_names: - allowed.add(name) - selected_tools = _filter_tools(tools, allowed) - - deny_rules = _read_only_deny_rules() - permission_mw = _build_permission_middleware( - deny_rules, origin="subagent_connector_negotiator" - ) - - spec: dict = { - "name": "connector_negotiator", - "description": ( - "Coordinate read-only chains across connectors (Slack → Linear, " - "Notion → Gmail, etc.). Returns a structured summary with the " - "evidence chain. Cannot mutate connector state." - ), - "system_prompt": CONNECTOR_NEGOTIATOR_SYSTEM_PROMPT, - "tools": selected_tools, - "middleware": _wrap_with_subagent_essentials( - [permission_mw], - agent_tools=selected_tools, - extra_middleware=extra_middleware, - ), - "skills": default_skills_sources(), - } - if model is not None: - spec["model"] = model - return spec # type: ignore[return-value] - - -def build_specialized_subagents( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> list[SubAgent]: - """Return the canonical list of specialized subagents to register. - - Order matters only for the order they appear in the ``task`` tool - description — most useful first. - """ - return [ - build_explore_subagent( - tools=tools, model=model, extra_middleware=extra_middleware - ), - build_report_writer_subagent( - tools=tools, model=model, extra_middleware=extra_middleware - ), - build_linear_specialist_subagent( - tools=tools, model=model, extra_middleware=extra_middleware - ), - build_slack_specialist_subagent( - tools=tools, model=model, extra_middleware=extra_middleware - ), - build_connector_negotiator_subagent( - tools=tools, model=model, extra_middleware=extra_middleware - ), - ] diff --git a/surfsense_backend/app/agents/new_chat/subagents/constants.py b/surfsense_backend/app/agents/new_chat/subagents/constants.py deleted file mode 100644 index cb1da499b..000000000 --- a/surfsense_backend/app/agents/new_chat/subagents/constants.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Shared constants for provider subagent safety policies.""" - -from __future__ import annotations - -# Generic mutation-deny patterns for read-only specialist roles. -WRITE_TOOL_DENY_PATTERNS: tuple[str, ...] = ( - "*create*", - "*update*", - "*delete*", - "*send*", - "*write*", - "*edit*", - "*move*", - "*mkdir*", - "*upload*", - "edit_file", - "write_file", - "move_file", - "mkdir", - "update_memory", - "update_memory_team", - "update_memory_private", -) - -# Tools that mutate virtual KB filesystem or parent/global chat state. -# Provider specialists should not mutate these surfaces directly. -NON_PROVIDER_STATE_MUTATION_DENY: frozenset[str] = frozenset( - { - # Exact tool names from shared deny patterns. - *{name for name in WRITE_TOOL_DENY_PATTERNS if "*" not in name}, - # Additional non-provider state mutation controls. - "write_todos", - "task", - } -) diff --git a/surfsense_backend/app/agents/new_chat/subagents/providers/linear.py b/surfsense_backend/app/agents/new_chat/subagents/providers/linear.py deleted file mode 100644 index da332fe28..000000000 --- a/surfsense_backend/app/agents/new_chat/subagents/providers/linear.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Linear provider specialist subagent. - -This file is intentionally standalone so provider specialists can be reviewed -and evolved independently (one provider per file). -""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -from app.agents.new_chat.permissions import Rule, Ruleset -from app.agents.new_chat.subagents.constants import NON_PROVIDER_STATE_MUTATION_DENY -from app.services.mcp_oauth.registry import ( - LINEAR_MCP_READONLY_TOOL_NAMES, - linear_mcp_original_tool_name, -) - -if TYPE_CHECKING: - from deepagents import SubAgent - from langchain_core.language_models import BaseChatModel - from langchain_core.tools import BaseTool - - -# Read vs write Linear MCP tools are defined in -# ``app.services.mcp_oauth.registry`` (``LINEAR_MCP_READONLY_TOOL_NAMES`` / -# ``LINEAR_MCP_WRITE_TOOL_NAMES``). Any other Linear-domain tool requires approval. - -LINEAR_SYSTEM_PROMPT = """You are the linear_specialist subagent for SurfSense. - -Role: -- You are the Linear domain specialist. Handle Linear-only requests accurately. - -Primary objective: -- Resolve the user's Linear task and return a concise, auditable result. - -Routing boundary: -- Use this subagent for Linear-domain tasks (issues, status, assignees, labels, - teams, and project references). -- If the task is primarily non-Linear or cross-connector orchestration, return - status=needs_input and hand control back to the parent with the exact next hop. - -Execution steps: -1) Verify Linear access first (use get_connected_accounts if needed). -2) Prefer read/list tools first to gather current issue facts before concluding. -3) Track key identifiers in your reasoning: issue ID, issue key, team ID, label ID. -4) If required identifiers are missing, ask the parent for exactly what is missing. -5) Return a compact result with findings + evidence references. - -Output format: -- status: success | needs_input | blocked | error -- summary: one short paragraph -- evidence: bullet list of concrete IDs / issue keys used -- next_step: one sentence (only when blocked or needs_input) - -Constraints: -- Do not invent issue keys, IDs, or workflow state names. -- Mutating Linear operations are allowed only with explicit approval. -- If Linear connector access is unavailable, stop and return status=blocked. -""" - - -def _select_linear_tools(tools: Sequence[BaseTool]) -> list[BaseTool]: - """Keep Linear tools plus minimal shared read utilities.""" - allowed_exact = { - "get_connected_accounts", - "read_file", - "ls", - "glob", - "grep", - } - selected: list[BaseTool] = [] - for tool in tools: - if tool.name in allowed_exact: - selected.append(tool) - continue - if linear_mcp_original_tool_name(tool.name) is not None: - selected.append(tool) - continue - if tool.name.startswith("linear_") or tool.name.endswith("_linear_issue"): - selected.append(tool) - return selected - - -def _is_linear_readonly_tool_name(name: str) -> bool: - """Return True when a tool name maps to a read-only Linear MCP operation.""" - base = linear_mcp_original_tool_name(name) - return base is not None and base in LINEAR_MCP_READONLY_TOOL_NAMES - - -def _is_linear_domain_tool_name(name: str) -> bool: - """Return True for Linear-domain tools handled by this specialist.""" - if linear_mcp_original_tool_name(name) is not None: - return True - return name.startswith("linear_") or name.endswith("_linear_issue") - - -def _permission_middleware(*, selected_tools: Sequence[BaseTool]) -> Any: - """Permission policy for Linear specialist.""" - from app.agents.new_chat.middleware.permission import PermissionMiddleware - - ask_tools = sorted( - { - tool.name - for tool in selected_tools - if _is_linear_domain_tool_name(tool.name) - and not _is_linear_readonly_tool_name(tool.name) - } - ) - rules: list[Rule] = [Rule(permission="*", pattern="*", action="allow")] - rules.extend( - Rule(permission=name, pattern="*", action="deny") - for name in NON_PROVIDER_STATE_MUTATION_DENY - ) - rules.extend(Rule(permission=name, pattern="*", action="ask") for name in ask_tools) - return PermissionMiddleware( - rulesets=[Ruleset(rules=rules, origin="subagent_linear_specialist")] - ) - - -def _wrap_subagent_middleware( - *, - selected_tools: Sequence[BaseTool], - extra_middleware: Sequence[Any] | None, -) -> list[Any]: - """Apply standard middleware chain used by other subagents.""" - from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware - - from app.agents.new_chat.middleware import DedupHITLToolCallsMiddleware - - return [ - *(extra_middleware or []), - _permission_middleware(selected_tools=selected_tools), - PatchToolCallsMiddleware(), - DedupHITLToolCallsMiddleware(agent_tools=list(selected_tools)), - ] - - -def build_linear_specialist_subagent( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> SubAgent: - """Build the ``linear_specialist`` provider subagent spec.""" - selected_tools = _select_linear_tools(tools) - spec: dict[str, Any] = { - "name": "linear_specialist", - "description": ( - "Linear operations specialist for issue and workflow requests, " - "with strict evidence tracking and approval-gated mutating operations." - ), - "system_prompt": LINEAR_SYSTEM_PROMPT, - "tools": selected_tools, - "middleware": _wrap_subagent_middleware( - selected_tools=selected_tools, - extra_middleware=extra_middleware, - ), - } - if model is not None: - spec["model"] = model - return spec # type: ignore[return-value] diff --git a/surfsense_backend/app/agents/new_chat/subagents/providers/slack.py b/surfsense_backend/app/agents/new_chat/subagents/providers/slack.py deleted file mode 100644 index 90ca80152..000000000 --- a/surfsense_backend/app/agents/new_chat/subagents/providers/slack.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Slack provider specialist subagent. - -This file is intentionally standalone so provider specialists can be reviewed -and evolved independently (one provider per file). -""" - -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -from app.agents.new_chat.permissions import Rule, Ruleset -from app.agents.new_chat.subagents.constants import NON_PROVIDER_STATE_MUTATION_DENY - -if TYPE_CHECKING: - from deepagents import SubAgent - from langchain_core.language_models import BaseChatModel - from langchain_core.tools import BaseTool - - -# Official references: -# - https://docs.slack.dev/ai/slack-mcp-server -# - https://www.npmjs.com/package/@modelcontextprotocol/server-slack -# -# Policy: only known read-only Slack tools are auto-allowed. Any other -# ``slack_*`` tool is treated as mutating and requires explicit approval. -SLACK_READONLY_TOOL_NAMES: frozenset[str] = frozenset( - { - # Slack-hosted MCP read tools - "slack_search_channels", - "slack_read_channel", - "slack_read_thread", - "slack_read_canvas", - "slack_read_user_profile", - # modelcontextprotocol/server-slack read tools - "slack_list_channels", - "slack_get_channel_history", - "slack_get_thread_replies", - "slack_get_users", - "slack_get_user_profile", - } -) - -SLACK_SYSTEM_PROMPT = """You are the slack_specialist subagent for SurfSense. - -Role: -- You are the Slack domain specialist. Handle Slack-only requests accurately. - -Primary objective: -- Resolve the user's Slack task and return a concise, auditable result. - -Routing boundary: -- Use this subagent for Slack-domain tasks (channels, threads, users, messages, - and Slack canvases). -- If the task is primarily non-Slack or cross-connector orchestration, return - status=needs_input and hand control back to the parent with the exact next hop. - -Execution steps: -1) Verify Slack access first (use get_connected_accounts if needed). -2) Prefer read/list tools first to gather facts before concluding. -3) Track key identifiers in your reasoning: channel ID, message ts, thread ts, user ID. -4) If required identifiers are missing, ask the parent for exactly what is missing. -5) Return a compact result with findings + evidence references. - -Output format: -- status: success | needs_input | blocked | error -- summary: one short paragraph -- evidence: bullet list of concrete IDs / timestamps used -- next_step: one sentence (only when blocked or needs_input) - -Constraints: -- Do not invent Slack IDs, channels, users, or message content. -- Mutating Slack operations are allowed only with explicit approval. -- If Slack connector access is unavailable, stop and return status=blocked. -""" - - -def _select_slack_tools(tools: Sequence[BaseTool]) -> list[BaseTool]: - """Keep Slack tools plus minimal shared read utilities.""" - allowed_exact = { - "get_connected_accounts", - "read_file", - "ls", - "glob", - "grep", - } - slack_prefix = "slack_" - selected: list[BaseTool] = [] - for tool in tools: - if tool.name in allowed_exact: - selected.append(tool) - continue - if tool.name.startswith(slack_prefix): - selected.append(tool) - return selected - - -def _permission_middleware(*, selected_tools: Sequence[BaseTool]) -> Any: - """Permission policy for Slack specialist. - - Intent: - - Allow Slack-domain operations by default. - - Gate Slack mutating operations behind approval (`ask`). - - Hard-deny non-Slack state mutations, especially KB virtual filesystem - mutation and parent-context mutation tools. - """ - from app.agents.new_chat.middleware.permission import PermissionMiddleware - - ask_tools = sorted( - { - tool.name - for tool in selected_tools - if tool.name.startswith("slack_") - and tool.name not in SLACK_READONLY_TOOL_NAMES - } - ) - rules: list[Rule] = [Rule(permission="*", pattern="*", action="allow")] - rules.extend( - Rule(permission=name, pattern="*", action="deny") - for name in NON_PROVIDER_STATE_MUTATION_DENY - ) - rules.extend(Rule(permission=name, pattern="*", action="ask") for name in ask_tools) - return PermissionMiddleware( - rulesets=[Ruleset(rules=rules, origin="subagent_slack_specialist")] - ) - - -def _wrap_subagent_middleware( - *, - selected_tools: Sequence[BaseTool], - extra_middleware: Sequence[Any] | None, -) -> list[Any]: - """Apply standard middleware chain used by other subagents.""" - from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware - - from app.agents.new_chat.middleware import DedupHITLToolCallsMiddleware - - return [ - *(extra_middleware or []), - _permission_middleware(selected_tools=selected_tools), - PatchToolCallsMiddleware(), - DedupHITLToolCallsMiddleware(agent_tools=list(selected_tools)), - ] - - -def build_slack_specialist_subagent( - *, - tools: Sequence[BaseTool], - model: BaseChatModel | None = None, - extra_middleware: Sequence[Any] | None = None, -) -> SubAgent: - """Build the ``slack_specialist`` provider subagent spec.""" - selected_tools = _select_slack_tools(tools) - spec: dict[str, Any] = { - "name": "slack_specialist", - "description": ( - "Slack operations specialist for any Slack-domain request " - "(channels, threads, users, and messages), with strict evidence " - "tracking and approval-gated mutating operations." - ), - "system_prompt": SLACK_SYSTEM_PROMPT, - "tools": selected_tools, - "middleware": _wrap_subagent_middleware( - selected_tools=selected_tools, - extra_middleware=extra_middleware, - ), - } - if model is not None: - spec["model"] = model - return spec # type: ignore[return-value] diff --git a/surfsense_backend/app/agents/new_chat/tools/__init__.py b/surfsense_backend/app/agents/new_chat/tools/__init__.py deleted file mode 100644 index 4b5ae3706..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Tools module for SurfSense deep agent. - -This module contains all the tools available to the SurfSense agent. -To add a new tool, see the documentation in registry.py. - -Available tools: -- generate_podcast: Generate audio podcasts from content -- generate_video_presentation: Generate video presentations with slides and narration -- generate_image: Generate images from text descriptions using AI models -- scrape_webpage: Extract content from webpages -- update_memory: Update the user's / team's memory document -""" - -# Registry exports -# Tool factory exports (for direct use) -from .generate_image import create_generate_image_tool -from .knowledge_base import ( - CONNECTOR_DESCRIPTIONS, - format_documents_for_context, - search_knowledge_base_async, -) -from .podcast import create_generate_podcast_tool -from .registry import ( - BUILTIN_TOOLS, - ToolDefinition, - build_tools, - get_all_tool_names, - get_default_enabled_tools, - get_tool_by_name, -) -from .scrape_webpage import create_scrape_webpage_tool -from .update_memory import create_update_memory_tool, create_update_team_memory_tool -from .video_presentation import create_generate_video_presentation_tool - -__all__ = [ - # Registry - "BUILTIN_TOOLS", - # Knowledge base utilities - "CONNECTOR_DESCRIPTIONS", - "ToolDefinition", - "build_tools", - # Tool factories - "create_generate_image_tool", - "create_generate_podcast_tool", - "create_generate_video_presentation_tool", - "create_scrape_webpage_tool", - "create_update_memory_tool", - "create_update_team_memory_tool", - "format_documents_for_context", - "get_all_tool_names", - "get_default_enabled_tools", - "get_tool_by_name", - "search_knowledge_base_async", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/confluence/__init__.py b/surfsense_backend/app/agents/new_chat/tools/confluence/__init__.py deleted file mode 100644 index 3bf80b61b..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/confluence/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Confluence tools for creating, updating, and deleting pages.""" - -from .create_page import create_create_confluence_page_tool -from .delete_page import create_delete_confluence_page_tool -from .update_page import create_update_confluence_page_tool - -__all__ = [ - "create_create_confluence_page_tool", - "create_delete_confluence_page_tool", - "create_update_confluence_page_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/confluence/create_page.py b/surfsense_backend/app/agents/new_chat/tools/confluence/create_page.py deleted file mode 100644 index c56db1528..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/confluence/create_page.py +++ /dev/null @@ -1,232 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.db import async_session_maker -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_confluence_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the create_confluence_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_confluence_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_confluence_page( - title: str, - content: str | None = None, - space_id: str | None = None, - ) -> dict[str, Any]: - """Create a new page in Confluence. - - Use this tool when the user explicitly asks to create a new Confluence page. - - Args: - title: Title of the page. - content: Optional HTML/storage format content for the page body. - space_id: Optional Confluence space ID to create the page in. - - Returns: - Dictionary with status, page_id, and message. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info(f"create_confluence_page called: title='{title}'") - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - return { - "status": "auth_error", - "message": "All connected Confluence accounts need re-authentication.", - "connector_type": "confluence", - } - - result = request_approval( - action_type="confluence_page_creation", - tool_name="create_confluence_page", - params={ - "title": title, - "content": content, - "space_id": space_id, - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_title = result.params.get("title", title) - final_content = result.params.get("content", content) or "" - final_space_id = result.params.get("space_id", space_id) - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_title or not final_title.strip(): - return {"status": "error", "message": "Page title cannot be empty."} - if not final_space_id: - return {"status": "error", "message": "A space must be selected."} - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Confluence connector found.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=actual_connector_id - ) - api_result = await client.create_page( - space_id=final_space_id, - title=final_title, - body=final_content, - ) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - _conn = connector - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - page_id = str(api_result.get("id", "")) - page_links = ( - api_result.get("_links", {}) if isinstance(api_result, dict) else {} - ) - page_url = "" - if page_links.get("base") and page_links.get("webui"): - page_url = f"{page_links['base']}{page_links['webui']}" - - kb_message_suffix = "" - try: - from app.services.confluence import ConfluenceKBSyncService - - kb_service = ConfluenceKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - page_id=page_id, - page_title=final_title, - space_id=final_space_id, - body_content=final_content, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "page_id": page_id, - "page_url": page_url, - "message": f"Confluence page '{final_title}' created successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error creating Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the page.", - } - - return create_confluence_page diff --git a/surfsense_backend/app/agents/new_chat/tools/confluence/delete_page.py b/surfsense_backend/app/agents/new_chat/tools/confluence/delete_page.py deleted file mode 100644 index d4cd5032f..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/confluence/delete_page.py +++ /dev/null @@ -1,213 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.db import async_session_maker -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_confluence_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the delete_confluence_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured delete_confluence_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_confluence_page( - page_title_or_id: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete a Confluence page. - - Use this tool when the user asks to delete or remove a Confluence page. - - Args: - page_title_or_id: The page title or ID to identify the page. - delete_from_kb: Whether to also remove from the knowledge base. - - Returns: - Dictionary with status, message, and deleted_from_kb. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message to the user. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"delete_confluence_page called: page_title_or_id='{page_title_or_id}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_deletion_context( - search_space_id, user_id, page_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "confluence", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - page_data = context["page"] - page_id = page_data["page_id"] - page_title = page_data.get("page_title", "") - document_id = page_data["document_id"] - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="confluence_page_deletion", - tool_name="delete_confluence_page", - params={ - "page_id": page_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this page.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - await client.delete_page(final_page_id) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - - message = f"Confluence page '{page_title}' deleted successfully." - if deleted_from_kb: - message += " Also removed from the knowledge base." - - return { - "status": "success", - "page_id": final_page_id, - "deleted_from_kb": deleted_from_kb, - "message": message, - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error deleting Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while deleting the page.", - } - - return delete_confluence_page diff --git a/surfsense_backend/app/agents/new_chat/tools/confluence/update_page.py b/surfsense_backend/app/agents/new_chat/tools/confluence/update_page.py deleted file mode 100644 index 51c205e00..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/confluence/update_page.py +++ /dev/null @@ -1,240 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.confluence_history import ConfluenceHistoryConnector -from app.db import async_session_maker -from app.services.confluence import ConfluenceToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_confluence_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the update_confluence_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured update_confluence_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_confluence_page( - page_title_or_id: str, - new_title: str | None = None, - new_content: str | None = None, - ) -> dict[str, Any]: - """Update an existing Confluence page. - - Use this tool when the user asks to modify or edit a Confluence page. - - Args: - page_title_or_id: The page title or ID to identify the page. - new_title: Optional new title for the page. - new_content: Optional new HTML/storage format content. - - Returns: - Dictionary with status and message. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message to the user. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"update_confluence_page called: page_title_or_id='{page_title_or_id}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Confluence tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = ConfluenceToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, page_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "confluence", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - page_data = context["page"] - page_id = page_data["page_id"] - current_title = page_data["page_title"] - current_body = page_data.get("body", "") - current_version = page_data.get("version", 1) - document_id = page_data.get("document_id") - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="confluence_page_update", - tool_name="update_confluence_page", - params={ - "page_id": page_id, - "document_id": document_id, - "new_title": new_title, - "new_content": new_content, - "version": current_version, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_title = result.params.get("new_title", new_title) or current_title - final_content = result.params.get("new_content", new_content) - if final_content is None: - final_content = current_body - final_version = result.params.get("version", current_version) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_document_id = result.params.get("document_id", document_id) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this page.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.CONFLUENCE_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Confluence connector is invalid.", - } - - try: - client = ConfluenceHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - api_result = await client.update_page( - page_id=final_page_id, - title=final_title, - body=final_content, - version_number=final_version + 1, - ) - await client.close() - except Exception as api_err: - if ( - "http 403" in str(api_err).lower() - or "status code 403" in str(api_err).lower() - ): - try: - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Confluence account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - page_links = ( - api_result.get("_links", {}) if isinstance(api_result, dict) else {} - ) - page_url = "" - if page_links.get("base") and page_links.get("webui"): - page_url = f"{page_links['base']}{page_links['webui']}" - - kb_message_suffix = "" - if final_document_id: - try: - from app.services.confluence import ConfluenceKBSyncService - - kb_service = ConfluenceKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=final_document_id, - page_id=final_page_id, - user_id=user_id, - search_space_id=search_space_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - except Exception as kb_err: - logger.warning(f"KB sync after update failed: {kb_err}") - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - - return { - "status": "success", - "page_id": final_page_id, - "page_url": page_url, - "message": f"Confluence page '{final_title}' updated successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error updating Confluence page: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while updating the page.", - } - - return update_confluence_page diff --git a/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py b/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py deleted file mode 100644 index 6420a90e6..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Connected-accounts discovery tool. - -Lets the LLM discover which accounts are connected for a given service -(e.g. "jira", "linear", "slack") and retrieve the metadata it needs to -call action tools — such as Jira's ``cloudId``. - -The tool returns **only** non-sensitive fields explicitly listed in the -service's ``account_metadata_keys`` (see ``registry.py``), plus the -always-present ``display_name`` and ``connector_id``. -""" - -import logging -from typing import Any - -from langchain_core.tools import StructuredTool -from pydantic import BaseModel, Field -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker -from app.services.mcp_oauth.registry import MCP_SERVICES - -logger = logging.getLogger(__name__) - -_SERVICE_KEY_BY_CONNECTOR_TYPE: dict[str, str] = { - cfg.connector_type: key for key, cfg in MCP_SERVICES.items() -} - - -class GetConnectedAccountsInput(BaseModel): - service: str = Field( - description=( - "Service key to look up connected accounts for. " - "Valid values: " + ", ".join(sorted(MCP_SERVICES.keys())) - ), - ) - - -def _extract_display_name(connector: SearchSourceConnector) -> str: - """Best-effort human-readable label for a connector.""" - cfg = connector.config or {} - if cfg.get("display_name"): - return cfg["display_name"] - if cfg.get("base_url"): - return f"{connector.name} ({cfg['base_url']})" - if cfg.get("organization_name"): - return f"{connector.name} ({cfg['organization_name']})" - return connector.name - - -def create_get_connected_accounts_tool( - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> StructuredTool: - """Factory function to create the get_connected_accounts tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to scope account discovery to. - user_id: User ID to scope account discovery to. - - Returns: - Configured StructuredTool for connected-accounts discovery. - """ - del db_session # per-call session — see docstring - - async def _run(service: str) -> list[dict[str, Any]]: - svc_cfg = MCP_SERVICES.get(service) - if not svc_cfg: - return [ - { - "error": f"Unknown service '{service}'. Valid: {', '.join(sorted(MCP_SERVICES.keys()))}" - } - ] - - try: - connector_type = SearchSourceConnectorType(svc_cfg.connector_type) - except ValueError: - return [{"error": f"Connector type '{svc_cfg.connector_type}' not found."}] - - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == connector_type, - ) - ) - connectors = result.scalars().all() - - if not connectors: - return [ - { - "error": f"No {svc_cfg.name} accounts connected. Ask the user to connect one in settings." - } - ] - - is_multi = len(connectors) > 1 - - accounts: list[dict[str, Any]] = [] - for conn in connectors: - cfg = conn.config or {} - entry: dict[str, Any] = { - "connector_id": conn.id, - "display_name": _extract_display_name(conn), - "service": service, - } - if is_multi: - entry["tool_prefix"] = f"{service}_{conn.id}" - for key in svc_cfg.account_metadata_keys: - if key in cfg: - entry[key] = cfg[key] - accounts.append(entry) - - return accounts - - return StructuredTool( - name="get_connected_accounts", - description=( - "Discover which accounts are connected for a service (e.g. jira, linear, slack, clickup, airtable). " - "Returns display names and service-specific metadata the action tools need " - "(e.g. Jira's cloudId). Call this BEFORE using a service's action tools when " - "you need an account identifier or are unsure which account to use." - ), - coroutine=_run, - args_schema=GetConnectedAccountsInput, - metadata={"hitl": False}, - ) diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/__init__.py b/surfsense_backend/app/agents/new_chat/tools/discord/__init__.py deleted file mode 100644 index b4eaec1f0..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/discord/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.discord.list_channels import ( - create_list_discord_channels_tool, -) -from app.agents.new_chat.tools.discord.read_messages import ( - create_read_discord_messages_tool, -) -from app.agents.new_chat.tools.discord.send_message import ( - create_send_discord_message_tool, -) - -__all__ = [ - "create_list_discord_channels_tool", - "create_read_discord_messages_tool", - "create_send_discord_message_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py b/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py deleted file mode 100644 index c345f8a5e..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Shared auth helper for Discord agent tools (REST API, not gateway bot).""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.config import config -from app.db import SearchSourceConnector, SearchSourceConnectorType -from app.utils.oauth_security import TokenEncryption - -DISCORD_API = "https://discord.com/api/v10" - - -async def get_discord_connector( - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DISCORD_CONNECTOR, - ) - ) - return result.scalars().first() - - -def get_bot_token(connector: SearchSourceConnector) -> str: - """Extract and decrypt the bot token from connector config.""" - cfg = dict(connector.config) - if cfg.get("_token_encrypted") and config.SECRET_KEY: - enc = TokenEncryption(config.SECRET_KEY) - if cfg.get("bot_token"): - cfg["bot_token"] = enc.decrypt_token(cfg["bot_token"]) - token = cfg.get("bot_token") - if not token: - raise ValueError("Discord bot token not found in connector config.") - return token - - -def get_guild_id(connector: SearchSourceConnector) -> str | None: - return connector.config.get("guild_id") diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py b/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py deleted file mode 100644 index 01159a261..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py +++ /dev/null @@ -1,107 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector, get_guild_id - -logger = logging.getLogger(__name__) - - -def create_list_discord_channels_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the list_discord_channels tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured list_discord_channels tool - """ - del db_session # per-call session — see docstring - - @tool - async def list_discord_channels() -> dict[str, Any]: - """List text channels in the connected Discord server. - - Returns: - Dictionary with status and a list of channels (id, name). - """ - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - connector = await get_discord_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - guild_id = get_guild_id(connector) - if not guild_id: - return { - "status": "error", - "message": "No guild ID in Discord connector config.", - } - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.get( - f"{DISCORD_API}/guilds/{guild_id}/channels", - headers={"Authorization": f"Bot {token}"}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - # Type 0 = text channel - channels = [ - {"id": ch["id"], "name": ch["name"]} - for ch in resp.json() - if ch.get("type") == 0 - ] - return { - "status": "success", - "guild_id": guild_id, - "channels": channels, - "total": len(channels), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Discord channels: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Discord channels."} - - return list_discord_channels diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py b/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py deleted file mode 100644 index 88d6cdd49..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py +++ /dev/null @@ -1,120 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector - -logger = logging.getLogger(__name__) - - -def create_read_discord_messages_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the read_discord_messages tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured read_discord_messages tool - """ - del db_session # per-call session — see docstring - - @tool - async def read_discord_messages( - channel_id: str, - limit: int = 25, - ) -> dict[str, Any]: - """Read recent messages from a Discord text channel. - - Args: - channel_id: The Discord channel ID (from list_discord_channels). - limit: Number of messages to fetch (default 25, max 50). - - Returns: - Dictionary with status and a list of messages including - id, author, content, timestamp. - """ - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - limit = min(limit, 50) - - try: - async with async_session_maker() as db_session: - connector = await get_discord_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.get( - f"{DISCORD_API}/channels/{channel_id}/messages", - headers={"Authorization": f"Bot {token}"}, - params={"limit": limit}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Bot lacks permission to read this channel.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - messages = [ - { - "id": m["id"], - "author": m.get("author", {}).get("username", "Unknown"), - "content": m.get("content", ""), - "timestamp": m.get("timestamp", ""), - } - for m in resp.json() - ] - - return { - "status": "success", - "channel_id": channel_id, - "messages": messages, - "total": len(messages), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Discord messages: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Discord messages."} - - return read_discord_messages diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py b/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py deleted file mode 100644 index 5fe6fde35..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py +++ /dev/null @@ -1,136 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker - -from ._auth import DISCORD_API, get_bot_token, get_discord_connector - -logger = logging.getLogger(__name__) - - -def create_send_discord_message_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the send_discord_message tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured send_discord_message tool - """ - del db_session # per-call session — see docstring - - @tool - async def send_discord_message( - channel_id: str, - content: str, - ) -> dict[str, Any]: - """Send a message to a Discord text channel. - - Args: - channel_id: The Discord channel ID (from list_discord_channels). - content: The message text (max 2000 characters). - - Returns: - Dictionary with status, message_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Discord tool not properly configured.", - } - - if len(content) > 2000: - return { - "status": "error", - "message": "Message exceeds Discord's 2000-character limit.", - } - - try: - async with async_session_maker() as db_session: - connector = await get_discord_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Discord connector found."} - - result = request_approval( - action_type="discord_send_message", - tool_name="send_discord_message", - params={"channel_id": channel_id, "content": content}, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Message was not sent.", - } - - final_content = result.params.get("content", content) - final_channel = result.params.get("channel_id", channel_id) - - token = get_bot_token(connector) - - async with httpx.AsyncClient() as client: - resp = await client.post( - f"{DISCORD_API}/channels/{final_channel}/messages", - headers={ - "Authorization": f"Bot {token}", - "Content-Type": "application/json", - }, - json={"content": final_content}, - timeout=15.0, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Discord bot token is invalid.", - "connector_type": "discord", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Bot lacks permission to send messages in this channel.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Discord API error: {resp.status_code}", - } - - msg_data = resp.json() - return { - "status": "success", - "message_id": msg_data.get("id"), - "message": f"Message sent to channel {final_channel}.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error sending Discord message: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to send Discord message."} - - return send_discord_message diff --git a/surfsense_backend/app/agents/new_chat/tools/dropbox/__init__.py b/surfsense_backend/app/agents/new_chat/tools/dropbox/__init__.py deleted file mode 100644 index 836b9ee41..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/dropbox/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.dropbox.create_file import ( - create_create_dropbox_file_tool, -) -from app.agents.new_chat.tools.dropbox.trash_file import ( - create_delete_dropbox_file_tool, -) - -__all__ = [ - "create_create_dropbox_file_tool", - "create_delete_dropbox_file_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/dropbox/create_file.py b/surfsense_backend/app/agents/new_chat/tools/dropbox/create_file.py deleted file mode 100644 index 7aae034cc..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/dropbox/create_file.py +++ /dev/null @@ -1,299 +0,0 @@ -import logging -import os -import tempfile -from pathlib import Path -from typing import Any, Literal - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.dropbox.client import DropboxClient -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker - -logger = logging.getLogger(__name__) - -DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - -_FILE_TYPE_LABELS = { - "paper": "Dropbox Paper (.paper)", - "docx": "Word Document (.docx)", -} - -_SUPPORTED_TYPES = [ - {"value": "paper", "label": "Dropbox Paper (.paper)"}, - {"value": "docx", "label": "Word Document (.docx)"}, -] - - -def _ensure_extension(name: str, file_type: str) -> str: - """Strip any existing extension and append the correct one.""" - stem = Path(name).stem - ext = ".paper" if file_type == "paper" else ".docx" - return f"{stem}{ext}" - - -def _markdown_to_docx(markdown_text: str) -> bytes: - """Convert a markdown string to DOCX bytes using pypandoc.""" - import pypandoc - - fd, tmp_path = tempfile.mkstemp(suffix=".docx") - os.close(fd) - try: - pypandoc.convert_text( - markdown_text, - "docx", - format="gfm", - extra_args=["--standalone"], - outputfile=tmp_path, - ) - with open(tmp_path, "rb") as f: - return f.read() - finally: - os.unlink(tmp_path) - - -def create_create_dropbox_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_dropbox_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_dropbox_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_dropbox_file( - name: str, - file_type: Literal["paper", "docx"] = "paper", - content: str | None = None, - ) -> dict[str, Any]: - """Create a new document in Dropbox. - - Use this tool when the user explicitly asks to create a new document - in Dropbox. The user MUST specify a topic before you call this tool. - - Args: - name: The document title (without extension). - file_type: Either "paper" (Dropbox Paper, default) or "docx" (Word document). - content: Optional initial content as markdown. - - Returns: - Dictionary with status, file_id, name, web_url, and message. - """ - logger.info( - f"create_dropbox_file called: name='{name}', file_type='{file_type}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Dropbox tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DROPBOX_CONNECTOR, - ) - ) - connectors = result.scalars().all() - - if not connectors: - return { - "status": "error", - "message": "No Dropbox connector found. Please connect Dropbox in your workspace settings.", - } - - accounts = [] - for c in connectors: - cfg = c.config or {} - accounts.append( - { - "id": c.id, - "name": c.name, - "user_email": cfg.get("user_email"), - "auth_expired": cfg.get("auth_expired", False), - } - ) - - if all(a.get("auth_expired") for a in accounts): - return { - "status": "auth_error", - "message": "All connected Dropbox accounts need re-authentication.", - "connector_type": "dropbox", - } - - parent_folders: dict[int, list[dict[str, str]]] = {} - for acc in accounts: - cid = acc["id"] - if acc.get("auth_expired"): - parent_folders[cid] = [] - continue - try: - client = DropboxClient(session=db_session, connector_id=cid) - items, err = await client.list_folder("") - if err: - logger.warning( - "Failed to list folders for connector %s: %s", cid, err - ) - parent_folders[cid] = [] - else: - parent_folders[cid] = [ - { - "folder_path": item.get("path_lower", ""), - "name": item["name"], - } - for item in items - if item.get(".tag") == "folder" and item.get("name") - ] - except Exception: - logger.warning( - "Error fetching folders for connector %s", - cid, - exc_info=True, - ) - parent_folders[cid] = [] - - context: dict[str, Any] = { - "accounts": accounts, - "parent_folders": parent_folders, - "supported_types": _SUPPORTED_TYPES, - } - - result = request_approval( - action_type="dropbox_file_creation", - tool_name="create_dropbox_file", - params={ - "name": name, - "file_type": file_type, - "content": content, - "connector_id": None, - "parent_folder_path": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_name = result.params.get("name", name) - final_file_type = result.params.get("file_type", file_type) - final_content = result.params.get("content", content) - final_connector_id = result.params.get("connector_id") - final_parent_folder_path = result.params.get("parent_folder_path") - - if not final_name or not final_name.strip(): - return {"status": "error", "message": "File name cannot be empty."} - - final_name = _ensure_extension(final_name, final_file_type) - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DROPBOX_CONNECTOR, - ) - ) - connector = result.scalars().first() - else: - connector = connectors[0] - - if not connector: - return { - "status": "error", - "message": "Selected Dropbox connector is invalid.", - } - - client = DropboxClient(session=db_session, connector_id=connector.id) - - parent_path = final_parent_folder_path or "" - file_path = ( - f"{parent_path}/{final_name}" if parent_path else f"/{final_name}" - ) - - if final_file_type == "paper": - created = await client.create_paper_doc( - file_path, final_content or "" - ) - file_id = created.get("file_id", "") - web_url = created.get("url", "") - else: - docx_bytes = _markdown_to_docx(final_content or "") - created = await client.upload_file( - file_path, docx_bytes, mode="add", autorename=True - ) - file_id = created.get("id", "") - web_url = "" - - logger.info(f"Dropbox file created: id={file_id}, name={final_name}") - - kb_message_suffix = "" - try: - from app.services.dropbox import DropboxKBSyncService - - kb_service = DropboxKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - file_id=file_id, - file_name=final_name, - file_path=file_path, - web_url=web_url, - content=final_content, - connector_id=connector.id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "file_id": file_id, - "name": final_name, - "web_url": web_url, - "message": f"Successfully created '{final_name}' in Dropbox.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error creating Dropbox file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the file. Please try again.", - } - - return create_dropbox_file diff --git a/surfsense_backend/app/agents/new_chat/tools/dropbox/trash_file.py b/surfsense_backend/app/agents/new_chat/tools/dropbox/trash_file.py deleted file mode 100644 index 0e59e49db..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/dropbox/trash_file.py +++ /dev/null @@ -1,301 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy import String, and_, cast, func -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.dropbox.client import DropboxClient -from app.db import ( - Document, - DocumentType, - SearchSourceConnector, - SearchSourceConnectorType, - async_session_maker, -) - -logger = logging.getLogger(__name__) - - -def create_delete_dropbox_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the delete_dropbox_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured delete_dropbox_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_dropbox_file( - file_name: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete a file from Dropbox. - - Use this tool when the user explicitly asks to delete, remove, or trash - a file in Dropbox. - - Args: - file_name: The exact name of the file to delete. - delete_from_kb: Whether to also remove the file from the knowledge base. - Default is False. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - file_id: Dropbox file ID (if success) - - deleted_from_kb: whether the document was removed from the knowledge base - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the file name or check if it has been indexed. - """ - logger.info( - f"delete_dropbox_file called: file_name='{file_name}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Dropbox tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - doc_result = await db_session.execute( - select(Document) - .join( - SearchSourceConnector, - Document.connector_id == SearchSourceConnector.id, - ) - .filter( - and_( - Document.search_space_id == search_space_id, - Document.document_type == DocumentType.DROPBOX_FILE, - func.lower(Document.title) == func.lower(file_name), - SearchSourceConnector.user_id == user_id, - ) - ) - .order_by(Document.updated_at.desc().nullslast()) - .limit(1) - ) - document = doc_result.scalars().first() - - if not document: - doc_result = await db_session.execute( - select(Document) - .join( - SearchSourceConnector, - Document.connector_id == SearchSourceConnector.id, - ) - .filter( - and_( - Document.search_space_id == search_space_id, - Document.document_type == DocumentType.DROPBOX_FILE, - func.lower( - cast( - Document.document_metadata["dropbox_file_name"], - String, - ) - ) - == func.lower(file_name), - SearchSourceConnector.user_id == user_id, - ) - ) - .order_by(Document.updated_at.desc().nullslast()) - .limit(1) - ) - document = doc_result.scalars().first() - - if not document: - return { - "status": "not_found", - "message": ( - f"File '{file_name}' not found in your indexed Dropbox files. " - "This could mean: (1) the file doesn't exist, (2) it hasn't been indexed yet, " - "or (3) the file name is different." - ), - } - - if not document.connector_id: - return { - "status": "error", - "message": "Document has no associated connector.", - } - - meta = document.document_metadata or {} - file_path = meta.get("dropbox_path") - file_id = meta.get("dropbox_file_id") - document_id = document.id - - if not file_path: - return { - "status": "error", - "message": "File path is missing. Please re-index the file.", - } - - conn_result = await db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.id == document.connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DROPBOX_CONNECTOR, - ) - ) - ) - connector = conn_result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Dropbox connector not found or access denied.", - } - - cfg = connector.config or {} - if cfg.get("auth_expired"): - return { - "status": "auth_error", - "message": "Dropbox account needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "dropbox", - } - - context = { - "file": { - "file_id": file_id, - "file_path": file_path, - "name": file_name, - "document_id": document_id, - }, - "account": { - "id": connector.id, - "name": connector.name, - "user_email": cfg.get("user_email"), - }, - } - - result = request_approval( - action_type="dropbox_file_trash", - tool_name="delete_dropbox_file", - params={ - "file_path": file_path, - "connector_id": connector.id, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_file_path = result.params.get("file_path", file_path) - final_connector_id = result.params.get("connector_id", connector.id) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - if final_connector_id != connector.id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id - == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.DROPBOX_CONNECTOR, - ) - ) - ) - validated_connector = result.scalars().first() - if not validated_connector: - return { - "status": "error", - "message": "Selected Dropbox connector is invalid or has been disconnected.", - } - actual_connector_id = validated_connector.id - else: - actual_connector_id = connector.id - - logger.info( - f"Deleting Dropbox file: path='{final_file_path}', connector={actual_connector_id}" - ) - - client = DropboxClient( - session=db_session, connector_id=actual_connector_id - ) - await client.delete_file(final_file_path) - - logger.info(f"Dropbox file deleted: path={final_file_path}") - - trash_result: dict[str, Any] = { - "status": "success", - "file_id": file_id, - "message": f"Successfully deleted '{file_name}' from Dropbox.", - } - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - doc = doc_result.scalars().first() - if doc: - await db_session.delete(doc) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - trash_result["warning"] = ( - f"File deleted, but failed to remove from knowledge base: {e!s}" - ) - - trash_result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - trash_result["message"] = ( - f"{trash_result.get('message', '')} (also removed from knowledge base)" - ) - - return trash_result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error deleting Dropbox file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while deleting the file. Please try again.", - } - - return delete_dropbox_file diff --git a/surfsense_backend/app/agents/new_chat/tools/generate_image.py b/surfsense_backend/app/agents/new_chat/tools/generate_image.py deleted file mode 100644 index 9e287ac51..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/generate_image.py +++ /dev/null @@ -1,280 +0,0 @@ -""" -Image generation tool for the SurfSense agent. - -This module provides a tool that generates images using litellm.aimage_generation() -and returns the result directly in a format the frontend Image component can render. - -Config resolution: -1. Uses the search space's image_generation_config_id preference -2. Falls back to Auto mode (router load balancing) if available -3. Supports global YAML configs (negative IDs) and user DB configs (positive IDs) -""" - -import hashlib -import logging -from typing import Any - -from langchain_core.tools import tool -from litellm import aimage_generation -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.config import config -from app.db import ( - ImageGeneration, - ImageGenerationConfig, - SearchSpace, - shielded_async_session, -) -from app.services.image_gen_router_service import ( - IMAGE_GEN_AUTO_MODE_ID, - ImageGenRouterService, - is_image_gen_auto_mode, -) -from app.services.provider_api_base import resolve_api_base -from app.utils.signed_image_urls import generate_image_token - -logger = logging.getLogger(__name__) - -# Provider mapping (same as routes) -_PROVIDER_MAP = { - "OPENAI": "openai", - "AZURE_OPENAI": "azure", - "GOOGLE": "gemini", - "VERTEX_AI": "vertex_ai", - "BEDROCK": "bedrock", - "RECRAFT": "recraft", - "OPENROUTER": "openrouter", - "XINFERENCE": "xinference", - "NSCALE": "nscale", -} - - -def _resolve_provider_prefix(provider: str, custom_provider: str | None) -> str: - if custom_provider: - return custom_provider - return _PROVIDER_MAP.get(provider.upper(), provider.lower()) - - -def _build_model_string( - provider: str, model_name: str, custom_provider: str | None -) -> str: - prefix = _resolve_provider_prefix(provider, custom_provider) - return f"{prefix}/{model_name}" - - -def _get_global_image_gen_config(config_id: int) -> dict | None: - """Get a global image gen config by negative ID.""" - for cfg in config.GLOBAL_IMAGE_GEN_CONFIGS: - if cfg.get("id") == config_id: - return cfg - return None - - -def create_generate_image_tool( - search_space_id: int, - db_session: AsyncSession, -): - """ - Factory function to create the generate_image tool. - - Args: - search_space_id: The search space ID (for config resolution) - db_session: Reserved for compatibility with the tool registry. - The streaming task's ``AsyncSession`` is shared by every tool; - because AsyncSession is not concurrency-safe, parallel tool calls - would interleave flushes (e.g. podcast + image in the same step) - and poison the transaction. This tool opens its own session. - """ - del db_session # use a fresh per-call session, see below - - @tool - async def generate_image( - prompt: str, - n: int = 1, - ) -> dict[str, Any]: - """ - Generate an image from a text description using AI image models. - - Use this tool when the user asks you to create, generate, draw, or make an image. - The generated image will be displayed directly in the chat. - - Args: - prompt: A detailed text description of the image to generate. - Be specific about subject, style, colors, composition, and mood. - n: Number of images to generate (1-4). Default: 1 - - Returns: - A dictionary containing the generated image(s) for display in the chat. - """ - try: - # Use a per-call session so concurrent tool calls don't share an - # AsyncSession (which is not concurrency-safe). The streaming - # task's session is shared across every tool; without isolation, - # autoflushes from a concurrent writer poison this tool too. - async with shielded_async_session() as session: - result = await session.execute( - select(SearchSpace).filter(SearchSpace.id == search_space_id) - ) - search_space = result.scalars().first() - if not search_space: - return {"error": "Search space not found"} - - config_id = ( - search_space.image_generation_config_id or IMAGE_GEN_AUTO_MODE_ID - ) - - # Build generation kwargs - # NOTE: size, quality, and style are intentionally NOT passed. - # Different models support different values for these params - # (e.g. DALL-E 3 wants "hd"/"standard" for quality while - # gpt-image-1 wants "high"/"medium"/"low"; size options also - # differ). Letting the model use its own defaults avoids errors. - gen_kwargs: dict[str, Any] = {} - if n is not None and n > 1: - gen_kwargs["n"] = n - - # Call litellm based on config type - if is_image_gen_auto_mode(config_id): - if not ImageGenRouterService.is_initialized(): - return { - "error": "No image generation models configured. " - "Please add an image model in Settings > Image Models." - } - response = await ImageGenRouterService.aimage_generation( - prompt=prompt, model="auto", **gen_kwargs - ) - elif config_id < 0: - cfg = _get_global_image_gen_config(config_id) - if not cfg: - return { - "error": f"Image generation config {config_id} not found" - } - - provider_prefix = _resolve_provider_prefix( - cfg.get("provider", ""), cfg.get("custom_provider") - ) - model_string = f"{provider_prefix}/{cfg['model_name']}" - gen_kwargs["api_key"] = cfg.get("api_key") - api_base = resolve_api_base( - provider=cfg.get("provider"), - provider_prefix=provider_prefix, - config_api_base=cfg.get("api_base"), - ) - if api_base: - gen_kwargs["api_base"] = api_base - if cfg.get("api_version"): - gen_kwargs["api_version"] = cfg["api_version"] - if cfg.get("litellm_params"): - gen_kwargs.update(cfg["litellm_params"]) - - response = await aimage_generation( - prompt=prompt, model=model_string, **gen_kwargs - ) - else: - # Positive ID = user-created ImageGenerationConfig - cfg_result = await session.execute( - select(ImageGenerationConfig).filter( - ImageGenerationConfig.id == config_id - ) - ) - db_cfg = cfg_result.scalars().first() - if not db_cfg: - return { - "error": f"Image generation config {config_id} not found" - } - - provider_prefix = _resolve_provider_prefix( - db_cfg.provider.value, db_cfg.custom_provider - ) - model_string = f"{provider_prefix}/{db_cfg.model_name}" - gen_kwargs["api_key"] = db_cfg.api_key - api_base = resolve_api_base( - provider=db_cfg.provider.value, - provider_prefix=provider_prefix, - config_api_base=db_cfg.api_base, - ) - if api_base: - gen_kwargs["api_base"] = api_base - if db_cfg.api_version: - gen_kwargs["api_version"] = db_cfg.api_version - if db_cfg.litellm_params: - gen_kwargs.update(db_cfg.litellm_params) - - response = await aimage_generation( - prompt=prompt, model=model_string, **gen_kwargs - ) - - # Parse the response and store in DB - response_dict = ( - response.model_dump() - if hasattr(response, "model_dump") - else dict(response) - ) - - # Generate a random access token for this image - access_token = generate_image_token() - - # Save to image_generations table for history - db_image_gen = ImageGeneration( - prompt=prompt, - model=getattr(response, "_hidden_params", {}).get("model"), - n=n, - image_generation_config_id=config_id, - response_data=response_dict, - search_space_id=search_space_id, - access_token=access_token, - ) - session.add(db_image_gen) - await session.commit() - await session.refresh(db_image_gen) - db_image_gen_id = db_image_gen.id - - # Extract image URLs from response - images = response_dict.get("data", []) - if not images: - return {"error": "No images were generated"} - - first_image = images[0] - revised_prompt = first_image.get("revised_prompt", prompt) - - # Resolve image URL: - # - If the API returned a URL, use it directly. - # - If the API returned b64_json (e.g. gpt-image-1), serve the - # image through our backend endpoint to avoid bloating the - # LLM context with megabytes of base64 data. - if first_image.get("url"): - image_url = first_image["url"] - elif first_image.get("b64_json"): - backend_url = config.BACKEND_URL or "http://localhost:8000" - image_url = ( - f"{backend_url}/api/v1/image-generations/" - f"{db_image_gen_id}/image?token={access_token}" - ) - else: - return {"error": "No displayable image data in the response"} - - image_id = f"image-{hashlib.md5(image_url.encode()).hexdigest()[:12]}" - - return { - "id": image_id, - "assetId": image_url, - "src": image_url, - "alt": revised_prompt or prompt, - "title": "Generated Image", - "description": revised_prompt if revised_prompt != prompt else None, - "domain": "ai-generated", - "ratio": "auto", - "generated": True, - "prompt": prompt, - "image_count": len(images), - } - - except Exception as e: - logger.exception("Image generation failed in tool") - return { - "error": f"Image generation failed: {e!s}", - "prompt": prompt, - } - - return generate_image diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/__init__.py b/surfsense_backend/app/agents/new_chat/tools/gmail/__init__.py deleted file mode 100644 index 294840122..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from app.agents.new_chat.tools.gmail.create_draft import ( - create_create_gmail_draft_tool, -) -from app.agents.new_chat.tools.gmail.read_email import ( - create_read_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.search_emails import ( - create_search_gmail_tool, -) -from app.agents.new_chat.tools.gmail.send_email import ( - create_send_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.trash_email import ( - create_trash_gmail_email_tool, -) -from app.agents.new_chat.tools.gmail.update_draft import ( - create_update_gmail_draft_tool, -) - -__all__ = [ - "create_create_gmail_draft_tool", - "create_read_gmail_email_tool", - "create_search_gmail_tool", - "create_send_gmail_email_tool", - "create_trash_gmail_email_tool", - "create_update_gmail_draft_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/composio_helpers.py b/surfsense_backend/app/agents/new_chat/tools/gmail/composio_helpers.py deleted file mode 100644 index 0ca1191a4..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/composio_helpers.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any - -from app.db import SearchSourceConnector -from app.services.composio_service import ComposioService - - -def split_recipients(value: str | None) -> list[str]: - if not value: - return [] - return [recipient.strip() for recipient in value.split(",") if recipient.strip()] - - -def unwrap_composio_data(data: Any) -> Any: - if isinstance(data, dict): - inner = data.get("data", data) - if isinstance(inner, dict): - return inner.get("response_data", inner) - return inner - return data - - -async def execute_composio_gmail_tool( - connector: SearchSourceConnector, - user_id: str, - tool_name: str, - params: dict[str, Any], -) -> tuple[Any, str | None]: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return None, "Composio connected account ID not found for this Gmail connector." - - result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name=tool_name, - params=params, - entity_id=f"surfsense_{user_id}", - ) - if not result.get("success"): - return None, result.get("error", "Unknown Composio Gmail error") - - return unwrap_composio_data(result.get("data")), None diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/create_draft.py b/surfsense_backend/app/agents/new_chat/tools/gmail/create_draft.py deleted file mode 100644 index c88b48d2d..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/create_draft.py +++ /dev/null @@ -1,361 +0,0 @@ -import asyncio -import base64 -import logging -from datetime import datetime -from email.mime.text import MIMEText -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.gmail import GmailToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_gmail_draft_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_gmail_draft tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_gmail_draft tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_gmail_draft( - to: str, - subject: str, - body: str, - cc: str | None = None, - bcc: str | None = None, - ) -> dict[str, Any]: - """Create a draft email in Gmail. - - Use when the user asks to draft, compose, or prepare an email without - sending it. - - Args: - to: Recipient email address. - subject: Email subject line. - body: Email body content. - cc: Optional CC recipient(s), comma-separated. - bcc: Optional BCC recipient(s), comma-separated. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - draft_id: Gmail draft ID (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment and do NOT retry or suggest alternatives. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry the action. - - Examples: - - "Draft an email to alice@example.com about the meeting" - - "Compose a reply to Bob about the project update" - """ - logger.info(f"create_gmail_draft called: to='{to}', subject='{subject}'") - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Gmail tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning("All Gmail accounts have expired authentication") - return { - "status": "auth_error", - "message": "All connected Gmail accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "gmail", - } - - logger.info( - f"Requesting approval for creating Gmail draft: to='{to}', subject='{subject}'" - ) - result = request_approval( - action_type="gmail_draft_creation", - tool_name="create_gmail_draft", - params={ - "to": to, - "subject": subject, - "body": body, - "cc": cc, - "bcc": bcc, - "connector_id": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The draft was not created. Do not ask again or suggest alternatives.", - } - - final_to = result.params.get("to", to) - final_subject = result.params.get("subject", subject) - final_body = result.params.get("body", body) - final_cc = result.params.get("cc", cc) - final_bcc = result.params.get("bcc", bcc) - final_connector_id = result.params.get("connector_id") - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _gmail_types = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, - ] - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Gmail connector is invalid or has been disconnected.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Gmail connector found. Please connect Gmail in your workspace settings.", - } - actual_connector_id = connector.id - - logger.info( - f"Creating Gmail draft: to='{final_to}', subject='{final_subject}', connector={actual_connector_id}" - ) - - is_composio_gmail = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ) - if is_composio_gmail: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Gmail connector.", - } - else: - from google.oauth2.credentials import Credentials - - from app.config import config - from app.utils.oauth_security import TokenEncryption - - config_data = dict(connector.config) - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and config.SECRET_KEY: - token_encryption = TokenEncryption(config.SECRET_KEY) - if config_data.get("token"): - config_data["token"] = token_encryption.decrypt_token( - config_data["token"] - ) - if config_data.get("refresh_token"): - config_data["refresh_token"] = ( - token_encryption.decrypt_token( - config_data["refresh_token"] - ) - ) - if config_data.get("client_secret"): - config_data["client_secret"] = ( - token_encryption.decrypt_token( - config_data["client_secret"] - ) - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - message = MIMEText(final_body) - message["to"] = final_to - message["subject"] = final_subject - if final_cc: - message["cc"] = final_cc - if final_bcc: - message["bcc"] = final_bcc - raw = base64.urlsafe_b64encode(message.as_bytes()).decode() - - try: - if is_composio_gmail: - from app.agents.new_chat.tools.gmail.composio_helpers import ( - execute_composio_gmail_tool, - split_recipients, - ) - - created, error = await execute_composio_gmail_tool( - connector, - user_id, - "GMAIL_CREATE_EMAIL_DRAFT", - { - "user_id": "me", - "recipient_email": final_to, - "subject": final_subject, - "body": final_body, - "cc": split_recipients(final_cc), - "bcc": split_recipients(final_bcc), - "is_html": False, - }, - ) - if error: - raise RuntimeError(error) - if not isinstance(created, dict): - created = {} - else: - from googleapiclient.discovery import build - - gmail_service = build("gmail", "v1", credentials=creds) - created = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - gmail_service.users() - .drafts() - .create(userId="me", body={"message": {"raw": raw}}) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Gmail account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info(f"Gmail draft created: id={created.get('id')}") - - kb_message_suffix = "" - try: - from app.services.gmail import GmailKBSyncService - - kb_service = GmailKBSyncService(db_session) - draft_message = created.get("message", {}) - kb_result = await kb_service.sync_after_create( - message_id=draft_message.get("id", ""), - thread_id=draft_message.get("threadId", ""), - subject=final_subject, - sender="me", - date_str=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - body_text=final_body, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - draft_id=created.get("id"), - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This draft will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This draft will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "draft_id": created.get("id"), - "message": f"Successfully created Gmail draft with subject '{final_subject}'.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating Gmail draft: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the draft. Please try again.", - } - - return create_gmail_draft diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py b/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py deleted file mode 100644 index 464713591..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py +++ /dev/null @@ -1,172 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker - -logger = logging.getLogger(__name__) - -_GMAIL_TYPES = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, -] - - -def create_read_gmail_email_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the read_gmail_email tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured read_gmail_email tool - """ - del db_session # per-call session — see docstring - - @tool - async def read_gmail_email(message_id: str) -> dict[str, Any]: - """Read the full content of a specific Gmail email by its message ID. - - Use after search_gmail to get the complete body of an email. - - Args: - message_id: The Gmail message ID (from search_gmail results). - - Returns: - Dictionary with status and the full email content formatted as markdown. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Gmail tool not properly configured."} - - try: - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_GMAIL_TYPES), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Gmail connector found. Please connect Gmail in your workspace settings.", - } - - if ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ): - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found.", - } - - from app.agents.new_chat.tools.gmail.search_emails import ( - _format_gmail_summary, - ) - from app.services.composio_service import ComposioService - - service = ComposioService() - detail, error = await service.get_gmail_message_detail( - connected_account_id=cca_id, - entity_id=f"surfsense_{user_id}", - message_id=message_id, - ) - if error: - return {"status": "error", "message": error} - if not detail: - return { - "status": "not_found", - "message": f"Email with ID '{message_id}' not found.", - } - - summary = _format_gmail_summary(detail) - content = ( - f"# {summary['subject']}\n\n" - f"**From:** {summary['from']}\n" - f"**To:** {summary['to']}\n" - f"**Date:** {summary['date']}\n\n" - f"## Message Content\n\n" - f"{detail.get('messageText') or detail.get('snippet') or ''}\n\n" - f"## Message Details\n\n" - f"- **Message ID:** {summary['message_id']}\n" - f"- **Thread ID:** {summary['thread_id']}\n" - ) - return { - "status": "success", - "message_id": summary["message_id"] or message_id, - "content": content, - } - - from app.agents.new_chat.tools.gmail.search_emails import ( - _build_credentials, - ) - - creds = _build_credentials(connector) - - from app.connectors.google_gmail_connector import GoogleGmailConnector - - gmail = GoogleGmailConnector( - credentials=creds, - session=db_session, - user_id=user_id, - connector_id=connector.id, - ) - - detail, error = await gmail.get_message_details(message_id) - if error: - if ( - "re-authenticate" in error.lower() - or "authentication failed" in error.lower() - ): - return { - "status": "auth_error", - "message": error, - "connector_type": "gmail", - } - return {"status": "error", "message": error} - - if not detail: - return { - "status": "not_found", - "message": f"Email with ID '{message_id}' not found.", - } - - content = gmail.format_message_to_markdown(detail) - - return { - "status": "success", - "message_id": message_id, - "content": content, - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Gmail email: %s", e, exc_info=True) - return { - "status": "error", - "message": "Failed to read email. Please try again.", - } - - return read_gmail_email diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py b/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py deleted file mode 100644 index 3ce154c53..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py +++ /dev/null @@ -1,260 +0,0 @@ -import logging -from datetime import datetime -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker - -logger = logging.getLogger(__name__) - -_GMAIL_TYPES = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, -] - -_token_encryption_cache: object | None = None - - -def _get_token_encryption(): - global _token_encryption_cache - if _token_encryption_cache is None: - from app.config import config - from app.utils.oauth_security import TokenEncryption - - if not config.SECRET_KEY: - raise RuntimeError("SECRET_KEY not configured for token decryption.") - _token_encryption_cache = TokenEncryption(config.SECRET_KEY) - return _token_encryption_cache - - -def _build_credentials(connector: SearchSourceConnector): - """Build Google OAuth Credentials from a connector's stored config. - - Handles both native OAuth connectors (with encrypted tokens) and - Composio-backed connectors. Shared by Gmail and Calendar tools. - """ - from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES - - if connector.connector_type in COMPOSIO_GOOGLE_CONNECTOR_TYPES: - raise ValueError("Composio connectors must use Composio tool execution.") - - from google.oauth2.credentials import Credentials - - cfg = dict(connector.config) - if cfg.get("_token_encrypted"): - enc = _get_token_encryption() - for key in ("token", "refresh_token", "client_secret"): - if cfg.get(key): - cfg[key] = enc.decrypt_token(cfg[key]) - - exp = (cfg.get("expiry") or "").replace("Z", "") - return Credentials( - token=cfg.get("token"), - refresh_token=cfg.get("refresh_token"), - token_uri=cfg.get("token_uri"), - client_id=cfg.get("client_id"), - client_secret=cfg.get("client_secret"), - scopes=cfg.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - -def _gmail_headers(message: dict[str, Any]) -> dict[str, str]: - headers = message.get("payload", {}).get("headers", []) - return { - header.get("name", "").lower(): header.get("value", "") - for header in headers - if isinstance(header, dict) - } - - -def _format_gmail_summary(message: dict[str, Any]) -> dict[str, Any]: - headers = _gmail_headers(message) - return { - "message_id": message.get("id") or message.get("messageId"), - "thread_id": message.get("threadId"), - "subject": message.get("subject") or headers.get("subject", "No Subject"), - "from": message.get("sender") or headers.get("from", "Unknown"), - "to": message.get("to") or headers.get("to", ""), - "date": message.get("messageTimestamp") or headers.get("date", ""), - "snippet": message.get("snippet") or message.get("messageText", "")[:300], - "labels": message.get("labelIds", []), - } - - -async def _search_composio_gmail( - connector: SearchSourceConnector, - user_id: str, - query: str, - max_results: int, -) -> dict[str, Any]: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found.", - } - - from app.services.composio_service import ComposioService - - service = ComposioService() - messages, _next_token, _estimate, error = await service.get_gmail_messages( - connected_account_id=cca_id, - entity_id=f"surfsense_{user_id}", - query=query, - max_results=max_results, - ) - if error: - return {"status": "error", "message": error} - - emails = [_format_gmail_summary(message) for message in messages] - return { - "status": "success", - "emails": emails, - "total": len(emails), - "message": "No emails found." if not emails else None, - } - - -def create_search_gmail_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the search_gmail tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured search_gmail tool - """ - del db_session # per-call session — see docstring - - @tool - async def search_gmail( - query: str, - max_results: int = 10, - ) -> dict[str, Any]: - """Search emails in the user's Gmail inbox using Gmail search syntax. - - Args: - query: Gmail search query, same syntax as the Gmail search bar. - Examples: "from:alice@example.com", "subject:meeting", - "is:unread", "after:2024/01/01 before:2024/02/01", - "has:attachment", "in:sent". - max_results: Number of emails to return (default 10, max 20). - - Returns: - Dictionary with status and a list of email summaries including - message_id, subject, from, date, snippet. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Gmail tool not properly configured."} - - max_results = min(max_results, 20) - - try: - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_GMAIL_TYPES), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Gmail connector found. Please connect Gmail in your workspace settings.", - } - - if ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ): - return await _search_composio_gmail( - connector, str(user_id), query, max_results - ) - - creds = _build_credentials(connector) - - from app.connectors.google_gmail_connector import GoogleGmailConnector - - gmail = GoogleGmailConnector( - credentials=creds, - session=db_session, - user_id=user_id, - connector_id=connector.id, - ) - - messages_list, error = await gmail.get_messages_list( - max_results=max_results, query=query - ) - if error: - if ( - "re-authenticate" in error.lower() - or "authentication failed" in error.lower() - ): - return { - "status": "auth_error", - "message": error, - "connector_type": "gmail", - } - return {"status": "error", "message": error} - - if not messages_list: - return { - "status": "success", - "emails": [], - "total": 0, - "message": "No emails found.", - } - - emails = [] - for msg in messages_list: - detail, err = await gmail.get_message_details(msg["id"]) - if err: - continue - headers = { - h["name"].lower(): h["value"] - for h in detail.get("payload", {}).get("headers", []) - } - emails.append( - { - "message_id": detail.get("id"), - "thread_id": detail.get("threadId"), - "subject": headers.get("subject", "No Subject"), - "from": headers.get("from", "Unknown"), - "to": headers.get("to", ""), - "date": headers.get("date", ""), - "snippet": detail.get("snippet", ""), - "labels": detail.get("labelIds", []), - } - ) - - return {"status": "success", "emails": emails, "total": len(emails)} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error searching Gmail: %s", e, exc_info=True) - return { - "status": "error", - "message": "Failed to search Gmail. Please try again.", - } - - return search_gmail diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/send_email.py b/surfsense_backend/app/agents/new_chat/tools/gmail/send_email.py deleted file mode 100644 index 4d5aa3bcc..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/send_email.py +++ /dev/null @@ -1,363 +0,0 @@ -import asyncio -import base64 -import logging -from datetime import datetime -from email.mime.text import MIMEText -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.gmail import GmailToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_send_gmail_email_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the send_gmail_email tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured send_gmail_email tool - """ - del db_session # per-call session — see docstring - - @tool - async def send_gmail_email( - to: str, - subject: str, - body: str, - cc: str | None = None, - bcc: str | None = None, - ) -> dict[str, Any]: - """Send an email via Gmail. - - Use when the user explicitly asks to send an email. This sends the - email immediately - it cannot be unsent. - - Args: - to: Recipient email address. - subject: Email subject line. - body: Email body content. - cc: Optional CC recipient(s), comma-separated. - bcc: Optional BCC recipient(s), comma-separated. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - message_id: Gmail message ID (if success) - - thread_id: Gmail thread ID (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment and do NOT retry or suggest alternatives. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry the action. - - Examples: - - "Send an email to alice@example.com about the meeting" - - "Email Bob the project update" - """ - logger.info(f"send_gmail_email called: to='{to}', subject='{subject}'") - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Gmail tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning("All Gmail accounts have expired authentication") - return { - "status": "auth_error", - "message": "All connected Gmail accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "gmail", - } - - logger.info( - f"Requesting approval for sending Gmail email: to='{to}', subject='{subject}'" - ) - result = request_approval( - action_type="gmail_email_send", - tool_name="send_gmail_email", - params={ - "to": to, - "subject": subject, - "body": body, - "cc": cc, - "bcc": bcc, - "connector_id": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The email was not sent. Do not ask again or suggest alternatives.", - } - - final_to = result.params.get("to", to) - final_subject = result.params.get("subject", subject) - final_body = result.params.get("body", body) - final_cc = result.params.get("cc", cc) - final_bcc = result.params.get("bcc", bcc) - final_connector_id = result.params.get("connector_id") - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _gmail_types = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, - ] - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Gmail connector is invalid or has been disconnected.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Gmail connector found. Please connect Gmail in your workspace settings.", - } - actual_connector_id = connector.id - - logger.info( - f"Sending Gmail email: to='{final_to}', subject='{final_subject}', connector={actual_connector_id}" - ) - - is_composio_gmail = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ) - if is_composio_gmail: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Gmail connector.", - } - else: - from google.oauth2.credentials import Credentials - - from app.config import config - from app.utils.oauth_security import TokenEncryption - - config_data = dict(connector.config) - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and config.SECRET_KEY: - token_encryption = TokenEncryption(config.SECRET_KEY) - if config_data.get("token"): - config_data["token"] = token_encryption.decrypt_token( - config_data["token"] - ) - if config_data.get("refresh_token"): - config_data["refresh_token"] = ( - token_encryption.decrypt_token( - config_data["refresh_token"] - ) - ) - if config_data.get("client_secret"): - config_data["client_secret"] = ( - token_encryption.decrypt_token( - config_data["client_secret"] - ) - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - message = MIMEText(final_body) - message["to"] = final_to - message["subject"] = final_subject - if final_cc: - message["cc"] = final_cc - if final_bcc: - message["bcc"] = final_bcc - raw = base64.urlsafe_b64encode(message.as_bytes()).decode() - - try: - if is_composio_gmail: - from app.agents.new_chat.tools.gmail.composio_helpers import ( - execute_composio_gmail_tool, - split_recipients, - ) - - sent, error = await execute_composio_gmail_tool( - connector, - user_id, - "GMAIL_SEND_EMAIL", - { - "user_id": "me", - "recipient_email": final_to, - "subject": final_subject, - "body": final_body, - "cc": split_recipients(final_cc), - "bcc": split_recipients(final_bcc), - "is_html": False, - }, - ) - if error: - raise RuntimeError(error) - if not isinstance(sent, dict): - sent = {} - else: - from googleapiclient.discovery import build - - gmail_service = build("gmail", "v1", credentials=creds) - sent = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - gmail_service.users() - .messages() - .send(userId="me", body={"raw": raw}) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Gmail account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info( - f"Gmail email sent: id={sent.get('id')}, threadId={sent.get('threadId')}" - ) - - kb_message_suffix = "" - try: - from app.services.gmail import GmailKBSyncService - - kb_service = GmailKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - message_id=sent.get("id", ""), - thread_id=sent.get("threadId", ""), - subject=final_subject, - sender="me", - date_str=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - body_text=final_body, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This email will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after send failed: {kb_err}") - kb_message_suffix = " This email will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "message_id": sent.get("id"), - "thread_id": sent.get("threadId"), - "message": f"Successfully sent email to '{final_to}' with subject '{final_subject}'.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error sending Gmail email: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while sending the email. Please try again.", - } - - return send_gmail_email diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/trash_email.py b/surfsense_backend/app/agents/new_chat/tools/gmail/trash_email.py deleted file mode 100644 index 95f5b4e6c..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/trash_email.py +++ /dev/null @@ -1,344 +0,0 @@ -import asyncio -import logging -from datetime import datetime -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.gmail import GmailToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_trash_gmail_email_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the trash_gmail_email tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured trash_gmail_email tool - """ - del db_session # per-call session — see docstring - - @tool - async def trash_gmail_email( - email_subject_or_id: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Move an email or draft to trash in Gmail. - - Use when the user asks to delete, remove, or trash an email or draft. - - Args: - email_subject_or_id: The exact subject line or message ID of the - email to trash (as it appears in the inbox). - delete_from_kb: Whether to also remove the email from the knowledge base. - Default is False. - Set to True to remove from both Gmail and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - message_id: Gmail message ID (if success) - - deleted_from_kb: whether the document was removed from the knowledge base - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the email subject or check if it has been indexed. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry this tool. - Examples: - - "Delete the email about 'Meeting Cancelled'" - - "Trash the email from Bob about the project" - """ - logger.info( - f"trash_gmail_email called: email_subject_or_id='{email_subject_or_id}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Gmail tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_trash_context( - search_space_id, user_id, email_subject_or_id - ) - - if "error" in context: - error_msg = context["error"] - if "not found" in error_msg.lower(): - logger.warning(f"Email not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - logger.error(f"Failed to fetch trash context: {error_msg}") - return {"status": "error", "message": error_msg} - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Gmail account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Gmail account for this email needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "gmail", - } - - email = context["email"] - message_id = email["message_id"] - document_id = email.get("document_id") - connector_id_from_context = context["account"]["id"] - - if not message_id: - return { - "status": "error", - "message": "Message ID is missing from the indexed document. Please re-index the email and try again.", - } - - logger.info( - f"Requesting approval for trashing Gmail email: '{email_subject_or_id}' (message_id={message_id}, delete_from_kb={delete_from_kb})" - ) - result = request_approval( - action_type="gmail_email_trash", - tool_name="trash_gmail_email", - params={ - "message_id": message_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The email was not trashed. Do not ask again or suggest alternatives.", - } - - final_message_id = result.params.get("message_id", message_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this email.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _gmail_types = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, - ] - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Gmail connector is invalid or has been disconnected.", - } - - logger.info( - f"Trashing Gmail email: message_id='{final_message_id}', connector={final_connector_id}" - ) - - is_composio_gmail = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ) - if is_composio_gmail: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Gmail connector.", - } - else: - from google.oauth2.credentials import Credentials - - from app.config import config - from app.utils.oauth_security import TokenEncryption - - config_data = dict(connector.config) - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and config.SECRET_KEY: - token_encryption = TokenEncryption(config.SECRET_KEY) - if config_data.get("token"): - config_data["token"] = token_encryption.decrypt_token( - config_data["token"] - ) - if config_data.get("refresh_token"): - config_data["refresh_token"] = ( - token_encryption.decrypt_token( - config_data["refresh_token"] - ) - ) - if config_data.get("client_secret"): - config_data["client_secret"] = ( - token_encryption.decrypt_token( - config_data["client_secret"] - ) - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - try: - if is_composio_gmail: - from app.agents.new_chat.tools.gmail.composio_helpers import ( - execute_composio_gmail_tool, - ) - - _trashed, error = await execute_composio_gmail_tool( - connector, - user_id, - "GMAIL_MOVE_TO_TRASH", - {"user_id": "me", "message_id": final_message_id}, - ) - if error: - raise RuntimeError(error) - else: - from googleapiclient.discovery import build - - gmail_service = build("gmail", "v1", credentials=creds) - await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - gmail_service.users() - .messages() - .trash(userId="me", id=final_message_id) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {connector.id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - if not connector.config.get("auth_expired"): - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - connector.id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": connector.id, - "message": "This Gmail account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info(f"Gmail email trashed: message_id={final_message_id}") - - trash_result: dict[str, Any] = { - "status": "success", - "message_id": final_message_id, - "message": f"Successfully moved email '{email.get('subject', email_subject_or_id)}' to trash.", - } - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - trash_result["warning"] = ( - f"Email trashed, but failed to remove from knowledge base: {e!s}" - ) - - trash_result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - trash_result["message"] = ( - f"{trash_result.get('message', '')} (also removed from knowledge base)" - ) - - return trash_result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error trashing Gmail email: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while trashing the email. Please try again.", - } - - return trash_gmail_email diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/update_draft.py b/surfsense_backend/app/agents/new_chat/tools/gmail/update_draft.py deleted file mode 100644 index 129b7defb..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/update_draft.py +++ /dev/null @@ -1,495 +0,0 @@ -import asyncio -import base64 -import logging -from datetime import datetime -from email.mime.text import MIMEText -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.gmail import GmailToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_gmail_draft_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the update_gmail_draft tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured update_gmail_draft tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_gmail_draft( - draft_subject_or_id: str, - body: str, - to: str | None = None, - subject: str | None = None, - cc: str | None = None, - bcc: str | None = None, - ) -> dict[str, Any]: - """Update an existing Gmail draft. - - Use when the user asks to modify, edit, or add content to an existing - email draft. This replaces the draft content with the new version. - The user will be able to review and edit the content before it is applied. - - If the user simply wants to "edit" a draft without specifying exact changes, - generate the body yourself using your best understanding of the conversation - context. The user will review and can freely edit the content in the approval - card before confirming. - - IMPORTANT: This tool is ONLY for modifying Gmail draft content, NOT for - deleting/trashing drafts (use trash_gmail_email instead), Notion pages, - calendar events, or any other content type. - - Args: - draft_subject_or_id: The exact subject line of the draft to update - (as it appears in Gmail drafts). - body: The full updated body content for the draft. Generate this - yourself based on the user's request and conversation context. - to: Optional new recipient email address (keeps original if omitted). - subject: Optional new subject line (keeps original if omitted). - cc: Optional CC recipient(s), comma-separated. - bcc: Optional BCC recipient(s), comma-separated. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - draft_id: Gmail draft ID (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the draft subject or check if it has been indexed. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry the action. - - Examples: - - "Update the Kurseong Plan draft with the new itinerary details" - - "Edit my draft about the project proposal and change the recipient" - - "Let me edit the meeting notes draft" (call with current body content so user can edit in the approval card) - """ - logger.info( - f"update_gmail_draft called: draft_subject_or_id='{draft_subject_or_id}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Gmail tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GmailToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, draft_subject_or_id - ) - - if "error" in context: - error_msg = context["error"] - if "not found" in error_msg.lower(): - logger.warning(f"Draft not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - logger.error(f"Failed to fetch update context: {error_msg}") - return {"status": "error", "message": error_msg} - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Gmail account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Gmail account for this draft needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "gmail", - } - - email = context["email"] - message_id = email["message_id"] - document_id = email.get("document_id") - connector_id_from_context = account["id"] - draft_id_from_context = context.get("draft_id") - - original_subject = email.get("subject", draft_subject_or_id) - final_subject_default = subject if subject else original_subject - final_to_default = to if to else "" - - logger.info( - f"Requesting approval for updating Gmail draft: '{original_subject}' " - f"(message_id={message_id}, draft_id={draft_id_from_context})" - ) - result = request_approval( - action_type="gmail_draft_update", - tool_name="update_gmail_draft", - params={ - "message_id": message_id, - "draft_id": draft_id_from_context, - "to": final_to_default, - "subject": final_subject_default, - "body": body, - "cc": cc, - "bcc": bcc, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The draft was not updated. Do not ask again or suggest alternatives.", - } - - final_to = result.params.get("to", final_to_default) - final_subject = result.params.get("subject", final_subject_default) - final_body = result.params.get("body", body) - final_cc = result.params.get("cc", cc) - final_bcc = result.params.get("bcc", bcc) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_draft_id = result.params.get("draft_id", draft_id_from_context) - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this draft.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _gmail_types = [ - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, - ] - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_gmail_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Gmail connector is invalid or has been disconnected.", - } - - logger.info( - f"Updating Gmail draft: subject='{final_subject}', connector={final_connector_id}" - ) - - is_composio_gmail = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR - ) - if is_composio_gmail: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Gmail connector.", - } - else: - from google.oauth2.credentials import Credentials - - from app.config import config - from app.utils.oauth_security import TokenEncryption - - config_data = dict(connector.config) - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and config.SECRET_KEY: - token_encryption = TokenEncryption(config.SECRET_KEY) - if config_data.get("token"): - config_data["token"] = token_encryption.decrypt_token( - config_data["token"] - ) - if config_data.get("refresh_token"): - config_data["refresh_token"] = ( - token_encryption.decrypt_token( - config_data["refresh_token"] - ) - ) - if config_data.get("client_secret"): - config_data["client_secret"] = ( - token_encryption.decrypt_token( - config_data["client_secret"] - ) - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - # Resolve draft_id if not already available - if not final_draft_id: - logger.info( - f"draft_id not in metadata, looking up via drafts.list for message_id={message_id}" - ) - if is_composio_gmail: - final_draft_id = await _find_composio_draft_id_by_message( - connector, user_id, message_id - ) - else: - from googleapiclient.discovery import build - - gmail_service = build("gmail", "v1", credentials=creds) - final_draft_id = await _find_draft_id_by_message( - gmail_service, message_id - ) - - if not final_draft_id: - return { - "status": "error", - "message": ( - "Could not find this draft in Gmail. " - "It may have already been sent or deleted." - ), - } - - message = MIMEText(final_body) - if final_to: - message["to"] = final_to - message["subject"] = final_subject - if final_cc: - message["cc"] = final_cc - if final_bcc: - message["bcc"] = final_bcc - raw = base64.urlsafe_b64encode(message.as_bytes()).decode() - - try: - if is_composio_gmail: - from app.agents.new_chat.tools.gmail.composio_helpers import ( - execute_composio_gmail_tool, - split_recipients, - ) - - updated, error = await execute_composio_gmail_tool( - connector, - user_id, - "GMAIL_UPDATE_DRAFT", - { - "user_id": "me", - "draft_id": final_draft_id, - "recipient_email": final_to, - "subject": final_subject, - "body": final_body, - "cc": split_recipients(final_cc), - "bcc": split_recipients(final_bcc), - "is_html": False, - }, - ) - if error: - raise RuntimeError(error) - if not isinstance(updated, dict): - updated = {} - else: - from googleapiclient.discovery import build - - gmail_service = build("gmail", "v1", credentials=creds) - updated = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - gmail_service.users() - .drafts() - .update( - userId="me", - id=final_draft_id, - body={"message": {"raw": raw}}, - ) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {connector.id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - if not connector.config.get("auth_expired"): - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - connector.id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": connector.id, - "message": "This Gmail account needs additional permissions. Please re-authenticate in connector settings.", - } - if isinstance(api_err, HttpError) and api_err.resp.status == 404: - return { - "status": "error", - "message": "Draft no longer exists in Gmail. It may have been sent or deleted.", - } - raise - - logger.info(f"Gmail draft updated: id={updated.get('id')}") - - kb_message_suffix = "" - if document_id: - try: - from sqlalchemy.future import select as sa_select - from sqlalchemy.orm.attributes import flag_modified - - from app.db import Document - - doc_result = await db_session.execute( - sa_select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - document.source_markdown = final_body - document.title = final_subject - meta = dict(document.document_metadata or {}) - meta["subject"] = final_subject - meta["draft_id"] = updated.get("id", final_draft_id) - updated_msg = updated.get("message", {}) - if updated_msg.get("id"): - meta["message_id"] = updated_msg["id"] - document.document_metadata = meta - flag_modified(document, "document_metadata") - await db_session.commit() - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - logger.info( - f"KB document {document_id} updated for draft {final_draft_id}" - ) - else: - kb_message_suffix = " This draft will be fully updated in your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB update after draft edit failed: {kb_err}") - await db_session.rollback() - kb_message_suffix = " This draft will be fully updated in your knowledge base in the next scheduled sync." - - return { - "status": "success", - "draft_id": updated.get("id"), - "message": f"Successfully updated Gmail draft with subject '{final_subject}'.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error updating Gmail draft: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while updating the draft. Please try again.", - } - - return update_gmail_draft - - -async def _find_draft_id_by_message(gmail_service: Any, message_id: str) -> str | None: - """Look up a draft's ID by its message ID via the Gmail API.""" - try: - page_token = None - while True: - kwargs: dict[str, Any] = {"userId": "me", "maxResults": 100} - if page_token: - kwargs["pageToken"] = page_token - - response = await asyncio.get_event_loop().run_in_executor( - None, - lambda kwargs=kwargs: ( - gmail_service.users().drafts().list(**kwargs).execute() - ), - ) - - for draft in response.get("drafts", []): - if draft.get("message", {}).get("id") == message_id: - return draft["id"] - - page_token = response.get("nextPageToken") - if not page_token: - break - - return None - except Exception as e: - logger.warning(f"Failed to look up draft by message_id: {e}") - return None - - -async def _find_composio_draft_id_by_message( - connector: Any, user_id: str, message_id: str -) -> str | None: - from app.agents.new_chat.tools.gmail.composio_helpers import ( - execute_composio_gmail_tool, - ) - - page_token = "" - while True: - params: dict[str, Any] = { - "user_id": "me", - "max_results": 100, - "verbose": False, - } - if page_token: - params["page_token"] = page_token - - data, error = await execute_composio_gmail_tool( - connector, user_id, "GMAIL_LIST_DRAFTS", params - ) - if error or not isinstance(data, dict): - return None - - for draft in data.get("drafts", []): - if draft.get("message", {}).get("id") == message_id: - return draft.get("id") - - page_token = data.get("nextPageToken") or data.get("next_page_token") or "" - if not page_token: - return None diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/__init__.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/__init__.py deleted file mode 100644 index 13d4c06cb..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from app.agents.new_chat.tools.google_calendar.create_event import ( - create_create_calendar_event_tool, -) -from app.agents.new_chat.tools.google_calendar.delete_event import ( - create_delete_calendar_event_tool, -) -from app.agents.new_chat.tools.google_calendar.search_events import ( - create_search_calendar_events_tool, -) -from app.agents.new_chat.tools.google_calendar.update_event import ( - create_update_calendar_event_tool, -) - -__all__ = [ - "create_create_calendar_event_tool", - "create_delete_calendar_event_tool", - "create_search_calendar_events_tool", - "create_update_calendar_event_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/create_event.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/create_event.py deleted file mode 100644 index dec92cc8b..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/create_event.py +++ /dev/null @@ -1,382 +0,0 @@ -import asyncio -import logging -from datetime import datetime -from typing import Any - -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.google_calendar import GoogleCalendarToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_calendar_event_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_calendar_event tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_calendar_event tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_calendar_event( - summary: str, - start_datetime: str, - end_datetime: str, - description: str | None = None, - location: str | None = None, - attendees: list[str] | None = None, - ) -> dict[str, Any]: - """Create a new event on Google Calendar. - - Use when the user asks to schedule, create, or add a calendar event. - Ask for event details if not provided. - - Args: - summary: The event title. - start_datetime: Start time in ISO 8601 format (e.g. "2026-03-20T10:00:00"). - end_datetime: End time in ISO 8601 format (e.g. "2026-03-20T11:00:00"). - description: Optional event description. - location: Optional event location. - attendees: Optional list of attendee email addresses. - - Returns: - Dictionary with: - - status: "success", "rejected", "auth_error", or "error" - - event_id: Google Calendar event ID (if success) - - html_link: URL to open the event (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment and do NOT retry or suggest alternatives. - - Examples: - - "Schedule a meeting with John tomorrow at 10am" - - "Create a calendar event for the team standup" - """ - logger.info( - f"create_calendar_event called: summary='{summary}', start='{start_datetime}', end='{end_datetime}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Google Calendar tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GoogleCalendarToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning( - "All Google Calendar accounts have expired authentication" - ) - return { - "status": "auth_error", - "message": "All connected Google Calendar accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "google_calendar", - } - - logger.info( - f"Requesting approval for creating calendar event: summary='{summary}'" - ) - result = request_approval( - action_type="google_calendar_event_creation", - tool_name="create_calendar_event", - params={ - "summary": summary, - "start_datetime": start_datetime, - "end_datetime": end_datetime, - "description": description, - "location": location, - "attendees": attendees, - "timezone": context.get("timezone"), - "connector_id": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The event was not created. Do not ask again or suggest alternatives.", - } - - final_summary = result.params.get("summary", summary) - final_start_datetime = result.params.get( - "start_datetime", start_datetime - ) - final_end_datetime = result.params.get("end_datetime", end_datetime) - final_description = result.params.get("description", description) - final_location = result.params.get("location", location) - final_attendees = result.params.get("attendees", attendees) - final_connector_id = result.params.get("connector_id") - - if not final_summary or not final_summary.strip(): - return { - "status": "error", - "message": "Event summary cannot be empty.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _calendar_types = [ - SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, - ] - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_calendar_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Google Calendar connector is invalid or has been disconnected.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_calendar_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Google Calendar connector found. Please connect Google Calendar in your workspace settings.", - } - actual_connector_id = connector.id - - logger.info( - f"Creating calendar event: summary='{final_summary}', connector={actual_connector_id}" - ) - - is_composio_calendar = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR - ) - if is_composio_calendar: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this connector.", - } - else: - config_data = dict(connector.config) - - from app.config import config as app_config - from app.utils.oauth_security import TokenEncryption - - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and app_config.SECRET_KEY: - token_encryption = TokenEncryption(app_config.SECRET_KEY) - for key in ("token", "refresh_token", "client_secret"): - if config_data.get(key): - config_data[key] = token_encryption.decrypt_token( - config_data[key] - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - tz = context.get("timezone", "UTC") - event_body: dict[str, Any] = { - "summary": final_summary, - "start": {"dateTime": final_start_datetime, "timeZone": tz}, - "end": {"dateTime": final_end_datetime, "timeZone": tz}, - } - if final_description: - event_body["description"] = final_description - if final_location: - event_body["location"] = final_location - if final_attendees: - event_body["attendees"] = [ - {"email": e.strip()} for e in final_attendees if e.strip() - ] - - try: - if is_composio_calendar: - from app.services.composio_service import ComposioService - - composio_params = { - "calendar_id": "primary", - "summary": final_summary, - "start_datetime": final_start_datetime, - "end_datetime": final_end_datetime, - "timezone": tz, - "attendees": final_attendees or [], - } - if final_description: - composio_params["description"] = final_description - if final_location: - composio_params["location"] = final_location - - composio_result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name="GOOGLECALENDAR_CREATE_EVENT", - params=composio_params, - entity_id=f"surfsense_{user_id}", - ) - if not composio_result.get("success"): - raise RuntimeError( - composio_result.get( - "error", "Unknown Composio Calendar error" - ) - ) - created = composio_result.get("data", {}) - if isinstance(created, dict): - created = created.get("data", created) - if isinstance(created, dict): - created = created.get("response_data", created) - else: - service = await asyncio.get_event_loop().run_in_executor( - None, lambda: build("calendar", "v3", credentials=creds) - ) - created = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - service.events() - .insert(calendarId="primary", body=event_body) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Google Calendar account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info( - f"Calendar event created: id={created.get('id')}, summary={created.get('summary')}" - ) - - kb_message_suffix = "" - try: - from app.services.google_calendar import GoogleCalendarKBSyncService - - kb_service = GoogleCalendarKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - event_id=created.get("id"), - event_summary=final_summary, - calendar_id="primary", - start_time=final_start_datetime, - end_time=final_end_datetime, - location=final_location, - html_link=created.get("htmlLink"), - description=final_description, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This event will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This event will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "event_id": created.get("id"), - "html_link": created.get("htmlLink"), - "message": f"Successfully created '{final_summary}' on Google Calendar.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating calendar event: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the event. Please try again.", - } - - return create_calendar_event diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/delete_event.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/delete_event.py deleted file mode 100644 index e7e891b08..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/delete_event.py +++ /dev/null @@ -1,340 +0,0 @@ -import asyncio -import logging -from datetime import datetime -from typing import Any - -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.google_calendar import GoogleCalendarToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_calendar_event_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the delete_calendar_event tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured delete_calendar_event tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_calendar_event( - event_title_or_id: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete a Google Calendar event. - - Use when the user asks to delete, remove, or cancel a calendar event. - - Args: - event_title_or_id: The exact title or event ID of the event to delete. - delete_from_kb: Whether to also remove the event from the knowledge base. - Default is False. - Set to True to remove from both Google Calendar and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", "auth_error", or "error" - - event_id: Google Calendar event ID (if success) - - deleted_from_kb: whether the document was removed from the knowledge base - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the event name or check if it has been indexed. - Examples: - - "Delete the team standup event" - - "Cancel my dentist appointment on Friday" - """ - logger.info( - f"delete_calendar_event called: event_ref='{event_title_or_id}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Google Calendar tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GoogleCalendarToolMetadataService(db_session) - context = await metadata_service.get_deletion_context( - search_space_id, user_id, event_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if "not found" in error_msg.lower(): - logger.warning(f"Event not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - logger.error(f"Failed to fetch deletion context: {error_msg}") - return {"status": "error", "message": error_msg} - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Google Calendar account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Google Calendar account for this event needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "google_calendar", - } - - event = context["event"] - event_id = event["event_id"] - document_id = event.get("document_id") - connector_id_from_context = context["account"]["id"] - - if not event_id: - return { - "status": "error", - "message": "Event ID is missing from the indexed document. Please re-index the event and try again.", - } - - logger.info( - f"Requesting approval for deleting calendar event: '{event_title_or_id}' (event_id={event_id}, delete_from_kb={delete_from_kb})" - ) - result = request_approval( - action_type="google_calendar_event_deletion", - tool_name="delete_calendar_event", - params={ - "event_id": event_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The event was not deleted. Do not ask again or suggest alternatives.", - } - - final_event_id = result.params.get("event_id", event_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this event.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _calendar_types = [ - SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, - ] - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_calendar_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Google Calendar connector is invalid or has been disconnected.", - } - - actual_connector_id = connector.id - - logger.info( - f"Deleting calendar event: event_id='{final_event_id}', connector={actual_connector_id}" - ) - - is_composio_calendar = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR - ) - if is_composio_calendar: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this connector.", - } - else: - config_data = dict(connector.config) - - from app.config import config as app_config - from app.utils.oauth_security import TokenEncryption - - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and app_config.SECRET_KEY: - token_encryption = TokenEncryption(app_config.SECRET_KEY) - for key in ("token", "refresh_token", "client_secret"): - if config_data.get(key): - config_data[key] = token_encryption.decrypt_token( - config_data[key] - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - try: - if is_composio_calendar: - from app.services.composio_service import ComposioService - - composio_result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name="GOOGLECALENDAR_DELETE_EVENT", - params={ - "calendar_id": "primary", - "event_id": final_event_id, - }, - entity_id=f"surfsense_{user_id}", - ) - if not composio_result.get("success"): - raise RuntimeError( - composio_result.get( - "error", "Unknown Composio Calendar error" - ) - ) - else: - service = await asyncio.get_event_loop().run_in_executor( - None, lambda: build("calendar", "v3", credentials=creds) - ) - await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - service.events() - .delete(calendarId="primary", eventId=final_event_id) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Google Calendar account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info(f"Calendar event deleted: event_id={final_event_id}") - - delete_result: dict[str, Any] = { - "status": "success", - "event_id": final_event_id, - "message": f"Successfully deleted the calendar event '{event.get('summary', event_title_or_id)}'.", - } - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - delete_result["warning"] = ( - f"Event deleted, but failed to remove from knowledge base: {e!s}" - ) - - delete_result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - delete_result["message"] = ( - f"{delete_result.get('message', '')} (also removed from knowledge base)" - ) - - return delete_result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error deleting calendar event: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while deleting the event. Please try again.", - } - - return delete_calendar_event diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py deleted file mode 100644 index e5f18f675..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.agents.new_chat.tools.gmail.search_emails import _build_credentials -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker - -logger = logging.getLogger(__name__) - -_CALENDAR_TYPES = [ - SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, -] - - -def _to_calendar_boundary(value: str, *, is_end: bool) -> str: - if "T" in value: - return value - time = "23:59:59" if is_end else "00:00:00" - return f"{value}T{time}Z" - - -def _format_calendar_events(events_raw: list[dict[str, Any]]) -> list[dict[str, Any]]: - events = [] - for ev in events_raw: - start = ev.get("start", {}) - end = ev.get("end", {}) - attendees_raw = ev.get("attendees", []) - events.append( - { - "event_id": ev.get("id"), - "summary": ev.get("summary", "No Title"), - "start": start.get("dateTime") or start.get("date", ""), - "end": end.get("dateTime") or end.get("date", ""), - "location": ev.get("location", ""), - "description": ev.get("description", ""), - "html_link": ev.get("htmlLink", ""), - "attendees": [a.get("email", "") for a in attendees_raw[:10]], - "status": ev.get("status", ""), - } - ) - return events - - -def create_search_calendar_events_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the search_calendar_events tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured search_calendar_events tool - """ - del db_session # per-call session — see docstring - - @tool - async def search_calendar_events( - start_date: str, - end_date: str, - max_results: int = 25, - ) -> dict[str, Any]: - """Search Google Calendar events within a date range. - - Args: - start_date: Start date in YYYY-MM-DD format (e.g. "2026-04-01"). - end_date: End date in YYYY-MM-DD format (e.g. "2026-04-30"). - max_results: Maximum number of events to return (default 25, max 50). - - Returns: - Dictionary with status and a list of events including - event_id, summary, start, end, location, attendees. - """ - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Calendar tool not properly configured.", - } - - max_results = min(max_results, 50) - - try: - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_CALENDAR_TYPES), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Google Calendar connector found. Please connect Google Calendar in your workspace settings.", - } - - if ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR - ): - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this connector.", - } - - from app.services.composio_service import ComposioService - - events_raw, error = await ComposioService().get_calendar_events( - connected_account_id=cca_id, - entity_id=f"surfsense_{user_id}", - time_min=_to_calendar_boundary(start_date, is_end=False), - time_max=_to_calendar_boundary(end_date, is_end=True), - max_results=max_results, - ) - if not events_raw and not error: - error = "No events found in the specified date range." - else: - creds = _build_credentials(connector) - - from app.connectors.google_calendar_connector import ( - GoogleCalendarConnector, - ) - - cal = GoogleCalendarConnector( - credentials=creds, - session=db_session, - user_id=user_id, - connector_id=connector.id, - ) - - events_raw, error = await cal.get_all_primary_calendar_events( - start_date=start_date, - end_date=end_date, - max_results=max_results, - ) - - if error: - if ( - "re-authenticate" in error.lower() - or "authentication failed" in error.lower() - ): - return { - "status": "auth_error", - "message": error, - "connector_type": "google_calendar", - } - if "no events found" in error.lower(): - return { - "status": "success", - "events": [], - "total": 0, - "message": error, - } - return {"status": "error", "message": error} - - events = _format_calendar_events(events_raw) - - return {"status": "success", "events": events, "total": len(events)} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error searching calendar events: %s", e, exc_info=True) - return { - "status": "error", - "message": "Failed to search calendar events. Please try again.", - } - - return search_calendar_events diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/update_event.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/update_event.py deleted file mode 100644 index b8561fee6..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/update_event.py +++ /dev/null @@ -1,419 +0,0 @@ -import asyncio -import logging -from datetime import datetime -from typing import Any - -from google.oauth2.credentials import Credentials -from googleapiclient.discovery import build -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker -from app.services.google_calendar import GoogleCalendarToolMetadataService - -logger = logging.getLogger(__name__) - - -def _is_date_only(value: str) -> bool: - """Return True when *value* looks like a bare date (YYYY-MM-DD) with no time component.""" - return len(value) <= 10 and "T" not in value - - -def _build_time_body(value: str, context: dict[str, Any] | Any) -> dict[str, str]: - """Build a Google Calendar start/end body using ``date`` for all-day - events and ``dateTime`` for timed events.""" - if _is_date_only(value): - return {"date": value} - tz = context.get("timezone", "UTC") if isinstance(context, dict) else "UTC" - return {"dateTime": value, "timeZone": tz} - - -def create_update_calendar_event_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the update_calendar_event tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured update_calendar_event tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_calendar_event( - event_title_or_id: str, - new_summary: str | None = None, - new_start_datetime: str | None = None, - new_end_datetime: str | None = None, - new_description: str | None = None, - new_location: str | None = None, - new_attendees: list[str] | None = None, - ) -> dict[str, Any]: - """Update an existing Google Calendar event. - - Use when the user asks to modify, reschedule, or change a calendar event. - - Args: - event_title_or_id: The exact title or event ID of the event to update. - new_summary: New event title (if changing). - new_start_datetime: New start time in ISO 8601 format (if rescheduling). - new_end_datetime: New end time in ISO 8601 format (if rescheduling). - new_description: New event description (if changing). - new_location: New event location (if changing). - new_attendees: New list of attendee email addresses (if changing). - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", "auth_error", or "error" - - event_id: Google Calendar event ID (if success) - - html_link: URL to open the event (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the event name or check if it has been indexed. - Examples: - - "Reschedule the team standup to 3pm" - - "Change the location of my dentist appointment" - """ - logger.info(f"update_calendar_event called: event_ref='{event_title_or_id}'") - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Google Calendar tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GoogleCalendarToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, event_title_or_id - ) - - if "error" in context: - error_msg = context["error"] - if "not found" in error_msg.lower(): - logger.warning(f"Event not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - logger.error(f"Failed to fetch update context: {error_msg}") - return {"status": "error", "message": error_msg} - - if context.get("auth_expired"): - logger.warning("Google Calendar account has expired authentication") - return { - "status": "auth_error", - "message": "The Google Calendar account for this event needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "google_calendar", - } - - event = context["event"] - event_id = event["event_id"] - document_id = event.get("document_id") - connector_id_from_context = context["account"]["id"] - - if not event_id: - return { - "status": "error", - "message": "Event ID is missing from the indexed document. Please re-index the event and try again.", - } - - logger.info( - f"Requesting approval for updating calendar event: '{event_title_or_id}' (event_id={event_id})" - ) - result = request_approval( - action_type="google_calendar_event_update", - tool_name="update_calendar_event", - params={ - "event_id": event_id, - "document_id": document_id, - "connector_id": connector_id_from_context, - "new_summary": new_summary, - "new_start_datetime": new_start_datetime, - "new_end_datetime": new_end_datetime, - "new_description": new_description, - "new_location": new_location, - "new_attendees": new_attendees, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The event was not updated. Do not ask again or suggest alternatives.", - } - - final_event_id = result.params.get("event_id", event_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_new_summary = result.params.get("new_summary", new_summary) - final_new_start_datetime = result.params.get( - "new_start_datetime", new_start_datetime - ) - final_new_end_datetime = result.params.get( - "new_end_datetime", new_end_datetime - ) - final_new_description = result.params.get( - "new_description", new_description - ) - final_new_location = result.params.get("new_location", new_location) - final_new_attendees = result.params.get("new_attendees", new_attendees) - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this event.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _calendar_types = [ - SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, - ] - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_calendar_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Google Calendar connector is invalid or has been disconnected.", - } - - actual_connector_id = connector.id - - logger.info( - f"Updating calendar event: event_id='{final_event_id}', connector={actual_connector_id}" - ) - - is_composio_calendar = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR - ) - if is_composio_calendar: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this connector.", - } - else: - config_data = dict(connector.config) - - from app.config import config as app_config - from app.utils.oauth_security import TokenEncryption - - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and app_config.SECRET_KEY: - token_encryption = TokenEncryption(app_config.SECRET_KEY) - for key in ("token", "refresh_token", "client_secret"): - if config_data.get(key): - config_data[key] = token_encryption.decrypt_token( - config_data[key] - ) - - exp = config_data.get("expiry", "") - if exp: - exp = exp.replace("Z", "") - - creds = Credentials( - token=config_data.get("token"), - refresh_token=config_data.get("refresh_token"), - token_uri=config_data.get("token_uri"), - client_id=config_data.get("client_id"), - client_secret=config_data.get("client_secret"), - scopes=config_data.get("scopes", []), - expiry=datetime.fromisoformat(exp) if exp else None, - ) - - update_body: dict[str, Any] = {} - if final_new_summary is not None: - update_body["summary"] = final_new_summary - if final_new_start_datetime is not None: - update_body["start"] = _build_time_body( - final_new_start_datetime, context - ) - if final_new_end_datetime is not None: - update_body["end"] = _build_time_body( - final_new_end_datetime, context - ) - if final_new_description is not None: - update_body["description"] = final_new_description - if final_new_location is not None: - update_body["location"] = final_new_location - if final_new_attendees is not None: - update_body["attendees"] = [ - {"email": e.strip()} for e in final_new_attendees if e.strip() - ] - - if not update_body: - return { - "status": "error", - "message": "No changes specified. Please provide at least one field to update.", - } - - try: - if is_composio_calendar: - from app.services.composio_service import ComposioService - - composio_params: dict[str, Any] = { - "calendar_id": "primary", - "event_id": final_event_id, - } - if final_new_summary is not None: - composio_params["summary"] = final_new_summary - if final_new_start_datetime is not None: - composio_params["start_time"] = final_new_start_datetime - if final_new_end_datetime is not None: - composio_params["end_time"] = final_new_end_datetime - if final_new_description is not None: - composio_params["description"] = final_new_description - if final_new_location is not None: - composio_params["location"] = final_new_location - if final_new_attendees is not None: - composio_params["attendees"] = [ - e.strip() for e in final_new_attendees if e.strip() - ] - if not _is_date_only( - final_new_start_datetime or final_new_end_datetime or "" - ): - composio_params["timezone"] = context.get("timezone", "UTC") - - composio_result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name="GOOGLECALENDAR_PATCH_EVENT", - params=composio_params, - entity_id=f"surfsense_{user_id}", - ) - if not composio_result.get("success"): - raise RuntimeError( - composio_result.get( - "error", "Unknown Composio Calendar error" - ) - ) - updated = composio_result.get("data", {}) - if isinstance(updated, dict): - updated = updated.get("data", updated) - if isinstance(updated, dict): - updated = updated.get("response_data", updated) - else: - service = await asyncio.get_event_loop().run_in_executor( - None, lambda: build("calendar", "v3", credentials=creds) - ) - updated = await asyncio.get_event_loop().run_in_executor( - None, - lambda: ( - service.events() - .patch( - calendarId="primary", - eventId=final_event_id, - body=update_body, - ) - .execute() - ), - ) - except Exception as api_err: - from googleapiclient.errors import HttpError - - if isinstance(api_err, HttpError) and api_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {api_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Google Calendar account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info(f"Calendar event updated: event_id={final_event_id}") - - kb_message_suffix = "" - if document_id is not None: - try: - from app.services.google_calendar import ( - GoogleCalendarKBSyncService, - ) - - kb_service = GoogleCalendarKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=document_id, - event_id=final_event_id, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " The knowledge base will be updated in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after update failed: {kb_err}") - kb_message_suffix = " The knowledge base will be updated in the next scheduled sync." - - return { - "status": "success", - "event_id": final_event_id, - "html_link": updated.get("htmlLink"), - "message": f"Successfully updated the calendar event.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error updating calendar event: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while updating the event. Please try again.", - } - - return update_calendar_event diff --git a/surfsense_backend/app/agents/new_chat/tools/google_drive/__init__.py b/surfsense_backend/app/agents/new_chat/tools/google_drive/__init__.py deleted file mode 100644 index 9c63bceb1..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_drive/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.google_drive.create_file import ( - create_create_google_drive_file_tool, -) -from app.agents.new_chat.tools.google_drive.trash_file import ( - create_delete_google_drive_file_tool, -) - -__all__ = [ - "create_create_google_drive_file_tool", - "create_delete_google_drive_file_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/google_drive/create_file.py b/surfsense_backend/app/agents/new_chat/tools/google_drive/create_file.py deleted file mode 100644 index 66199ca67..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_drive/create_file.py +++ /dev/null @@ -1,340 +0,0 @@ -import logging -from typing import Any, Literal - -from googleapiclient.errors import HttpError -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.google_drive.client import GoogleDriveClient -from app.connectors.google_drive.file_types import GOOGLE_DOC, GOOGLE_SHEET -from app.db import async_session_maker -from app.services.google_drive import GoogleDriveToolMetadataService - -logger = logging.getLogger(__name__) - -_MIME_MAP: dict[str, str] = { - "google_doc": GOOGLE_DOC, - "google_sheet": GOOGLE_SHEET, -} - - -def create_create_google_drive_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_google_drive_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Google Drive connector - user_id: User ID for fetching user-specific context - - Returns: - Configured create_google_drive_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_google_drive_file( - name: str, - file_type: Literal["google_doc", "google_sheet"], - content: str | None = None, - ) -> dict[str, Any]: - """Create a new Google Doc or Google Sheet in Google Drive. - - Use this tool when the user explicitly asks to create a new document - or spreadsheet in Google Drive. The user MUST specify a topic before - you call this tool. If the request does not contain a topic (e.g. - "create a drive doc" or "make a Google Sheet"), ask what the file - should be about. Never call this tool without a clear topic from the user. - - Args: - name: The file name (without extension). - file_type: Either "google_doc" or "google_sheet". - content: Optional initial content. Generate from the user's topic. - For google_doc, provide markdown text. For google_sheet, provide CSV-formatted text. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - file_id: Google Drive file ID (if success) - - name: File name (if success) - - web_view_link: URL to open the file (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment and do NOT retry or suggest alternatives. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry the action. - - Examples: - - "Create a Google Doc with today's meeting notes" - - "Create a spreadsheet for the 2026 budget" - """ - logger.info( - f"create_google_drive_file called: name='{name}', type='{file_type}'" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Google Drive tool not properly configured. Please contact support.", - } - - if file_type not in _MIME_MAP: - return { - "status": "error", - "message": f"Unsupported file type '{file_type}'. Use 'google_doc' or 'google_sheet'.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GoogleDriveToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning( - "All Google Drive accounts have expired authentication" - ) - return { - "status": "auth_error", - "message": "All connected Google Drive accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "google_drive", - } - - logger.info( - f"Requesting approval for creating Google Drive file: name='{name}', type='{file_type}'" - ) - result = request_approval( - action_type="google_drive_file_creation", - tool_name="create_google_drive_file", - params={ - "name": name, - "file_type": file_type, - "content": content, - "connector_id": None, - "parent_folder_id": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The file was not created. Do not ask again or suggest alternatives.", - } - - final_name = result.params.get("name", name) - final_file_type = result.params.get("file_type", file_type) - final_content = result.params.get("content", content) - final_connector_id = result.params.get("connector_id") - final_parent_folder_id = result.params.get("parent_folder_id") - - if not final_name or not final_name.strip(): - return {"status": "error", "message": "File name cannot be empty."} - - mime_type = _MIME_MAP.get(final_file_type) - if not mime_type: - return { - "status": "error", - "message": f"Unsupported file type '{final_file_type}'.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _drive_types = [ - SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, - ] - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_drive_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Google Drive connector is invalid or has been disconnected.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_drive_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Google Drive connector found. Please connect Google Drive in your workspace settings.", - } - actual_connector_id = connector.id - - logger.info( - f"Creating Google Drive file: name='{final_name}', type='{final_file_type}', connector={actual_connector_id}" - ) - - is_composio_drive = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR - ) - if is_composio_drive: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Drive connector.", - } - client = GoogleDriveClient( - session=db_session, - connector_id=actual_connector_id, - ) - try: - if is_composio_drive: - from app.services.composio_service import ComposioService - - params: dict[str, Any] = { - "name": final_name, - "mimeType": mime_type, - "fields": "id,name,webViewLink,mimeType", - } - if final_parent_folder_id: - params["parents"] = [final_parent_folder_id] - if final_content: - params["description"] = final_content[:4096] - - result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name="GOOGLEDRIVE_CREATE_FILE", - params=params, - entity_id=f"surfsense_{user_id}", - ) - if not result.get("success"): - raise RuntimeError( - result.get("error", "Unknown Composio Drive error") - ) - created = result.get("data", {}) - if isinstance(created, dict): - created = created.get("data", created) - if isinstance(created, dict): - created = created.get("response_data", created) - if not isinstance(created, dict): - created = {} - else: - created = await client.create_file( - name=final_name, - mime_type=mime_type, - parent_folder_id=final_parent_folder_id, - content=final_content, - ) - except HttpError as http_err: - if http_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {actual_connector_id}: {http_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - _res = await db_session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == actual_connector_id - ) - ) - _conn = _res.scalar_one_or_none() - if _conn and not _conn.config.get("auth_expired"): - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - actual_connector_id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Google Drive account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info( - f"Google Drive file created: id={created.get('id')}, name={created.get('name')}" - ) - - kb_message_suffix = "" - try: - from app.services.google_drive import GoogleDriveKBSyncService - - kb_service = GoogleDriveKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - file_id=created.get("id"), - file_name=created.get("name", final_name), - mime_type=mime_type, - web_view_link=created.get("webViewLink"), - content=final_content, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "file_id": created.get("id"), - "name": created.get("name"), - "web_view_link": created.get("webViewLink"), - "message": f"Successfully created '{created.get('name')}' in Google Drive.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating Google Drive file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the file. Please try again.", - } - - return create_google_drive_file diff --git a/surfsense_backend/app/agents/new_chat/tools/google_drive/trash_file.py b/surfsense_backend/app/agents/new_chat/tools/google_drive/trash_file.py deleted file mode 100644 index b3c9240d8..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/google_drive/trash_file.py +++ /dev/null @@ -1,299 +0,0 @@ -import logging -from typing import Any - -from googleapiclient.errors import HttpError -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.google_drive.client import GoogleDriveClient -from app.db import async_session_maker -from app.services.google_drive import GoogleDriveToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_google_drive_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the delete_google_drive_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Google Drive connector - user_id: User ID for fetching user-specific context - - Returns: - Configured delete_google_drive_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_google_drive_file( - file_name: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Move a Google Drive file to trash. - - Use this tool when the user explicitly asks to delete, remove, or trash - a file in Google Drive. - - Args: - file_name: The exact name of the file to trash (as it appears in Drive). - delete_from_kb: Whether to also remove the file from the knowledge base. - Default is False. - Set to True to remove from both Google Drive and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - file_id: Google Drive file ID (if success) - - deleted_from_kb: whether the document was removed from the knowledge base - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the file name or check if it has been indexed. - - If status is "insufficient_permissions", the connector lacks the required OAuth scope. - Inform the user they need to re-authenticate and do NOT retry this tool. - Examples: - - "Delete the 'Meeting Notes' file from Google Drive" - - "Trash the 'Old Budget' spreadsheet" - """ - logger.info( - f"delete_google_drive_file called: file_name='{file_name}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "Google Drive tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = GoogleDriveToolMetadataService(db_session) - context = await metadata_service.get_trash_context( - search_space_id, user_id, file_name - ) - - if "error" in context: - error_msg = context["error"] - if "not found" in error_msg.lower(): - logger.warning(f"File not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - logger.error(f"Failed to fetch trash context: {error_msg}") - return {"status": "error", "message": error_msg} - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Google Drive account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Google Drive account for this file needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "google_drive", - } - - file = context["file"] - file_id = file["file_id"] - document_id = file.get("document_id") - connector_id_from_context = context["account"]["id"] - - if not file_id: - return { - "status": "error", - "message": "File ID is missing from the indexed document. Please re-index the file and try again.", - } - - logger.info( - f"Requesting approval for deleting Google Drive file: '{file_name}' (file_id={file_id}, delete_from_kb={delete_from_kb})" - ) - result = request_approval( - action_type="google_drive_file_trash", - tool_name="delete_google_drive_file", - params={ - "file_id": file_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. The file was not trashed. Do not ask again or suggest alternatives.", - } - - final_file_id = result.params.get("file_id", file_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this file.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - _drive_types = [ - SearchSourceConnectorType.GOOGLE_DRIVE_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, - ] - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type.in_(_drive_types), - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Google Drive connector is invalid or has been disconnected.", - } - - logger.info( - f"Deleting Google Drive file: file_id='{final_file_id}', connector={final_connector_id}" - ) - - is_composio_drive = ( - connector.connector_type - == SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR - ) - if is_composio_drive: - cca_id = connector.config.get("composio_connected_account_id") - if not cca_id: - return { - "status": "error", - "message": "Composio connected account ID not found for this Drive connector.", - } - - client = GoogleDriveClient( - session=db_session, - connector_id=connector.id, - ) - try: - if is_composio_drive: - from app.services.composio_service import ComposioService - - result = await ComposioService().execute_tool( - connected_account_id=cca_id, - tool_name="GOOGLEDRIVE_TRASH_FILE", - params={"file_id": final_file_id}, - entity_id=f"surfsense_{user_id}", - ) - if not result.get("success"): - raise RuntimeError( - result.get("error", "Unknown Composio Drive error") - ) - else: - await client.trash_file(file_id=final_file_id) - except HttpError as http_err: - if http_err.resp.status == 403: - logger.warning( - f"Insufficient permissions for connector {connector.id}: {http_err}" - ) - try: - from sqlalchemy.orm.attributes import flag_modified - - if not connector.config.get("auth_expired"): - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - connector.id, - exc_info=True, - ) - return { - "status": "insufficient_permissions", - "connector_id": connector.id, - "message": "This Google Drive account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - logger.info( - f"Google Drive file deleted (moved to trash): file_id={final_file_id}" - ) - - trash_result: dict[str, Any] = { - "status": "success", - "file_id": final_file_id, - "message": f"Successfully moved '{file['name']}' to trash.", - } - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - trash_result["warning"] = ( - f"File moved to trash, but failed to remove from knowledge base: {e!s}" - ) - - trash_result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - trash_result["message"] = ( - f"{trash_result.get('message', '')} (also removed from knowledge base)" - ) - - return trash_result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error deleting Google Drive file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while trashing the file. Please try again.", - } - - return delete_google_drive_file diff --git a/surfsense_backend/app/agents/new_chat/tools/jira/__init__.py b/surfsense_backend/app/agents/new_chat/tools/jira/__init__.py deleted file mode 100644 index 768738118..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/jira/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Jira tools for creating, updating, and deleting issues.""" - -from .create_issue import create_create_jira_issue_tool -from .delete_issue import create_delete_jira_issue_tool -from .update_issue import create_update_jira_issue_tool - -__all__ = [ - "create_create_jira_issue_tool", - "create_delete_jira_issue_tool", - "create_update_jira_issue_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/jira/create_issue.py b/surfsense_backend/app/agents/new_chat/tools/jira/create_issue.py deleted file mode 100644 index 0b04f1642..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/jira/create_issue.py +++ /dev/null @@ -1,248 +0,0 @@ -import asyncio -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.jira_history import JiraHistoryConnector -from app.db import async_session_maker -from app.services.jira import JiraToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_jira_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the create_jira_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. Per-call sessions also - keep the request's outer transaction free of long-running Jira API - blocking. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Jira connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured create_jira_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_jira_issue( - project_key: str, - summary: str, - issue_type: str = "Task", - description: str | None = None, - priority: str | None = None, - ) -> dict[str, Any]: - """Create a new issue in Jira. - - Use this tool when the user explicitly asks to create a new Jira issue/ticket. - - Args: - project_key: The Jira project key (e.g. "PROJ", "ENG"). - summary: Short, descriptive issue title. - issue_type: Issue type (default "Task"). Others: "Bug", "Story", "Epic". - description: Optional description body for the issue. - priority: Optional priority name (e.g. "High", "Medium", "Low"). - - Returns: - Dictionary with status, issue_key, and message. - - IMPORTANT: - - If status is "rejected", the user declined. Do NOT retry. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"create_jira_issue called: project_key='{project_key}', summary='{summary}'" - ) - - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Jira tool not properly configured."} - - try: - async with async_session_maker() as db_session: - metadata_service = JiraToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - return {"status": "error", "message": context["error"]} - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - return { - "status": "auth_error", - "message": "All connected Jira accounts need re-authentication.", - "connector_type": "jira", - } - - result = request_approval( - action_type="jira_issue_creation", - tool_name="create_jira_issue", - params={ - "project_key": project_key, - "summary": summary, - "issue_type": issue_type, - "description": description, - "priority": priority, - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_project_key = result.params.get("project_key", project_key) - final_summary = result.params.get("summary", summary) - final_issue_type = result.params.get("issue_type", issue_type) - final_description = result.params.get("description", description) - final_priority = result.params.get("priority", priority) - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_summary or not final_summary.strip(): - return { - "status": "error", - "message": "Issue summary cannot be empty.", - } - if not final_project_key: - return {"status": "error", "message": "A project must be selected."} - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Jira connector found.", - } - actual_connector_id = connector.id - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Jira connector is invalid.", - } - - try: - jira_history = JiraHistoryConnector( - session=db_session, connector_id=actual_connector_id - ) - jira_client = await jira_history._get_jira_client() - api_result = await asyncio.to_thread( - jira_client.create_issue, - project_key=final_project_key, - summary=final_summary, - issue_type=final_issue_type, - description=final_description, - priority=final_priority, - ) - except Exception as api_err: - if "status code 403" in str(api_err).lower(): - try: - _conn = connector - _conn.config = {**_conn.config, "auth_expired": True} - flag_modified(_conn, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": actual_connector_id, - "message": "This Jira account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - issue_key = api_result.get("key", "") - issue_url = ( - f"{jira_history._base_url}/browse/{issue_key}" - if jira_history._base_url and issue_key - else "" - ) - - kb_message_suffix = "" - try: - from app.services.jira import JiraKBSyncService - - kb_service = JiraKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - issue_id=issue_key, - issue_identifier=issue_key, - issue_title=final_summary, - description=final_description, - state="To Do", - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This issue will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This issue will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "issue_key": issue_key, - "issue_url": issue_url, - "message": f"Jira issue {issue_key} created successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error creating Jira issue: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the issue.", - } - - return create_jira_issue diff --git a/surfsense_backend/app/agents/new_chat/tools/jira/delete_issue.py b/surfsense_backend/app/agents/new_chat/tools/jira/delete_issue.py deleted file mode 100644 index c41aedad9..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/jira/delete_issue.py +++ /dev/null @@ -1,210 +0,0 @@ -import asyncio -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.jira_history import JiraHistoryConnector -from app.db import async_session_maker -from app.services.jira import JiraToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_jira_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the delete_jira_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Jira connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured delete_jira_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_jira_issue( - issue_title_or_key: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete a Jira issue. - - Use this tool when the user asks to delete or remove a Jira issue. - - Args: - issue_title_or_key: The issue key (e.g. "PROJ-42") or title. - delete_from_kb: Whether to also remove from the knowledge base. - - Returns: - Dictionary with status, message, and deleted_from_kb. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message to the user. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"delete_jira_issue called: issue_title_or_key='{issue_title_or_key}'" - ) - - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Jira tool not properly configured."} - - try: - async with async_session_maker() as db_session: - metadata_service = JiraToolMetadataService(db_session) - context = await metadata_service.get_deletion_context( - search_space_id, user_id, issue_title_or_key - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "jira", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - issue_data = context["issue"] - issue_key = issue_data["issue_id"] - document_id = issue_data["document_id"] - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="jira_issue_deletion", - tool_name="delete_jira_issue", - params={ - "issue_key": issue_key, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_issue_key = result.params.get("issue_key", issue_key) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this issue.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Jira connector is invalid.", - } - - try: - jira_history = JiraHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - jira_client = await jira_history._get_jira_client() - await asyncio.to_thread(jira_client.delete_issue, final_issue_key) - except Exception as api_err: - if "status code 403" in str(api_err).lower(): - try: - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Jira account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - - message = f"Jira issue {final_issue_key} deleted successfully." - if deleted_from_kb: - message += " Also removed from the knowledge base." - - return { - "status": "success", - "issue_key": final_issue_key, - "deleted_from_kb": deleted_from_kb, - "message": message, - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error deleting Jira issue: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while deleting the issue.", - } - - return delete_jira_issue diff --git a/surfsense_backend/app/agents/new_chat/tools/jira/update_issue.py b/surfsense_backend/app/agents/new_chat/tools/jira/update_issue.py deleted file mode 100644 index 0fd7b28b3..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/jira/update_issue.py +++ /dev/null @@ -1,255 +0,0 @@ -import asyncio -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.jira_history import JiraHistoryConnector -from app.db import async_session_maker -from app.services.jira import JiraToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_jira_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the update_jira_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Jira connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured update_jira_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_jira_issue( - issue_title_or_key: str, - new_summary: str | None = None, - new_description: str | None = None, - new_priority: str | None = None, - ) -> dict[str, Any]: - """Update an existing Jira issue. - - Use this tool when the user asks to modify, edit, or update a Jira issue. - - Args: - issue_title_or_key: The issue key (e.g. "PROJ-42") or title to identify the issue. - new_summary: Optional new title/summary for the issue. - new_description: Optional new description. - new_priority: Optional new priority name. - - Returns: - Dictionary with status and message. - - IMPORTANT: - - If status is "rejected", do NOT retry. - - If status is "not_found", relay the message and ask user to verify. - - If status is "insufficient_permissions", inform user to re-authenticate. - """ - logger.info( - f"update_jira_issue called: issue_title_or_key='{issue_title_or_key}'" - ) - - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Jira tool not properly configured."} - - try: - async with async_session_maker() as db_session: - metadata_service = JiraToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, issue_title_or_key - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "jira", - } - if "not found" in error_msg.lower(): - return {"status": "not_found", "message": error_msg} - return {"status": "error", "message": error_msg} - - issue_data = context["issue"] - issue_key = issue_data["issue_id"] - document_id = issue_data.get("document_id") - connector_id_from_context = context.get("account", {}).get("id") - - result = request_approval( - action_type="jira_issue_update", - tool_name="update_jira_issue", - params={ - "issue_key": issue_key, - "document_id": document_id, - "new_summary": new_summary, - "new_description": new_description, - "new_priority": new_priority, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_issue_key = result.params.get("issue_key", issue_key) - final_summary = result.params.get("new_summary", new_summary) - final_description = result.params.get( - "new_description", new_description - ) - final_priority = result.params.get("new_priority", new_priority) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_document_id = result.params.get("document_id", document_id) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if not final_connector_id: - return { - "status": "error", - "message": "No connector found for this issue.", - } - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Jira connector is invalid.", - } - - fields: dict[str, Any] = {} - if final_summary: - fields["summary"] = final_summary - if final_description is not None: - fields["description"] = { - "type": "doc", - "version": 1, - "content": [ - { - "type": "paragraph", - "content": [ - {"type": "text", "text": final_description} - ], - } - ], - } - if final_priority: - fields["priority"] = {"name": final_priority} - - if not fields: - return {"status": "error", "message": "No changes specified."} - - try: - jira_history = JiraHistoryConnector( - session=db_session, connector_id=final_connector_id - ) - jira_client = await jira_history._get_jira_client() - await asyncio.to_thread( - jira_client.update_issue, final_issue_key, fields - ) - except Exception as api_err: - if "status code 403" in str(api_err).lower(): - try: - connector.config = { - **connector.config, - "auth_expired": True, - } - flag_modified(connector, "config") - await db_session.commit() - except Exception: - pass - return { - "status": "insufficient_permissions", - "connector_id": final_connector_id, - "message": "This Jira account needs additional permissions. Please re-authenticate in connector settings.", - } - raise - - issue_url = ( - f"{jira_history._base_url}/browse/{final_issue_key}" - if jira_history._base_url and final_issue_key - else "" - ) - - kb_message_suffix = "" - if final_document_id: - try: - from app.services.jira import JiraKBSyncService - - kb_service = JiraKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=final_document_id, - issue_id=final_issue_key, - user_id=user_id, - search_space_id=search_space_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - except Exception as kb_err: - logger.warning(f"KB sync after update failed: {kb_err}") - kb_message_suffix = ( - " The knowledge base will be updated in the next sync." - ) - - return { - "status": "success", - "issue_key": final_issue_key, - "issue_url": issue_url, - "message": f"Jira issue {final_issue_key} updated successfully.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error updating Jira issue: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while updating the issue.", - } - - return update_jira_issue diff --git a/surfsense_backend/app/agents/new_chat/tools/linear/__init__.py b/surfsense_backend/app/agents/new_chat/tools/linear/__init__.py deleted file mode 100644 index 31acf1e2a..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/linear/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Linear tools for creating, updating, and deleting issues.""" - -from .create_issue import create_create_linear_issue_tool -from .delete_issue import create_delete_linear_issue_tool -from .update_issue import create_update_linear_issue_tool - -__all__ = [ - "create_create_linear_issue_tool", - "create_delete_linear_issue_tool", - "create_update_linear_issue_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/linear/create_issue.py b/surfsense_backend/app/agents/new_chat/tools/linear/create_issue.py deleted file mode 100644 index f897bee7a..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/linear/create_issue.py +++ /dev/null @@ -1,266 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.linear_connector import LinearAPIError, LinearConnector -from app.db import async_session_maker -from app.services.linear import LinearToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_linear_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the create_linear_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Linear connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured create_linear_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_linear_issue( - title: str, - description: str | None = None, - ) -> dict[str, Any]: - """Create a new issue in Linear. - - Use this tool when the user explicitly asks to create, add, or file - a new issue / ticket / task in Linear. The user MUST describe the issue - before you call this tool. If the request is vague, ask what the issue - should be about. Never call this tool without a clear topic from the user. - - Args: - title: Short, descriptive issue title. Infer from the user's request. - description: Optional markdown body for the issue. Generate from context. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - issue_id: Linear issue UUID (if success) - - identifier: Human-readable ID like "ENG-42" (if success) - - url: URL to the created issue (if success) - - message: Result message - - IMPORTANT: If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I won't create the issue.") - and move on. Do NOT retry, troubleshoot, or suggest alternatives. - - Examples: - - "Create a Linear issue for the login bug" - - "File a ticket about the payment timeout problem" - - "Add an issue for the broken search feature" - """ - logger.info(f"create_linear_issue called: title='{title}'") - - if search_space_id is None or user_id is None: - logger.error( - "Linear tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Linear tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = LinearToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return {"status": "error", "message": context["error"]} - - workspaces = context.get("workspaces", []) - if workspaces and all(w.get("auth_expired") for w in workspaces): - logger.warning("All Linear accounts have expired authentication") - return { - "status": "auth_error", - "message": "All connected Linear accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "linear", - } - - logger.info(f"Requesting approval for creating Linear issue: '{title}'") - result = request_approval( - action_type="linear_issue_creation", - tool_name="create_linear_issue", - params={ - "title": title, - "description": description, - "team_id": None, - "state_id": None, - "assignee_id": None, - "priority": None, - "label_ids": [], - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - logger.info("Linear issue creation rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_title = result.params.get("title", title) - final_description = result.params.get("description", description) - final_team_id = result.params.get("team_id") - final_state_id = result.params.get("state_id") - final_assignee_id = result.params.get("assignee_id") - final_priority = result.params.get("priority") - final_label_ids = result.params.get("label_ids") or [] - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_title or not final_title.strip(): - logger.error("Title is empty or contains only whitespace") - return { - "status": "error", - "message": "Issue title cannot be empty.", - } - if not final_team_id: - return { - "status": "error", - "message": "A team must be selected to create an issue.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LINEAR_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "No Linear connector found. Please connect Linear in your workspace settings.", - } - actual_connector_id = connector.id - logger.info(f"Found Linear connector: id={actual_connector_id}") - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LINEAR_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - return { - "status": "error", - "message": "Selected Linear connector is invalid or has been disconnected.", - } - logger.info(f"Validated Linear connector: id={actual_connector_id}") - - logger.info( - f"Creating Linear issue with final params: title='{final_title}'" - ) - linear_client = LinearConnector( - session=db_session, connector_id=actual_connector_id - ) - result = await linear_client.create_issue( - team_id=final_team_id, - title=final_title, - description=final_description, - state_id=final_state_id, - assignee_id=final_assignee_id, - priority=final_priority, - label_ids=final_label_ids if final_label_ids else None, - ) - - if result.get("status") == "error": - logger.error( - f"Failed to create Linear issue: {result.get('message')}" - ) - return {"status": "error", "message": result.get("message")} - - logger.info( - f"Linear issue created: {result.get('identifier')} - {result.get('title')}" - ) - - kb_message_suffix = "" - try: - from app.services.linear import LinearKBSyncService - - kb_service = LinearKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - issue_id=result.get("id"), - issue_identifier=result.get("identifier", ""), - issue_title=result.get("title", final_title), - issue_url=result.get("url"), - description=final_description, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This issue will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This issue will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "issue_id": result.get("id"), - "identifier": result.get("identifier"), - "url": result.get("url"), - "message": (result.get("message", "") + kb_message_suffix), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating Linear issue: {e}", exc_info=True) - if isinstance(e, ValueError | LinearAPIError): - message = str(e) - else: - message = ( - "Something went wrong while creating the issue. Please try again." - ) - return {"status": "error", "message": message} - - return create_linear_issue diff --git a/surfsense_backend/app/agents/new_chat/tools/linear/delete_issue.py b/surfsense_backend/app/agents/new_chat/tools/linear/delete_issue.py deleted file mode 100644 index c5039a8eb..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/linear/delete_issue.py +++ /dev/null @@ -1,256 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.linear_connector import LinearAPIError, LinearConnector -from app.db import async_session_maker -from app.services.linear import LinearToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_linear_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the delete_linear_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Linear connector - user_id: User ID for finding the correct Linear connector - connector_id: Optional specific connector ID (if known) - - Returns: - Configured delete_linear_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_linear_issue( - issue_ref: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Archive (delete) a Linear issue. - - Use this tool when the user asks to delete, remove, or archive a Linear issue. - Note that Linear archives issues rather than permanently deleting them - (they can be restored from the archive). - - - Args: - issue_ref: The issue to delete. Can be the issue title (e.g. "Fix login bug"), - the identifier (e.g. "ENG-42"), or the full document title - (e.g. "ENG-42: Fix login bug"). - delete_from_kb: Whether to also remove the issue from the knowledge base. - Default is False. Set to True to remove from both Linear - and the knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - identifier: Human-readable ID like "ENG-42" (if success) - - message: Success or error message - - deleted_from_kb: Whether the issue was also removed from the knowledge base (if success) - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I won't delete the issue.") - and move on. Do NOT ask for alternatives or troubleshoot. - - If status is "not_found", inform the user conversationally using the exact message - provided. Do NOT treat this as an error. Simply relay the message and ask the user - to verify the issue title or identifier, or check if it has been indexed. - Examples: - - "Delete the 'Fix login bug' Linear issue" - - "Archive ENG-42" - - "Remove the 'Old payment flow' issue from Linear" - """ - logger.info( - f"delete_linear_issue called: issue_ref='{issue_ref}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - logger.error( - "Linear tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Linear tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = LinearToolMetadataService(db_session) - context = await metadata_service.get_delete_context( - search_space_id, user_id, issue_ref - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - logger.warning(f"Auth expired for delete context: {error_msg}") - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "linear", - } - if "not found" in error_msg.lower(): - logger.warning(f"Issue not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - else: - logger.error(f"Failed to fetch delete context: {error_msg}") - return {"status": "error", "message": error_msg} - - issue_id = context["issue"]["id"] - issue_identifier = context["issue"].get("identifier", "") - document_id = context["issue"]["document_id"] - connector_id_from_context = context.get("workspace", {}).get("id") - - logger.info( - f"Requesting approval for deleting Linear issue: '{issue_ref}' " - f"(id={issue_id}, delete_from_kb={delete_from_kb})" - ) - result = request_approval( - action_type="linear_issue_deletion", - tool_name="delete_linear_issue", - params={ - "issue_id": issue_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - logger.info("Linear issue deletion rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_issue_id = result.params.get("issue_id", issue_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - logger.info( - f"Deleting Linear issue with final params: issue_id={final_issue_id}, " - f"connector_id={final_connector_id}, delete_from_kb={final_delete_from_kb}" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if final_connector_id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LINEAR_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "Selected Linear connector is invalid or has been disconnected.", - } - actual_connector_id = connector.id - logger.info(f"Validated Linear connector: id={actual_connector_id}") - else: - logger.error("No connector found for this issue") - return { - "status": "error", - "message": "No connector found for this issue.", - } - - linear_client = LinearConnector( - session=db_session, connector_id=actual_connector_id - ) - - result = await linear_client.archive_issue(issue_id=final_issue_id) - - logger.info( - f"archive_issue result: {result.get('status')} - {result.get('message', '')}" - ) - - deleted_from_kb = False - if ( - result.get("status") == "success" - and final_delete_from_kb - and document_id - ): - try: - from app.db import Document - - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - result["warning"] = ( - f"Issue archived in Linear, but failed to remove from knowledge base: {e!s}" - ) - - if result.get("status") == "success": - result["deleted_from_kb"] = deleted_from_kb - if issue_identifier: - result["message"] = ( - f"Issue {issue_identifier} archived successfully." - ) - if deleted_from_kb: - result["message"] = ( - f"{result.get('message', '')} Also removed from the knowledge base." - ) - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error deleting Linear issue: {e}", exc_info=True) - if isinstance(e, ValueError | LinearAPIError): - message = str(e) - else: - message = ( - "Something went wrong while deleting the issue. Please try again." - ) - return {"status": "error", "message": message} - - return delete_linear_issue diff --git a/surfsense_backend/app/agents/new_chat/tools/linear/update_issue.py b/surfsense_backend/app/agents/new_chat/tools/linear/update_issue.py deleted file mode 100644 index d610ce2b7..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/linear/update_issue.py +++ /dev/null @@ -1,327 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.linear_connector import LinearAPIError, LinearConnector -from app.db import async_session_maker -from app.services.linear import LinearKBSyncService, LinearToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_linear_issue_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """Factory function to create the update_linear_issue tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Linear connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured update_linear_issue tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_linear_issue( - issue_ref: str, - new_title: str | None = None, - new_description: str | None = None, - new_state_name: str | None = None, - new_assignee_email: str | None = None, - new_priority: int | None = None, - new_label_names: list[str] | None = None, - ) -> dict[str, Any]: - """Update an existing Linear issue that has been indexed in the knowledge base. - - Use this tool when the user asks to modify, change, or update a Linear issue — - for example, changing its status, reassigning it, updating its title or description, - adjusting its priority, or changing its labels. - - Only issues already indexed in the knowledge base can be updated. - - Args: - issue_ref: The issue to update. Can be the issue title (e.g. "Fix login bug"), - the identifier (e.g. "ENG-42"), or the full document title - (e.g. "ENG-42: Fix login bug"). Matched case-insensitively. - new_title: New title for the issue (optional). - new_description: New markdown body for the issue (optional). - new_state_name: New workflow state name (e.g. "In Progress", "Done"). - Matched case-insensitively against the team's states. - new_assignee_email: Email address of the new assignee. - Matched case-insensitively against the team's members. - new_priority: New priority (0 = No Priority, 1 = Urgent, 2 = High, - 3 = Medium, 4 = Low). - new_label_names: New set of label names to apply. - Matched case-insensitively against the team's labels. - Unrecognised names are silently skipped. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - identifier: Human-readable ID like "ENG-42" (if success) - - url: URL to the updated issue (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I didn't update the issue.") - and move on. Do NOT ask for alternatives or troubleshoot. - - If status is "not_found", inform the user conversationally using the exact message - provided. Do NOT treat this as an error. Simply relay the message and ask the user - to verify the issue title or identifier, or check if it has been indexed. - - Examples: - - "Mark the 'Fix login bug' issue as done" - - "Assign ENG-42 to john@company.com" - - "Change the priority of 'Payment timeout' to urgent" - """ - logger.info(f"update_linear_issue called: issue_ref='{issue_ref}'") - - if search_space_id is None or user_id is None: - logger.error( - "Linear tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Linear tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = LinearToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, issue_ref - ) - - if "error" in context: - error_msg = context["error"] - if context.get("auth_expired"): - logger.warning(f"Auth expired for update context: {error_msg}") - return { - "status": "auth_error", - "message": error_msg, - "connector_id": context.get("connector_id"), - "connector_type": "linear", - } - if "not found" in error_msg.lower(): - logger.warning(f"Issue not found: {error_msg}") - return {"status": "not_found", "message": error_msg} - else: - logger.error(f"Failed to fetch update context: {error_msg}") - return {"status": "error", "message": error_msg} - - issue_id = context["issue"]["id"] - document_id = context["issue"]["document_id"] - connector_id_from_context = context.get("workspace", {}).get("id") - - team = context.get("team", {}) - new_state_id = _resolve_state(team, new_state_name) - new_assignee_id = _resolve_assignee(team, new_assignee_email) - new_label_ids = _resolve_labels(team, new_label_names) - - logger.info( - f"Requesting approval for updating Linear issue: '{issue_ref}' (id={issue_id})" - ) - result = request_approval( - action_type="linear_issue_update", - tool_name="update_linear_issue", - params={ - "issue_id": issue_id, - "document_id": document_id, - "new_title": new_title, - "new_description": new_description, - "new_state_id": new_state_id, - "new_assignee_id": new_assignee_id, - "new_priority": new_priority, - "new_label_ids": new_label_ids, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - logger.info("Linear issue update rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_issue_id = result.params.get("issue_id", issue_id) - final_document_id = result.params.get("document_id", document_id) - final_new_title = result.params.get("new_title", new_title) - final_new_description = result.params.get( - "new_description", new_description - ) - final_new_state_id = result.params.get("new_state_id", new_state_id) - final_new_assignee_id = result.params.get( - "new_assignee_id", new_assignee_id - ) - final_new_priority = result.params.get("new_priority", new_priority) - final_new_label_ids: list[str] | None = result.params.get( - "new_label_ids", new_label_ids - ) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - - if not final_connector_id: - logger.error("No connector found for this issue") - return { - "status": "error", - "message": "No connector found for this issue.", - } - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LINEAR_CONNECTOR, - ) - ) - connector = result.scalars().first() - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "Selected Linear connector is invalid or has been disconnected.", - } - logger.info(f"Validated Linear connector: id={final_connector_id}") - - logger.info( - f"Updating Linear issue with final params: issue_id={final_issue_id}" - ) - linear_client = LinearConnector( - session=db_session, connector_id=final_connector_id - ) - updated_issue = await linear_client.update_issue( - issue_id=final_issue_id, - title=final_new_title, - description=final_new_description, - state_id=final_new_state_id, - assignee_id=final_new_assignee_id, - priority=final_new_priority, - label_ids=final_new_label_ids, - ) - - if updated_issue.get("status") == "error": - logger.error( - f"Failed to update Linear issue: {updated_issue.get('message')}" - ) - return { - "status": "error", - "message": updated_issue.get("message"), - } - - logger.info( - f"update_issue result: {updated_issue.get('identifier')} - {updated_issue.get('title')}" - ) - - if final_document_id is not None: - logger.info( - f"Updating knowledge base for document {final_document_id}..." - ) - kb_service = LinearKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=final_document_id, - issue_id=final_issue_id, - user_id=user_id, - search_space_id=search_space_id, - ) - if kb_result["status"] == "success": - logger.info( - f"Knowledge base successfully updated for issue {final_issue_id}" - ) - kb_message = " Your knowledge base has also been updated." - elif kb_result["status"] == "not_indexed": - kb_message = " This issue will be added to your knowledge base in the next scheduled sync." - else: - logger.warning( - f"KB update failed for issue {final_issue_id}: {kb_result.get('message')}" - ) - kb_message = " Your knowledge base will be updated in the next scheduled sync." - else: - kb_message = "" - - identifier = updated_issue.get("identifier") - default_msg = f"Issue {identifier} updated successfully." - return { - "status": "success", - "identifier": identifier, - "url": updated_issue.get("url"), - "message": f"{updated_issue.get('message', default_msg)}{kb_message}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error updating Linear issue: {e}", exc_info=True) - if isinstance(e, ValueError | LinearAPIError): - message = str(e) - else: - message = ( - "Something went wrong while updating the issue. Please try again." - ) - return {"status": "error", "message": message} - - return update_linear_issue - - -def _resolve_state(team: dict, state_name: str | None) -> str | None: - if not state_name: - return None - name_lower = state_name.lower() - for state in team.get("states", []): - if state.get("name", "").lower() == name_lower: - return state["id"] - return None - - -def _resolve_assignee(team: dict, assignee_email: str | None) -> str | None: - if not assignee_email: - return None - email_lower = assignee_email.lower() - for member in team.get("members", []): - if member.get("email", "").lower() == email_lower: - return member["id"] - return None - - -def _resolve_labels(team: dict, label_names: list[str] | None) -> list[str] | None: - if label_names is None: - return None - if not label_names: - return [] - name_set = {n.lower() for n in label_names} - return [ - label["id"] - for label in team.get("labels", []) - if label.get("name", "").lower() in name_set - ] diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/__init__.py b/surfsense_backend/app/agents/new_chat/tools/luma/__init__.py deleted file mode 100644 index 255119bee..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/luma/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.luma.create_event import ( - create_create_luma_event_tool, -) -from app.agents.new_chat.tools.luma.list_events import ( - create_list_luma_events_tool, -) -from app.agents.new_chat.tools.luma.read_event import ( - create_read_luma_event_tool, -) - -__all__ = [ - "create_create_luma_event_tool", - "create_list_luma_events_tool", - "create_read_luma_event_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py b/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py deleted file mode 100644 index 37deb1525..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Shared auth helper for Luma agent tools.""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType - -LUMA_API = "https://public-api.luma.com/v1" - - -async def get_luma_connector( - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.LUMA_CONNECTOR, - ) - ) - return result.scalars().first() - - -def get_api_key(connector: SearchSourceConnector) -> str: - """Extract the API key from connector config (handles both key names).""" - key = connector.config.get("api_key") or connector.config.get("LUMA_API_KEY") - if not key: - raise ValueError("Luma API key not found in connector config.") - return key - - -def luma_headers(api_key: str) -> dict[str, str]: - return { - "Content-Type": "application/json", - "x-luma-api-key": api_key, - } diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py b/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py deleted file mode 100644 index 65c177d7a..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py +++ /dev/null @@ -1,150 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_create_luma_event_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_luma_event tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_luma_event tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_luma_event( - name: str, - start_at: str, - end_at: str, - description: str | None = None, - timezone: str = "UTC", - ) -> dict[str, Any]: - """Create a new event on Luma. - - Args: - name: The event title. - start_at: Start time in ISO 8601 format (e.g. "2026-05-01T18:00:00"). - end_at: End time in ISO 8601 format (e.g. "2026-05-01T20:00:00"). - description: Optional event description (markdown supported). - timezone: Timezone string (default "UTC", e.g. "America/New_York"). - - Returns: - Dictionary with status, event_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - try: - async with async_session_maker() as db_session: - connector = await get_luma_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - result = request_approval( - action_type="luma_create_event", - tool_name="create_luma_event", - params={ - "name": name, - "start_at": start_at, - "end_at": end_at, - "description": description, - "timezone": timezone, - }, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Event was not created.", - } - - final_name = result.params.get("name", name) - final_start = result.params.get("start_at", start_at) - final_end = result.params.get("end_at", end_at) - final_desc = result.params.get("description", description) - final_tz = result.params.get("timezone", timezone) - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - body: dict[str, Any] = { - "name": final_name, - "start_at": final_start, - "end_at": final_end, - "timezone": final_tz, - } - if final_desc: - body["description_md"] = final_desc - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.post( - f"{LUMA_API}/event/create", - headers=headers, - json=body, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Luma Plus subscription required to create events via API.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Luma API error: {resp.status_code} — {resp.text[:200]}", - } - - data = resp.json() - event_id = data.get("api_id") or data.get("event", {}).get("api_id") - - return { - "status": "success", - "event_id": event_id, - "message": f"Event '{final_name}' created on Luma.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error creating Luma event: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to create Luma event."} - - return create_luma_event diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py b/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py deleted file mode 100644 index 6885c2049..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py +++ /dev/null @@ -1,133 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_list_luma_events_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the list_luma_events tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured list_luma_events tool - """ - del db_session # per-call session — see docstring - - @tool - async def list_luma_events( - max_results: int = 25, - ) -> dict[str, Any]: - """List upcoming and recent Luma events. - - Args: - max_results: Maximum events to return (default 25, max 50). - - Returns: - Dictionary with status and a list of events including - event_id, name, start_at, end_at, location, url. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - max_results = min(max_results, 50) - - try: - async with async_session_maker() as db_session: - connector = await get_luma_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - all_entries: list[dict] = [] - cursor = None - - async with httpx.AsyncClient(timeout=20.0) as client: - while len(all_entries) < max_results: - params: dict[str, Any] = { - "limit": min(100, max_results - len(all_entries)) - } - if cursor: - params["cursor"] = cursor - - resp = await client.get( - f"{LUMA_API}/calendar/list-events", - headers=headers, - params=params, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Luma API error: {resp.status_code}", - } - - data = resp.json() - entries = data.get("entries", []) - if not entries: - break - all_entries.extend(entries) - - next_cursor = data.get("next_cursor") - if not next_cursor: - break - cursor = next_cursor - - events = [] - for entry in all_entries[:max_results]: - ev = entry.get("event", {}) - geo = ev.get("geo_info", {}) - events.append( - { - "event_id": entry.get("api_id"), - "name": ev.get("name", "Untitled"), - "start_at": ev.get("start_at", ""), - "end_at": ev.get("end_at", ""), - "timezone": ev.get("timezone", ""), - "location": geo.get("name", ""), - "url": ev.get("url", ""), - "visibility": ev.get("visibility", ""), - } - ) - - return {"status": "success", "events": events, "total": len(events)} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Luma events: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Luma events."} - - return list_luma_events diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py b/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py deleted file mode 100644 index a8484e9c0..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import LUMA_API, get_api_key, get_luma_connector, luma_headers - -logger = logging.getLogger(__name__) - - -def create_read_luma_event_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the read_luma_event tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured read_luma_event tool - """ - del db_session # per-call session — see docstring - - @tool - async def read_luma_event(event_id: str) -> dict[str, Any]: - """Read detailed information about a specific Luma event. - - Args: - event_id: The Luma event API ID (from list_luma_events). - - Returns: - Dictionary with status and full event details including - description, attendees count, meeting URL. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Luma tool not properly configured."} - - try: - async with async_session_maker() as db_session: - connector = await get_luma_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Luma connector found."} - - api_key = get_api_key(connector) - headers = luma_headers(api_key) - - async with httpx.AsyncClient(timeout=15.0) as client: - resp = await client.get( - f"{LUMA_API}/events/{event_id}", - headers=headers, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Luma API key is invalid.", - "connector_type": "luma", - } - if resp.status_code == 404: - return { - "status": "not_found", - "message": f"Event '{event_id}' not found.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Luma API error: {resp.status_code}", - } - - data = resp.json() - ev = data.get("event", data) - geo = ev.get("geo_info", {}) - - event_detail = { - "event_id": event_id, - "name": ev.get("name", ""), - "description": ev.get("description", ""), - "start_at": ev.get("start_at", ""), - "end_at": ev.get("end_at", ""), - "timezone": ev.get("timezone", ""), - "location_name": geo.get("name", ""), - "address": geo.get("address", ""), - "url": ev.get("url", ""), - "meeting_url": ev.get("meeting_url", ""), - "visibility": ev.get("visibility", ""), - "cover_url": ev.get("cover_url", ""), - } - - return {"status": "success", "event": event_detail} - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Luma event: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Luma event."} - - return read_luma_event diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py b/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py deleted file mode 100644 index 6ce825dca..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Notion tools for creating, updating, and deleting pages.""" - -from .create_page import create_create_notion_page_tool -from .delete_page import create_delete_notion_page_tool -from .update_page import create_update_notion_page_tool - -__all__ = [ - "create_create_notion_page_tool", - "create_delete_notion_page_tool", - "create_update_notion_page_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py deleted file mode 100644 index 6ec95e9f0..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py +++ /dev/null @@ -1,258 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.db import async_session_maker -from app.services.notion import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_create_notion_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the create_notion_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker`. This is critical for the compiled-agent - cache: the compiled graph (and therefore this closure) is reused - across HTTP requests, so capturing a per-request session here would - surface stale/closed sessions on cache hits. Per-call sessions also - keep the request's outer transaction free of long-running Notion API - blocking. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Notion connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured create_notion_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_notion_page( - title: str, - content: str | None = None, - ) -> dict[str, Any]: - """Create a new page in Notion with the given title and content. - - Use this tool when the user asks you to create, save, or publish - something to Notion. The page will be created in the user's - configured Notion workspace. The user MUST specify a topic before you - call this tool. If the request does not contain a topic (e.g. "create a - notion page"), ask what the page should be about. Never call this tool - without a clear topic from the user. - - Args: - title: The title of the Notion page. - content: Optional markdown content for the page body (supports headings, lists, paragraphs). - Generate this yourself based on the user's topic. - - Returns: - Dictionary with: - - status: "success", "rejected", or "error" - - page_id: Created page ID (if success) - - url: URL to the created page (if success) - - title: Page title (if success) - - message: Result message - - IMPORTANT: If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I didn't create the page.") - and move on. Do NOT troubleshoot or suggest alternatives. - - Examples: - - "Create a Notion page about our Q2 roadmap" - - "Save a summary of today's discussion to Notion" - """ - logger.info(f"create_notion_page called: title='{title}'") - - if search_space_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_creation_context( - search_space_id, user_id - ) - - if "error" in context: - logger.error( - f"Failed to fetch creation context: {context['error']}" - ) - return { - "status": "error", - "message": context["error"], - } - - accounts = context.get("accounts", []) - if accounts and all(a.get("auth_expired") for a in accounts): - logger.warning("All Notion accounts have expired authentication") - return { - "status": "auth_error", - "message": "All connected Notion accounts need re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "notion", - } - - logger.info(f"Requesting approval for creating Notion page: '{title}'") - result = request_approval( - action_type="notion_page_creation", - tool_name="create_notion_page", - params={ - "title": title, - "content": content, - "parent_page_id": None, - "connector_id": connector_id, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page creation rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_title = result.params.get("title", title) - final_content = result.params.get("content", content) - final_parent_page_id = result.params.get("parent_page_id") - final_connector_id = result.params.get("connector_id", connector_id) - - if not final_title or not final_title.strip(): - logger.error("Title is empty or contains only whitespace") - return { - "status": "error", - "message": "Page title cannot be empty. Please provide a valid title.", - } - - logger.info( - f"Creating Notion page with final params: title='{final_title}'" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - actual_connector_id = final_connector_id - if actual_connector_id is None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.warning( - f"No Notion connector found for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "No Notion connector found. Please connect Notion in your workspace settings.", - } - - actual_connector_id = connector.id - logger.info(f"Found Notion connector: id={actual_connector_id}") - else: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == actual_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={actual_connector_id} for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - } - logger.info(f"Validated Notion connector: id={actual_connector_id}") - - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - result = await notion_connector.create_page( - title=final_title, - content=final_content, - parent_page_id=final_parent_page_id, - ) - logger.info( - f"create_page result: {result.get('status')} - {result.get('message', '')}" - ) - - if result.get("status") == "success": - kb_message_suffix = "" - try: - from app.services.notion import NotionKBSyncService - - kb_service = NotionKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - page_id=result.get("page_id"), - page_title=result.get("title", final_title), - page_url=result.get("url"), - content=final_content, - connector_id=actual_connector_id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This page will be added to your knowledge base in the next scheduled sync." - - result["message"] = result.get("message", "") + kb_message_suffix - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error creating Notion page: {e}", exc_info=True) - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while creating the page. Please try again." - ) - return {"status": "error", "message": message} - - return create_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py deleted file mode 100644 index 7b85da4c2..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py +++ /dev/null @@ -1,273 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.db import async_session_maker -from app.services.notion.tool_metadata_service import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_delete_notion_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the delete_notion_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Notion connector - user_id: User ID for finding the correct Notion connector - connector_id: Optional specific connector ID (if known) - - Returns: - Configured delete_notion_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_notion_page( - page_title: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Delete (archive) a Notion page. - - Use this tool when the user asks you to delete, remove, or archive - a Notion page. Note that Notion doesn't permanently delete pages, - it archives them (they can be restored from trash). - - Args: - page_title: The title of the Notion page to delete. - delete_from_kb: Whether to also remove the page from the knowledge base. - Default is False. - Set to True to permanently remove from both Notion and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - page_id: Deleted page ID (if success) - - message: Success or error message - - deleted_from_kb: Whether the page was also removed from knowledge base (if success) - - Examples: - - "Delete the 'Meeting Notes' Notion page" - - "Remove the 'Old Project Plan' Notion page" - - "Archive the 'Draft Ideas' Notion page" - """ - logger.info( - f"delete_notion_page called: page_title='{page_title}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - } - - try: - async with async_session_maker() as db_session: - # Get page context (page_id, account, title) from indexed data - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_delete_context( - search_space_id, user_id, page_title - ) - - if "error" in context: - error_msg = context["error"] - # Check if it's a "not found" error (softer handling for LLM) - if "not found" in error_msg.lower(): - logger.warning(f"Page not found: {error_msg}") - return { - "status": "not_found", - "message": error_msg, - } - else: - logger.error(f"Failed to fetch delete context: {error_msg}") - return { - "status": "error", - "message": error_msg, - } - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Notion account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Notion account for this page needs re-authentication. Please re-authenticate in your connector settings.", - } - - page_id = context.get("page_id") - connector_id_from_context = account.get("id") - document_id = context.get("document_id") - - logger.info( - f"Requesting approval for deleting Notion page: '{page_title}' (page_id={page_id}, delete_from_kb={delete_from_kb})" - ) - - result = request_approval( - action_type="notion_page_deletion", - tool_name="delete_notion_page", - params={ - "page_id": page_id, - "connector_id": connector_id_from_context, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page deletion rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - logger.info( - f"Deleting Notion page with final params: page_id={final_page_id}, connector_id={final_connector_id}, delete_from_kb={final_delete_from_kb}" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - # Validate the connector - if final_connector_id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - } - actual_connector_id = connector.id - logger.info(f"Validated Notion connector: id={actual_connector_id}") - else: - logger.error("No connector found for this page") - return { - "status": "error", - "message": "No connector found for this page.", - } - - # Create connector instance - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - # Delete the page from Notion - result = await notion_connector.delete_page(page_id=final_page_id) - logger.info( - f"delete_page result: {result.get('status')} - {result.get('message', '')}" - ) - - # If deletion was successful and user wants to delete from KB - deleted_from_kb = False - if ( - result.get("status") == "success" - and final_delete_from_kb - and document_id - ): - try: - from sqlalchemy.future import select - - from app.db import Document - - # Get the document - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - document = doc_result.scalars().first() - - if document: - await db_session.delete(document) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - result["warning"] = ( - f"Page deleted from Notion, but failed to remove from knowledge base: {e!s}" - ) - - # Update result with KB deletion status - if result.get("status") == "success": - result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - result["message"] = ( - f"{result.get('message', '')} (also removed from knowledge base)" - ) - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error deleting Notion page: {e}", exc_info=True) - error_str = str(e).lower() - if isinstance(e, NotionAPIError) and ( - "401" in error_str or "unauthorized" in error_str - ): - return { - "status": "auth_error", - "message": str(e), - "connector_id": connector_id_from_context - if "connector_id_from_context" in dir() - else None, - "connector_type": "notion", - } - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while deleting the page. Please try again." - ) - return {"status": "error", "message": message} - - return delete_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py deleted file mode 100644 index df757476a..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py +++ /dev/null @@ -1,276 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.notion_history import NotionAPIError, NotionHistoryConnector -from app.db import async_session_maker -from app.services.notion import NotionToolMetadataService - -logger = logging.getLogger(__name__) - - -def create_update_notion_page_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, - connector_id: int | None = None, -): - """ - Factory function to create the update_notion_page tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache (see - ``create_create_notion_page_tool`` for the full rationale). - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - search_space_id: Search space ID to find the Notion connector - user_id: User ID for fetching user-specific context - connector_id: Optional specific connector ID (if known) - - Returns: - Configured update_notion_page tool - """ - del db_session # per-call session — see docstring - - @tool - async def update_notion_page( - page_title: str, - content: str | None = None, - ) -> dict[str, Any]: - """Update an existing Notion page by appending new content. - - Use this tool when the user asks you to add content to, modify, or update - a Notion page. The new content will be appended to the existing page content. - The user MUST specify what to add before you call this tool. If the - request is vague, ask what content they want added. - - Args: - page_title: The title of the Notion page to update. - content: Optional markdown content to append to the page body (supports headings, lists, paragraphs). - Generate this yourself based on the user's request. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - page_id: Updated page ID (if success) - - url: URL to the updated page (if success) - - title: Current page title (if success) - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined the action. - Respond with a brief acknowledgment (e.g., "Understood, I didn't update the page.") - and move on. Do NOT ask for alternatives or troubleshoot. - - If status is "not_found", inform the user conversationally using the exact message provided. - Example: "I couldn't find the page '[page_title]' in your indexed Notion pages. [message details]" - Do NOT treat this as an error. Do NOT invent information. Simply relay the message and - ask the user to verify the page title or check if it's been indexed. - Examples: - - "Add today's meeting notes to the 'Meeting Notes' Notion page" - - "Update the 'Project Plan' page with a status update on phase 1" - """ - logger.info( - f"update_notion_page called: page_title='{page_title}', content_length={len(content) if content else 0}" - ) - - if search_space_id is None or user_id is None: - logger.error( - "Notion tool not properly configured - missing required parameters" - ) - return { - "status": "error", - "message": "Notion tool not properly configured. Please contact support.", - } - - if not content or not content.strip(): - logger.error(f"Empty content provided for page '{page_title}'") - return { - "status": "error", - "message": "Content is required to update the page. Please provide the actual content you want to add.", - } - - try: - async with async_session_maker() as db_session: - metadata_service = NotionToolMetadataService(db_session) - context = await metadata_service.get_update_context( - search_space_id, user_id, page_title - ) - - if "error" in context: - error_msg = context["error"] - # Check if it's a "not found" error (softer handling for LLM) - if "not found" in error_msg.lower(): - logger.warning(f"Page not found: {error_msg}") - return { - "status": "not_found", - "message": error_msg, - } - else: - logger.error(f"Failed to fetch update context: {error_msg}") - return { - "status": "error", - "message": error_msg, - } - - account = context.get("account", {}) - if account.get("auth_expired"): - logger.warning( - "Notion account %s has expired authentication", - account.get("id"), - ) - return { - "status": "auth_error", - "message": "The Notion account for this page needs re-authentication. Please re-authenticate in your connector settings.", - } - - page_id = context.get("page_id") - document_id = context.get("document_id") - connector_id_from_context = context.get("account", {}).get("id") - - logger.info( - f"Requesting approval for updating Notion page: '{page_title}' (page_id={page_id})" - ) - result = request_approval( - action_type="notion_page_update", - tool_name="update_notion_page", - params={ - "page_id": page_id, - "content": content, - "connector_id": connector_id_from_context, - }, - context=context, - ) - - if result.rejected: - logger.info("Notion page update rejected by user") - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_page_id = result.params.get("page_id", page_id) - final_content = result.params.get("content", content) - final_connector_id = result.params.get( - "connector_id", connector_id_from_context - ) - - logger.info( - f"Updating Notion page with final params: page_id={final_page_id}, has_content={final_content is not None}" - ) - - from sqlalchemy.future import select - - from app.db import SearchSourceConnector, SearchSourceConnectorType - - if final_connector_id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.NOTION_CONNECTOR, - ) - ) - connector = result.scalars().first() - - if not connector: - logger.error( - f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" - ) - return { - "status": "error", - "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", - } - actual_connector_id = connector.id - logger.info(f"Validated Notion connector: id={actual_connector_id}") - else: - logger.error("No connector found for this page") - return { - "status": "error", - "message": "No connector found for this page.", - } - - notion_connector = NotionHistoryConnector( - session=db_session, - connector_id=actual_connector_id, - ) - - result = await notion_connector.update_page( - page_id=final_page_id, - content=final_content, - ) - logger.info( - f"update_page result: {result.get('status')} - {result.get('message', '')}" - ) - - if result.get("status") == "success" and document_id is not None: - from app.services.notion import NotionKBSyncService - - logger.info( - f"Updating knowledge base for document {document_id}..." - ) - kb_service = NotionKBSyncService(db_session) - kb_result = await kb_service.sync_after_update( - document_id=document_id, - appended_content=final_content, - user_id=user_id, - search_space_id=search_space_id, - appended_block_ids=result.get("appended_block_ids"), - ) - - if kb_result["status"] == "success": - result["message"] = ( - f"{result['message']}. Your knowledge base has also been updated." - ) - logger.info( - f"Knowledge base successfully updated for page {final_page_id}" - ) - elif kb_result["status"] == "not_indexed": - result["message"] = ( - f"{result['message']}. This page will be added to your knowledge base in the next scheduled sync." - ) - else: - result["message"] = ( - f"{result['message']}. Your knowledge base will be updated in the next scheduled sync." - ) - logger.warning( - f"KB update failed for page {final_page_id}: {kb_result['message']}" - ) - - return result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - - logger.error(f"Error updating Notion page: {e}", exc_info=True) - error_str = str(e).lower() - if isinstance(e, NotionAPIError) and ( - "401" in error_str or "unauthorized" in error_str - ): - return { - "status": "auth_error", - "message": str(e), - "connector_id": connector_id_from_context - if "connector_id_from_context" in dir() - else None, - "connector_type": "notion", - } - if isinstance(e, ValueError | NotionAPIError): - message = str(e) - else: - message = ( - "Something went wrong while updating the page. Please try again." - ) - return {"status": "error", "message": message} - - return update_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/onedrive/__init__.py b/surfsense_backend/app/agents/new_chat/tools/onedrive/__init__.py deleted file mode 100644 index 8edb4857e..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/onedrive/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from app.agents.new_chat.tools.onedrive.create_file import ( - create_create_onedrive_file_tool, -) -from app.agents.new_chat.tools.onedrive.trash_file import ( - create_delete_onedrive_file_tool, -) - -__all__ = [ - "create_create_onedrive_file_tool", - "create_delete_onedrive_file_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/onedrive/create_file.py b/surfsense_backend/app/agents/new_chat/tools/onedrive/create_file.py deleted file mode 100644 index 5f199a41b..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/onedrive/create_file.py +++ /dev/null @@ -1,274 +0,0 @@ -import logging -import os -import tempfile -from pathlib import Path -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.onedrive.client import OneDriveClient -from app.db import SearchSourceConnector, SearchSourceConnectorType, async_session_maker - -logger = logging.getLogger(__name__) - -DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - - -def _ensure_docx_extension(name: str) -> str: - """Strip any existing extension and append .docx.""" - stem = Path(name).stem - return f"{stem}.docx" - - -def _markdown_to_docx(markdown_text: str) -> bytes: - """Convert a markdown string to DOCX bytes using pypandoc.""" - import pypandoc - - fd, tmp_path = tempfile.mkstemp(suffix=".docx") - os.close(fd) - try: - pypandoc.convert_text( - markdown_text, - "docx", - format="gfm", - extra_args=["--standalone"], - outputfile=tmp_path, - ) - with open(tmp_path, "rb") as f: - return f.read() - finally: - os.unlink(tmp_path) - - -def create_create_onedrive_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the create_onedrive_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured create_onedrive_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def create_onedrive_file( - name: str, - content: str | None = None, - ) -> dict[str, Any]: - """Create a new Word document (.docx) in Microsoft OneDrive. - - Use this tool when the user explicitly asks to create a new document - in OneDrive. The user MUST specify a topic before you call this tool. - - The file is always saved as a .docx Word document. Provide content as - markdown and it will be automatically converted to a formatted Word file. - - Args: - name: The document title (without extension). Extension will be set to .docx automatically. - content: Optional initial content as markdown. Will be converted to a formatted Word document. - - Returns: - Dictionary with status, file_id, name, web_url, and message. - """ - logger.info(f"create_onedrive_file called: name='{name}'") - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "OneDrive tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ONEDRIVE_CONNECTOR, - ) - ) - connectors = result.scalars().all() - - if not connectors: - return { - "status": "error", - "message": "No OneDrive connector found. Please connect OneDrive in your workspace settings.", - } - - accounts = [] - for c in connectors: - cfg = c.config or {} - accounts.append( - { - "id": c.id, - "name": c.name, - "user_email": cfg.get("user_email"), - "auth_expired": cfg.get("auth_expired", False), - } - ) - - if all(a.get("auth_expired") for a in accounts): - return { - "status": "auth_error", - "message": "All connected OneDrive accounts need re-authentication.", - "connector_type": "onedrive", - } - - parent_folders: dict[int, list[dict[str, str]]] = {} - for acc in accounts: - cid = acc["id"] - if acc.get("auth_expired"): - parent_folders[cid] = [] - continue - try: - client = OneDriveClient(session=db_session, connector_id=cid) - items, err = await client.list_children("root") - if err: - logger.warning( - "Failed to list folders for connector %s: %s", cid, err - ) - parent_folders[cid] = [] - else: - parent_folders[cid] = [ - {"folder_id": item["id"], "name": item["name"]} - for item in items - if item.get("folder") is not None - and item.get("id") - and item.get("name") - ] - except Exception: - logger.warning( - "Error fetching folders for connector %s", - cid, - exc_info=True, - ) - parent_folders[cid] = [] - - context: dict[str, Any] = { - "accounts": accounts, - "parent_folders": parent_folders, - } - - result = request_approval( - action_type="onedrive_file_creation", - tool_name="create_onedrive_file", - params={ - "name": name, - "content": content, - "connector_id": None, - "parent_folder_id": None, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_name = result.params.get("name", name) - final_content = result.params.get("content", content) - final_connector_id = result.params.get("connector_id") - final_parent_folder_id = result.params.get("parent_folder_id") - - if not final_name or not final_name.strip(): - return {"status": "error", "message": "File name cannot be empty."} - - final_name = _ensure_docx_extension(final_name) - - if final_connector_id is not None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ONEDRIVE_CONNECTOR, - ) - ) - connector = result.scalars().first() - else: - connector = connectors[0] - - if not connector: - return { - "status": "error", - "message": "Selected OneDrive connector is invalid.", - } - - docx_bytes = _markdown_to_docx(final_content or "") - - client = OneDriveClient(session=db_session, connector_id=connector.id) - created = await client.create_file( - name=final_name, - parent_id=final_parent_folder_id, - content=docx_bytes, - mime_type=DOCX_MIME, - ) - - logger.info( - f"OneDrive file created: id={created.get('id')}, name={created.get('name')}" - ) - - kb_message_suffix = "" - try: - from app.services.onedrive import OneDriveKBSyncService - - kb_service = OneDriveKBSyncService(db_session) - kb_result = await kb_service.sync_after_create( - file_id=created.get("id"), - file_name=created.get("name", final_name), - mime_type=DOCX_MIME, - web_url=created.get("webUrl"), - content=final_content, - connector_id=connector.id, - search_space_id=search_space_id, - user_id=user_id, - ) - if kb_result["status"] == "success": - kb_message_suffix = ( - " Your knowledge base has also been updated." - ) - else: - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - except Exception as kb_err: - logger.warning(f"KB sync after create failed: {kb_err}") - kb_message_suffix = " This file will be added to your knowledge base in the next scheduled sync." - - return { - "status": "success", - "file_id": created.get("id"), - "name": created.get("name"), - "web_url": created.get("webUrl"), - "message": f"Successfully created '{created.get('name')}' in OneDrive.{kb_message_suffix}", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error creating OneDrive file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while creating the file. Please try again.", - } - - return create_onedrive_file diff --git a/surfsense_backend/app/agents/new_chat/tools/onedrive/trash_file.py b/surfsense_backend/app/agents/new_chat/tools/onedrive/trash_file.py deleted file mode 100644 index 4857ea988..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/onedrive/trash_file.py +++ /dev/null @@ -1,305 +0,0 @@ -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy import String, and_, cast, func -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.agents.new_chat.tools.hitl import request_approval -from app.connectors.onedrive.client import OneDriveClient -from app.db import ( - Document, - DocumentType, - SearchSourceConnector, - SearchSourceConnectorType, - async_session_maker, -) - -logger = logging.getLogger(__name__) - - -def create_delete_onedrive_file_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the delete_onedrive_file tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured delete_onedrive_file tool - """ - del db_session # per-call session — see docstring - - @tool - async def delete_onedrive_file( - file_name: str, - delete_from_kb: bool = False, - ) -> dict[str, Any]: - """Move a OneDrive file to the recycle bin. - - Use this tool when the user explicitly asks to delete, remove, or trash - a file in OneDrive. - - Args: - file_name: The exact name of the file to trash. - delete_from_kb: Whether to also remove the file from the knowledge base. - Default is False. - Set to True to remove from both OneDrive and knowledge base. - - Returns: - Dictionary with: - - status: "success", "rejected", "not_found", or "error" - - file_id: OneDrive file ID (if success) - - deleted_from_kb: whether the document was removed from the knowledge base - - message: Result message - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Respond with a brief - acknowledgment and do NOT retry or suggest alternatives. - - If status is "not_found", relay the exact message to the user and ask them - to verify the file name or check if it has been indexed. - """ - logger.info( - f"delete_onedrive_file called: file_name='{file_name}', delete_from_kb={delete_from_kb}" - ) - - if search_space_id is None or user_id is None: - return { - "status": "error", - "message": "OneDrive tool not properly configured.", - } - - try: - async with async_session_maker() as db_session: - doc_result = await db_session.execute( - select(Document) - .join( - SearchSourceConnector, - Document.connector_id == SearchSourceConnector.id, - ) - .filter( - and_( - Document.search_space_id == search_space_id, - Document.document_type == DocumentType.ONEDRIVE_FILE, - func.lower(Document.title) == func.lower(file_name), - SearchSourceConnector.user_id == user_id, - ) - ) - .order_by(Document.updated_at.desc().nullslast()) - .limit(1) - ) - document = doc_result.scalars().first() - - if not document: - doc_result = await db_session.execute( - select(Document) - .join( - SearchSourceConnector, - Document.connector_id == SearchSourceConnector.id, - ) - .filter( - and_( - Document.search_space_id == search_space_id, - Document.document_type == DocumentType.ONEDRIVE_FILE, - func.lower( - cast( - Document.document_metadata[ - "onedrive_file_name" - ], - String, - ) - ) - == func.lower(file_name), - SearchSourceConnector.user_id == user_id, - ) - ) - .order_by(Document.updated_at.desc().nullslast()) - .limit(1) - ) - document = doc_result.scalars().first() - - if not document: - return { - "status": "not_found", - "message": ( - f"File '{file_name}' not found in your indexed OneDrive files. " - "This could mean: (1) the file doesn't exist, (2) it hasn't been indexed yet, " - "or (3) the file name is different." - ), - } - - if not document.connector_id: - return { - "status": "error", - "message": "Document has no associated connector.", - } - - meta = document.document_metadata or {} - file_id = meta.get("onedrive_file_id") - document_id = document.id - - if not file_id: - return { - "status": "error", - "message": "File ID is missing. Please re-index the file.", - } - - conn_result = await db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.id == document.connector_id, - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ONEDRIVE_CONNECTOR, - ) - ) - ) - connector = conn_result.scalars().first() - if not connector: - return { - "status": "error", - "message": "OneDrive connector not found or access denied.", - } - - cfg = connector.config or {} - if cfg.get("auth_expired"): - return { - "status": "auth_error", - "message": "OneDrive account needs re-authentication. Please re-authenticate in your connector settings.", - "connector_type": "onedrive", - } - - context = { - "file": { - "file_id": file_id, - "name": file_name, - "document_id": document_id, - "web_url": meta.get("web_url"), - }, - "account": { - "id": connector.id, - "name": connector.name, - "user_email": cfg.get("user_email"), - }, - } - - result = request_approval( - action_type="onedrive_file_trash", - tool_name="delete_onedrive_file", - params={ - "file_id": file_id, - "connector_id": connector.id, - "delete_from_kb": delete_from_kb, - }, - context=context, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Do not retry or suggest alternatives.", - } - - final_file_id = result.params.get("file_id", file_id) - final_connector_id = result.params.get("connector_id", connector.id) - final_delete_from_kb = result.params.get( - "delete_from_kb", delete_from_kb - ) - - if final_connector_id != connector.id: - result = await db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.id == final_connector_id, - SearchSourceConnector.search_space_id - == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.ONEDRIVE_CONNECTOR, - ) - ) - ) - validated_connector = result.scalars().first() - if not validated_connector: - return { - "status": "error", - "message": "Selected OneDrive connector is invalid or has been disconnected.", - } - actual_connector_id = validated_connector.id - else: - actual_connector_id = connector.id - - logger.info( - f"Deleting OneDrive file: file_id='{final_file_id}', connector={actual_connector_id}" - ) - - client = OneDriveClient( - session=db_session, connector_id=actual_connector_id - ) - await client.trash_file(final_file_id) - - logger.info( - f"OneDrive file deleted (moved to recycle bin): file_id={final_file_id}" - ) - - trash_result: dict[str, Any] = { - "status": "success", - "file_id": final_file_id, - "message": f"Successfully moved '{file_name}' to the recycle bin.", - } - - deleted_from_kb = False - if final_delete_from_kb and document_id: - try: - doc_result = await db_session.execute( - select(Document).filter(Document.id == document_id) - ) - doc = doc_result.scalars().first() - if doc: - await db_session.delete(doc) - await db_session.commit() - deleted_from_kb = True - logger.info( - f"Deleted document {document_id} from knowledge base" - ) - else: - logger.warning(f"Document {document_id} not found in KB") - except Exception as e: - logger.error(f"Failed to delete document from KB: {e}") - await db_session.rollback() - trash_result["warning"] = ( - f"File moved to recycle bin, but failed to remove from knowledge base: {e!s}" - ) - - trash_result["deleted_from_kb"] = deleted_from_kb - if deleted_from_kb: - trash_result["message"] = ( - f"{trash_result.get('message', '')} (also removed from knowledge base)" - ) - - return trash_result - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error(f"Error deleting OneDrive file: {e}", exc_info=True) - return { - "status": "error", - "message": "Something went wrong while trashing the file. Please try again.", - } - - return delete_onedrive_file diff --git a/surfsense_backend/app/agents/new_chat/tools/podcast.py b/surfsense_backend/app/agents/new_chat/tools/podcast.py deleted file mode 100644 index 83ac98768..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/podcast.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Podcast generation tool for the SurfSense agent. - -This module provides a factory function for creating the generate_podcast tool -that submits a Celery task for background podcast generation. The tool then -polls the podcast row until it reaches a terminal status (READY/FAILED) and -returns that status. The wait is bounded by the chat's HTTP / process -lifetime; see app.agents.shared.deliverable_wait for details. -""" - -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.shared.deliverable_wait import wait_for_deliverable -from app.db import Podcast, PodcastStatus, shielded_async_session - -logger = logging.getLogger(__name__) - - -def create_generate_podcast_tool( - search_space_id: int, - db_session: AsyncSession, - thread_id: int | None = None, -): - """ - Factory function to create the generate_podcast tool with injected dependencies. - - Pre-creates podcast record with pending status so podcast_id is available - immediately for frontend polling. - - Args: - search_space_id: The user's search space ID - db_session: Reserved for future read-side use; the row is written via a - fresh, tool-local session so parallel tool calls (e.g. podcast + - video presentation in the same agent step) don't share an - ``AsyncSession`` (which is not concurrency-safe). - thread_id: The chat thread ID for associating the podcast - - Returns: - A configured tool function for generating podcasts - """ - del db_session # writes use a fresh tool-local session, see below - - @tool - async def generate_podcast( - source_content: str, - podcast_title: str = "SurfSense Podcast", - user_prompt: str | None = None, - ) -> dict[str, Any]: - """ - Generate a podcast from the provided content. - - Use this tool when the user asks to create, generate, or make a podcast. - Common triggers include phrases like: - - "Give me a podcast about this" - - "Create a podcast from this conversation" - - "Generate a podcast summary" - - "Make a podcast about..." - - "Turn this into a podcast" - - Args: - source_content: The text content to convert into a podcast. - podcast_title: Title for the podcast (default: "SurfSense Podcast") - user_prompt: Optional instructions for podcast style, tone, or format. - - Returns: - A dictionary containing: - - status: PodcastStatus value (pending, generating, or failed) - - podcast_id: The podcast ID for polling (when status is pending or generating) - - title: The podcast title - - message: Status message (or "error" field if status is failed) - """ - try: - # Open a fresh session per call. The streaming task's session is - # shared between every tool, and ``AsyncSession`` is NOT safe for - # concurrent use: when the LLM emits parallel tool calls, two - # concurrent ``add()`` / ``commit()`` paths interleave and the - # second one hits "Session.add() during flush" → the transaction - # is poisoned for both tools. - async with shielded_async_session() as session: - podcast = Podcast( - title=podcast_title, - status=PodcastStatus.PENDING, - search_space_id=search_space_id, - thread_id=thread_id, - ) - session.add(podcast) - await session.commit() - await session.refresh(podcast) - podcast_id = podcast.id - - from app.tasks.celery_tasks.podcast_tasks import ( - generate_content_podcast_task, - ) - - task = generate_content_podcast_task.delay( - podcast_id=podcast_id, - source_content=source_content, - search_space_id=search_space_id, - user_prompt=user_prompt, - ) - - logger.info( - "[generate_podcast] Created podcast %s, task: %s", - podcast_id, - task.id, - ) - - # Wait until the Celery worker flips the row to a terminal - # state. No internal budget — see deliverable_wait module. - terminal_status, columns, elapsed = await wait_for_deliverable( - model=Podcast, - row_id=podcast_id, - columns=[Podcast.status, Podcast.file_location], - terminal_statuses={PodcastStatus.READY, PodcastStatus.FAILED}, - ) - - if terminal_status == PodcastStatus.READY: - file_location = columns[1] if columns else None - logger.info( - "[generate_podcast] Podcast %s READY in %.2fs (file=%s)", - podcast_id, - elapsed, - file_location, - ) - return { - "status": PodcastStatus.READY.value, - "podcast_id": podcast_id, - "title": podcast_title, - "file_location": file_location, - "message": ("Podcast generated and saved to your podcast panel."), - } - - # Only other terminal state is FAILED. - logger.warning( - "[generate_podcast] Podcast %s FAILED in %.2fs", - podcast_id, - elapsed, - ) - return { - "status": PodcastStatus.FAILED.value, - "podcast_id": podcast_id, - "title": podcast_title, - "error": ("Background worker reported FAILED status for this podcast."), - } - - except Exception as e: - error_message = str(e) - logger.exception("[generate_podcast] Error: %s", error_message) - return { - "status": PodcastStatus.FAILED.value, - "error": error_message, - "title": podcast_title, - "podcast_id": None, - } - - return generate_podcast diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py deleted file mode 100644 index 6f011e372..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ /dev/null @@ -1,962 +0,0 @@ -"""Tools registry for SurfSense deep agent. - -This module provides a registry pattern for managing tools in the SurfSense agent. -It makes it easy for OSS contributors to add new tools by: -1. Creating a tool factory function in a new file in this directory -2. Registering the tool in the BUILTIN_TOOLS list below - -Example of adding a new tool: ------------------------------- -1. Create your tool file (e.g., `tools/my_tool.py`): - - from langchain_core.tools import tool - from sqlalchemy.ext.asyncio import AsyncSession - - def create_my_tool(search_space_id: int, db_session: AsyncSession): - @tool - async def my_tool(param: str) -> dict: - '''My tool description.''' - # Your implementation - return {"result": "success"} - return my_tool - -2. Import and register in this file: - - from .my_tool import create_my_tool - - # Add to BUILTIN_TOOLS list: - ToolDefinition( - name="my_tool", - description="Description of what your tool does", - factory=lambda deps: create_my_tool( - search_space_id=deps["search_space_id"], - db_session=deps["db_session"], - ), - requires=["search_space_id", "db_session"], - ), -""" - -import logging -from collections.abc import Callable -from dataclasses import dataclass, field -from typing import Any - -from langchain_core.tools import BaseTool - -from app.agents.new_chat.middleware.dedup_tool_calls import ( - wrap_dedup_key_by_arg_name, -) -from app.db import ChatVisibility - -from .confluence import ( - create_create_confluence_page_tool, - create_delete_confluence_page_tool, - create_update_confluence_page_tool, -) -from .connected_accounts import create_get_connected_accounts_tool -from .discord import ( - create_list_discord_channels_tool, - create_read_discord_messages_tool, - create_send_discord_message_tool, -) -from .dropbox import ( - create_create_dropbox_file_tool, - create_delete_dropbox_file_tool, -) -from .generate_image import create_generate_image_tool -from .gmail import ( - create_create_gmail_draft_tool, - create_read_gmail_email_tool, - create_search_gmail_tool, - create_send_gmail_email_tool, - create_trash_gmail_email_tool, - create_update_gmail_draft_tool, -) -from .google_calendar import ( - create_create_calendar_event_tool, - create_delete_calendar_event_tool, - create_search_calendar_events_tool, - create_update_calendar_event_tool, -) -from .google_drive import ( - create_create_google_drive_file_tool, - create_delete_google_drive_file_tool, -) -from .luma import ( - create_create_luma_event_tool, - create_list_luma_events_tool, - create_read_luma_event_tool, -) -from .mcp_tool import load_mcp_tools -from .notion import ( - create_create_notion_page_tool, - create_delete_notion_page_tool, - create_update_notion_page_tool, -) -from .onedrive import ( - create_create_onedrive_file_tool, - create_delete_onedrive_file_tool, -) -from .podcast import create_generate_podcast_tool -from .report import create_generate_report_tool -from .resume import create_generate_resume_tool -from .scrape_webpage import create_scrape_webpage_tool -from .teams import ( - create_list_teams_channels_tool, - create_read_teams_messages_tool, - create_send_teams_message_tool, -) -from .update_memory import create_update_memory_tool, create_update_team_memory_tool -from .video_presentation import create_generate_video_presentation_tool -from .web_search import create_web_search_tool - -logger = logging.getLogger(__name__) - -# ============================================================================= -# Tool Definition -# ============================================================================= - - -@dataclass -class ToolDefinition: - """Definition of a tool that can be added to the agent. - - Attributes: - name: Unique identifier for the tool - description: Human-readable description of what the tool does - factory: Callable that creates the tool. Receives a dict of dependencies. - requires: List of dependency names this tool needs (e.g., "search_space_id", "db_session") - enabled_by_default: Whether the tool is enabled when no explicit config is provided - required_connector: Searchable type string (e.g. ``"LINEAR_CONNECTOR"``) - that must be in ``available_connectors`` for the tool to be enabled. - dedup_key: Optional callable that maps a tool's ``args`` dict to a - string signature used by :class:`DedupHITLToolCallsMiddleware` - to drop duplicate calls within a single LLM response. - reverse: Optional callable that, given the tool's ``(args, result)``, - returns a ``ReverseDescriptor`` describing the inverse tool - invocation. Consumed by the snapshot/revert pipeline. - - """ - - name: str - description: str - factory: Callable[[dict[str, Any]], BaseTool] - requires: list[str] = field(default_factory=list) - enabled_by_default: bool = True - hidden: bool = False - required_connector: str | None = None - dedup_key: Callable[[dict[str, Any]], str] | None = None - reverse: Callable[[dict[str, Any], Any], dict[str, Any]] | None = None - - -# ============================================================================= -# Deferred-import factories -# ============================================================================= -# Used for tools whose impls live under ``multi_agent_chat``. Importing those -# at module-load time would cycle (``multi_agent_chat`` middleware imports -# this registry). The import inside the factory runs only when -# ``build_tools`` is called, by which point ``multi_agent_chat`` is fully -# initialised. - - -def _build_create_automation_tool(deps: dict[str, Any]) -> BaseTool: - from app.agents.multi_agent_chat.main_agent.tools.automation import ( - create_create_automation_tool, - ) - - return create_create_automation_tool( - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - llm=deps["llm"], - ) - - -# ============================================================================= -# Built-in Tools Registry -# ============================================================================= - -# Registry of all built-in tools -# Contributors: Add your new tools here! -BUILTIN_TOOLS: list[ToolDefinition] = [ - # Podcast generation tool - ToolDefinition( - name="generate_podcast", - description="Generate an audio podcast from provided content", - factory=lambda deps: create_generate_podcast_tool( - search_space_id=deps["search_space_id"], - db_session=deps["db_session"], - thread_id=deps["thread_id"], - ), - requires=["search_space_id", "db_session", "thread_id"], - ), - # Video presentation generation tool - ToolDefinition( - name="generate_video_presentation", - description="Generate a video presentation with slides and narration from provided content", - factory=lambda deps: create_generate_video_presentation_tool( - search_space_id=deps["search_space_id"], - db_session=deps["db_session"], - thread_id=deps["thread_id"], - ), - requires=["search_space_id", "db_session", "thread_id"], - ), - # Report generation tool (inline, short-lived sessions for DB ops) - # Supports internal KB search via source_strategy so the agent does not - # need a separate search step before generating. - ToolDefinition( - name="generate_report", - description="Generate a structured report from provided content and export it", - factory=lambda deps: create_generate_report_tool( - search_space_id=deps["search_space_id"], - thread_id=deps["thread_id"], - connector_service=deps.get("connector_service"), - available_connectors=deps.get("available_connectors"), - available_document_types=deps.get("available_document_types"), - ), - requires=["search_space_id", "thread_id"], - # connector_service, available_connectors, and available_document_types - # are optional — when missing, source_strategy="kb_search" degrades - # gracefully to "provided" - ), - # Resume generation tool (Typst-based, uses rendercv package) - ToolDefinition( - name="generate_resume", - description="Generate a professional resume as a Typst document", - factory=lambda deps: create_generate_resume_tool( - search_space_id=deps["search_space_id"], - thread_id=deps["thread_id"], - ), - requires=["search_space_id", "thread_id"], - ), - # Generate image tool - creates images using AI models (DALL-E, GPT Image, etc.) - ToolDefinition( - name="generate_image", - description="Generate images from text descriptions using AI image models", - factory=lambda deps: create_generate_image_tool( - search_space_id=deps["search_space_id"], - db_session=deps["db_session"], - ), - requires=["search_space_id", "db_session"], - ), - # Web scraping tool - extracts content from webpages - ToolDefinition( - name="scrape_webpage", - description="Scrape and extract the main content from a webpage", - factory=lambda deps: create_scrape_webpage_tool( - firecrawl_api_key=deps.get("firecrawl_api_key"), - ), - requires=[], # firecrawl_api_key is optional - ), - # Web search tool — real-time web search via SearXNG + user-configured engines - ToolDefinition( - name="web_search", - description="Search the web for real-time information using configured search engines", - factory=lambda deps: create_web_search_tool( - search_space_id=deps.get("search_space_id"), - available_connectors=deps.get("available_connectors"), - ), - requires=[], - ), - # ========================================================================= - # SERVICE ACCOUNT DISCOVERY - # Generic tool for the LLM to discover connected accounts and resolve - # service-specific identifiers (e.g. Jira cloudId, Slack team, etc.) - # ========================================================================= - ToolDefinition( - name="get_connected_accounts", - description="Discover connected accounts for a service and their metadata", - factory=lambda deps: create_get_connected_accounts_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - ), - # ========================================================================= - # AUTOMATION AUTHORING - single HITL tool. The tool takes an NL ``intent`` - # from the main agent, drafts the full AutomationCreate JSON via a focused - # sub-LLM, surfaces it on an approval card, and persists on approval. The - # factory defers its import because the impl lives under ``multi_agent_chat`` - # and that package transitively pulls this registry via middleware; - # deferring to ``build_tools`` call-time breaks the cycle without a - # parallel registry. - # ========================================================================= - ToolDefinition( - name="create_automation", - description="Draft an automation from an NL intent; user approves the card; tool saves", - factory=_build_create_automation_tool, - requires=["search_space_id", "user_id", "llm"], - ), - # ========================================================================= - # MEMORY TOOL - single update_memory, private or team by thread_visibility - # ========================================================================= - ToolDefinition( - name="update_memory", - description="Save important long-term facts, preferences, and instructions to the (personal or team) memory", - factory=lambda deps: ( - create_update_team_memory_tool( - search_space_id=deps["search_space_id"], - db_session=deps["db_session"], - llm=deps.get("llm"), - ) - if deps["thread_visibility"] == ChatVisibility.SEARCH_SPACE - else create_update_memory_tool( - user_id=deps["user_id"], - db_session=deps["db_session"], - llm=deps.get("llm"), - ) - ), - requires=[ - "user_id", - "search_space_id", - "db_session", - "thread_visibility", - "llm", - ], - ), - # ========================================================================= - # NOTION TOOLS - create, update, delete pages - # Auto-disabled when no Notion connector is configured (see chat_deepagent.py) - # ========================================================================= - ToolDefinition( - name="create_notion_page", - description="Create a new page in the user's Notion workspace", - factory=lambda deps: create_create_notion_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="NOTION_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("title"), - ), - ToolDefinition( - name="update_notion_page", - description="Append new content to an existing Notion page", - factory=lambda deps: create_update_notion_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="NOTION_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("page_title"), - ), - ToolDefinition( - name="delete_notion_page", - description="Delete an existing Notion page", - factory=lambda deps: create_delete_notion_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="NOTION_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("page_title"), - ), - # ========================================================================= - # GOOGLE DRIVE TOOLS - create files, delete files - # Auto-disabled when no Google Drive connector is configured (see chat_deepagent.py) - # ========================================================================= - ToolDefinition( - name="create_google_drive_file", - description="Create a new Google Doc or Google Sheet in Google Drive", - factory=lambda deps: create_create_google_drive_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_DRIVE_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - ToolDefinition( - name="delete_google_drive_file", - description="Move an indexed Google Drive file to trash", - factory=lambda deps: create_delete_google_drive_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_DRIVE_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - # ========================================================================= - # DROPBOX TOOLS - create and trash files - # Auto-disabled when no Dropbox connector is configured (see chat_deepagent.py) - # ========================================================================= - ToolDefinition( - name="create_dropbox_file", - description="Create a new file in Dropbox", - factory=lambda deps: create_create_dropbox_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="DROPBOX_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - ToolDefinition( - name="delete_dropbox_file", - description="Delete a file from Dropbox", - factory=lambda deps: create_delete_dropbox_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="DROPBOX_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - # ========================================================================= - # ONEDRIVE TOOLS - create and trash files - # Auto-disabled when no OneDrive connector is configured (see chat_deepagent.py) - # ========================================================================= - ToolDefinition( - name="create_onedrive_file", - description="Create a new file in Microsoft OneDrive", - factory=lambda deps: create_create_onedrive_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="ONEDRIVE_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - ToolDefinition( - name="delete_onedrive_file", - description="Move a OneDrive file to the recycle bin", - factory=lambda deps: create_delete_onedrive_file_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="ONEDRIVE_FILE", - dedup_key=wrap_dedup_key_by_arg_name("file_name"), - ), - # ========================================================================= - # GOOGLE CALENDAR TOOLS - search, create, update, delete events - # Auto-disabled when no Google Calendar connector is configured - # ========================================================================= - ToolDefinition( - name="search_calendar_events", - description="Search Google Calendar events within a date range", - factory=lambda deps: create_search_calendar_events_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_CALENDAR_CONNECTOR", - ), - ToolDefinition( - name="create_calendar_event", - description="Create a new event on Google Calendar", - factory=lambda deps: create_create_calendar_event_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_CALENDAR_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("title"), - ), - ToolDefinition( - name="update_calendar_event", - description="Update an existing indexed Google Calendar event", - factory=lambda deps: create_update_calendar_event_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_CALENDAR_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("event_title_or_id"), - ), - ToolDefinition( - name="delete_calendar_event", - description="Delete an existing indexed Google Calendar event", - factory=lambda deps: create_delete_calendar_event_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_CALENDAR_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("event_title_or_id"), - ), - # ========================================================================= - # GMAIL TOOLS - search, read, create drafts, update drafts, send, trash - # Auto-disabled when no Gmail connector is configured - # ========================================================================= - ToolDefinition( - name="search_gmail", - description="Search emails in Gmail using Gmail search syntax", - factory=lambda deps: create_search_gmail_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - ), - ToolDefinition( - name="read_gmail_email", - description="Read the full content of a specific Gmail email", - factory=lambda deps: create_read_gmail_email_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - ), - ToolDefinition( - name="create_gmail_draft", - description="Create a draft email in Gmail", - factory=lambda deps: create_create_gmail_draft_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("subject"), - ), - ToolDefinition( - name="send_gmail_email", - description="Send an email via Gmail", - factory=lambda deps: create_send_gmail_email_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("subject"), - ), - ToolDefinition( - name="trash_gmail_email", - description="Move an indexed email to trash in Gmail", - factory=lambda deps: create_trash_gmail_email_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("email_subject_or_id"), - ), - ToolDefinition( - name="update_gmail_draft", - description="Update an existing Gmail draft", - factory=lambda deps: create_update_gmail_draft_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="GOOGLE_GMAIL_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("draft_subject_or_id"), - ), - # ========================================================================= - # CONFLUENCE TOOLS - create, update, delete pages - # Auto-disabled when no Confluence connector is configured (see chat_deepagent.py) - # ========================================================================= - ToolDefinition( - name="create_confluence_page", - description="Create a new page in the user's Confluence space", - factory=lambda deps: create_create_confluence_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="CONFLUENCE_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("title"), - ), - ToolDefinition( - name="update_confluence_page", - description="Update an existing indexed Confluence page", - factory=lambda deps: create_update_confluence_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="CONFLUENCE_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("page_title_or_id"), - ), - ToolDefinition( - name="delete_confluence_page", - description="Delete an existing indexed Confluence page", - factory=lambda deps: create_delete_confluence_page_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="CONFLUENCE_CONNECTOR", - dedup_key=wrap_dedup_key_by_arg_name("page_title_or_id"), - ), - # ========================================================================= - # DISCORD TOOLS - list channels, read messages, send messages - # Auto-disabled when no Discord connector is configured - # ========================================================================= - ToolDefinition( - name="list_discord_channels", - description="List text channels in the connected Discord server", - factory=lambda deps: create_list_discord_channels_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="DISCORD_CONNECTOR", - ), - ToolDefinition( - name="read_discord_messages", - description="Read recent messages from a Discord text channel", - factory=lambda deps: create_read_discord_messages_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="DISCORD_CONNECTOR", - ), - ToolDefinition( - name="send_discord_message", - description="Send a message to a Discord text channel", - factory=lambda deps: create_send_discord_message_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="DISCORD_CONNECTOR", - ), - # ========================================================================= - # TEAMS TOOLS - list channels, read messages, send messages - # Auto-disabled when no Teams connector is configured - # ========================================================================= - ToolDefinition( - name="list_teams_channels", - description="List Microsoft Teams and their channels", - factory=lambda deps: create_list_teams_channels_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="TEAMS_CONNECTOR", - ), - ToolDefinition( - name="read_teams_messages", - description="Read recent messages from a Microsoft Teams channel", - factory=lambda deps: create_read_teams_messages_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="TEAMS_CONNECTOR", - ), - ToolDefinition( - name="send_teams_message", - description="Send a message to a Microsoft Teams channel", - factory=lambda deps: create_send_teams_message_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="TEAMS_CONNECTOR", - ), - # ========================================================================= - # LUMA TOOLS - list events, read event details, create events - # Auto-disabled when no Luma connector is configured - # ========================================================================= - ToolDefinition( - name="list_luma_events", - description="List upcoming and recent Luma events", - factory=lambda deps: create_list_luma_events_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="LUMA_CONNECTOR", - ), - ToolDefinition( - name="read_luma_event", - description="Read detailed information about a specific Luma event", - factory=lambda deps: create_read_luma_event_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="LUMA_CONNECTOR", - ), - ToolDefinition( - name="create_luma_event", - description="Create a new event on Luma", - factory=lambda deps: create_create_luma_event_tool( - db_session=deps["db_session"], - search_space_id=deps["search_space_id"], - user_id=deps["user_id"], - ), - requires=["db_session", "search_space_id", "user_id"], - required_connector="LUMA_CONNECTOR", - ), -] - - -# ============================================================================= -# Registry Functions -# ============================================================================= - - -def get_tool_by_name(name: str) -> ToolDefinition | None: - """Get a tool definition by its name.""" - for tool_def in BUILTIN_TOOLS: - if tool_def.name == name: - return tool_def - return None - - -def get_connector_gated_tools( - available_connectors: list[str] | None, -) -> list[str]: - """Return tool names to disable""" - available = set() if available_connectors is None else set(available_connectors) - - disabled: list[str] = [] - for tool_def in BUILTIN_TOOLS: - if tool_def.required_connector and tool_def.required_connector not in available: - disabled.append(tool_def.name) - return disabled - - -def get_all_tool_names() -> list[str]: - """Get names of all registered tools.""" - return [tool_def.name for tool_def in BUILTIN_TOOLS] - - -def get_default_enabled_tools() -> list[str]: - """Get names of tools that are enabled by default (excludes hidden tools).""" - return [tool_def.name for tool_def in BUILTIN_TOOLS if tool_def.enabled_by_default] - - -def build_tools( - dependencies: dict[str, Any], - enabled_tools: list[str] | None = None, - disabled_tools: list[str] | None = None, - additional_tools: list[BaseTool] | None = None, -) -> list[BaseTool]: - """Build the list of tools for the agent. - - Args: - dependencies: Dict containing all possible dependencies: - - search_space_id: The search space ID - - db_session: Database session - - connector_service: Connector service instance - - firecrawl_api_key: Optional Firecrawl API key - enabled_tools: Explicit list of tool names to enable. If None, uses defaults. - disabled_tools: List of tool names to disable (applied after enabled_tools). - additional_tools: Extra tools to add (e.g., custom tools not in registry). - - Returns: - List of configured tool instances ready for the agent. - - Example: - # Use all default tools - tools = build_tools(deps) - - # Use only specific tools - tools = build_tools(deps, enabled_tools=["generate_report"]) - - # Use defaults but disable podcast - tools = build_tools(deps, disabled_tools=["generate_podcast"]) - - # Add custom tools - tools = build_tools(deps, additional_tools=[my_custom_tool]) - - """ - # Determine which tools to enable - if enabled_tools is not None: - tool_names_to_use = set(enabled_tools) - else: - tool_names_to_use = set(get_default_enabled_tools()) - - # Apply disabled list - if disabled_tools: - tool_names_to_use -= set(disabled_tools) - - # Build the tools (skip hidden/WIP tools unconditionally) - tools: list[BaseTool] = [] - for tool_def in BUILTIN_TOOLS: - if tool_def.hidden or tool_def.name not in tool_names_to_use: - continue - - # Check that all required dependencies are provided - missing_deps = [dep for dep in tool_def.requires if dep not in dependencies] - if missing_deps: - msg = f"Tool '{tool_def.name}' requires dependencies: {missing_deps}" - raise ValueError( - msg, - ) - - # Create the tool - tool = tool_def.factory(dependencies) - # Propagate the registry-level metadata so middleware (e.g. - # ``DedupHITLToolCallsMiddleware``) and the action-log/revert - # pipeline can pick the resolvers up via ``tool.metadata`` without - # re-importing :data:`BUILTIN_TOOLS`. - if tool_def.dedup_key is not None or tool_def.reverse is not None: - existing_meta = getattr(tool, "metadata", None) or {} - merged_meta = dict(existing_meta) - if tool_def.dedup_key is not None: - merged_meta.setdefault("dedup_key", tool_def.dedup_key) - if tool_def.reverse is not None: - merged_meta.setdefault("reverse", tool_def.reverse) - try: - tool.metadata = merged_meta - except Exception: - logger.debug( - "Tool %s rejected metadata mutation; relying on registry lookup", - tool_def.name, - ) - tools.append(tool) - - # Add any additional custom tools - if additional_tools: - tools.extend(additional_tools) - - return tools - - -async def build_tools_async( - dependencies: dict[str, Any], - enabled_tools: list[str] | None = None, - disabled_tools: list[str] | None = None, - additional_tools: list[BaseTool] | None = None, - include_mcp_tools: bool = True, -) -> list[BaseTool]: - """Async version of build_tools that also loads MCP tools from database. - - Design Note: - This function exists because MCP tools require database queries to load - user configs, while built-in tools are created synchronously from static - code. - - Alternative: We could make build_tools() itself async and always query - the database, but that would force async everywhere even when only using - built-in tools. The current design keeps the simple case (static tools - only) synchronous while supporting dynamic database-loaded tools through - this async wrapper. - - Phase 1.3: built-in tool construction (CPU; runs in a thread pool to - avoid event-loop stalls) and MCP tool loading (HTTP/DB I/O; runs on - the event loop) are kicked off concurrently. Cold-path savings are - bounded by the slower of the two — typically MCP at ~200ms-1.7s — - so the parallelization recovers the ~50-200ms previously spent - serially on built-in construction. - - Args: - dependencies: Dict containing all possible dependencies - enabled_tools: Explicit list of tool names to enable. If None, uses defaults. - disabled_tools: List of tool names to disable (applied after enabled_tools). - additional_tools: Extra tools to add (e.g., custom tools not in registry). - include_mcp_tools: Whether to load user's MCP tools from database. - - Returns: - List of configured tool instances ready for the agent, including MCP tools. - - """ - import asyncio - import time - - _perf_log = logging.getLogger("surfsense.perf") - _perf_log.setLevel(logging.DEBUG) - - can_load_mcp = ( - include_mcp_tools - and "db_session" in dependencies - and "search_space_id" in dependencies - ) - - # Built-in tool construction is synchronous + CPU-only. Off-loop it so - # MCP's HTTP/DB I/O can fire concurrently. ``build_tools`` is pure - # function over its inputs — safe to thread-shift. - _t0 = time.perf_counter() - builtin_task = asyncio.create_task( - asyncio.to_thread( - build_tools, dependencies, enabled_tools, disabled_tools, additional_tools - ) - ) - - mcp_task: asyncio.Task | None = None - if can_load_mcp: - mcp_task = asyncio.create_task( - load_mcp_tools( - dependencies["db_session"], - dependencies["search_space_id"], - ) - ) - - # Surface failures from each task independently so a flaky MCP - # endpoint never poisons built-in tool registration. ``return_exceptions`` - # gives us per-task exceptions instead of dropping the second result - # when the first raises. - if mcp_task is not None: - builtin_result, mcp_result = await asyncio.gather( - builtin_task, mcp_task, return_exceptions=True - ) - else: - builtin_result = await builtin_task - mcp_result = None - - if isinstance(builtin_result, BaseException): - raise builtin_result # built-in registration failure is non-recoverable - tools: list[BaseTool] = builtin_result - _perf_log.info( - "[build_tools_async] Built-in tools in %.3fs (%d tools, parallel)", - time.perf_counter() - _t0, - len(tools), - ) - - if mcp_task is not None: - if isinstance(mcp_result, BaseException): - # ``return_exceptions=True`` captures the exception out-of-band, - # so ``sys.exc_info()`` is empty here. Pass the captured - # exception via ``exc_info=`` to get a real traceback. - logging.error( - "Failed to load MCP tools: %s", mcp_result, exc_info=mcp_result - ) - else: - mcp_tools = mcp_result or [] - _perf_log.info( - "[build_tools_async] MCP tools loaded in %.3fs (%d tools, parallel)", - time.perf_counter() - _t0, - len(mcp_tools), - ) - tools.extend(mcp_tools) - logging.info( - "Registered %d MCP tools: %s", - len(mcp_tools), - [t.name for t in mcp_tools], - ) - - logging.info( - "Total tools for agent: %d — %s", - len(tools), - [t.name for t in tools], - ) - - return tools diff --git a/surfsense_backend/app/agents/new_chat/tools/report.py b/surfsense_backend/app/agents/new_chat/tools/report.py deleted file mode 100644 index 6bc1b7d57..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/report.py +++ /dev/null @@ -1,1084 +0,0 @@ -""" -Report generation tool for the SurfSense agent. - -This module provides a factory function for creating the generate_report tool -that generates a structured Markdown report inline (no Celery). The LLM is -called within the tool, the result is saved to the database, and the tool -returns immediately with a ready status. - -Uses short-lived database sessions to avoid holding connections during long -LLM calls (30-120+ seconds). Each DB operation (read config, save report) -opens and closes its own session, ensuring no connection is held idle during -the LLM API call. - -Generation strategies: - - Single-shot generation for all new reports - - Section-level revision for targeted edits (preserves unchanged sections) - - Full-document revision as fallback for global changes - -Source strategies (how source content is collected): - - "provided" — Use only the supplied source_content (default, backward-compat) - - "conversation" — Same as "provided"; agent passes conversation summary - - "kb_search" — Tool searches knowledge base internally with targeted queries - - "auto" — Use source_content if sufficient, else search KB as fallback -""" - -import asyncio -import json -import logging -import re -from typing import Any - -from langchain_core.callbacks import dispatch_custom_event -from langchain_core.messages import HumanMessage -from langchain_core.tools import tool - -from app.db import Report, shielded_async_session -from app.services.connector_service import ConnectorService -from app.services.llm_service import get_document_summary_llm - -logger = logging.getLogger(__name__) - -# ─── Shared Formatting Rules ──────────────────────────────────────────────── -# Reusable formatting instructions appended to section-level and review prompts. - -_FORMATTING_RULES = """\ -- IMPORTANT: Output raw Markdown directly. Do NOT wrap the entire output in a \ -code fence (e.g. ```markdown, ````markdown, or any backtick fence). Individual \ -code examples and diagrams inside the report should still use fenced code blocks, \ -but the report itself must NOT be enclosed in one. -- Maintain proper Markdown formatting throughout. -- When including code examples, ALWAYS format them as proper fenced code blocks \ -with the correct language identifier (e.g. ```java, ```python). Code inside code \ -blocks MUST have proper line breaks and indentation — NEVER put multiple statements \ -on a single line. Each statement, brace, and logical block must be on its own line \ -with correct indentation. -- When including Mermaid diagrams, use ```mermaid fenced code blocks. Each Mermaid \ -statement MUST be on its own line — NEVER use semicolons to join multiple statements \ -on one line. For line breaks inside node labels, use
(NOT
). -- When including mathematical formulas or equations, ALWAYS use LaTeX notation. \ -NEVER use backtick code spans or Unicode symbols for math.""" - -# ─── Standard Report Footer ───────────────────────────────────────────────── -# Appended to every generated report after content generation. - -_REPORT_FOOTER = "Powered by SurfSense AI." - -# ─── Prompt: Single-Shot Report Generation ─────────────────────────────────── - -_REPORT_PROMPT = """You are an expert report writer. Generate a comprehensive Markdown report. - -**Topic:** {topic} -**Report Style:** {report_style} -{user_instructions_section} -{previous_version_section} - -**Source Content:** -{source_content} - ---- - -{length_instruction} - -Write a well-structured Markdown report with a # title, executive summary, organized sections, and conclusion. Cite facts from the source content. Be thorough and professional. - -{formatting_rules} -""" - -# ─── Prompt: Full-Document Revision (fallback when section-level fails) ────── - -_REVISION_PROMPT = """You are an expert report editor. Apply ONLY the requested changes — do NOT rewrite from scratch. - -**Topic:** {topic} -**Report Style:** {report_style} -**Modification Instructions:** {user_instructions_section} - -**Source Content (use if relevant):** -{source_content} - ---- - -**EXISTING REPORT:** - -{previous_report_content} - ---- - -{length_instruction} - -Preserve all structure and content not affected by the modification. - -{formatting_rules} -""" - -# ─── Prompt: Section-Level Revision — Identify Affected Sections ───────────── - -_IDENTIFY_SECTIONS_PROMPT = """You are analyzing a Markdown report to determine which sections need modification based on the user's request. - -**User's Modification Request:** {user_instructions} - -**Report Sections (indexed starting at 0):** -{sections_listing} - ---- - -Determine which sections need to be modified, added, or removed to fulfill the user's request. - -Return ONLY a JSON object with these fields: -- "modify": Array of section indices (0-based) that need content changes -- "add": Array of objects like {{"after_index": 2, "heading": "## New Section Title", "description": "What this section should cover"}} for new sections to insert -- "remove": Array of section indices to remove entirely (use sparingly) -- "reasoning": A brief explanation of your decisions - -Guidelines: -- If the change is GLOBAL (e.g., "change the tone", "make the whole report shorter", "translate to Spanish"), include ALL section indices in "modify". -- If the change is TARGETED (e.g., "expand the budget section", "fix the conclusion"), include ONLY the affected section indices. -- For "add a section about X", use the "add" field with the appropriate insertion point. -- Prefer modifying over removing+adding when possible. - -Return ONLY valid JSON, no markdown fences: -""" - -# ─── Prompt: Section-Level Revision — Revise a Single Section ──────────────── - -_REVISE_SECTION_PROMPT = """Revise ONLY this section based on the instructions. If the instructions don't apply, return it UNCHANGED. - -**Modification Instructions:** {user_instructions} - -**Current Section:** -{section_content} - -**Context (surrounding sections — for coherence only, do NOT output them):** -{context_sections} - -**Source Content:** -{source_content} - ---- - -Keep the same heading and heading level. Preserve content not affected by the modification. -{formatting_rules} -""" - -# ─── Prompt: New Section Generation (for section-level add) ───────────────── - -_NEW_SECTION_PROMPT = """You are an expert report writer. Write a new section to be inserted into an existing report. - -**Report Topic:** {topic} -**Report Style:** {report_style} -**Section Heading:** {heading} -**Section Goal:** {description} -**User Instructions:** {user_instructions} - -**Surrounding Context:** -{context_sections} - -**Source Content:** -{source_content} - ---- - -**Rules:** -1. Write ONLY this section, starting with the heading "{heading}". -2. Ensure the section flows naturally with the surrounding context. -3. Be comprehensive — cover the topic described above. -{formatting_rules} - -Write the new section now: -""" - - -# ─── Utility Functions ────────────────────────────────────────────────────── - - -def _strip_wrapping_code_fences(text: str) -> str: - """Remove wrapping code fences that LLMs often add around Markdown output. - - Handles patterns like: - ```markdown\\n...content...\\n``` - ````markdown\\n...content...\\n```` - ```md\\n...content...\\n``` - ```\\n...content...\\n``` - ```json\\n...content...\\n``` - Supports 3 or more backticks (LLMs escalate when content has triple-backtick blocks). - """ - stripped = text.strip() - # Match opening fence with 3+ backticks and optional language tag - m = re.match(r"^(`{3,})(?:markdown|md|json)?\s*\n", stripped) - if m: - fence = m.group(1) # e.g. "```" or "````" - if stripped.endswith(fence): - stripped = stripped[m.end() :] # remove opening fence - stripped = stripped[: -len(fence)].rstrip() # remove closing fence - return stripped - - -def _extract_metadata(content: str) -> dict[str, Any]: - """Extract metadata from generated Markdown content.""" - # Count section headings - headings = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE) - - # Word count - word_count = len(content.split()) - - # Character count - char_count = len(content) - - return { - "status": "ready", - "word_count": word_count, - "char_count": char_count, - "section_count": len(headings), - } - - -def _parse_sections(content: str) -> list[dict[str, str]]: - """Parse Markdown content into sections split by # and ## headings. - - Returns a list of dicts: [{"heading": "## Title", "body": "content..."}, ...] - Content before the first heading is captured with heading="". - ### and deeper headings are kept inside their parent ## section's body. - """ - lines = content.split("\n") - sections: list[dict[str, str]] = [] - current_heading = "" - current_body_lines: list[str] = [] - in_code_block = False - - for line in lines: - # Track code blocks to avoid matching headings inside them - stripped = line.strip() - if stripped.startswith("```"): - in_code_block = not in_code_block - - # Only split on # or ## headings (not ### or deeper) and only outside code blocks - is_section_heading = ( - not in_code_block - and re.match(r"^#{1,2}\s+", line) - and not re.match(r"^#{3,}\s+", line) - ) - - if is_section_heading: - # Save previous section - if current_heading or current_body_lines: - sections.append( - { - "heading": current_heading, - "body": "\n".join(current_body_lines).strip(), - } - ) - current_heading = line.strip() - current_body_lines = [] - else: - current_body_lines.append(line) - - # Save last section - if current_heading or current_body_lines: - sections.append( - { - "heading": current_heading, - "body": "\n".join(current_body_lines).strip(), - } - ) - - return sections - - -def _stitch_sections(sections: list[dict[str, str]]) -> str: - """Stitch parsed sections back into a single Markdown string.""" - parts = [] - for section in sections: - if section["heading"]: - parts.append(section["heading"]) - if section["body"]: - parts.append(section["body"]) - return "\n\n".join(parts) - - -# ─── Async Generation Helpers ─────────────────────────────────────────────── - - -async def _revise_with_sections( - llm: Any, - parent_content: str, - user_instructions: str, - source_content: str, - topic: str, - report_style: str, -) -> str | None: - """Section-level revision: identify affected sections and revise only those. - - Unchanged sections are kept byte-for-byte identical. - Returns the revised content, or None to trigger full-document revision fallback. - """ - # Parse report into sections - sections = _parse_sections(parent_content) - if len(sections) < 2: - logger.info( - "[generate_report] Too few sections for section-level revision, using full revision" - ) - return None - - # Build a sections listing for the LLM - sections_listing = "" - for i, sec in enumerate(sections): - heading = sec["heading"] or "(preamble — content before first heading)" - body_preview = ( - sec["body"][:200] + "..." if len(sec["body"]) > 200 else sec["body"] - ) - sections_listing += f"\n[{i}] {heading}\n Preview: {body_preview}\n" - - # Step 1: Ask LLM which sections need modification - identify_prompt = _IDENTIFY_SECTIONS_PROMPT.format( - user_instructions=user_instructions, - sections_listing=sections_listing, - ) - - try: - response = await llm.ainvoke([HumanMessage(content=identify_prompt)]) - raw = response.content - if not raw or not isinstance(raw, str): - return None - - raw = _strip_wrapping_code_fences(raw).strip() - json_match = re.search(r"\{[\s\S]*\}", raw) - if json_match: - raw = json_match.group(0) - - plan = json.loads(raw) - modify_indices: list[int] = plan.get("modify", []) - add_sections: list[dict[str, Any]] = plan.get("add", []) - remove_indices: list[int] = plan.get("remove", []) - reasoning = plan.get("reasoning", "") - - logger.info( - f"[generate_report] Section-level revision plan: " - f"modify={modify_indices}, add={len(add_sections)}, " - f"remove={remove_indices}, reasoning={reasoning}" - ) - except Exception: - logger.warning( - "[generate_report] Failed to identify sections for revision, " - "falling back to full revision", - exc_info=True, - ) - return None - - # If ALL sections need modification, full revision is more efficient and coherent - if len(modify_indices) >= len(sections): - logger.info( - "[generate_report] All sections need modification, deferring to full revision" - ) - return None - - # Compute total operations for progress tracking - total_ops = len(modify_indices) + len(add_sections) - current_op = 0 - - # Emit plan summary - parts = [] - if modify_indices: - parts.append( - f"modifying {len(modify_indices)} section{'s' if len(modify_indices) > 1 else ''}" - ) - if add_sections: - parts.append( - f"adding {len(add_sections)} new section{'s' if len(add_sections) > 1 else ''}" - ) - if remove_indices: - parts.append( - f"removing {len(remove_indices)} section{'s' if len(remove_indices) > 1 else ''}" - ) - plan_summary = ", ".join(parts) if parts else "no changes needed" - - dispatch_custom_event( - "report_progress", - { - "phase": "revision_plan", - "message": plan_summary.capitalize(), - "modify_count": len(modify_indices), - "add_count": len(add_sections), - "remove_count": len(remove_indices), - "total_ops": total_ops, - }, - ) - - # Step 2: Revise only the affected sections - revised_sections = list(sections) # shallow copy — unmodified sections stay as-is - - for idx in modify_indices: - if idx < 0 or idx >= len(sections): - continue - - current_op += 1 - sec = sections[idx] - - # Extract plain section name (strip markdown heading markers) - section_name = ( - re.sub(r"^#+\s*", "", sec["heading"]).strip() - if sec["heading"] - else "Preamble" - ) - dispatch_custom_event( - "report_progress", - { - "phase": "revising_section", - "message": f"Revising: {section_name} ({current_op}/{total_ops})...", - }, - ) - - section_content = ( - f"{sec['heading']}\n\n{sec['body']}" if sec["heading"] else sec["body"] - ) - - # Build context from surrounding sections - context_parts = [] - if idx > 0: - prev = sections[idx - 1] - prev_preview = prev["body"][:300] + ( - "..." if len(prev["body"]) > 300 else "" - ) - context_parts.append( - f"**Previous section:** {prev['heading']}\n{prev_preview}" - ) - if idx < len(sections) - 1: - nxt = sections[idx + 1] - nxt_preview = nxt["body"][:300] + ("..." if len(nxt["body"]) > 300 else "") - context_parts.append(f"**Next section:** {nxt['heading']}\n{nxt_preview}") - context = ( - "\n\n".join(context_parts) if context_parts else "(No surrounding sections)" - ) - - revise_prompt = _REVISE_SECTION_PROMPT.format( - user_instructions=user_instructions, - section_content=section_content, - context_sections=context, - source_content=source_content[:40000], - formatting_rules=_FORMATTING_RULES, - ) - - resp = await llm.ainvoke([HumanMessage(content=revise_prompt)]) - revised_text = resp.content - if revised_text and isinstance(revised_text, str): - revised_text = _strip_wrapping_code_fences(revised_text).strip() - # Parse the LLM output back into heading + body - revised_parsed = _parse_sections(revised_text) - if revised_parsed: - revised_sections[idx] = revised_parsed[0] - else: - revised_sections[idx] = { - "heading": sec["heading"], - "body": revised_text, - } - - logger.info(f"[generate_report] Revised section [{idx}]: {sec['heading']}") - - # Step 3: Handle new section additions (insert in reverse order to preserve indices) - for add_info in sorted( - add_sections, - key=lambda x: x.get("after_index", len(revised_sections) - 1), - reverse=True, - ): - current_op += 1 - after_idx = add_info.get("after_index", len(revised_sections) - 1) - heading = add_info.get("heading", "## New Section") - description = add_info.get("description", "") - - # Extract plain section name for progress display - plain_heading = re.sub(r"^#+\s*", "", heading).strip() - dispatch_custom_event( - "report_progress", - { - "phase": "adding_section", - "message": f"Adding: {plain_heading} ({current_op}/{total_ops})...", - }, - ) - - # Build context from the surrounding sections at the insertion point - ctx_parts = [] - if 0 <= after_idx < len(revised_sections): - before_sec = revised_sections[after_idx] - ctx_parts.append( - f"**Section before:** {before_sec['heading']}\n{before_sec['body'][:300]}" - ) - insert_idx = min(after_idx + 1, len(revised_sections)) - if insert_idx < len(revised_sections): - after_sec = revised_sections[insert_idx] - ctx_parts.append( - f"**Section after:** {after_sec['heading']}\n{after_sec['body'][:300]}" - ) - - new_prompt = _NEW_SECTION_PROMPT.format( - topic=topic, - report_style=report_style, - heading=heading, - description=description, - user_instructions=user_instructions, - context_sections="\n\n".join(ctx_parts) if ctx_parts else "(None)", - source_content=source_content[:30000], - formatting_rules=_FORMATTING_RULES, - ) - - resp = await llm.ainvoke([HumanMessage(content=new_prompt)]) - new_content = resp.content - if new_content and isinstance(new_content, str): - new_content = _strip_wrapping_code_fences(new_content).strip() - new_parsed = _parse_sections(new_content) - if new_parsed: - revised_sections.insert(insert_idx, new_parsed[0]) - else: - revised_sections.insert( - insert_idx, - { - "heading": heading, - "body": new_content, - }, - ) - - logger.info( - f"[generate_report] Added new section after [{after_idx}]: {heading}" - ) - - # Step 4: Handle removals (reverse order to preserve indices) - for idx in sorted(remove_indices, reverse=True): - if 0 <= idx < len(revised_sections): - logger.info( - f"[generate_report] Removed section [{idx}]: " - f"{revised_sections[idx]['heading']}" - ) - revised_sections.pop(idx) - - return _stitch_sections(revised_sections) - - -# ─── Tool Factory ─────────────────────────────────────────────────────────── - - -def create_generate_report_tool( - search_space_id: int, - thread_id: int | None = None, - connector_service: ConnectorService | None = None, - available_connectors: list[str] | None = None, - available_document_types: list[str] | None = None, -): - """ - Factory function to create the generate_report tool with injected dependencies. - - The tool generates a Markdown report inline using the search space's - document summary LLM, saves it to the database, and returns immediately. - - Uses short-lived database sessions for each DB operation so no connection - is held during the long LLM API call. - - Generation strategies: - - New reports: single-shot generation (1 LLM call) - - Revisions (targeted edits): section-level (unchanged sections preserved) - - Revisions (global changes): full-document revision fallback - - Source strategies: - - "provided"/"conversation": use only the supplied source_content - - "kb_search": search the knowledge base internally using targeted queries - - "auto": use source_content if sufficient, otherwise fall back to KB search - - Args: - search_space_id: The user's search space ID - thread_id: The chat thread ID for associating the report - connector_service: Optional connector service for internal KB search. - When provided, the tool can search the knowledge base internally - (used by the "kb_search" and "auto" source strategies). - available_connectors: Optional list of connector types available in the - search space (used to scope internal KB searches). - - Returns: - A configured tool function for generating reports - """ - - @tool - async def generate_report( - topic: str, - source_content: str = "", - source_strategy: str = "provided", - search_queries: list[str] | None = None, - report_style: str = "detailed", - user_instructions: str | None = None, - parent_report_id: int | None = None, - ) -> dict[str, Any]: - """ - Generate a structured Markdown report artifact from provided content. - - Use this tool when the user asks to create, generate, write, produce, - draft, or summarize into a report-style deliverable. - - Trigger classes include: - - Direct trigger words WITH creation/modification verb: report, - document, memo, letter, template, article, guide, blog post, - one-pager, briefing, comprehensive guide. - - Creation-intent phrases: "write a report", "generate a document", - "draft a summary", "create an executive summary". - - Modification-intent phrases: "revise the report", "update the - report", "make it shorter", "add a section about X", "expand the - budget section", "rewrite in formal tone". - - IMPORTANT — what does NOT count as "asking for a report": - - Questions or discussion about a report or its topic are NOT report - requests. Respond to these conversationally in chat. - Examples: "What other examples to put there?", "What else could be - added?", "Can you explain section 2?", "Is the data accurate?", - "What's missing?", "How could this be improved?", "What other - topics are related?" - - Quick summary requests, explanations, or follow-up questions. - - The test: Does the message contain a creation/modification VERB - (write, create, generate, draft, add, revise, update, expand, - rewrite, make) directed at producing a deliverable? If no verb - → answer in chat. - - FORMAT/EXPORT RULE: - - Always generate the report content in Markdown. - - If the user requests DOCX/Word/PDF or another file format, export - from the generated Markdown report. - - SOURCE STRATEGY (how to collect source material): - - source_strategy="conversation" — The conversation already has - enough context (prior Q&A, filesystem exploration, pasted text, - uploaded files, scraped webpages). Pass a thorough summary as - source_content. - - source_strategy="kb_search" — Search the knowledge base - internally. Provide 1-5 targeted search_queries. The tool - handles searching internally — do NOT manually read and dump - /documents/ files into source_content. - - source_strategy="provided" — Use only what is in source_content - (default, backward-compatible). - - source_strategy="auto" — Use source_content if it has enough - material; otherwise fall back to internal KB search using - search_queries. - - CONVERSATION REUSE (HIGH PRIORITY): - - If the user has been asking questions in this chat and the - conversation contains substantive answers/discussion on the - topic, prefer source_strategy="conversation" with a thorough - summary of the full chat history as source_content. - - The user's prior questions and your answers ARE the source - material. Do NOT redundantly search the knowledge base for - information that is already in the chat. - - VERSIONING — parent_report_id: - - Set parent_report_id when the user wants to MODIFY, REVISE, - IMPROVE, UPDATE, EXPAND, or ADD CONTENT TO an existing report - that was already generated in this conversation. - - This includes both explicit AND implicit modification requests. - If the user references the existing report using words like "it", - "this", "here", "the report", or clearly refers to a previously - generated report, treat it as a revision request. - - The value must be the report_id from a previous generate_report - result in this same conversation. - - Do NOT set parent_report_id when: - * The user asks for a report on a completely NEW/DIFFERENT topic - * The user says "generate another report" (new report, not revision) - * There is no prior report to reference - - Examples of when to SET parent_report_id: - User: "Make that report shorter" → parent_report_id = - User: "Add a cost analysis section to the report" → parent_report_id = - User: "Rewrite the report in a more formal tone" → parent_report_id = - User: "I want more details about pricing in here" → parent_report_id = - User: "Include more examples" → parent_report_id = - User: "Can you also cover nutrition in this?" → parent_report_id = - User: "Make it more detailed" → parent_report_id = - User: "Not bad, but expand on the budget section" → parent_report_id = - User: "Also mention the competitor landscape" → parent_report_id = - - Examples of when to LEAVE parent_report_id as None: - User: "Generate a report on climate change" → None (new topic) - User: "Write me a report about the budget" → None (new topic) - User: "Create another report, this time about marketing" → None - User: "Now write one about travel trends in Europe" → None (new topic) - - Args: - topic: Short title for the report (max ~8 words). - source_content: Text to base the report on. Can be empty when - using source_strategy="kb_search". - source_strategy: How to collect source material. One of - "provided", "conversation", "kb_search", or "auto". - search_queries: When source_strategy is "kb_search" or "auto", - provide 1-5 targeted search queries for the knowledge base. - These should be specific, not just the topic repeated. - report_style: "detailed", "deep_research", or "brief". - user_instructions: Optional focus or modification instructions. - When revising (parent_report_id set), describe WHAT TO CHANGE. - parent_report_id: ID of a previous report to revise (creates new - version in the same version group). - - Returns: - Dict with status, report_id, title, word_count, and message. - """ - # Initialize version tracking variables (used by _save_failed_report closure) - parent_report_content: str | None = None - report_group_id: int | None = None - - async def _save_failed_report(error_msg: str) -> int | None: - """Persist a failed report row using a short-lived session.""" - try: - async with shielded_async_session() as session: - failed_report = Report( - title=topic, - content=None, - report_metadata={ - "status": "failed", - "error_message": error_msg, - }, - report_style=report_style, - search_space_id=search_space_id, - thread_id=thread_id, - report_group_id=report_group_id, - ) - session.add(failed_report) - await session.commit() - await session.refresh(failed_report) - # If this is a new group (v1 failed), set group to self - if not failed_report.report_group_id: - failed_report.report_group_id = failed_report.id - await session.commit() - logger.info( - f"[generate_report] Saved failed report {failed_report.id}: {error_msg}" - ) - return failed_report.id - except Exception: - logger.exception( - "[generate_report] Could not persist failed report row" - ) - return None - - try: - # ── Phase 1: READ (short-lived session) ────────────────────── - # Fetch parent report and LLM config, then close the session - # so no DB connection is held during the long LLM call. - async with shielded_async_session() as read_session: - if parent_report_id: - parent_report = await read_session.get(Report, parent_report_id) - if parent_report: - report_group_id = parent_report.report_group_id - parent_report_content = parent_report.content - logger.info( - f"[generate_report] Creating new version from parent {parent_report_id} " - f"(group {report_group_id})" - ) - else: - logger.warning( - f"[generate_report] parent_report_id={parent_report_id} not found, " - "creating standalone report" - ) - - llm = await get_document_summary_llm(read_session, search_space_id) - # read_session closed — connection returned to pool - - if not llm: - error_msg = ( - "No LLM configured. Please configure a language model in Settings." - ) - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": topic, - } - - # Build the user instructions string - user_instructions_section = "" - if user_instructions: - user_instructions_section = ( - f"**Additional Instructions:** {user_instructions}" - ) - - # ── Phase 1b: SOURCE COLLECTION (smart KB search) ──────────── - # Decide whether to augment source_content with KB search results. - effective_source = source_content or "" - - strategy = (source_strategy or "provided").lower().strip() - - needs_kb_search = False - if strategy == "kb_search": - needs_kb_search = True - elif strategy == "auto": - # Heuristic: if source_content has fewer than 200 words, - # it's likely insufficient — augment with KB search. - word_count_estimate = len(effective_source.split()) - if word_count_estimate < 200: - needs_kb_search = True - logger.info( - f"[generate_report] auto strategy: source has ~{word_count_estimate} words, " - "triggering KB search" - ) - # "provided" and "conversation" → use source_content as-is - - if needs_kb_search and connector_service and search_queries: - query_count = min(len(search_queries), 5) - dispatch_custom_event( - "report_progress", - { - "phase": "kb_search", - "message": f"Searching knowledge base ({query_count} queries)...", - }, - ) - logger.info( - f"[generate_report] Running internal KB search with " - f"{query_count} queries: {search_queries[:5]}" - ) - try: - from .knowledge_base import search_knowledge_base_async - - # Run all queries in parallel, each with its own session - async def _run_single_query(q: str) -> str: - async with shielded_async_session() as kb_session: - kb_connector_svc = ConnectorService( - kb_session, search_space_id - ) - return await search_knowledge_base_async( - query=q, - search_space_id=search_space_id, - db_session=kb_session, - connector_service=kb_connector_svc, - top_k=10, - available_connectors=available_connectors, - available_document_types=available_document_types, - ) - - kb_results = await asyncio.gather( - *[_run_single_query(q) for q in search_queries[:5]] - ) - - # Merge non-empty results into source_content - kb_text_parts = [r for r in kb_results if r and r.strip()] - if kb_text_parts: - kb_combined = "\n\n---\n\n".join(kb_text_parts) - if effective_source.strip(): - effective_source = ( - effective_source - + "\n\n--- Knowledge Base Search Results ---\n\n" - + kb_combined - ) - else: - effective_source = kb_combined - - # Count docs found (rough: count tags) - doc_count = kb_combined.count("") - dispatch_custom_event( - "report_progress", - { - "phase": "kb_search_done", - "message": f"Found {doc_count} relevant documents" - if doc_count - else f"Found results from {len(kb_text_parts)} queries", - }, - ) - logger.info( - f"[generate_report] KB search added ~{len(kb_combined)} chars " - f"from {len(kb_text_parts)} queries" - ) - else: - dispatch_custom_event( - "report_progress", - { - "phase": "kb_search_done", - "message": "No results found in knowledge base", - }, - ) - logger.info("[generate_report] KB search returned no results") - - except Exception as e: - logger.warning( - f"[generate_report] Internal KB search failed: {e}. " - "Proceeding with existing source_content." - ) - elif needs_kb_search and not connector_service: - logger.warning( - "[generate_report] KB search requested but connector_service " - "not available. Using source_content as-is." - ) - elif needs_kb_search and not search_queries: - logger.warning( - "[generate_report] KB search requested but no search_queries " - "provided. Using source_content as-is." - ) - - capped_source = effective_source[:100000] # Cap source content - - # Length constraint — only when user explicitly asks for brevity - length_instruction = "" - if report_style == "brief": - length_instruction = ( - "**LENGTH CONSTRAINT (MANDATORY):** The user wants a SHORT report. " - "Keep it concise — aim for ~400 words (~1 page) unless a different " - "length is specified in the Additional Instructions above. " - "Prioritize brevity over thoroughness. Do NOT write a long report." - ) - - # ── Phase 2: LLM GENERATION (no DB connection held) ────────── - - report_content: str | None = None - - if parent_report_content: - # ─── REVISION MODE ─────────────────────────────────────── - # Strategy: Try section-level revision first (preserves - # unchanged sections byte-for-byte). Falls back to full- - # document revision if section identification fails or if - # all sections need changes. - dispatch_custom_event( - "report_progress", - { - "phase": "revision_start", - "message": "Analyzing sections to modify...", - }, - ) - logger.info( - "[generate_report] Revision mode — attempting section-level revision" - ) - report_content = await _revise_with_sections( - llm=llm, - parent_content=parent_report_content, - user_instructions=user_instructions - or "Improve and refine the report.", - source_content=capped_source, - topic=topic, - report_style=report_style, - ) - - if report_content is None: - # Fallback: full-document revision - dispatch_custom_event( - "report_progress", - {"phase": "writing", "message": "Rewriting your full report"}, - ) - logger.info( - "[generate_report] Section-level revision deferred, " - "using full-document revision" - ) - prompt = _REVISION_PROMPT.format( - topic=topic, - report_style=report_style, - user_instructions_section=user_instructions_section - or "Improve and refine the report.", - source_content=capped_source, - previous_report_content=parent_report_content, - length_instruction=length_instruction, - formatting_rules=_FORMATTING_RULES, - ) - response = await llm.ainvoke([HumanMessage(content=prompt)]) - report_content = response.content - - else: - # ─── NEW REPORT MODE ───────────────────────────────────── - # Single-shot generation: one LLM call produces the full - # report. Fast, globally coherent, and cost-efficient. - dispatch_custom_event( - "report_progress", - {"phase": "writing", "message": "Writing your report"}, - ) - logger.info( - "[generate_report] New report — using single-shot generation" - ) - prompt = _REPORT_PROMPT.format( - topic=topic, - report_style=report_style, - user_instructions_section=user_instructions_section, - previous_version_section="", - source_content=capped_source, - length_instruction=length_instruction, - formatting_rules=_FORMATTING_RULES, - ) - response = await llm.ainvoke([HumanMessage(content=prompt)]) - report_content = response.content - - # ── Validate LLM output ────────────────────────────────────── - - if not report_content or not isinstance(report_content, str): - error_msg = "LLM returned empty or invalid content" - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": topic, - } - - # LLMs often wrap output in ```markdown ... ``` fences — strip them - report_content = _strip_wrapping_code_fences(report_content) - - if not report_content: - error_msg = "LLM returned empty or invalid content" - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": topic, - } - - # Strip any existing footer(s) carried over from parent version(s) - while report_content.rstrip().endswith(_REPORT_FOOTER): - idx = report_content.rstrip().rfind(_REPORT_FOOTER) - report_content = report_content[:idx].rstrip() - if report_content.rstrip().endswith("---"): - report_content = report_content.rstrip()[:-3].rstrip() - - # Append exactly one standard disclaimer - report_content += "\n\n---\n\n" + _REPORT_FOOTER - - # Extract metadata (includes "status": "ready") - metadata = _extract_metadata(report_content) - - # ── Phase 3: WRITE (short-lived session) ───────────────────── - # Save the report to the database, then close the session. - async with shielded_async_session() as write_session: - report = Report( - title=topic, - content=report_content, - report_metadata=metadata, - report_style=report_style, - search_space_id=search_space_id, - thread_id=thread_id, - report_group_id=report_group_id, - ) - write_session.add(report) - await write_session.commit() - await write_session.refresh(report) - - # If this is a brand-new report (v1), set report_group_id = own id - if not report.report_group_id: - report.report_group_id = report.id - await write_session.commit() - - saved_report_id = report.id - saved_group_id = report.report_group_id - # write_session closed — connection returned to pool - - logger.info( - f"[generate_report] Created report {saved_report_id} " - f"(group={saved_group_id}): " - f"{metadata.get('word_count', 0)} words, " - f"{metadata.get('section_count', 0)} sections" - ) - - return { - "status": "ready", - "report_id": saved_report_id, - "title": topic, - "word_count": metadata.get("word_count", 0), - "is_revision": bool(parent_report_content), - "report_markdown": report_content, - "message": f"Report generated successfully: {topic}", - } - - except Exception as e: - error_message = str(e) - logger.exception(f"[generate_report] Error: {error_message}") - report_id = await _save_failed_report(error_message) - - return { - "status": "failed", - "error": error_message, - "report_id": report_id, - "title": topic, - } - - return generate_report diff --git a/surfsense_backend/app/agents/new_chat/tools/resume.py b/surfsense_backend/app/agents/new_chat/tools/resume.py deleted file mode 100644 index 4abe48ba6..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/resume.py +++ /dev/null @@ -1,812 +0,0 @@ -""" -Resume generation tool for the SurfSense agent. - -Generates a structured resume as Typst source code using the rendercv package. -The LLM outputs only the content body (= heading, sections, entries) while -the template header (import + show rule) is hardcoded and prepended by the -backend. This eliminates LLM errors in the complex configuration block. - -Templates are stored in a registry so new designs can be added by defining -a new entry in _TEMPLATES. - -Uses the same short-lived session pattern as generate_report so no DB -connection is held during the long LLM call. -""" - -import io -import logging -import re -from datetime import UTC, datetime -from typing import Any - -import pypdf -import typst -from langchain_core.callbacks import dispatch_custom_event -from langchain_core.messages import HumanMessage -from langchain_core.tools import tool - -from app.db import Report, shielded_async_session -from app.services.llm_service import get_document_summary_llm - -logger = logging.getLogger(__name__) - - -# ─── Template Registry ─────────────────────────────────────────────────────── -# Each template defines: -# header - Typst import + show rule with {name}, {year}, {month}, {day} placeholders -# component_reference - component docs shown to the LLM -# rules - generation rules for the LLM - -_TEMPLATES: dict[str, dict[str, str]] = { - "classic": { - "header": """\ -#import "@preview/rendercv:0.3.0": * - -#show: rendercv.with( - name: "{name}", - title: "{name} - Resume", - footer: context {{ [#emph[{name} -- #str(here().page())\\/#str(counter(page).final().first())]] }}, - top-note: [ #emph[Last updated in {month_name} {year}] ], - locale-catalog-language: "en", - text-direction: ltr, - page-size: "us-letter", - page-top-margin: 0.7in, - page-bottom-margin: 0.7in, - page-left-margin: 0.7in, - page-right-margin: 0.7in, - page-show-footer: false, - page-show-top-note: true, - colors-body: rgb(0, 0, 0), - colors-name: rgb(0, 0, 0), - colors-headline: rgb(0, 0, 0), - colors-connections: rgb(0, 0, 0), - colors-section-titles: rgb(0, 0, 0), - colors-links: rgb(0, 0, 0), - colors-footer: rgb(128, 128, 128), - colors-top-note: rgb(128, 128, 128), - typography-line-spacing: 0.6em, - typography-alignment: "justified", - typography-date-and-location-column-alignment: right, - typography-font-family-body: "XCharter", - typography-font-family-name: "XCharter", - typography-font-family-headline: "XCharter", - typography-font-family-connections: "XCharter", - typography-font-family-section-titles: "XCharter", - typography-font-size-body: 10pt, - typography-font-size-name: 25pt, - typography-font-size-headline: 10pt, - typography-font-size-connections: 10pt, - typography-font-size-section-titles: 1.2em, - typography-small-caps-name: false, - typography-small-caps-headline: false, - typography-small-caps-connections: false, - typography-small-caps-section-titles: false, - typography-bold-name: false, - typography-bold-headline: false, - typography-bold-connections: false, - typography-bold-section-titles: true, - links-underline: true, - links-show-external-link-icon: false, - header-alignment: center, - header-photo-width: 3.5cm, - header-space-below-name: 0.7cm, - header-space-below-headline: 0.7cm, - header-space-below-connections: 0.7cm, - header-connections-hyperlink: true, - header-connections-show-icons: false, - header-connections-display-urls-instead-of-usernames: true, - header-connections-separator: "|", - header-connections-space-between-connections: 0.5cm, - section-titles-type: "with_full_line", - section-titles-line-thickness: 0.5pt, - section-titles-space-above: 0.5cm, - section-titles-space-below: 0.3cm, - sections-allow-page-break: true, - sections-space-between-text-based-entries: 0.15cm, - sections-space-between-regular-entries: 0.42cm, - entries-date-and-location-width: 4.15cm, - entries-side-space: 0cm, - entries-space-between-columns: 0.1cm, - entries-allow-page-break: false, - entries-short-second-row: false, - entries-degree-width: 1cm, - entries-summary-space-left: 0cm, - entries-summary-space-above: 0.08cm, - entries-highlights-bullet: text(13pt, [\\u{2022}], baseline: -0.6pt), - entries-highlights-nested-bullet: text(13pt, [\\u{2022}], baseline: -0.6pt), - entries-highlights-space-left: 0cm, - entries-highlights-space-above: 0.08cm, - entries-highlights-space-between-items: 0.02cm, - entries-highlights-space-between-bullet-and-text: 0.3em, - date: datetime( - year: {year}, - month: {month}, - day: {day}, - ), -) - -""", - "component_reference": """\ -Available components (use ONLY these): - -= Full Name // Top-level heading — person's full name - -#connections( // Contact info row (pipe-separated) - [City, Country], - [#link("mailto:email@example.com", icon: false, if-underline: false, if-color: false)[email\\@example.com]], - [#link("https://linkedin.com/in/user", icon: false, if-underline: false, if-color: false)[linkedin.com\\/in\\/user]], - [#link("https://github.com/user", icon: false, if-underline: false, if-color: false)[github.com\\/user]], -) - -== Section Title // Section heading (arbitrary name) - -#regular-entry( // Work experience, projects, publications, etc. - [ - #strong[Role/Title], Company Name -- Location - ], - [ - Start -- End - ], - main-column-second-row: [ - - Achievement or responsibility - - Another bullet point - ], -) - -#education-entry( // Education entries - [ - #strong[Institution], Degree in Field -- Location - ], - [ - Start -- End - ], - main-column-second-row: [ - - GPA, honours, relevant coursework - ], -) - -#summary([Short paragraph summary]) // Optional summary inside an entry -#content-area([Free-form content]) // Freeform text block - -For skills sections, use one bullet per category label: -- #strong[Category:] item1, item2, item3 - -For simple list sections (e.g. Honors), use plain bullet points: -- Item one -- Item two -""", - "rules": """\ -RULES: -- Do NOT include any #import or #show lines. Start directly with = Full Name. -- Output ONLY valid Typst content. No explanatory text before or after. -- Do NOT wrap output in ```typst code fences. -- The = heading MUST use the person's COMPLETE full name exactly as provided. NEVER shorten or abbreviate. -- Escape @ symbols inside link labels with a backslash: email\\@example.com -- Escape forward slashes in link display text: linkedin.com\\/in\\/user -- Every section MUST use == heading. -- Use #regular-entry() for experience, projects, publications, certifications, and similar entries. -- Use #education-entry() for education. -- For skills sections, use one bullet line per category with a bold label. -- Keep content professional, concise, and achievement-oriented. -- Use action verbs for bullet points (Led, Built, Designed, Reduced, etc.). -- This template works for ALL professions — adapt sections to the user's field. -- Default behavior should prioritize concise one-page content. -""", - }, -} - -DEFAULT_TEMPLATE = "classic" -MIN_RESUME_PAGES = 1 -MAX_RESUME_PAGES = 5 -MAX_COMPRESSION_ATTEMPTS = 2 - - -# ─── Template Helpers ───────────────────────────────────────────────────────── - - -def _get_template(template_id: str | None = None) -> dict[str, str]: - """Get a template by ID, falling back to default.""" - return _TEMPLATES.get(template_id or DEFAULT_TEMPLATE, _TEMPLATES[DEFAULT_TEMPLATE]) - - -_MONTH_NAMES = [ - "", - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec", -] - - -def _build_header(template: dict[str, str], name: str) -> str: - """Build the template header with the person's name and current date.""" - now = datetime.now(tz=UTC) - return ( - template["header"] - .replace("{name}", name) - .replace("{year}", str(now.year)) - .replace("{month}", str(now.month)) - .replace("{day}", str(now.day)) - .replace("{month_name}", _MONTH_NAMES[now.month]) - ) - - -def _strip_header(full_source: str) -> str: - """Strip the import + show rule from stored source to get the body only. - - Finds the closing parenthesis of the rendercv.with(...) block by tracking - nesting depth, then returns everything after it. - """ - show_match = re.search(r"#show:\s*rendercv\.with\(", full_source) - if not show_match: - return full_source - - start = show_match.end() - depth = 1 - i = start - while i < len(full_source) and depth > 0: - if full_source[i] == "(": - depth += 1 - elif full_source[i] == ")": - depth -= 1 - i += 1 - - return full_source[i:].lstrip("\n") - - -def _extract_name(body: str) -> str | None: - """Extract the person's full name from the = heading in the body.""" - match = re.search(r"^=\s+(.+)$", body, re.MULTILINE) - return match.group(1).strip() if match else None - - -def _strip_imports(body: str) -> str: - """Remove any #import or #show lines the LLM might accidentally include.""" - lines = body.split("\n") - cleaned: list[str] = [] - skip_show = False - depth = 0 - - for line in lines: - stripped = line.strip() - - if stripped.startswith("#import"): - continue - - if skip_show: - depth += stripped.count("(") - stripped.count(")") - if depth <= 0: - skip_show = False - continue - - if stripped.startswith("#show:") and "rendercv" in stripped: - depth = stripped.count("(") - stripped.count(")") - if depth > 0: - skip_show = True - continue - - cleaned.append(line) - - result = "\n".join(cleaned).strip() - return result - - -def _build_llm_reference(template: dict[str, str]) -> str: - """Build the LLM prompt reference from a template.""" - return f"""\ -You MUST output valid Typst content for a resume. -Do NOT include any #import or #show lines — those are handled automatically. -Start directly with the = Full Name heading. - -{template["component_reference"]} - -{template["rules"]}""" - - -# ─── Prompts ───────────────────────────────────────────────────────────────── - -_RESUME_PROMPT = """\ -You are an expert resume writer. Generate professional resume content as Typst markup. - -{llm_reference} - -**User Information:** -{user_info} - -**Target Maximum Pages:** {max_pages} - -{user_instructions_section} - -Generate the resume content now (starting with = Full Name): -""" - -_REVISION_PROMPT = """\ -You are an expert resume editor. Modify the existing resume according to the instructions. -Apply ONLY the requested changes — do NOT rewrite sections that are not affected. - -{llm_reference} - -**Target Maximum Pages:** {max_pages} - -**Modification Instructions:** {user_instructions} - -**EXISTING RESUME CONTENT:** - -{previous_content} - ---- - -Output the complete, updated resume content with the changes applied (starting with = Full Name): -""" - -_FIX_COMPILE_PROMPT = """\ -The resume content you generated failed to compile. Fix the error while preserving all content. - -{llm_reference} - -**Compilation Error:** -{error} - -**Full Typst Source (for context — error line numbers refer to this):** -{full_source} - -**Your content starts after the template header. Output ONLY the content portion \ -(starting with = Full Name), NOT the #import or #show rule:** -""" - -_COMPRESS_TO_PAGE_LIMIT_PROMPT = """\ -The resume compiles, but it exceeds the maximum allowed page count. -Compress the resume while preserving high-impact accomplishments and role relevance. - -{llm_reference} - -**Target Maximum Pages:** {max_pages} -**Current Page Count:** {actual_pages} -**Compression Attempt:** {attempt_number} - -Compression priorities (in this order): -1) Keep recent, high-impact, role-relevant bullets. -2) Remove low-impact or redundant bullets. -3) Shorten verbose wording while preserving meaning. -4) Trim older or less relevant details before recent ones. - -Return the complete updated Typst content (starting with = Full Name), and keep it at or below the target pages. - -**EXISTING RESUME CONTENT:** -{previous_content} -""" - - -# ─── Helpers ───────────────────────────────────────────────────────────────── - - -def _strip_typst_fences(text: str) -> str: - """Remove wrapping ```typst ... ``` fences that LLMs sometimes add.""" - stripped = text.strip() - m = re.match(r"^(`{3,})(?:typst|typ)?\s*\n", stripped) - if m: - fence = m.group(1) - if stripped.endswith(fence): - stripped = stripped[m.end() :] - stripped = stripped[: -len(fence)].rstrip() - return stripped - - -def _compile_typst(source: str) -> bytes: - """Compile Typst source to PDF bytes. Raises on failure.""" - return typst.compile(source.encode("utf-8")) - - -def _count_pdf_pages(pdf_bytes: bytes) -> int: - """Count the number of pages in compiled PDF bytes.""" - with io.BytesIO(pdf_bytes) as pdf_stream: - reader = pypdf.PdfReader(pdf_stream) - return len(reader.pages) - - -def _validate_max_pages(max_pages: int) -> int: - """Validate and normalize max_pages input.""" - if MIN_RESUME_PAGES <= max_pages <= MAX_RESUME_PAGES: - return max_pages - msg = ( - f"max_pages must be between {MIN_RESUME_PAGES} and " - f"{MAX_RESUME_PAGES}. Received: {max_pages}" - ) - raise ValueError(msg) - - -# ─── Tool Factory ─────────────────────────────────────────────────────────── - - -def create_generate_resume_tool( - search_space_id: int, - thread_id: int | None = None, -): - """ - Factory function to create the generate_resume tool. - - Generates a Typst-based resume, validates it via compilation, - and stores the source in the Report table with content_type='typst'. - The LLM generates only the content body; the template header is - prepended by the backend. - """ - - @tool - async def generate_resume( - user_info: str, - user_instructions: str | None = None, - parent_report_id: int | None = None, - max_pages: int = 1, - ) -> dict[str, Any]: - """ - Generate a professional resume as a Typst document. - - Use this tool when the user asks to create, build, generate, write, - or draft a resume or CV. Also use it when the user wants to modify, - update, or revise an existing resume generated in this conversation. - - Trigger phrases include: - - "build me a resume", "create my resume", "generate a CV" - - "update my resume", "change my title", "add my new job" - - "make my resume more concise", "reformat my resume" - - Do NOT use this tool for: - - General questions about resumes or career advice - - Reviewing or critiquing a resume without changes - - Cover letters (use generate_report instead) - - VERSIONING — parent_report_id: - - Set parent_report_id when the user wants to MODIFY an existing - resume that was already generated in this conversation. - - Leave as None for new resumes. - - Args: - user_info: The user's resume content — work experience, - education, skills, contact info, etc. Can be structured - or unstructured text. - user_instructions: Optional style or content preferences - (e.g. "emphasize leadership", "keep it to one page", - "use a modern style"). For revisions, describe what to change. - parent_report_id: ID of a previous resume to revise (creates - new version in the same version group). - max_pages: Maximum number of pages for the generated resume. - Defaults to 1. Allowed range: 1-5. - - Returns: - Dict with status, report_id, title, and content_type. - """ - report_group_id: int | None = None - parent_content: str | None = None - - template = _get_template() - llm_reference = _build_llm_reference(template) - - async def _save_failed_report(error_msg: str) -> int | None: - try: - async with shielded_async_session() as session: - failed = Report( - title="Resume", - content=None, - content_type="typst", - report_metadata={ - "status": "failed", - "error_message": error_msg, - }, - report_style="resume", - search_space_id=search_space_id, - thread_id=thread_id, - report_group_id=report_group_id, - ) - session.add(failed) - await session.commit() - await session.refresh(failed) - if not failed.report_group_id: - failed.report_group_id = failed.id - await session.commit() - logger.info( - f"[generate_resume] Saved failed report {failed.id}: {error_msg}" - ) - return failed.id - except Exception: - logger.exception( - "[generate_resume] Could not persist failed report row" - ) - return None - - try: - try: - validated_max_pages = _validate_max_pages(max_pages) - except ValueError as e: - error_msg = str(e) - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - # ── Phase 1: READ ───────────────────────────────────────────── - async with shielded_async_session() as read_session: - if parent_report_id: - parent_report = await read_session.get(Report, parent_report_id) - if parent_report: - report_group_id = parent_report.report_group_id - parent_content = parent_report.content - logger.info( - f"[generate_resume] Revising from parent {parent_report_id} " - f"(group {report_group_id})" - ) - - llm = await get_document_summary_llm(read_session, search_space_id) - - if not llm: - error_msg = ( - "No LLM configured. Please configure a language model in Settings." - ) - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - # ── Phase 2: LLM GENERATION ─────────────────────────────────── - - user_instructions_section = "" - if user_instructions: - user_instructions_section = ( - f"**Additional Instructions:** {user_instructions}" - ) - - if parent_content: - dispatch_custom_event( - "report_progress", - {"phase": "writing", "message": "Updating your resume"}, - ) - parent_body = _strip_header(parent_content) - prompt = _REVISION_PROMPT.format( - llm_reference=llm_reference, - max_pages=validated_max_pages, - user_instructions=user_instructions - or "Improve and refine the resume.", - previous_content=parent_body, - ) - else: - dispatch_custom_event( - "report_progress", - {"phase": "writing", "message": "Building your resume"}, - ) - prompt = _RESUME_PROMPT.format( - llm_reference=llm_reference, - user_info=user_info, - max_pages=validated_max_pages, - user_instructions_section=user_instructions_section, - ) - - response = await llm.ainvoke([HumanMessage(content=prompt)]) - body = response.content - - if not body or not isinstance(body, str): - error_msg = "LLM returned empty or invalid content" - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - body = _strip_typst_fences(body) - body = _strip_imports(body) - - # ── Phase 3: ASSEMBLE + COMPILE ─────────────────────────────── - dispatch_custom_event( - "report_progress", - {"phase": "compiling", "message": "Compiling resume..."}, - ) - - name = _extract_name(body) or "Resume" - typst_source = "" - actual_pages = 0 - compression_attempts = 0 - target_page_met = False - - for compression_round in range(MAX_COMPRESSION_ATTEMPTS + 1): - header = _build_header(template, name) - typst_source = header + body - compile_error: str | None = None - pdf_bytes: bytes | None = None - - for compile_attempt in range(2): - try: - pdf_bytes = _compile_typst(typst_source) - compile_error = None - break - except Exception as e: - compile_error = str(e) - logger.warning( - "[generate_resume] Compile attempt %s failed: %s", - compile_attempt + 1, - compile_error, - ) - - if compile_attempt == 0: - dispatch_custom_event( - "report_progress", - { - "phase": "fixing", - "message": "Fixing compilation issue...", - }, - ) - fix_prompt = _FIX_COMPILE_PROMPT.format( - llm_reference=llm_reference, - error=compile_error, - full_source=typst_source, - ) - fix_response = await llm.ainvoke( - [HumanMessage(content=fix_prompt)] - ) - if fix_response.content and isinstance( - fix_response.content, str - ): - body = _strip_typst_fences(fix_response.content) - body = _strip_imports(body) - name = _extract_name(body) or name - header = _build_header(template, name) - typst_source = header + body - - if compile_error or not pdf_bytes: - error_msg = ( - "Typst compilation failed after 2 attempts: " - f"{compile_error or 'Unknown compile error'}" - ) - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - actual_pages = _count_pdf_pages(pdf_bytes) - if actual_pages <= validated_max_pages: - target_page_met = True - break - - if compression_round >= MAX_COMPRESSION_ATTEMPTS: - break - - compression_attempts += 1 - dispatch_custom_event( - "report_progress", - { - "phase": "compressing", - "message": f"Condensing resume to {validated_max_pages} page(s)...", - }, - ) - compress_prompt = _COMPRESS_TO_PAGE_LIMIT_PROMPT.format( - llm_reference=llm_reference, - max_pages=validated_max_pages, - actual_pages=actual_pages, - attempt_number=compression_attempts, - previous_content=body, - ) - compress_response = await llm.ainvoke( - [HumanMessage(content=compress_prompt)] - ) - if not compress_response.content or not isinstance( - compress_response.content, str - ): - error_msg = "LLM returned empty content while compressing resume" - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - body = _strip_typst_fences(compress_response.content) - body = _strip_imports(body) - name = _extract_name(body) or name - - if actual_pages > MAX_RESUME_PAGES: - error_msg = ( - "Resume exceeds hard page limit after compression retries. " - f"Hard limit: <= {MAX_RESUME_PAGES} page(s), actual: {actual_pages}." - ) - report_id = await _save_failed_report(error_msg) - return { - "status": "failed", - "error": error_msg, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - # ── Phase 4: SAVE ───────────────────────────────────────────── - dispatch_custom_event( - "report_progress", - {"phase": "saving", "message": "Saving your resume"}, - ) - - resume_title = f"{name} - Resume" if name != "Resume" else "Resume" - - metadata: dict[str, Any] = { - "status": "ready", - "word_count": len(typst_source.split()), - "char_count": len(typst_source), - "target_max_pages": validated_max_pages, - "actual_page_count": actual_pages, - "page_limit_enforced": True, - "compression_attempts": compression_attempts, - "target_page_met": target_page_met, - } - - async with shielded_async_session() as write_session: - report = Report( - title=resume_title, - content=typst_source, - content_type="typst", - report_metadata=metadata, - report_style="resume", - search_space_id=search_space_id, - thread_id=thread_id, - report_group_id=report_group_id, - ) - write_session.add(report) - await write_session.commit() - await write_session.refresh(report) - - if not report.report_group_id: - report.report_group_id = report.id - await write_session.commit() - - saved_id = report.id - - logger.info(f"[generate_resume] Created resume {saved_id}: {resume_title}") - - return { - "status": "ready", - "report_id": saved_id, - "title": resume_title, - "content_type": "typst", - "is_revision": bool(parent_content), - "message": ( - f"Resume generated successfully: {resume_title}" - if target_page_met - else ( - f"Resume generated, but could not fit the target of <= {validated_max_pages} " - f"page(s). Final length: {actual_pages} page(s)." - ) - ), - } - - except Exception as e: - error_message = str(e) - logger.exception(f"[generate_resume] Error: {error_message}") - report_id = await _save_failed_report(error_message) - return { - "status": "failed", - "error": error_message, - "report_id": report_id, - "title": "Resume", - "content_type": "typst", - } - - return generate_resume diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/__init__.py b/surfsense_backend/app/agents/new_chat/tools/teams/__init__.py deleted file mode 100644 index 60e2add49..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/teams/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from app.agents.new_chat.tools.teams.list_channels import ( - create_list_teams_channels_tool, -) -from app.agents.new_chat.tools.teams.read_messages import ( - create_read_teams_messages_tool, -) -from app.agents.new_chat.tools.teams.send_message import ( - create_send_teams_message_tool, -) - -__all__ = [ - "create_list_teams_channels_tool", - "create_read_teams_messages_tool", - "create_send_teams_message_tool", -] diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py b/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py deleted file mode 100644 index 4345bb476..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Shared auth helper for Teams agent tools (Microsoft Graph REST API).""" - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.db import SearchSourceConnector, SearchSourceConnectorType - -GRAPH_API = "https://graph.microsoft.com/v1.0" - - -async def get_teams_connector( - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> SearchSourceConnector | None: - result = await db_session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.TEAMS_CONNECTOR, - ) - ) - return result.scalars().first() - - -async def get_access_token( - db_session: AsyncSession, - connector: SearchSourceConnector, -) -> str: - """Get a valid Microsoft Graph access token, refreshing if expired.""" - from app.connectors.teams_connector import TeamsConnector - - tc = TeamsConnector( - session=db_session, - connector_id=connector.id, - ) - return await tc._get_valid_token() diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py b/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py deleted file mode 100644 index 0fc52b5c7..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py +++ /dev/null @@ -1,114 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_list_teams_channels_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the list_teams_channels tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured list_teams_channels tool - """ - del db_session # per-call session — see docstring - - @tool - async def list_teams_channels() -> dict[str, Any]: - """List all Microsoft Teams and their channels the user has access to. - - Returns: - Dictionary with status and a list of teams, each containing - team_id, team_name, and a list of channels (id, name). - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - try: - async with async_session_maker() as db_session: - connector = await get_teams_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - token = await get_access_token(db_session, connector) - headers = {"Authorization": f"Bearer {token}"} - - async with httpx.AsyncClient(timeout=20.0) as client: - teams_resp = await client.get( - f"{GRAPH_API}/me/joinedTeams", headers=headers - ) - - if teams_resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if teams_resp.status_code != 200: - return { - "status": "error", - "message": f"Graph API error: {teams_resp.status_code}", - } - - teams_data = teams_resp.json().get("value", []) - result_teams = [] - - async with httpx.AsyncClient(timeout=20.0) as client: - for team in teams_data: - team_id = team["id"] - ch_resp = await client.get( - f"{GRAPH_API}/teams/{team_id}/channels", - headers=headers, - ) - channels = [] - if ch_resp.status_code == 200: - channels = [ - {"id": ch["id"], "name": ch.get("displayName", "")} - for ch in ch_resp.json().get("value", []) - ] - result_teams.append( - { - "team_id": team_id, - "team_name": team.get("displayName", ""), - "channels": channels, - } - ) - - return { - "status": "success", - "teams": result_teams, - "total_teams": len(result_teams), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error listing Teams channels: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to list Teams channels."} - - return list_teams_channels diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py b/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py deleted file mode 100644 index 0ebda021e..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py +++ /dev/null @@ -1,125 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import async_session_maker - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_read_teams_messages_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the read_teams_messages tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured read_teams_messages tool - """ - del db_session # per-call session — see docstring - - @tool - async def read_teams_messages( - team_id: str, - channel_id: str, - limit: int = 25, - ) -> dict[str, Any]: - """Read recent messages from a Microsoft Teams channel. - - Args: - team_id: The team ID (from list_teams_channels). - channel_id: The channel ID (from list_teams_channels). - limit: Number of messages to fetch (default 25, max 50). - - Returns: - Dictionary with status and a list of messages including - id, sender, content, timestamp. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - limit = min(limit, 50) - - try: - async with async_session_maker() as db_session: - connector = await get_teams_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - token = await get_access_token(db_session, connector) - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.get( - f"{GRAPH_API}/teams/{team_id}/channels/{channel_id}/messages", - headers={"Authorization": f"Bearer {token}"}, - params={"$top": limit}, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if resp.status_code == 403: - return { - "status": "error", - "message": "Insufficient permissions to read this channel.", - } - if resp.status_code != 200: - return { - "status": "error", - "message": f"Graph API error: {resp.status_code}", - } - - raw_msgs = resp.json().get("value", []) - messages = [] - for m in raw_msgs: - sender = m.get("from", {}) - user_info = sender.get("user", {}) if sender else {} - body = m.get("body", {}) - messages.append( - { - "id": m.get("id"), - "sender": user_info.get("displayName", "Unknown"), - "content": body.get("content", ""), - "content_type": body.get("contentType", "text"), - "timestamp": m.get("createdDateTime", ""), - } - ) - - return { - "status": "success", - "team_id": team_id, - "channel_id": channel_id, - "messages": messages, - "total": len(messages), - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error reading Teams messages: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read Teams messages."} - - return read_teams_messages diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py b/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py deleted file mode 100644 index 6f40d27e1..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py +++ /dev/null @@ -1,136 +0,0 @@ -import logging -from typing import Any - -import httpx -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.tools.hitl import request_approval -from app.db import async_session_maker - -from ._auth import GRAPH_API, get_access_token, get_teams_connector - -logger = logging.getLogger(__name__) - - -def create_send_teams_message_tool( - db_session: AsyncSession | None = None, - search_space_id: int | None = None, - user_id: str | None = None, -): - """ - Factory function to create the send_teams_message tool. - - The tool acquires its own short-lived ``AsyncSession`` per call via - :data:`async_session_maker` so the closure is safe to share across - HTTP requests by the compiled-agent cache. Capturing a per-request - session here would surface stale/closed sessions on cache hits. - - Args: - db_session: Reserved for registry compatibility. Per-call sessions - are opened via :data:`async_session_maker` inside the tool body. - - Returns: - Configured send_teams_message tool - """ - del db_session # per-call session — see docstring - - @tool - async def send_teams_message( - team_id: str, - channel_id: str, - content: str, - ) -> dict[str, Any]: - """Send a message to a Microsoft Teams channel. - - Requires the ChannelMessage.Send OAuth scope. If the user gets a - permission error, they may need to re-authenticate with updated scopes. - - Args: - team_id: The team ID (from list_teams_channels). - channel_id: The channel ID (from list_teams_channels). - content: The message text (HTML supported). - - Returns: - Dictionary with status, message_id on success. - - IMPORTANT: - - If status is "rejected", the user explicitly declined. Do NOT retry. - """ - if search_space_id is None or user_id is None: - return {"status": "error", "message": "Teams tool not properly configured."} - - try: - async with async_session_maker() as db_session: - connector = await get_teams_connector( - db_session, search_space_id, user_id - ) - if not connector: - return {"status": "error", "message": "No Teams connector found."} - - result = request_approval( - action_type="teams_send_message", - tool_name="send_teams_message", - params={ - "team_id": team_id, - "channel_id": channel_id, - "content": content, - }, - context={"connector_id": connector.id}, - ) - - if result.rejected: - return { - "status": "rejected", - "message": "User declined. Message was not sent.", - } - - final_content = result.params.get("content", content) - final_team = result.params.get("team_id", team_id) - final_channel = result.params.get("channel_id", channel_id) - - token = await get_access_token(db_session, connector) - - async with httpx.AsyncClient(timeout=20.0) as client: - resp = await client.post( - f"{GRAPH_API}/teams/{final_team}/channels/{final_channel}/messages", - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - json={"body": {"content": final_content}}, - ) - - if resp.status_code == 401: - return { - "status": "auth_error", - "message": "Teams token expired. Please re-authenticate.", - "connector_type": "teams", - } - if resp.status_code == 403: - return { - "status": "insufficient_permissions", - "message": "Missing ChannelMessage.Send permission. Please re-authenticate with updated scopes.", - } - if resp.status_code not in (200, 201): - return { - "status": "error", - "message": f"Graph API error: {resp.status_code} — {resp.text[:200]}", - } - - msg_data = resp.json() - return { - "status": "success", - "message_id": msg_data.get("id"), - "message": "Message sent to Teams channel.", - } - - except Exception as e: - from langgraph.errors import GraphInterrupt - - if isinstance(e, GraphInterrupt): - raise - logger.error("Error sending Teams message: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to send Teams message."} - - return send_teams_message diff --git a/surfsense_backend/app/agents/new_chat/tools/tool_response.py b/surfsense_backend/app/agents/new_chat/tools/tool_response.py deleted file mode 100644 index 8644ada5c..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/tool_response.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Standardised response dict factories for LangChain agent tools.""" - -from __future__ import annotations - -from typing import Any - - -class ToolResponse: - @staticmethod - def success(message: str, **data: Any) -> dict[str, Any]: - return {"status": "success", "message": message, **data} - - @staticmethod - def error(error: str, **data: Any) -> dict[str, Any]: - return {"status": "error", "error": error, **data} - - @staticmethod - def auth_error(service: str, **data: Any) -> dict[str, Any]: - return { - "status": "auth_error", - "error": ( - f"{service} authentication has expired or been revoked. " - "Please re-connect the integration in Settings → Connectors." - ), - **data, - } - - @staticmethod - def rejected(message: str = "Action was declined by the user.") -> dict[str, Any]: - return {"status": "rejected", "message": message} - - @staticmethod - def not_found(resource: str, identifier: str, **data: Any) -> dict[str, Any]: - return { - "status": "not_found", - "error": f"{resource} '{identifier}' was not found.", - **data, - } diff --git a/surfsense_backend/app/agents/new_chat/tools/video_presentation.py b/surfsense_backend/app/agents/new_chat/tools/video_presentation.py deleted file mode 100644 index 34f5183ca..000000000 --- a/surfsense_backend/app/agents/new_chat/tools/video_presentation.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -Video presentation generation tool for the SurfSense agent. - -This module provides a factory function for creating the generate_video_presentation -tool that submits a Celery task for background video presentation generation. The -tool then polls the row until it reaches a terminal status (READY/FAILED) and -returns that status. The wait is bounded by the chat's HTTP / process lifetime; -see app.agents.shared.deliverable_wait for details. -""" - -import logging -from typing import Any - -from langchain_core.tools import tool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.shared.deliverable_wait import wait_for_deliverable -from app.db import VideoPresentation, VideoPresentationStatus, shielded_async_session - -logger = logging.getLogger(__name__) - - -def create_generate_video_presentation_tool( - search_space_id: int, - db_session: AsyncSession, - thread_id: int | None = None, -): - """ - Factory function to create the generate_video_presentation tool with injected dependencies. - - Pre-creates video presentation record with pending status so the ID is available - immediately for frontend polling. The row is written via a fresh, tool-local - session so parallel tool calls (e.g. video + podcast in the same agent step) - don't share an ``AsyncSession`` (which is not concurrency-safe). - """ - del db_session # writes use a fresh tool-local session, see below - - @tool - async def generate_video_presentation( - source_content: str, - video_title: str = "SurfSense Presentation", - user_prompt: str | None = None, - ) -> dict[str, Any]: - """Generate a video presentation from the provided content. - - Use this tool when the user asks to create a video, presentation, slides, or slide deck. - - Args: - source_content: The text content to turn into a presentation. - video_title: Title for the presentation (default: "SurfSense Presentation") - user_prompt: Optional style/tone instructions. - """ - try: - # See podcast.py for the rationale: parallel tool calls share the - # streaming session, and AsyncSession is not concurrency-safe — - # interleaved flushes produce "Session.add() during flush" and - # poison the transaction for every concurrent tool. - async with shielded_async_session() as session: - video_pres = VideoPresentation( - title=video_title, - status=VideoPresentationStatus.PENDING, - search_space_id=search_space_id, - thread_id=thread_id, - ) - session.add(video_pres) - await session.commit() - await session.refresh(video_pres) - video_pres_id = video_pres.id - - from app.tasks.celery_tasks.video_presentation_tasks import ( - generate_video_presentation_task, - ) - - task = generate_video_presentation_task.delay( - video_presentation_id=video_pres_id, - source_content=source_content, - search_space_id=search_space_id, - user_prompt=user_prompt, - ) - - logger.info( - "[generate_video_presentation] Created video presentation %s, task: %s", - video_pres_id, - task.id, - ) - - # Wait until the Celery worker flips the row to a terminal - # state. No internal budget — see deliverable_wait module. - terminal_status, _columns, elapsed = await wait_for_deliverable( - model=VideoPresentation, - row_id=video_pres_id, - columns=[VideoPresentation.status], - terminal_statuses={ - VideoPresentationStatus.READY, - VideoPresentationStatus.FAILED, - }, - ) - - if terminal_status == VideoPresentationStatus.READY: - logger.info( - "[generate_video_presentation] %s READY in %.2fs", - video_pres_id, - elapsed, - ) - return { - "status": VideoPresentationStatus.READY.value, - "video_presentation_id": video_pres_id, - "title": video_title, - "message": "Video presentation generated and saved.", - } - - # Only other terminal state is FAILED. - logger.warning( - "[generate_video_presentation] %s FAILED in %.2fs", - video_pres_id, - elapsed, - ) - return { - "status": VideoPresentationStatus.FAILED.value, - "video_presentation_id": video_pres_id, - "title": video_title, - "error": ( - "Background worker reported FAILED status for this " - "video presentation." - ), - } - - except Exception as e: - error_message = str(e) - logger.exception("[generate_video_presentation] Error: %s", error_message) - return { - "status": VideoPresentationStatus.FAILED.value, - "error": error_message, - "title": video_title, - "video_presentation_id": None, - } - - return generate_video_presentation diff --git a/surfsense_backend/app/agents/podcaster/nodes.py b/surfsense_backend/app/agents/podcaster/nodes.py index 517d900a3..d1f140a44 100644 --- a/surfsense_backend/app/agents/podcaster/nodes.py +++ b/surfsense_backend/app/agents/podcaster/nodes.py @@ -24,38 +24,27 @@ from .utils import get_voice_for_provider async def create_podcast_transcript( state: State, config: RunnableConfig ) -> dict[str, Any]: - """Each node does work.""" - - # Get configuration from runnable config + """Generate the podcast transcript from the source content.""" configuration = Configuration.from_runnable_config(config) search_space_id = configuration.search_space_id user_prompt = configuration.user_prompt - # Get search space's document summary LLM llm = await get_agent_llm(state.db_session, search_space_id) if not llm: - error_message = ( - f"No document summary LLM configured for search space {search_space_id}" - ) + error_message = f"No agent LLM configured for search space {search_space_id}" print(error_message) raise RuntimeError(error_message) - # Get the prompt prompt = get_podcast_generation_prompt(user_prompt) - - # Create the messages messages = [ SystemMessage(content=prompt), HumanMessage( content=f"{state.source_content}" ), ] - - # Generate the podcast transcript llm_response = await llm.ainvoke(messages) - # Reasoning models (e.g. Kimi K2.5) may return content as a list of - # blocks including 'reasoning' entries. Normalise to a plain string. + # Reasoning models may return content as blocks; normalise to a string. content = strip_markdown_fences(extract_text_content(llm_response.content)) try: @@ -89,17 +78,13 @@ async def create_merged_podcast_audio( state: State, config: RunnableConfig ) -> dict[str, Any]: """Generate audio for each transcript and merge them into a single podcast file.""" - - # configuration = Configuration.from_runnable_config(config) - starting_transcript = PodcastTranscriptEntry( speaker_id=1, dialog="Welcome to Surfsense Podcast." ) transcript = state.podcast_transcript - # Merge the starting transcript with the podcast transcript - # Check if transcript is a PodcastTranscripts object or already a list + # transcript may be a PodcastTranscripts object or already a list. if hasattr(transcript, "podcast_transcripts"): transcript_entries = transcript.podcast_transcripts else: @@ -107,20 +92,16 @@ async def create_merged_podcast_audio( merged_transcript = [starting_transcript, *transcript_entries] - # Create a temporary directory for audio files temp_dir = Path("temp_audio") temp_dir.mkdir(exist_ok=True) - # Generate a unique session ID for this podcast session_id = str(uuid.uuid4()) output_path = f"podcasts/{session_id}_podcast.mp3" os.makedirs("podcasts", exist_ok=True) - # Generate audio for each transcript segment audio_files = [] async def generate_speech_for_segment(segment, index): - # Handle both dictionary and PodcastTranscriptEntry objects if hasattr(segment, "speaker_id"): speaker_id = segment.speaker_id dialog = segment.dialog @@ -128,20 +109,15 @@ async def create_merged_podcast_audio( speaker_id = segment.get("speaker_id", 0) dialog = segment.get("dialog", "") - # Select voice based on speaker_id voice = get_voice_for_provider(app_config.TTS_SERVICE, speaker_id) - # Generate a unique filename for this segment if app_config.TTS_SERVICE == "local/kokoro": - # Kokoro generates WAV files filename = f"{temp_dir}/{session_id}_{index}.wav" else: - # Other services generate MP3 files filename = f"{temp_dir}/{session_id}_{index}.mp3" try: if app_config.TTS_SERVICE == "local/kokoro": - # Use Kokoro TTS service kokoro_service = await get_kokoro_tts_service( lang_code="a" ) # American English @@ -170,7 +146,6 @@ async def create_merged_podcast_audio( timeout=600, ) - # Save the audio to a file - use proper streaming method with open(filename, "wb") as f: f.write(response.content) @@ -179,23 +154,17 @@ async def create_merged_podcast_audio( print(f"Error generating speech for segment {index}: {e!s}") raise - # Generate all audio files concurrently tasks = [ generate_speech_for_segment(segment, i) for i, segment in enumerate(merged_transcript) ] audio_files = await asyncio.gather(*tasks) - # Merge audio files using ffmpeg try: - # Create FFmpeg instance with the first input ffmpeg = FFmpeg().option("y") - - # Add each audio file as input for audio_file in audio_files: ffmpeg = ffmpeg.input(audio_file) - # Configure the concatenation and output filter_complex = [] for i in range(len(audio_files)): filter_complex.append(f"[{i}:0]") @@ -205,8 +174,6 @@ async def create_merged_podcast_audio( ) ffmpeg = ffmpeg.option("filter_complex", filter_complex_str) ffmpeg = ffmpeg.output(output_path, map="[outa]") - - # Execute FFmpeg await ffmpeg.execute() print(f"Successfully created podcast audio: {output_path}") @@ -215,7 +182,6 @@ async def create_merged_podcast_audio( print(f"Error merging audio files: {e!s}") raise finally: - # Clean up temporary files for audio_file in audio_files: try: os.remove(audio_file) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index 9bd637ba6..d3f5dce2a 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -23,7 +23,7 @@ from starlette.requests import Request as StarletteRequest from starlette.responses import Response as StarletteResponse from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware -from app.agents.new_chat.checkpointer import ( +from app.agents.chat.runtime.checkpointer import ( close_checkpointer, setup_checkpointer_tables, ) @@ -37,6 +37,18 @@ from app.config import ( ) from app.db import User, create_db_and_tables, get_async_session from app.exceptions import GENERIC_5XX_MESSAGE, ISSUES_URL, SurfSenseError +from app.gateway.byo_long_poll import ( + start_byo_long_poll_supervisors, + stop_byo_long_poll_supervisors, +) +from app.gateway.discord.intake import ( + start_discord_gateway_supervisor, + stop_discord_gateway_supervisor, +) +from app.gateway.inbox_worker import ( + start_gateway_inbox_worker, + stop_gateway_inbox_worker, +) from app.observability import metrics as ot_metrics from app.observability.bootstrap import init_otel, shutdown_otel from app.rate_limiter import get_real_client_ip, limiter @@ -475,7 +487,7 @@ async def _warm_agent_jit_caches() -> None: ) from langchain_core.tools import tool - from app.agents.new_chat.context import SurfSenseContextSchema + from app.agents.chat.shared.context import SurfSenseContextSchema # Minimal LLM stub. ``FakeListChatModel`` satisfies # ``BaseChatModel`` without any network or auth — perfect for @@ -559,6 +571,41 @@ async def _warm_agent_jit_caches() -> None: ) +async def _warm_embedding_model() -> None: + """Pre-load/JIT the embedding model so the first KB search is fast. + + With lazy KB retrieval (OpenCode-style), the main agent no longer embeds + on every turn — it calls the on-demand ``search_knowledge_base`` tool only + when it needs KB content, and that tool's first ``embed_texts`` call in a + fresh process pays the model's one-time load/JIT (local sentence-transformer + warm or API client init). Doing one throwaway embed at startup moves that + cost off the first real search. + + Safety: behind the embedding global lock (run in a worker thread), bounded + by the caller's ``asyncio.wait_for``, and non-fatal — on any failure we log + and swallow so the worst case is the first real search pays the cold cost. + """ + import time as _time + + logger = logging.getLogger(__name__) + t0 = _time.perf_counter() + try: + from app.utils.document_converters import embed_texts + + await asyncio.to_thread(embed_texts, ["warmup"]) + logger.info( + "[startup] Embedding model warmup completed in %.3fs", + _time.perf_counter() - t0, + ) + except Exception: + logger.warning( + "[startup] Embedding model warmup failed in %.3fs (non-fatal — first " + "KB search will pay the cold embed cost)", + _time.perf_counter() - t0, + exc_info=True, + ) + + @asynccontextmanager async def lifespan(app: FastAPI): # Tune GC: lower gen-2 threshold so long-lived garbage is collected @@ -589,14 +636,31 @@ async def lifespan(app: FastAPI): "first real request will pay the full compile cost." ) + # Phase 2 — embedding warmup so the first lazy ``search_knowledge_base`` + # call doesn't pay the cold embed-model load. Bounded + non-fatal. + try: + await asyncio.wait_for(asyncio.shield(_warm_embedding_model()), timeout=20) + except (TimeoutError, Exception): # pragma: no cover - defensive + logging.getLogger(__name__).warning( + "[startup] Embedding warmup hit timeout/error — skipping; " + "first KB search will pay the cold embed cost." + ) + register_session_hooks() log_system_snapshot("startup_complete") + await start_gateway_inbox_worker() + await start_byo_long_poll_supervisors() + await start_discord_gateway_supervisor() - yield - - _stop_openrouter_background_refresh() - await close_checkpointer() - shutdown_otel() + try: + yield + finally: + await stop_discord_gateway_supervisor() + await stop_byo_long_poll_supervisors() + await stop_gateway_inbox_worker() + _stop_openrouter_background_refresh() + await close_checkpointer() + shutdown_otel() def registration_allowed(): diff --git a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py index e3736cc95..4ef8c52bf 100644 --- a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py +++ b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py @@ -85,23 +85,16 @@ async def build_dependencies( connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( session, search_space_id=search_space_id ) - # Quick fix: use an in-memory checkpointer for automation runs. + # Per-task InMemorySaver: the shared Postgres checkpointer's connection + # pool binds connections to the loop that opened them, but Celery uses a + # fresh loop per task, so the next task hangs 30s on a dead-loop connection + # (`PoolTimeout`). InMemorySaver has no pool and dies with the task — fine + # while runs are one-shot (the checkpoint only spans one graph execution). # - # The shared Postgres checkpointer caches DB connections in a - # module-level pool. Each cached connection is bound to the asyncio - # loop that opened it. Celery throws away the loop after every task, - # so the pool ends up full of connections pointing to a dead loop, - # and the next Celery task (running on a fresh loop) can't use any - # of them — it hangs 30s and fails with - # `PoolTimeout: couldn't get a connection after 30.00 sec`. - # - # InMemorySaver has no cached connections, no loop binding — each - # Celery task creates one and drops it on exit. - # - # TODO(checkpointer): proper fix is to dispose the checkpointer - # pool around each Celery task in `run_async_celery_task`, the same - # way `_dispose_shared_db_engine` already does for the SQLAlchemy - # pool. Then this site can switch back to the shared checkpointer. + # TODO(checkpointer): when runs need durability (crash-resume or HITL + # interrupt/resume across tasks), dispose the checkpointer pool around each + # Celery task in `run_async_celery_task` — as `_dispose_shared_db_engine` + # already does for the SQLAlchemy pool — then use the shared checkpointer. checkpointer = InMemorySaver() return AgentDependencies( llm=llm, diff --git a/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py b/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py index 99e295f30..aa96e4f6e 100644 --- a/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py +++ b/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py @@ -10,9 +10,12 @@ from langchain_core.messages import HumanMessage from langgraph.types import Command from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.mention_resolver import resolve_mentions, substitute_in_text +from app.agents.chat.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.agents.chat.runtime.mention_resolver import ( + resolve_mentions, + substitute_in_text, +) +from app.agents.chat.shared.context import SurfSenseContextSchema from app.db import ChatVisibility, async_session_maker from app.schemas.new_chat import MentionedDocumentInfo diff --git a/surfsense_backend/app/automations/services/model_policy.py b/surfsense_backend/app/automations/services/model_policy.py index 88e9d5f28..7e3e46b61 100644 --- a/surfsense_backend/app/automations/services/model_policy.py +++ b/surfsense_backend/app/automations/services/model_policy.py @@ -39,7 +39,9 @@ def _is_premium_global(kind: ModelKind, config_id: int) -> bool: cfg: dict | None = None if kind == "llm": - from app.agents.new_chat.llm_config import load_global_llm_config_by_id + from app.agents.chat.runtime.llm_config import ( + load_global_llm_config_by_id, + ) cfg = load_global_llm_config_by_id(config_id) elif kind == "image": diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 99e34e8ca..0e852b801 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -1,7 +1,6 @@ """Celery application configuration and setup.""" import contextlib -import os import time from celery import Celery @@ -19,6 +18,8 @@ try: except ImportError: # pragma: no cover - optional OTel dependency trace = None # type: ignore[assignment] +from app.config import config + # Load environment variables load_dotenv() @@ -103,7 +104,7 @@ def init_worker(**kwargs): """Initialize the LLM Router and Image Gen Router when a Celery worker process starts. This ensures the Auto mode (LiteLLM Router) is available for background tasks - like document summarization and image generation. + like agent workflows and image generation. """ from app.observability.bootstrap import init_otel @@ -124,16 +125,16 @@ def init_worker(**kwargs): initialize_vision_llm_router() -# Get Celery configuration from environment -CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") -CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") -CELERY_TASK_DEFAULT_QUEUE = os.getenv("CELERY_TASK_DEFAULT_QUEUE", "surfsense") +# Celery configuration, sourced from the central Config singleton +CELERY_BROKER_URL = config.CELERY_BROKER_URL +CELERY_RESULT_BACKEND = config.CELERY_RESULT_BACKEND +CELERY_TASK_DEFAULT_QUEUE = config.CELERY_TASK_DEFAULT_QUEUE -# Get schedule checker interval from environment +# Schedule checker interval # Format: "" where unit is 'm' (minutes) or 'h' (hours) # Examples: "1m" (every minute), "5m" (every 5 minutes), "1h" (every hour) -SCHEDULE_CHECKER_INTERVAL = os.getenv("SCHEDULE_CHECKER_INTERVAL", "2m") -STRIPE_RECONCILIATION_INTERVAL = os.getenv("STRIPE_RECONCILIATION_INTERVAL", "10m") +SCHEDULE_CHECKER_INTERVAL = config.SCHEDULE_CHECKER_INTERVAL +STRIPE_RECONCILIATION_INTERVAL = config.STRIPE_RECONCILIATION_INTERVAL def parse_schedule_interval(interval: str) -> dict: @@ -188,6 +189,7 @@ celery_app = Celery( "app.tasks.celery_tasks.document_reindex_tasks", "app.tasks.celery_tasks.stale_notification_cleanup_task", "app.tasks.celery_tasks.stripe_reconciliation_task", + "app.tasks.celery_tasks.gateway_tasks", "app.automations.tasks.execute_run", "app.automations.triggers.builtin.schedule.selector", "app.automations.triggers.builtin.event.selector", @@ -245,6 +247,9 @@ celery_app.conf.update( "index_obsidian_attachment": {"queue": CONNECTORS_QUEUE}, # Everything else (document processing, podcasts, reindexing, # schedule checker, cleanup) stays on the default fast queue. + "gateway.reconcile_inbox": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, + "gateway.health_check": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, + "gateway.retention_sweep": {"queue": f"{CELERY_TASK_DEFAULT_QUEUE}.gateway"}, }, ) @@ -291,6 +296,21 @@ celery_app.conf.beat_schedule = { "expires": 60, }, }, + "gateway-reconcile-inbox": { + "task": "gateway.reconcile_inbox", + "schedule": crontab(minute="*"), + "options": {"expires": 60}, + }, + "gateway-health-check": { + "task": "gateway.health_check", + "schedule": crontab(minute="*/5"), + "options": {"expires": 120}, + }, + "gateway-retention-sweep": { + "task": "gateway.retention_sweep", + "schedule": crontab(hour="3", minute="17"), + "options": {"expires": 600}, + }, # Fire due automation schedule triggers (Beat entry owned by the schedule # trigger; see app.automations.triggers.builtin.schedule.source). **SCHEDULE_BEAT_SCHEDULE, diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 5643c048b..75af17d11 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -1,5 +1,7 @@ +import copy import os import shutil +from functools import lru_cache from pathlib import Path import yaml @@ -17,6 +19,37 @@ os.environ.setdefault("OR_APP_NAME", "SurfSense") os.environ.setdefault("OR_SITE_URL", "https://surfsense.com") +@lru_cache(maxsize=8) +def _read_global_config_yaml(path_str: str) -> dict: + """Read and parse ``global_llm_config.yaml`` once per resolved path. + + Cached so the seven ``load_*`` helpers (and their re-invocations during + startup) don't re-open and re-parse the same file repeatedly. Keyed on the + resolved path string so tests that monkeypatch ``BASE_DIR`` to a unique + ``tmp_path`` still get a fresh parse. Callers MUST treat the returned dict + as read-only and deep-copy any section they intend to mutate. + """ + f = Path(path_str) + if not f.exists(): + return {} + try: + with open(f, encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + except Exception as e: + print(f"Warning: Failed to read global_llm_config.yaml: {e}") + return {} + + +def _global_config_data() -> dict: + """Return the parsed global config YAML for the current ``BASE_DIR``. + + ``BASE_DIR`` is read at call time (not bound at import) so a + ``monkeypatch.setattr(config, "BASE_DIR", tmp_path)`` is honored. + """ + path = BASE_DIR / "app" / "config" / "global_llm_config.yaml" + return _read_global_config_yaml(str(path)) + + def is_ffmpeg_installed(): """ Check if ffmpeg is installed on the current system. @@ -35,17 +68,15 @@ def load_global_llm_configs(): Returns: list: List of global LLM config dictionaries, or empty list if file doesn't exist """ - # Try main config file first - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: # No global configs available return [] try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - configs = data.get("global_llm_configs", []) + # Deep-copy so the in-place mutations below (setdefault, scoring + # stamps) never leak into the cached YAML structure. + configs = copy.deepcopy(data.get("global_llm_configs", [])) # Lazy import keeps the `app.config` -> `app.services` edge one-way # and matches the `provider_api_base` pattern used elsewhere. @@ -145,18 +176,14 @@ def load_router_settings(): "cooldown_time": 60, } - # Try main config file first - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return default_settings try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - settings = data.get("router_settings", {}) - # Merge with defaults - return {**default_settings, **settings} + settings = data.get("router_settings", {}) + # Merge with defaults + return {**default_settings, **settings} except Exception as e: print(f"Warning: Failed to load router settings: {e}") return default_settings @@ -169,38 +196,32 @@ def load_global_image_gen_configs(): Returns: list: List of global image generation config dictionaries, or empty list """ - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return [] try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - configs = data.get("global_image_generation_configs", []) or [] - for cfg in configs: - if isinstance(cfg, dict): - cfg.setdefault("billing_tier", "free") - return configs + configs = copy.deepcopy(data.get("global_image_generation_configs", []) or []) + for cfg in configs: + if isinstance(cfg, dict): + cfg.setdefault("billing_tier", "free") + return configs except Exception as e: print(f"Warning: Failed to load global image generation configs: {e}") return [] def load_global_vision_llm_configs(): - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return [] try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - configs = data.get("global_vision_llm_configs", []) or [] - for cfg in configs: - if isinstance(cfg, dict): - cfg.setdefault("billing_tier", "free") - return configs + configs = copy.deepcopy(data.get("global_vision_llm_configs", []) or []) + for cfg in configs: + if isinstance(cfg, dict): + cfg.setdefault("billing_tier", "free") + return configs except Exception as e: print(f"Warning: Failed to load global vision LLM configs: {e}") return [] @@ -214,16 +235,13 @@ def load_vision_llm_router_settings(): "cooldown_time": 60, } - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return default_settings try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - settings = data.get("vision_llm_router_settings", {}) - return {**default_settings, **settings} + settings = data.get("vision_llm_router_settings", {}) + return {**default_settings, **settings} except Exception as e: print(f"Warning: Failed to load vision LLM router settings: {e}") return default_settings @@ -243,16 +261,13 @@ def load_image_gen_router_settings(): "cooldown_time": 60, } - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return default_settings try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - settings = data.get("image_generation_router_settings", {}) - return {**default_settings, **settings} + settings = data.get("image_generation_router_settings", {}) + return {**default_settings, **settings} except Exception as e: print(f"Warning: Failed to load image generation router settings: {e}") return default_settings @@ -268,49 +283,44 @@ def load_openrouter_integration_settings() -> dict | None: Returns: dict with settings if present and enabled, None otherwise """ - global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" - - if not global_config_file.exists(): + data = _global_config_data() + if not data: return None try: - with open(global_config_file, encoding="utf-8") as f: - data = yaml.safe_load(f) - settings = data.get("openrouter_integration") - if not settings or not settings.get("enabled"): - return None + # Deep-copy so the setdefault back-compat seeding below never mutates + # the cached YAML structure. + settings = copy.deepcopy(data.get("openrouter_integration")) + if not settings or not settings.get("enabled"): + return None - if "billing_tier" in settings: - print( - "Warning: openrouter_integration.billing_tier is deprecated; " - "tier is now derived per model from OpenRouter data " - "(':free' suffix or zero pricing). Remove this key." - ) + if "billing_tier" in settings: + print( + "Warning: openrouter_integration.billing_tier is deprecated; " + "tier is now derived per model from OpenRouter data " + "(':free' suffix or zero pricing). Remove this key." + ) - if "anonymous_enabled" in settings: - print( - "Warning: openrouter_integration.anonymous_enabled is " - "deprecated; use anonymous_enabled_paid and/or " - "anonymous_enabled_free instead. Both new flags have been " - "seeded from the legacy value for back-compat." - ) - settings.setdefault( - "anonymous_enabled_paid", settings["anonymous_enabled"] - ) - settings.setdefault( - "anonymous_enabled_free", settings["anonymous_enabled"] - ) + if "anonymous_enabled" in settings: + print( + "Warning: openrouter_integration.anonymous_enabled is " + "deprecated; use anonymous_enabled_paid and/or " + "anonymous_enabled_free instead. Both new flags have been " + "seeded from the legacy value for back-compat." + ) + settings.setdefault("anonymous_enabled_paid", settings["anonymous_enabled"]) + settings.setdefault("anonymous_enabled_free", settings["anonymous_enabled"]) - # Image generation + vision LLM emission are opt-in (issue L). - # OpenRouter's catalogue contains hundreds of image / vision - # capable models; auto-injecting all of them into every - # deployment would explode the model selector and surprise - # operators upgrading from prior versions. Default to False so - # admins must explicitly turn them on. - settings.setdefault("image_generation_enabled", False) - settings.setdefault("vision_enabled", False) + # Image generation + vision LLM emission are opt-in (issue L). + # OpenRouter's catalogue contains hundreds of image / vision + # capable models; auto-injecting all of them into every + # deployment would explode the model selector and surprise + # operators upgrading from prior versions. Default to False so + # admins must explicitly turn them on. + settings.setdefault("image_generation_enabled", False) + settings.setdefault("vision_enabled", False) - return settings + return settings except Exception as e: print(f"Warning: Failed to load OpenRouter integration settings: {e}") return None @@ -415,7 +425,9 @@ def initialize_llm_router(): static YAML configs and dynamic OpenRouter models. """ all_configs = config.GLOBAL_LLM_CONFIGS - router_settings = load_router_settings() + # Reuse the router settings already parsed at Config construction instead + # of re-reading the YAML here. + router_settings = config.ROUTER_SETTINGS if not all_configs: print("Info: No global LLM configs found, Auto mode will not be available") @@ -439,7 +451,10 @@ def initialize_image_gen_router(): This should be called during application startup. """ image_gen_configs = load_global_image_gen_configs() - router_settings = load_image_gen_router_settings() + # Reuse the router settings already parsed at Config construction. The + # *configs* list is intentionally re-read from YAML (it must exclude the + # OpenRouter-injected dynamic models held in config.GLOBAL_IMAGE_GEN_CONFIGS). + router_settings = config.IMAGE_GEN_ROUTER_SETTINGS if not image_gen_configs: print( @@ -462,7 +477,10 @@ def initialize_image_gen_router(): def initialize_vision_llm_router(): vision_configs = load_global_vision_llm_configs() - router_settings = load_vision_llm_router_settings() + # Reuse the router settings already parsed at Config construction. The + # *configs* list is intentionally re-read from YAML (it must exclude the + # OpenRouter-injected dynamic models held in config.GLOBAL_VISION_LLM_CONFIGS). + router_settings = config.VISION_LLM_ROUTER_SETTINGS if not vision_configs: print( @@ -524,16 +542,51 @@ class Config: DATABASE_URL = os.getenv("DATABASE_URL") # Celery / Redis - CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0") - CELERY_RESULT_BACKEND = os.getenv( - "CELERY_RESULT_BACKEND", "redis://localhost:6379/0" - ) + # Redis (single endpoint for Celery broker, result backend, and app cache). + # Legacy CELERY_BROKER_URL / CELERY_RESULT_BACKEND / REDIS_APP_URL still + # override individually when you need to split Redis across instances. + REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") + CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", REDIS_URL) + CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", REDIS_URL) CELERY_TASK_DEFAULT_QUEUE = os.getenv("CELERY_TASK_DEFAULT_QUEUE", "surfsense") REDIS_APP_URL = os.getenv("REDIS_APP_URL", CELERY_BROKER_URL) CONNECTOR_INDEXING_LOCK_TTL_SECONDS = int( os.getenv("CONNECTOR_INDEXING_LOCK_TTL_SECONDS", str(8 * 60 * 60)) ) + # Celery beat scheduling intervals (format: "", e.g. "2m", "1h") + SCHEDULE_CHECKER_INTERVAL = os.getenv("SCHEDULE_CHECKER_INTERVAL", "2m") + STRIPE_RECONCILIATION_INTERVAL = os.getenv("STRIPE_RECONCILIATION_INTERVAL", "10m") + + # File storage (local filesystem by default; Azure Blob optional) + FILE_STORAGE_BACKEND = os.getenv("FILE_STORAGE_BACKEND", "local").strip().lower() + AZURE_STORAGE_CONNECTION_STRING = os.getenv("AZURE_STORAGE_CONNECTION_STRING") + AZURE_STORAGE_CONTAINER = os.getenv("AZURE_STORAGE_CONTAINER") + FILE_STORAGE_LOCAL_PATH = os.getenv( + "FILE_STORAGE_LOCAL_PATH", str(BASE_DIR / ".local_object_store") + ) + + # Daytona sandbox (code execution / filesystem sandbox) + DAYTONA_SANDBOX_ENABLED = ( + os.getenv("DAYTONA_SANDBOX_ENABLED", "FALSE").upper() == "TRUE" + ) + DAYTONA_API_KEY = os.getenv("DAYTONA_API_KEY", "") + DAYTONA_API_URL = os.getenv("DAYTONA_API_URL", "https://app.daytona.io/api") + DAYTONA_TARGET = os.getenv("DAYTONA_TARGET", "us") + DAYTONA_SNAPSHOT_ID = os.getenv("DAYTONA_SNAPSHOT_ID") or None + SANDBOX_FILES_DIR = os.getenv("SANDBOX_FILES_DIR", "sandbox_files") + + # Agent cache (in-process LRU+TTL cache for built agents) + AGENT_CACHE_MAXSIZE = int(os.getenv("SURFSENSE_AGENT_CACHE_MAXSIZE", "256")) + AGENT_CACHE_TTL_SECONDS = float( + os.getenv("SURFSENSE_AGENT_CACHE_TTL_SECONDS", "1800") + ) + + # Connector discovery cache TTL + CONNECTOR_DISCOVERY_TTL_SECONDS = float( + os.getenv("SURFSENSE_CONNECTOR_DISCOVERY_TTL_SECONDS", "30") + ) + # Platform web search (SearXNG) SEARXNG_DEFAULT_HOST = os.getenv("SEARXNG_DEFAULT_HOST") @@ -541,6 +594,52 @@ class Config: # Backend URL to override the http to https in the OAuth redirect URI BACKEND_URL = os.getenv("BACKEND_URL") + # Messaging gateway (Telegram v1) + # Global master switch: when FALSE, no gateway supervisors/workers start and all + # gateway HTTP routes return 404, regardless of the per-channel flags below. + GATEWAY_ENABLED = os.getenv("GATEWAY_ENABLED", "TRUE").upper() == "TRUE" + TELEGRAM_SHARED_BOT_TOKEN = os.getenv("TELEGRAM_SHARED_BOT_TOKEN") + TELEGRAM_SHARED_BOT_USERNAME = os.getenv("TELEGRAM_SHARED_BOT_USERNAME") + TELEGRAM_WEBHOOK_SECRET = os.getenv("TELEGRAM_WEBHOOK_SECRET") + GATEWAY_BASE_URL = os.getenv("GATEWAY_BASE_URL", BACKEND_URL) + GATEWAY_TELEGRAM_INTAKE_MODE = os.getenv( + "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook" + ).lower() + if GATEWAY_TELEGRAM_INTAKE_MODE not in {"webhook", "longpoll", "disabled"}: + raise ValueError( + "GATEWAY_TELEGRAM_INTAKE_MODE must be one of: webhook, longpoll, disabled" + ) + WHATSAPP_SHARED_BUSINESS_TOKEN = os.getenv("WHATSAPP_SHARED_BUSINESS_TOKEN") + WHATSAPP_SHARED_PHONE_NUMBER_ID = os.getenv("WHATSAPP_SHARED_PHONE_NUMBER_ID") + WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER = os.getenv( + "WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER" + ) + WHATSAPP_SHARED_WABA_ID = os.getenv("WHATSAPP_SHARED_WABA_ID") + WHATSAPP_GRAPH_API_VERSION = os.getenv("WHATSAPP_GRAPH_API_VERSION", "v25.0") + WHATSAPP_WEBHOOK_VERIFY_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFY_TOKEN") + WHATSAPP_WEBHOOK_APP_SECRET = os.getenv("WHATSAPP_WEBHOOK_APP_SECRET") + WHATSAPP_BRIDGE_URL = os.getenv( + "WHATSAPP_BRIDGE_URL", "http://whatsapp-bridge:9929" + ) + GATEWAY_WHATSAPP_INTAKE_MODE = os.getenv( + "GATEWAY_WHATSAPP_INTAKE_MODE", "disabled" + ).lower() + if GATEWAY_WHATSAPP_INTAKE_MODE not in {"cloud", "baileys", "disabled"}: + raise ValueError( + "GATEWAY_WHATSAPP_INTAKE_MODE must be one of: cloud, baileys, disabled" + ) + GATEWAY_SLACK_CLIENT_ID = os.getenv("SLACK_CLIENT_ID") + GATEWAY_SLACK_CLIENT_SECRET = os.getenv("SLACK_CLIENT_SECRET") + GATEWAY_SLACK_ENABLED = ( + os.getenv("GATEWAY_SLACK_ENABLED", "FALSE").upper() == "TRUE" + ) + GATEWAY_SLACK_SIGNING_SECRET = os.getenv("GATEWAY_SLACK_SIGNING_SECRET") + GATEWAY_SLACK_REDIRECT_URI = os.getenv("GATEWAY_SLACK_REDIRECT_URI") + GATEWAY_DISCORD_ENABLED = ( + os.getenv("GATEWAY_DISCORD_ENABLED", "FALSE").upper() == "TRUE" + ) + GATEWAY_DISCORD_REDIRECT_URI = os.getenv("GATEWAY_DISCORD_REDIRECT_URI") + # Stripe checkout for pay-as-you-go page packs STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET") @@ -606,9 +705,6 @@ class Config: # Anonymous / no-login mode settings NOLOGIN_MODE_ENABLED = os.getenv("NOLOGIN_MODE_ENABLED", "FALSE").upper() == "TRUE" - MULTI_AGENT_CHAT_ENABLED = ( - os.getenv("MULTI_AGENT_CHAT_ENABLED", "FALSE").upper() == "TRUE" - ) ANON_TOKEN_LIMIT = int(os.getenv("ANON_TOKEN_LIMIT", "500000")) ANON_TOKEN_WARNING_THRESHOLD = int( os.getenv("ANON_TOKEN_WARNING_THRESHOLD", "400000") @@ -820,8 +916,13 @@ class Config: AZURE_DI_ENDPOINT = os.getenv("AZURE_DI_ENDPOINT") AZURE_DI_KEY = os.getenv("AZURE_DI_KEY") + # Proxy provider selection. Maps to a ProxyProvider implementation registered + # in app/utils/proxy/registry.py. Add new vendors there and switch via this var. + PROXY_PROVIDER = os.getenv("PROXY_PROVIDER", "anonymous_proxies") + # Residential Proxy Configuration (anonymous-proxies.net) # Used for web crawling and YouTube transcript fetching to avoid IP bans. + # Consumed by the "anonymous_proxies" proxy provider. RESIDENTIAL_PROXY_USERNAME = os.getenv("RESIDENTIAL_PROXY_USERNAME") RESIDENTIAL_PROXY_PASSWORD = os.getenv("RESIDENTIAL_PROXY_PASSWORD") RESIDENTIAL_PROXY_HOSTNAME = os.getenv("RESIDENTIAL_PROXY_HOSTNAME") diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 83d556754..1c09a91ac 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -236,17 +236,17 @@ global_llm_configs: use_default_system_instructions: true citations_enabled: true - # Example: MiniMax M2.5 - High-performance with 204K context window + # Example: MiniMax M3 - High-performance with 512K context window - id: -8 - name: "Global MiniMax M2.5" - description: "MiniMax M2.5 with 204K context window and competitive pricing" + name: "Global MiniMax M3" + description: "MiniMax M3 with 512K context window and competitive pricing" billing_tier: "free" anonymous_enabled: true seo_enabled: true - seo_slug: "minimax-m2.5" + seo_slug: "minimax-m3" quota_reserve_tokens: 4000 provider: "MINIMAX" - model_name: "MiniMax-M2.5" + model_name: "MiniMax-M3" api_key: "your-minimax-api-key-here" api_base: "https://api.minimax.io/v1" rpm: 60 diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 86c789b97..59392831d 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -141,7 +141,6 @@ async def download_and_process_file( task_logger: TaskLoggingService, log_entry: Log, connector_id: int | None = None, - enable_summary: bool = True, ) -> tuple[Any, str | None, dict[str, Any] | None]: """ Download Google Drive file and process using Surfsense file processors. @@ -215,8 +214,6 @@ async def download_and_process_file( "source_connector": "google_drive", }, } - # Include connector_id for de-indexing support - connector_info["enable_summary"] = enable_summary if connector_id is not None: connector_info["connector_id"] = connector_id diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py deleted file mode 100644 index 772b9404c..000000000 --- a/surfsense_backend/app/connectors/jira_connector.py +++ /dev/null @@ -1,648 +0,0 @@ -""" -Jira Connector Module - -A module for retrieving data from Jira. -Allows fetching issue lists and their comments, projects and more. -Supports both OAuth 2.0 (preferred) and legacy API token authentication. -""" - -import base64 -from datetime import datetime -from typing import Any - -import requests - - -class JiraConnector: - """Class for retrieving data from Jira.""" - - def __init__( - self, - base_url: str | None = None, - access_token: str | None = None, - cloud_id: str | None = None, - email: str | None = None, - api_token: str | None = None, - ): - """ - Initialize the JiraConnector class. - - Args: - base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') - access_token: OAuth 2.0 access token (preferred method) - cloud_id: Atlassian cloud ID (used with OAuth for API URL construction) - email: Jira account email address (legacy method, used with api_token) - api_token: Jira API token (legacy method, used with email) - """ - self.base_url = base_url.rstrip("/") if base_url else None - self.access_token = access_token - self.cloud_id = cloud_id - self.email = email - self.api_token = api_token - self.api_version = "3" # Jira Cloud API version - self._use_oauth = access_token is not None - - def set_oauth_credentials( - self, base_url: str, access_token: str, cloud_id: str | None = None - ) -> None: - """ - Set OAuth 2.0 credentials (preferred method). - - Args: - base_url: Jira instance base URL - access_token: OAuth 2.0 access token - cloud_id: Atlassian cloud ID (optional, used for API URL construction) - """ - self.base_url = base_url.rstrip("/") - self.access_token = access_token - self.cloud_id = cloud_id - self._use_oauth = True - - def set_credentials(self, base_url: str, email: str, api_token: str) -> None: - """ - Set the Jira credentials (legacy method using API token). - - Args: - base_url: Jira instance base URL - email: Jira account email address - api_token: Jira API token - """ - self.base_url = base_url.rstrip("/") - self.email = email - self.api_token = api_token - self._use_oauth = False - - def set_email(self, email: str) -> None: - """ - Set the Jira account email (legacy method). - - Args: - email: Jira account email address - """ - self.email = email - self._use_oauth = False - - def set_api_token(self, api_token: str) -> None: - """ - Set the Jira API token (legacy method). - - Args: - api_token: Jira API token - """ - self.api_token = api_token - self._use_oauth = False - - def get_headers(self) -> dict[str, str]: - """ - Get headers for Jira API requests. - - Uses OAuth Bearer token if available, otherwise falls back to Basic Auth. - - Returns: - Dictionary of headers - - Raises: - ValueError: If credentials have not been set - """ - if self._use_oauth: - # OAuth 2.0 authentication - if not self.base_url or not self.access_token: - raise ValueError( - "Jira OAuth credentials not initialized. Call set_oauth_credentials() first." - ) - - return { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.access_token}", - "Accept": "application/json", - } - else: - # Legacy Basic Auth - if not all([self.base_url, self.email, self.api_token]): - raise ValueError( - "Jira credentials not initialized. Call set_credentials() first." - ) - - # Create Basic Auth header using email:api_token - auth_str = f"{self.email}:{self.api_token}" - auth_bytes = auth_str.encode("utf-8") - auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii") - - return { - "Content-Type": "application/json", - "Authorization": auth_header, - "Accept": "application/json", - } - - def make_api_request( - self, - endpoint: str, - params: dict[str, Any] | None = None, - method: str = "GET", - json_payload: dict[str, Any] | None = None, - ) -> dict[str, Any]: - """ - Make a request to the Jira API. - - Args: - endpoint: API endpoint (without base URL) - params: Query parameters for the request (optional) - method: HTTP method (GET or POST) - json_payload: JSON payload for POST requests (optional) - - Returns: - Response data from the API - - Raises: - ValueError: If credentials have not been set - Exception: If the API request fails - """ - headers = self.get_headers() - - # Construct API URL based on authentication method - if self._use_oauth and self.cloud_id: - # Use Atlassian API gateway with cloud_id for OAuth - url = f"https://api.atlassian.com/ex/jira/{self.cloud_id}/rest/api/{self.api_version}/{endpoint}" - else: - # Use direct base URL (works for both OAuth and legacy) - url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}" - - method_upper = method.upper() - if method_upper == "POST": - response = requests.post( - url, headers=headers, json=json_payload, timeout=500 - ) - elif method_upper == "PUT": - response = requests.put( - url, headers=headers, json=json_payload, timeout=500 - ) - elif method_upper == "DELETE": - response = requests.delete(url, headers=headers, params=params, timeout=500) - else: - response = requests.get(url, headers=headers, params=params, timeout=500) - - if response.status_code in (200, 201, 204): - if response.status_code == 204 or not response.text: - return {"status": "success"} - return response.json() - else: - raise Exception( - f"API request failed with status code {response.status_code}: {response.text}" - ) - - def get_all_projects(self) -> dict[str, Any]: - """ - Fetch all projects from Jira. - - Returns: - List of project objects - - Raises: - ValueError: If credentials have not been set - Exception: If the API request fails - """ - return self.make_api_request("project/search") - - def get_all_issues(self, project_key: str | None = None) -> list[dict[str, Any]]: - """ - Fetch all issues from Jira. - - Args: - project_key: Optional project key to filter issues (e.g., 'PROJ') - - Returns: - List of issue objects - - Raises: - ValueError: If credentials have not been set - Exception: If the API request fails - """ - jql = "ORDER BY created DESC" - if project_key: - jql = f'project = "{project_key}" ' + jql - - fields = [ - "summary", - "description", - "status", - "assignee", - "reporter", - "created", - "updated", - "priority", - "issuetype", - "project", - ] - - all_issues = [] - start_at = 0 - max_results = 100 - - all_issues = [] - start_at = 0 - - while True: - json_payload = { - "jql": jql, - "fields": fields, # API accepts list - "maxResults": max_results, - "startAt": start_at, - } - result = self.make_api_request( - "search/jql", json_payload=json_payload, method="POST" - ) - - if not isinstance(result, dict) or "issues" not in result: - raise Exception("Invalid response from Jira API") - - issues = result["issues"] - all_issues.extend(issues) - - print(f"Fetched {len(issues)} issues (startAt={start_at})") - - total = result.get("total", 0) - if start_at + len(issues) >= total: - break - - start_at += len(issues) - - return all_issues - - def get_issues_by_date_range( - self, - start_date: str, - end_date: str, - include_comments: bool = True, - project_key: str | None = None, - ) -> tuple[list[dict[str, Any]], str | None]: - """ - Fetch issues within a date range. - - Args: - start_date: Start date in YYYY-MM-DD format - end_date: End date in YYYY-MM-DD format (inclusive) - include_comments: Whether to include comments in the response - project_key: Optional project key to filter issues - - Returns: - Tuple containing (issues list, error message or None) - """ - try: - # Build JQL query for date range - # Query issues that were either created OR updated within the date range - # Use end_date + 1 day with < operator to include the full end date - from datetime import datetime, timedelta - - # Parse end_date and add 1 day for inclusive end date - end_date_obj = datetime.strptime(end_date, "%Y-%m-%d") - end_date_next = (end_date_obj + timedelta(days=1)).strftime("%Y-%m-%d") - - # Check both created and updated dates to catch all relevant issues - # Use 'created' and 'updated' (standard JQL field names) - date_filter = ( - f"(created >= '{start_date}' AND created < '{end_date_next}') " - f"OR (updated >= '{start_date}' AND updated < '{end_date_next}')" - ) - - jql = f"{date_filter} ORDER BY created DESC" - if project_key: - jql = f'project = "{project_key}" AND ({date_filter}) ORDER BY created DESC' - - # Define fields to retrieve - fields = [ - "summary", - "description", - "status", - "assignee", - "reporter", - "created", - "updated", - "priority", - "issuetype", - "project", - ] - - if include_comments: - fields.append("comment") - - params = { - "jql": jql, - "fields": ",".join(fields), - "maxResults": 100, - "startAt": 0, - } - - all_issues = [] - start_at = 0 - - while True: - params["startAt"] = start_at - - result = self.make_api_request("search/jql", params) - - if not isinstance(result, dict) or "issues" not in result: - return [], "Invalid response from Jira API" - - issues = result["issues"] - all_issues.extend(issues) - - # Check if there are more issues to fetch - total = result.get("total", 0) - if start_at + len(issues) >= total: - break - - start_at += len(issues) - - if not all_issues: - return [], "No issues found in the specified date range." - - return all_issues, None - - except Exception as e: - return [], f"Error fetching issues: {e!s}" - - def get_myself(self) -> dict[str, Any]: - """Fetch the current user's profile (health check).""" - return self.make_api_request("myself") - - def get_projects(self) -> list[dict[str, Any]]: - """Fetch all projects the user has access to.""" - result = self.make_api_request("project/search") - return result.get("values", []) - - def get_issue_types(self) -> list[dict[str, Any]]: - """Fetch all issue types.""" - return self.make_api_request("issuetype") - - def get_priorities(self) -> list[dict[str, Any]]: - """Fetch all priority levels.""" - return self.make_api_request("priority") - - def get_issue(self, issue_id_or_key: str) -> dict[str, Any]: - """Fetch a single issue by ID or key.""" - return self.make_api_request(f"issue/{issue_id_or_key}") - - def create_issue( - self, - project_key: str, - summary: str, - issue_type: str = "Task", - description: str | None = None, - priority: str | None = None, - assignee_id: str | None = None, - ) -> dict[str, Any]: - """Create a new Jira issue.""" - fields: dict[str, Any] = { - "project": {"key": project_key}, - "summary": summary, - "issuetype": {"name": issue_type}, - } - if description: - fields["description"] = { - "type": "doc", - "version": 1, - "content": [ - { - "type": "paragraph", - "content": [{"type": "text", "text": description}], - } - ], - } - if priority: - fields["priority"] = {"name": priority} - if assignee_id: - fields["assignee"] = {"accountId": assignee_id} - - return self.make_api_request( - "issue", method="POST", json_payload={"fields": fields} - ) - - def update_issue( - self, issue_id_or_key: str, fields: dict[str, Any] - ) -> dict[str, Any]: - """Update an existing Jira issue fields.""" - return self.make_api_request( - f"issue/{issue_id_or_key}", - method="PUT", - json_payload={"fields": fields}, - ) - - def delete_issue(self, issue_id_or_key: str) -> dict[str, Any]: - """Delete a Jira issue.""" - return self.make_api_request(f"issue/{issue_id_or_key}", method="DELETE") - - def get_transitions(self, issue_id_or_key: str) -> list[dict[str, Any]]: - """Get available transitions for an issue (for status changes).""" - result = self.make_api_request(f"issue/{issue_id_or_key}/transitions") - return result.get("transitions", []) - - def transition_issue( - self, issue_id_or_key: str, transition_id: str - ) -> dict[str, Any]: - """Transition an issue to a new status.""" - return self.make_api_request( - f"issue/{issue_id_or_key}/transitions", - method="POST", - json_payload={"transition": {"id": transition_id}}, - ) - - def format_issue(self, issue: dict[str, Any]) -> dict[str, Any]: - """ - Format an issue for easier consumption. - - Args: - issue: The issue object from Jira API - - Returns: - Formatted issue dictionary - """ - fields = issue.get("fields", {}) - - # Extract basic issue details - formatted = { - "id": issue.get("id", ""), - "key": issue.get("key", ""), - "title": fields.get("summary", ""), - "description": fields.get("description", ""), - "status": ( - fields.get("status", {}).get("name", "Unknown") - if fields.get("status") - else "Unknown" - ), - "status_category": ( - fields.get("status", {}) - .get("statusCategory", {}) - .get("name", "Unknown") - if fields.get("status") - else "Unknown" - ), - "priority": ( - fields.get("priority", {}).get("name", "Unknown") - if fields.get("priority") - else "Unknown" - ), - "issue_type": ( - fields.get("issuetype", {}).get("name", "Unknown") - if fields.get("issuetype") - else "Unknown" - ), - "project": ( - fields.get("project", {}).get("key", "Unknown") - if fields.get("project") - else "Unknown" - ), - "created_at": fields.get("created", ""), - "updated_at": fields.get("updated", ""), - "reporter": ( - { - "account_id": ( - fields.get("reporter", {}).get("accountId", "") - if fields.get("reporter") - else "" - ), - "display_name": ( - fields.get("reporter", {}).get("displayName", "Unknown") - if fields.get("reporter") - else "Unknown" - ), - "email": ( - fields.get("reporter", {}).get("emailAddress", "") - if fields.get("reporter") - else "" - ), - } - if fields.get("reporter") - else {"account_id": "", "display_name": "Unknown", "email": ""} - ), - "assignee": ( - { - "account_id": fields.get("assignee", {}).get("accountId", ""), - "display_name": fields.get("assignee", {}).get( - "displayName", "Unknown" - ), - "email": fields.get("assignee", {}).get("emailAddress", ""), - } - if fields.get("assignee") - else None - ), - "comments": [], - } - - # Extract comments if available - if "comment" in fields and "comments" in fields["comment"]: - for comment in fields["comment"]["comments"]: - formatted_comment = { - "id": comment.get("id", ""), - "body": comment.get("body", ""), - "created_at": comment.get("created", ""), - "updated_at": comment.get("updated", ""), - "author": ( - { - "account_id": ( - comment.get("author", {}).get("accountId", "") - if comment.get("author") - else "" - ), - "display_name": ( - comment.get("author", {}).get("displayName", "Unknown") - if comment.get("author") - else "Unknown" - ), - "email": ( - comment.get("author", {}).get("emailAddress", "") - if comment.get("author") - else "" - ), - } - if comment.get("author") - else {"account_id": "", "display_name": "Unknown", "email": ""} - ), - } - formatted["comments"].append(formatted_comment) - - return formatted - - def format_issue_to_markdown(self, issue: dict[str, Any]) -> str: - """ - Convert an issue to markdown format. - - Args: - issue: The issue object (either raw or formatted) - - Returns: - Markdown string representation of the issue - """ - # Format the issue if it's not already formatted - if "key" not in issue: - issue = self.format_issue(issue) - - # Build the markdown content - markdown = ( - f"# {issue.get('key', 'No Key')}: {issue.get('title', 'No Title')}\n\n" - ) - - if issue.get("status"): - markdown += f"**Status:** {issue['status']}\n" - - if issue.get("priority"): - markdown += f"**Priority:** {issue['priority']}\n" - - if issue.get("issue_type"): - markdown += f"**Type:** {issue['issue_type']}\n" - - if issue.get("project"): - markdown += f"**Project:** {issue['project']}\n\n" - - if issue.get("assignee") and issue["assignee"].get("display_name"): - markdown += f"**Assignee:** {issue['assignee']['display_name']}\n" - - if issue.get("reporter") and issue["reporter"].get("display_name"): - markdown += f"**Reporter:** {issue['reporter']['display_name']}\n" - - if issue.get("created_at"): - created_date = self.format_date(issue["created_at"]) - markdown += f"**Created:** {created_date}\n" - - if issue.get("updated_at"): - updated_date = self.format_date(issue["updated_at"]) - markdown += f"**Updated:** {updated_date}\n\n" - - if issue.get("description"): - markdown += f"## Description\n\n{issue['description']}\n\n" - - if issue.get("comments"): - markdown += f"## Comments ({len(issue['comments'])})\n\n" - - for comment in issue["comments"]: - author_name = "Unknown" - if comment.get("author") and comment["author"].get("display_name"): - author_name = comment["author"]["display_name"] - - comment_date = "Unknown date" - if comment.get("created_at"): - comment_date = self.format_date(comment["created_at"]) - - markdown += f"### {author_name} ({comment_date})\n\n{comment.get('body', '')}\n\n---\n\n" - - return markdown - - @staticmethod - def format_date(iso_date: str) -> str: - """ - Format an ISO date string to a more readable format. - - Args: - iso_date: ISO format date string - - Returns: - Formatted date string - """ - if not iso_date or not isinstance(iso_date, str): - return "Unknown date" - - try: - # Jira dates are typically in format: 2023-01-01T12:00:00.000+0000 - dt = datetime.fromisoformat(iso_date.replace("Z", "+00:00")) - return dt.strftime("%Y-%m-%d %H:%M:%S") - except ValueError: - return iso_date diff --git a/surfsense_backend/app/connectors/jira_history.py b/surfsense_backend/app/connectors/jira_history.py deleted file mode 100644 index 30162964e..000000000 --- a/surfsense_backend/app/connectors/jira_history.py +++ /dev/null @@ -1,350 +0,0 @@ -""" -Jira OAuth Connector. - -Handles OAuth-based authentication and token refresh for Jira API access. -Supports both OAuth 2.0 (preferred) and legacy API token authentication. -""" - -import logging -from typing import Any - -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select - -from app.config import config -from app.connectors.jira_connector import JiraConnector -from app.db import SearchSourceConnector -from app.schemas.atlassian_auth_credentials import AtlassianAuthCredentialsBase -from app.utils.oauth_security import TokenEncryption - -logger = logging.getLogger(__name__) - - -class JiraHistoryConnector: - """ - Jira connector with OAuth support and automatic token refresh. - - This connector uses OAuth 2.0 access tokens to authenticate with the - Jira API. It automatically refreshes expired tokens when needed. - Also supports legacy API token authentication for backward compatibility. - """ - - def __init__( - self, - session: AsyncSession, - connector_id: int, - credentials: AtlassianAuthCredentialsBase | None = None, - ): - """ - Initialize the JiraHistoryConnector with auto-refresh capability. - - Args: - session: Database session for updating connector - connector_id: Connector ID for direct updates - credentials: Jira OAuth credentials (optional, will be loaded from DB if not provided) - """ - self._session = session - self._connector_id = connector_id - self._credentials = credentials - self._cloud_id: str | None = None - self._base_url: str | None = None - self._jira_client: JiraConnector | None = None - self._use_oauth = True - self._legacy_email: str | None = None - self._legacy_api_token: str | None = None - - async def _get_valid_token(self) -> str: - """ - Get valid Jira access token, refreshing if needed. - - Returns: - Valid access token - - Raises: - ValueError: If credentials are missing or invalid - Exception: If token refresh fails - """ - # Load credentials from DB if not provided - if self._credentials is None: - result = await self._session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == self._connector_id - ) - ) - connector = result.scalars().first() - - if not connector: - raise ValueError(f"Connector {self._connector_id} not found") - - config_data = connector.config.copy() - - # Check if using OAuth or legacy API token - is_oauth = config_data.get("_token_encrypted", False) or config_data.get( - "access_token" - ) - - if is_oauth: - # OAuth 2.0 authentication - # Check if access_token exists before processing - raw_access_token = config_data.get("access_token") - if not raw_access_token: - raise ValueError( - "Jira access token not found. " - "Please reconnect your Jira account." - ) - - if not config.SECRET_KEY: - raise ValueError( - "SECRET_KEY not configured but tokens are marked as encrypted" - ) - - try: - token_encryption = TokenEncryption(config.SECRET_KEY) - - # Decrypt access_token - if config_data.get("access_token"): - config_data["access_token"] = token_encryption.decrypt_token( - config_data["access_token"] - ) - logger.info( - f"Decrypted Jira access token for connector {self._connector_id}" - ) - - # Decrypt refresh_token if present - if config_data.get("refresh_token"): - config_data["refresh_token"] = token_encryption.decrypt_token( - config_data["refresh_token"] - ) - logger.info( - f"Decrypted Jira refresh token for connector {self._connector_id}" - ) - except Exception as e: - logger.error( - f"Failed to decrypt Jira credentials for connector {self._connector_id}: {e!s}" - ) - raise ValueError( - f"Failed to decrypt Jira credentials: {e!s}" - ) from e - - # Final validation after decryption - final_token = config_data.get("access_token") - if not final_token or ( - isinstance(final_token, str) and not final_token.strip() - ): - raise ValueError( - "Jira access token is invalid or empty. " - "Please reconnect your Jira account." - ) - - try: - self._credentials = AtlassianAuthCredentialsBase.from_dict( - config_data - ) - self._cloud_id = config_data.get("cloud_id") - self._base_url = config_data.get("base_url") - self._use_oauth = True - except Exception as e: - raise ValueError(f"Invalid Jira OAuth credentials: {e!s}") from e - else: - # Legacy API token authentication - self._legacy_email = config_data.get("JIRA_EMAIL") - self._legacy_api_token = config_data.get("JIRA_API_TOKEN") - self._base_url = config_data.get("JIRA_BASE_URL") - self._use_oauth = False - - if ( - not self._legacy_email - or not self._legacy_api_token - or not self._base_url - ): - raise ValueError("Jira credentials not found in connector config") - - # Check if token is expired and refreshable (only for OAuth) - if ( - self._use_oauth - and self._credentials.is_expired - and self._credentials.is_refreshable - ): - try: - logger.info( - f"Jira token expired for connector {self._connector_id}, refreshing..." - ) - - # Get connector for refresh - result = await self._session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.id == self._connector_id - ) - ) - connector = result.scalars().first() - - if not connector: - raise RuntimeError( - f"Connector {self._connector_id} not found; cannot refresh token." - ) - - # Lazy import to avoid circular dependency - from app.routes.jira_add_connector_route import refresh_jira_token - - connector = await refresh_jira_token(self._session, connector) - - # Reload credentials after refresh - config_data = connector.config.copy() - token_encrypted = config_data.get("_token_encrypted", False) - if token_encrypted and config.SECRET_KEY: - token_encryption = TokenEncryption(config.SECRET_KEY) - if config_data.get("access_token"): - config_data["access_token"] = token_encryption.decrypt_token( - config_data["access_token"] - ) - if config_data.get("refresh_token"): - config_data["refresh_token"] = token_encryption.decrypt_token( - config_data["refresh_token"] - ) - - self._credentials = AtlassianAuthCredentialsBase.from_dict(config_data) - self._cloud_id = config_data.get("cloud_id") - self._base_url = config_data.get("base_url") - - # Invalidate cached client so it's recreated with new token - self._jira_client = None - - logger.info( - f"Successfully refreshed Jira token for connector {self._connector_id}" - ) - except Exception as e: - logger.error( - f"Failed to refresh Jira token for connector {self._connector_id}: {e!s}" - ) - raise Exception( - f"Failed to refresh Jira OAuth credentials: {e!s}" - ) from e - - if self._use_oauth: - return self._credentials.access_token - else: - # For legacy auth, return empty string (not used for token-based auth) - return "" - - async def _get_jira_client(self) -> JiraConnector: - """ - Get or create JiraConnector with valid credentials. - - Returns: - JiraConnector instance - """ - if self._jira_client is None: - if self._use_oauth: - # Ensure we have valid token (will refresh if needed) - await self._get_valid_token() - - self._jira_client = JiraConnector( - base_url=self._base_url, - access_token=self._credentials.access_token, - cloud_id=self._cloud_id, - ) - else: - # Legacy API token authentication - self._jira_client = JiraConnector( - base_url=self._base_url, - email=self._legacy_email, - api_token=self._legacy_api_token, - ) - else: - # If OAuth, refresh token if expired before returning client - if self._use_oauth: - await self._get_valid_token() - # Update client with new token if it was refreshed - if self._credentials: - self._jira_client.set_oauth_credentials( - base_url=self._base_url or "", - access_token=self._credentials.access_token, - cloud_id=self._cloud_id, - ) - - return self._jira_client - - async def get_issues_by_date_range( - self, - start_date: str, - end_date: str, - include_comments: bool = True, - project_key: str | None = None, - ) -> tuple[list[dict[str, Any]], str | None]: - """ - Fetch issues within a date range. - This method wraps JiraConnector.get_issues_by_date_range() with automatic token refresh. - - Args: - start_date: Start date in YYYY-MM-DD format - end_date: End date in YYYY-MM-DD format (inclusive) - include_comments: Whether to include comments in the response - project_key: Optional project key to filter issues - - Returns: - Tuple containing (issues list, error message or None) - """ - # Ensure token is valid (will refresh if needed) - if self._use_oauth: - await self._get_valid_token() - - # Get client with valid credentials - client = await self._get_jira_client() - - # JiraConnector methods are synchronous, so we call them directly - # Token refresh has already been handled above - return client.get_issues_by_date_range( - start_date=start_date, - end_date=end_date, - include_comments=include_comments, - project_key=project_key, - ) - - def format_issue(self, issue: dict[str, Any]) -> dict[str, Any]: - """ - Format an issue for easier consumption. - Wraps JiraConnector.format_issue(). - - Args: - issue: The issue object from Jira API - - Returns: - Formatted issue dictionary - """ - # This is a synchronous method that doesn't need token refresh - # since it just formats data that's already been fetched - if self._jira_client is None: - # Create a minimal client just for formatting (doesn't need credentials) - self._jira_client = JiraConnector() - return self._jira_client.format_issue(issue) - - def format_issue_to_markdown(self, issue: dict[str, Any]) -> str: - """ - Convert an issue to markdown format. - Wraps JiraConnector.format_issue_to_markdown(). - - Args: - issue: The issue object (either raw or formatted) - - Returns: - Markdown string representation of the issue - """ - # This is a synchronous method that doesn't need token refresh - # since it just formats data that's already been fetched - if self._jira_client is None: - # Create a minimal client just for formatting (doesn't need credentials) - self._jira_client = JiraConnector() - return self._jira_client.format_issue_to_markdown(issue) - - async def close(self): - """Close any resources (currently no-op for JiraConnector).""" - # JiraConnector doesn't maintain persistent connections, so nothing to close - self._jira_client = None - - async def __aenter__(self): - """Async context manager entry.""" - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit.""" - await self.close() diff --git a/surfsense_backend/app/connectors/webcrawler_connector.py b/surfsense_backend/app/connectors/webcrawler_connector.py index 1b866dfc4..1d4ff1f58 100644 --- a/surfsense_backend/app/connectors/webcrawler_connector.py +++ b/surfsense_backend/app/connectors/webcrawler_connector.py @@ -1,31 +1,34 @@ """ WebCrawler Connector Module -A module for crawling web pages and extracting content using Firecrawl, -plain HTTP+Trafilatura, or Playwright. Provides a unified interface for -web scraping. +A module for crawling web pages and extracting content using Firecrawl or +Scrapling's tiered fetchers, with Trafilatura for HTML -> markdown extraction. +Provides a unified interface for web scraping. Fallback order: - 1. Firecrawl (if API key is configured) - 2. HTTP + Trafilatura (lightweight, works on any event loop) - 3. Playwright / Chromium (runs in a thread to avoid event-loop limitations) + 1. Firecrawl (if API key is configured) + 2. Scrapling AsyncFetcher (fast static HTTP, no browser subprocess) + 3. Scrapling DynamicFetcher (full browser, run in a thread) + 4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread) """ import asyncio import logging +import time from typing import Any -import httpx import trafilatura import validators -from fake_useragent import UserAgent from firecrawl import AsyncFirecrawlApp -from playwright.sync_api import sync_playwright +from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher -from app.utils.proxy_config import get_playwright_proxy, get_residential_proxy_url +from app.utils.proxy import get_proxy_url logger = logging.getLogger(__name__) +# Prefix for performance/timing log lines so they are easy to grep/filter. +_PERF = "[webcrawler][perf]" + class WebCrawlerConnector: """Class for crawling web pages and extracting content.""" @@ -36,8 +39,8 @@ class WebCrawlerConnector: Args: firecrawl_api_key: Firecrawl API key (optional). If provided, Firecrawl will be tried first - and Chromium will be used as fallback if Firecrawl fails. If not provided, - Chromium will be used directly. + and Scrapling will be used as fallback if Firecrawl fails. If not provided, + Scrapling fetchers are used directly. """ self.firecrawl_api_key = firecrawl_api_key self.use_firecrawl = bool(firecrawl_api_key) @@ -60,8 +63,9 @@ class WebCrawlerConnector: Fallback order: 1. Firecrawl (if API key configured) - 2. Plain HTTP + Trafilatura (lightweight, no subprocess) - 3. Playwright / Chromium (needs subprocess-capable event loop) + 2. Scrapling AsyncFetcher (fast static HTTP, no subprocess) + 3. Scrapling DynamicFetcher (full browser, run in a thread) + 4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread) Args: url: URL to crawl @@ -74,8 +78,8 @@ class WebCrawlerConnector: - metadata: Page metadata (title, description, etc.) - source: Original URL - crawler_type: Type of crawler used - # Validate URL """ + total_start = time.perf_counter() try: if not validators.url(url): return None, f"Invalid URL: {url}" @@ -84,48 +88,138 @@ class WebCrawlerConnector: # --- 1. Firecrawl (premium, if configured) --- if self.use_firecrawl: + tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Firecrawl for: {url}") - return await self._crawl_with_firecrawl(url, formats), None + result = await self._crawl_with_firecrawl(url, formats) + self._log_tier_outcome("firecrawl", url, tier_start, "success") + self._log_total(url, "firecrawl", total_start) + return result, None except Exception as exc: errors.append(f"Firecrawl: {exc!s}") - logger.warning(f"[webcrawler] Firecrawl failed for {url}: {exc!s}") + self._log_tier_outcome("firecrawl", url, tier_start, "error", exc) - # --- 2. HTTP + Trafilatura (no subprocess required) --- + # --- 2. Scrapling AsyncFetcher (fast static HTTP) --- + tier_start = time.perf_counter() try: - logger.info(f"[webcrawler] Using HTTP+Trafilatura for: {url}") - result = await self._crawl_with_http(url) + logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}") + result = await self._crawl_with_async_fetcher(url) if result: + self._log_tier_outcome( + "scrapling-static", url, tier_start, "success" + ) + self._log_total(url, "scrapling-static", total_start) return result, None - errors.append("HTTP+Trafilatura: empty extraction") + errors.append("Scrapling static: empty extraction") + self._log_tier_outcome("scrapling-static", url, tier_start, "empty") except Exception as exc: - errors.append(f"HTTP+Trafilatura: {exc!s}") - logger.warning( - f"[webcrawler] HTTP+Trafilatura failed for {url}: {exc!s}" + errors.append(f"Scrapling static: {exc!s}") + self._log_tier_outcome( + "scrapling-static", url, tier_start, "error", exc ) - # --- 3. Playwright / Chromium (full browser, last resort) --- + # --- 3. Scrapling DynamicFetcher (full browser) --- + tier_start = time.perf_counter() try: - logger.info(f"[webcrawler] Using Chromium+Trafilatura for: {url}") - return await self._crawl_with_chromium(url), None + logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}") + result = await self._crawl_with_dynamic(url) + if result: + self._log_tier_outcome( + "scrapling-dynamic", url, tier_start, "success" + ) + self._log_total(url, "scrapling-dynamic", total_start) + return result, None + errors.append("Scrapling dynamic: empty extraction") + self._log_tier_outcome("scrapling-dynamic", url, tier_start, "empty") except NotImplementedError: errors.append( - "Chromium: event loop does not support subprocesses " + "Scrapling dynamic: event loop does not support subprocesses " "(common on Windows with uvicorn --reload)" ) - logger.warning( - f"[webcrawler] Chromium unavailable for {url}: " - "current event loop does not support subprocesses" + self._log_tier_outcome( + "scrapling-dynamic", url, tier_start, "unavailable" ) except Exception as exc: - errors.append(f"Chromium: {exc!s}") - logger.warning(f"[webcrawler] Chromium failed for {url}: {exc!s}") + errors.append(f"Scrapling dynamic: {exc!s}") + self._log_tier_outcome( + "scrapling-dynamic", url, tier_start, "error", exc + ) + # --- 4. Scrapling StealthyFetcher (anti-bot, last resort) --- + tier_start = time.perf_counter() + try: + logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}") + result = await self._crawl_with_stealthy(url) + if result: + self._log_tier_outcome( + "scrapling-stealthy", url, tier_start, "success" + ) + self._log_total(url, "scrapling-stealthy", total_start) + return result, None + errors.append("Scrapling stealthy: empty extraction") + self._log_tier_outcome("scrapling-stealthy", url, tier_start, "empty") + except NotImplementedError: + errors.append( + "Scrapling stealthy: event loop does not support subprocesses " + "(common on Windows with uvicorn --reload)" + ) + self._log_tier_outcome( + "scrapling-stealthy", url, tier_start, "unavailable" + ) + except Exception as exc: + errors.append(f"Scrapling stealthy: {exc!s}") + self._log_tier_outcome( + "scrapling-stealthy", url, tier_start, "error", exc + ) + + self._log_total(url, "none", total_start) return None, f"All crawl methods failed for {url}. {'; '.join(errors)}" except Exception as e: + self._log_total(url, "error", total_start) return None, f"Error crawling URL {url}: {e!s}" + @staticmethod + def _log_tier_outcome( + tier: str, + url: str, + tier_start: float, + outcome: str, + exc: Exception | None = None, + ) -> None: + """Log how long a single tier took and how it ended.""" + elapsed_ms = (time.perf_counter() - tier_start) * 1000 + if outcome == "error": + logger.warning( + "%s tier=%s url=%s elapsed_ms=%.1f outcome=error error=%s", + _PERF, + tier, + url, + elapsed_ms, + exc, + ) + else: + logger.info( + "%s tier=%s url=%s elapsed_ms=%.1f outcome=%s", + _PERF, + tier, + url, + elapsed_ms, + outcome, + ) + + @staticmethod + def _log_total(url: str, selected: str, total_start: float) -> None: + """Log the total time spent across all attempted tiers.""" + total_ms = (time.perf_counter() - total_start) * 1000 + logger.info( + "%s url=%s selected=%s total_ms=%.1f", + _PERF, + url, + selected, + total_ms, + ) + async def _crawl_with_firecrawl( self, url: str, formats: list[str] | None = None ) -> dict[str, Any]: @@ -177,52 +271,172 @@ class WebCrawlerConnector: "crawler_type": "firecrawl", } - async def _crawl_with_http(self, url: str) -> dict[str, Any] | None: + async def _crawl_with_async_fetcher(self, url: str) -> dict[str, Any] | None: """ - Crawl URL using a plain HTTP request + Trafilatura content extraction. + Crawl URL using Scrapling's AsyncFetcher (static HTTP) + Trafilatura. - This method avoids launching a browser subprocess, making it safe to - call from any asyncio event loop (including Windows SelectorEventLoop - which does not support ``create_subprocess_exec``). - - Returns ``None`` when Trafilatura cannot extract meaningful content - (e.g. JS-rendered SPAs) so the caller can fall through to Chromium. + AsyncFetcher is httpx/curl_cffi based and does not launch a browser + subprocess, making it safe to call from any asyncio event loop. Returns + ``None`` when Trafilatura cannot extract meaningful content (e.g. JS + rendered SPAs) so the caller can fall through to the browser tiers. """ - ua = UserAgent() - user_agent = ua.random - proxy_url = get_residential_proxy_url() + fetch_start = time.perf_counter() + page = await AsyncFetcher.get( + url, + stealthy_headers=True, + proxy=get_proxy_url(), + timeout=20, + ) + fetch_ms = (time.perf_counter() - fetch_start) * 1000 - async with httpx.AsyncClient( - timeout=20.0, - follow_redirects=True, - proxy=proxy_url, - headers={ - "User-Agent": user_agent, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.9", - "Accept-Encoding": "gzip, deflate, br", - }, - ) as client: - response = await client.get(url) - response.raise_for_status() - raw_html = response.text - - if not raw_html or len(raw_html.strip()) == 0: + status = getattr(page, "status", None) + if status is not None and status >= 400: + logger.info( + "%s tier=scrapling-static url=%s fetch_ms=%.1f status=%s outcome=http_error", + _PERF, + url, + fetch_ms, + status, + ) return None - extracted_content = trafilatura.extract( - raw_html, - output_format="markdown", - include_comments=False, - include_tables=True, - include_images=True, - include_links=True, + return self._build_result( + page.html_content, + url, + "scrapling-static", + allow_raw_fallback=False, + fetch_ms=fetch_ms, + status=status, ) - if not extracted_content or len(extracted_content.strip()) == 0: + async def _crawl_with_dynamic(self, url: str) -> dict[str, Any] | None: + """ + Crawl URL using Scrapling's DynamicFetcher (full browser) + Trafilatura. + + Runs the sync fetch in a worker thread so it works on any event loop, + including Windows ``SelectorEventLoop`` which cannot spawn subprocesses. + """ + return await asyncio.to_thread(self._crawl_with_dynamic_sync, url) + + def _crawl_with_dynamic_sync(self, url: str) -> dict[str, Any] | None: + """Synchronous DynamicFetcher crawl executed in a worker thread.""" + fetch_start = time.perf_counter() + page = DynamicFetcher.fetch( + url, + headless=True, + network_idle=True, + timeout=30000, + proxy=get_proxy_url(), + ) + fetch_ms = (time.perf_counter() - fetch_start) * 1000 + return self._build_result( + page.html_content, + url, + "scrapling-dynamic", + allow_raw_fallback=False, + fetch_ms=fetch_ms, + status=getattr(page, "status", None), + ) + + async def _crawl_with_stealthy(self, url: str) -> dict[str, Any] | None: + """ + Crawl URL using Scrapling's StealthyFetcher (Camoufox) + Trafilatura. + + Last-resort tier with anti-bot features. Runs the sync fetch in a worker + thread for the same event-loop-safety reasons as DynamicFetcher. Falls + back to the raw HTML when Trafilatura extraction is empty. + """ + return await asyncio.to_thread(self._crawl_with_stealthy_sync, url) + + def _crawl_with_stealthy_sync(self, url: str) -> dict[str, Any] | None: + """Synchronous StealthyFetcher crawl executed in a worker thread.""" + fetch_start = time.perf_counter() + page = StealthyFetcher.fetch( + url, + headless=True, + network_idle=True, + block_ads=True, + proxy=get_proxy_url(), + ) + fetch_ms = (time.perf_counter() - fetch_start) * 1000 + return self._build_result( + page.html_content, + url, + "scrapling-stealthy", + allow_raw_fallback=True, + fetch_ms=fetch_ms, + status=getattr(page, "status", None), + ) + + def _build_result( + self, + raw_html: str | None, + url: str, + crawler_type: str, + *, + allow_raw_fallback: bool, + fetch_ms: float | None = None, + status: int | None = None, + ) -> dict[str, Any] | None: + """ + Extract markdown + metadata from raw HTML using Trafilatura. + + Args: + raw_html: Raw HTML source from a fetcher. + url: Original URL (used as the metadata source/title fallback). + crawler_type: Identifier of the tier that produced the HTML. + allow_raw_fallback: When True, return the raw HTML as content if + Trafilatura cannot extract anything (used by the last-resort + stealthy tier). When False, return ``None`` so the caller can + fall through to the next tier. + fetch_ms: Time spent fetching the page (for perf logging). + status: HTTP status code returned by the fetcher (for perf logging). + + Returns: + Result dict (content/metadata/crawler_type) or ``None``. + """ + html_len = len(raw_html) if raw_html else 0 + + if not raw_html or len(raw_html.strip()) == 0: + self._log_build( + crawler_type, url, fetch_ms, 0.0, status, html_len, 0, "empty_html" + ) return None - trafilatura_metadata = trafilatura.extract_metadata(raw_html) + extract_start = time.perf_counter() + extracted_content: str | None = None + trafilatura_metadata = None + + try: + extracted_content = trafilatura.extract( + raw_html, + output_format="markdown", + include_comments=False, + include_tables=True, + include_images=True, + include_links=True, + ) + trafilatura_metadata = trafilatura.extract_metadata(raw_html) + + if extracted_content and len(extracted_content.strip()) == 0: + extracted_content = None + except Exception: + extracted_content = None + + extract_ms = (time.perf_counter() - extract_start) * 1000 + + if not extracted_content and not allow_raw_fallback: + self._log_build( + crawler_type, + url, + fetch_ms, + extract_ms, + status, + html_len, + 0, + "no_extraction", + ) + return None metadata: dict[str, str] = {"source": url} if trafilatura_metadata: @@ -236,105 +450,51 @@ class WebCrawlerConnector: metadata["date"] = trafilatura_metadata.date metadata.setdefault("title", url) - return { - "content": extracted_content, - "metadata": metadata, - "crawler_type": "http", - } - - async def _crawl_with_chromium(self, url: str) -> dict[str, Any]: - """ - Crawl URL using Playwright with Trafilatura for content extraction. - Falls back to raw HTML if Trafilatura extraction fails. - - Runs the sync Playwright API in a thread so it works on any event - loop, including Windows ``SelectorEventLoop`` which cannot spawn - subprocesses. - - Args: - url: URL to crawl - - Returns: - Dict containing crawled content and metadata - - Raises: - Exception: If crawling fails - """ - return await asyncio.to_thread(self._crawl_with_chromium_sync, url) - - def _crawl_with_chromium_sync(self, url: str) -> dict[str, Any]: - """Synchronous Playwright crawl executed in a worker thread.""" - ua = UserAgent() - user_agent = ua.random - - playwright_proxy = get_playwright_proxy() - - with sync_playwright() as p: - launch_kwargs: dict = {"headless": True} - if playwright_proxy: - launch_kwargs["proxy"] = playwright_proxy - browser = p.chromium.launch(**launch_kwargs) - context = browser.new_context(user_agent=user_agent) - page = context.new_page() - - try: - page.goto(url, wait_until="domcontentloaded", timeout=30000) - raw_html = page.content() - page_title = page.title() - finally: - browser.close() - - if not raw_html: - raise ValueError(f"Failed to load content from {url}") - - base_metadata = {"title": page_title} if page_title else {} - - extracted_content = None - trafilatura_metadata = None - - try: - extracted_content = trafilatura.extract( - raw_html, - output_format="markdown", - include_comments=False, - include_tables=True, - include_images=True, - include_links=True, - ) - - trafilatura_metadata = trafilatura.extract_metadata(raw_html) - - if not extracted_content or len(extracted_content.strip()) == 0: - extracted_content = None - - except Exception: - extracted_content = None - - metadata = { - "source": url, - "title": ( - trafilatura_metadata.title - if trafilatura_metadata and trafilatura_metadata.title - else base_metadata.get("title", url) - ), - } - - if trafilatura_metadata: - if trafilatura_metadata.description: - metadata["description"] = trafilatura_metadata.description - if trafilatura_metadata.author: - metadata["author"] = trafilatura_metadata.author - if trafilatura_metadata.date: - metadata["date"] = trafilatura_metadata.date - - metadata.update(base_metadata) + content = extracted_content if extracted_content else raw_html + self._log_build( + crawler_type, + url, + fetch_ms, + extract_ms, + status, + html_len, + len(content), + "extracted" if extracted_content else "raw_fallback", + ) return { - "content": extracted_content if extracted_content else raw_html, + "content": content, "metadata": metadata, - "crawler_type": "chromium", + "crawler_type": crawler_type, } + @staticmethod + def _log_build( + crawler_type: str, + url: str, + fetch_ms: float | None, + extract_ms: float, + status: int | None, + html_len: int, + content_len: int, + outcome: str, + ) -> None: + """Emit a detailed perf line splitting fetch vs Trafilatura extraction.""" + fetch_repr = f"{fetch_ms:.1f}" if fetch_ms is not None else "n/a" + logger.info( + "%s tier=%s url=%s status=%s fetch_ms=%s extract_ms=%.1f " + "html_len=%d content_len=%d outcome=%s", + _PERF, + crawler_type, + url, + status, + fetch_repr, + extract_ms, + html_len, + content_len, + outcome, + ) + def format_to_structured_document( self, crawl_result: dict[str, Any], exclude_metadata: bool = False ) -> str: diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index d6ee9ff88..6117caecb 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -14,6 +14,7 @@ from sqlalchemy import ( TIMESTAMP, BigInteger, Boolean, + CheckConstraint, Column, Enum as SQLAlchemyEnum, ForeignKey, @@ -587,6 +588,58 @@ class ChatVisibility(StrEnum): # PUBLIC = "PUBLIC" # Reserved for future implementation +class ExternalChatPlatform(StrEnum): + TELEGRAM = "telegram" + WHATSAPP = "whatsapp" + SLACK = "slack" + DISCORD = "discord" + SIGNAL = "signal" + + +class ExternalChatAccountMode(StrEnum): + CLOUD_SHARED = "cloud_shared" + SELF_HOST_BYO = "self_host_byo" + + +class ExternalChatHealthStatus(StrEnum): + UNKNOWN = "unknown" + OK = "ok" + FAILING = "failing" + + +class ExternalChatBindingState(StrEnum): + PENDING = "pending" + BOUND = "bound" + REVOKED = "revoked" + SUSPENDED = "suspended" + + +class ExternalChatPeerKind(StrEnum): + DIRECT = "direct" + GROUP = "group" + CHANNEL = "channel" + UNKNOWN = "unknown" + + +class ExternalChatEventKind(StrEnum): + MESSAGE = "message" + EDITED_MESSAGE = "edited_message" + CALLBACK_QUERY = "callback_query" + OTHER = "other" + + +class ExternalChatEventStatus(StrEnum): + RECEIVED = "received" + PROCESSING = "processing" + PROCESSED = "processed" + IGNORED = "ignored" + FAILED = "failed" + + +def _enum_values(enum_cls): + return [item.value for item in enum_cls] + + class NewChatThread(BaseModel, TimestampMixin): """ Thread model for the new chat feature using assistant-ui. @@ -659,6 +712,18 @@ class NewChatThread(BaseModel, TimestampMixin): # agent_llm_id changes). Unindexed: all reads are by primary key. pinned_llm_config_id = Column(Integer, nullable=True) + # Surface metadata for first-party SurfSense and external chat threads. + # Zero publishes all chat-message sources; the UI can decide which surfaces to render. + source = Column( + Text, nullable=False, default="surfsense", server_default="surfsense" + ) + external_chat_binding_id = Column( + BigInteger, + ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + # Relationships search_space = relationship("SearchSpace", back_populates="new_chat_threads") created_by = relationship("User", back_populates="new_chat_threads") @@ -679,6 +744,11 @@ class NewChatThread(BaseModel, TimestampMixin): back_populates="thread", cascade="all, delete-orphan", ) + external_chat_binding = relationship( + "ExternalChatBinding", + foreign_keys=[external_chat_binding_id], + back_populates="threads", + ) class NewChatMessage(BaseModel, TimestampMixin): @@ -732,6 +802,13 @@ class NewChatMessage(BaseModel, TimestampMixin): # a message back to the LangGraph checkpoint that produced its turn. turn_id = Column(String(64), nullable=True, index=True) + # Mirrors the parent thread source for publication-level filtering. + # This denormalization avoids join-dependent logical replication rules. + source = Column( + Text, nullable=False, default="surfsense", server_default="surfsense" + ) + platform_metadata = Column(JSONB, nullable=True) + # Relationships thread = relationship("NewChatThread", back_populates="messages") author = relationship("User") @@ -748,6 +825,310 @@ class NewChatMessage(BaseModel, TimestampMixin): ) +class ExternalChatAccount(Base, TimestampMixin): + __tablename__ = "external_chat_accounts" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + platform = Column( + SQLAlchemyEnum( + ExternalChatPlatform, + name="external_chat_platform", + values_callable=_enum_values, + ), + nullable=False, + ) + mode = Column( + SQLAlchemyEnum( + ExternalChatAccountMode, + name="external_chat_account_mode", + values_callable=_enum_values, + ), + nullable=False, + ) + owner_user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=True + ) + owner_search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True + ) + is_system_account = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + encrypted_credentials = Column(Text, nullable=True) + bot_username = Column(String(255), nullable=True) + webhook_secret = Column(String(64), nullable=True) + cursor_state = Column( + JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb") + ) + health_status = Column( + SQLAlchemyEnum( + ExternalChatHealthStatus, + name="external_chat_health_status", + values_callable=_enum_values, + ), + nullable=False, + default=ExternalChatHealthStatus.UNKNOWN, + server_default=ExternalChatHealthStatus.UNKNOWN.value, + ) + last_health_check_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_reason = Column(Text, nullable=True) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + + owner = relationship("User", foreign_keys=[owner_user_id]) + owner_search_space = relationship( + "SearchSpace", foreign_keys=[owner_search_space_id] + ) + bindings = relationship( + "ExternalChatBinding", + back_populates="account", + cascade="all, delete-orphan", + ) + inbound_events = relationship( + "ExternalChatInboundEvent", + back_populates="account", + cascade="all, delete-orphan", + ) + + __table_args__ = ( + CheckConstraint( + "(is_system_account = true AND owner_user_id IS NULL) OR " + "(is_system_account = false AND owner_user_id IS NOT NULL)", + name="ck_external_chat_accounts_owner_shape", + ), + Index( + "uq_external_chat_accounts_owner_platform", + "owner_user_id", + "platform", + unique=True, + postgresql_where=text("is_system_account = false"), + ), + Index( + "uq_external_chat_accounts_system_platform", + "platform", + unique=True, + postgresql_where=text( + "is_system_account = true " + "AND NOT (cursor_state ? 'team_id') " + "AND NOT (cursor_state ? 'guild_id')" + ), + ), + Index( + "uq_external_chat_accounts_slack_team", + "platform", + text("(cursor_state ->> 'team_id')"), + unique=True, + postgresql_where=text( + "is_system_account = true AND cursor_state ? 'team_id'" + ), + ), + Index( + "uq_external_chat_accounts_discord_guild", + "platform", + text("(cursor_state ->> 'guild_id')"), + unique=True, + postgresql_where=text( + "is_system_account = true AND cursor_state ? 'guild_id'" + ), + ), + Index( + "uq_external_chat_accounts_webhook_secret", + "webhook_secret", + unique=True, + postgresql_where=text("webhook_secret IS NOT NULL"), + ), + ) + + +class ExternalChatBinding(Base, TimestampMixin): + __tablename__ = "external_chat_bindings" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + account_id = Column( + BigInteger, + ForeignKey("external_chat_accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + state = Column( + SQLAlchemyEnum( + ExternalChatBindingState, + name="external_chat_binding_state", + values_callable=_enum_values, + ), + nullable=False, + default=ExternalChatBindingState.PENDING, + server_default=ExternalChatBindingState.PENDING.value, + ) + pairing_code = Column(Text, nullable=True) + pairing_code_expires_at = Column(TIMESTAMP(timezone=True), nullable=True) + external_peer_id = Column(Text, nullable=True) + external_peer_kind = Column( + SQLAlchemyEnum( + ExternalChatPeerKind, + name="external_chat_peer_kind", + values_callable=_enum_values, + ), + nullable=False, + default=ExternalChatPeerKind.UNKNOWN, + server_default=ExternalChatPeerKind.UNKNOWN.value, + ) + external_thread_id = Column(Text, nullable=True) + external_display_name = Column(Text, nullable=True) + external_username = Column(Text, nullable=True) + external_metadata = Column( + JSONB, nullable=False, default=dict, server_default=text("'{}'::jsonb") + ) + new_chat_thread_id = Column( + Integer, + ForeignKey("new_chat_threads.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + revoked_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_at = Column(TIMESTAMP(timezone=True), nullable=True) + suspended_reason = Column(Text, nullable=True) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + + account = relationship("ExternalChatAccount", back_populates="bindings") + user = relationship("User", foreign_keys=[user_id]) + search_space = relationship("SearchSpace", foreign_keys=[search_space_id]) + new_chat_thread = relationship("NewChatThread", foreign_keys=[new_chat_thread_id]) + threads = relationship( + "NewChatThread", + back_populates="external_chat_binding", + foreign_keys="NewChatThread.external_chat_binding_id", + ) + inbound_events = relationship( + "ExternalChatInboundEvent", + back_populates="binding", + foreign_keys="ExternalChatInboundEvent.external_chat_binding_id", + ) + + __table_args__ = ( + Index( + "uq_external_chat_bindings_account_peer_active", + "account_id", + "external_peer_id", + unique=True, + postgresql_where=text( + "state IN ('bound', 'suspended') AND external_peer_id IS NOT NULL" + ), + ), + Index( + "uq_external_chat_bindings_pairing_code_pending", + "pairing_code", + unique=True, + postgresql_where=text("state = 'pending'"), + ), + Index("ix_external_chat_bindings_user_state", "user_id", "state"), + Index( + "ix_external_chat_bindings_search_space_state", "search_space_id", "state" + ), + ) + + +class ExternalChatInboundEvent(Base, TimestampMixin): + __tablename__ = "external_chat_inbound_events" + __allow_unmapped__ = True + + id = Column(BigInteger, primary_key=True, index=True) + account_id = Column( + BigInteger, + ForeignKey("external_chat_accounts.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + external_chat_binding_id = Column( + BigInteger, + ForeignKey("external_chat_bindings.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + platform = Column( + SQLAlchemyEnum( + ExternalChatPlatform, + name="external_chat_platform", + values_callable=_enum_values, + ), + nullable=False, + ) + event_dedupe_key = Column(Text, nullable=False) + external_event_id = Column(Text, nullable=True) + external_message_id = Column(Text, nullable=True) + event_kind = Column( + SQLAlchemyEnum( + ExternalChatEventKind, + name="external_chat_event_kind", + values_callable=_enum_values, + ), + nullable=False, + ) + raw_payload = Column(JSONB, nullable=True) + request_id = Column(String(64), nullable=True) + status = Column( + SQLAlchemyEnum( + ExternalChatEventStatus, + name="external_chat_event_status", + values_callable=_enum_values, + ), + nullable=False, + default=ExternalChatEventStatus.RECEIVED, + server_default=ExternalChatEventStatus.RECEIVED.value, + ) + attempt_count = Column(Integer, nullable=False, default=0, server_default="0") + last_error = Column(Text, nullable=True) + received_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + server_default=text("(now() AT TIME ZONE 'utc')"), + ) + processed_at = Column(TIMESTAMP(timezone=True), nullable=True) + + account = relationship("ExternalChatAccount", back_populates="inbound_events") + binding = relationship("ExternalChatBinding", back_populates="inbound_events") + + __table_args__ = ( + UniqueConstraint( + "account_id", + "event_dedupe_key", + name="uq_external_chat_inbound_account_dedupe_key", + ), + Index("ix_external_chat_inbound_status_received_at", "status", "received_at"), + Index( + "ix_external_chat_inbound_binding_received_at", + "external_chat_binding_id", + "received_at", + ), + Index( + "ix_external_chat_inbound_request_id", + "request_id", + postgresql_where=text("request_id IS NOT NULL"), + ), + ) + + class TokenUsage(BaseModel, TimestampMixin): """ Tracks LLM token consumption per assistant turn. @@ -1111,6 +1492,11 @@ class Document(BaseModel, TimestampMixin): chunks = relationship( "Chunk", back_populates="document", cascade="all, delete-orphan" ) + # Original upload + future derived artifacts (redacted, filled-form). + # Model lives in app.file_storage.persistence to keep that feature cohesive. + files = relationship( + "DocumentFile", back_populates="document", cascade="all, delete-orphan" + ) class DocumentVersion(BaseModel, TimestampMixin): @@ -1409,9 +1795,6 @@ class SearchSpace(BaseModel, TimestampMixin): agent_llm_id = Column( Integer, nullable=True, default=0 ) # For agent/chat operations, defaults to Auto mode - document_summary_llm_id = Column( - Integer, nullable=True, default=0 - ) # For document summarization, defaults to Auto mode image_generation_config_id = Column( Integer, nullable=True, default=0 ) # For image generation, defaults to Auto mode @@ -1579,12 +1962,6 @@ class SearchSourceConnector(BaseModel, TimestampMixin): last_indexed_at = Column(TIMESTAMP(timezone=True), nullable=True) config = Column(JSON, nullable=False) - # Summary generation (LLM-based) - disabled by default to save resources. - # When enabled, improves hybrid search quality at the cost of LLM calls. - enable_summary = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Vision LLM for image files - disabled by default to save cost/time. # When enabled, images are described via a vision language model instead # of falling back to the document parser. @@ -1688,60 +2065,6 @@ class Log(BaseModel, TimestampMixin): search_space = relationship("SearchSpace", back_populates="logs") -class Notification(BaseModel, TimestampMixin): - __tablename__ = "notifications" - __table_args__ = ( - # Composite index for unread-count queries that filter by - # (user_id, read, type) and order by created_at. - Index( - "ix_notifications_user_read_type_created", - "user_id", - "read", - "type", - "created_at", - ), - # Covers the common list query: user_id + search_space_id + created_at DESC - Index( - "ix_notifications_user_space_created", - "user_id", - "search_space_id", - "created_at", - ), - ) - - user_id = Column( - UUID(as_uuid=True), - ForeignKey("user.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - search_space_id = Column( - Integer, - ForeignKey("searchspaces.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - type = Column( - String(50), nullable=False, index=True - ) # 'connector_indexing', 'document_processing', etc. - title = Column(String(200), nullable=False) - message = Column(Text, nullable=False) - read = Column( - Boolean, nullable=False, default=False, server_default=text("false"), index=True - ) - notification_metadata = Column("metadata", JSONB, nullable=True, default={}) - updated_at = Column( - TIMESTAMP(timezone=True), - nullable=True, - default=lambda: datetime.now(UTC), - onupdate=lambda: datetime.now(UTC), - index=True, - ) - - user = relationship("User", back_populates="notifications") - search_space = relationship("SearchSpace", back_populates="notifications") - - class UserIncentiveTask(BaseModel, TimestampMixin): """ Tracks completed incentive tasks for users. @@ -2564,6 +2887,8 @@ from app.automations.persistence import ( # noqa: E402, F401 AutomationRun, AutomationTrigger, ) +from app.file_storage.persistence import DocumentFile # noqa: E402, F401 +from app.notifications.persistence import Notification # noqa: E402, F401 engine = create_async_engine( DATABASE_URL, @@ -2599,7 +2924,7 @@ async def shielded_async_session(): async def setup_indexes(): async with engine.begin() as conn: # Create indexes - # Document Summary Indexes + # Document embedding indexes await conn.execute( text( "CREATE INDEX IF NOT EXISTS document_vector_index ON documents USING hnsw (embedding public.vector_cosine_ops)" diff --git a/surfsense_backend/app/file_storage/__init__.py b/surfsense_backend/app/file_storage/__init__.py new file mode 100644 index 000000000..756f5c7b2 --- /dev/null +++ b/surfsense_backend/app/file_storage/__init__.py @@ -0,0 +1,15 @@ +"""Durable storage for original uploaded files (and future derived artifacts). + +Public surface: resolve the configured backend via :func:`get_storage_backend` +and persist/retrieve a document's files via :mod:`app.file_storage.service`. +""" + +from __future__ import annotations + +from app.file_storage.backends.base import StorageBackend +from app.file_storage.factory import get_storage_backend + +__all__ = [ + "StorageBackend", + "get_storage_backend", +] diff --git a/surfsense_backend/app/file_storage/api.py b/surfsense_backend/app/file_storage/api.py new file mode 100644 index 000000000..c649ba63d --- /dev/null +++ b/surfsense_backend/app/file_storage/api.py @@ -0,0 +1,89 @@ +"""HTTP routes for document file storage (metadata listing + original download).""" + +from __future__ import annotations + +from urllib.parse import quote + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import StreamingResponse +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import Document, Permission, User, get_async_session +from app.file_storage.persistence.enums import DocumentFileKind +from app.file_storage.schemas import DocumentFileRead +from app.file_storage.service import ( + get_document_file, + list_document_files, + open_document_file_stream, +) +from app.users import current_active_user +from app.utils.rbac import check_permission + +router = APIRouter() + + +async def _load_readable_document( + *, document_id: int, session: AsyncSession, user: User +) -> Document: + """Load a document the user may read, or raise 404/403.""" + document = ( + await session.execute(select(Document).where(Document.id == document_id)) + ).scalar_one_or_none() + if document is None: + raise HTTPException(status_code=404, detail="Document not found") + + await check_permission( + session, + user, + document.search_space_id, + Permission.DOCUMENTS_READ.value, + "You don't have permission to read documents in this search space", + ) + return document + + +def _content_disposition(filename: str) -> str: + """Build an attachment header safe for arbitrary filenames (RFC 5987).""" + fallback = filename.encode("ascii", "ignore").decode("ascii") or "download" + fallback = fallback.replace('"', "") + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename)}" + + +@router.get( + "/documents/{document_id}/files", + response_model=list[DocumentFileRead], +) +async def read_document_files( + document_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +) -> list[DocumentFileRead]: + """Return metadata for every stored file of a document (gates the UI).""" + await _load_readable_document(document_id=document_id, session=session, user=user) + records = await list_document_files(session, document_id=document_id) + return [DocumentFileRead.model_validate(r) for r in records] + + +@router.get("/documents/{document_id}/download-original") +async def download_original_document_file( + document_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +) -> StreamingResponse: + """Stream the document's original uploaded file.""" + await _load_readable_document(document_id=document_id, session=session, user=user) + + record = await get_document_file( + session, document_id=document_id, kind=DocumentFileKind.ORIGINAL + ) + if record is None: + raise HTTPException( + status_code=404, detail="No original file stored for this document" + ) + + return StreamingResponse( + open_document_file_stream(record), + media_type=record.mime_type or "application/octet-stream", + headers={"Content-Disposition": _content_disposition(record.original_filename)}, + ) diff --git a/surfsense_backend/app/file_storage/backends/__init__.py b/surfsense_backend/app/file_storage/backends/__init__.py new file mode 100644 index 000000000..9e396f21c --- /dev/null +++ b/surfsense_backend/app/file_storage/backends/__init__.py @@ -0,0 +1,7 @@ +"""Storage backend implementations behind the shared :class:`StorageBackend`.""" + +from __future__ import annotations + +from app.file_storage.backends.base import StorageBackend + +__all__ = ["StorageBackend"] diff --git a/surfsense_backend/app/file_storage/backends/azure.py b/surfsense_backend/app/file_storage/backends/azure.py new file mode 100644 index 000000000..b82a47f0c --- /dev/null +++ b/surfsense_backend/app/file_storage/backends/azure.py @@ -0,0 +1,53 @@ +"""Azure Blob Storage backend (the first production target).""" + +from __future__ import annotations + +import contextlib +from collections.abc import AsyncIterator + +from app.file_storage.backends.base import StorageBackend + + +class AzureBlobBackend(StorageBackend): + """Stores objects as blobs in an Azure Blob Storage container.""" + + backend_name = "azure" + + def __init__(self, *, connection_string: str, container: str) -> None: + self._connection_string = connection_string + self._container = container + + def _service(self): + from azure.storage.blob.aio import BlobServiceClient + + return BlobServiceClient.from_connection_string(self._connection_string) + + async def put( + self, key: str, data: bytes, *, content_type: str | None = None + ) -> None: + from azure.storage.blob import ContentSettings + + settings = ContentSettings(content_type=content_type) if content_type else None + async with self._service() as service: + blob = service.get_blob_client(self._container, key) + await blob.upload_blob(data, overwrite=True, content_settings=settings) + + async def open_stream(self, key: str) -> AsyncIterator[bytes]: + async with self._service() as service: + blob = service.get_blob_client(self._container, key) + downloader = await blob.download_blob() + async for chunk in downloader.chunks(): + yield chunk + + async def delete(self, key: str) -> None: + from azure.core.exceptions import ResourceNotFoundError + + async with self._service() as service: + blob = service.get_blob_client(self._container, key) + with contextlib.suppress(ResourceNotFoundError): + await blob.delete_blob() + + async def exists(self, key: str) -> bool: + async with self._service() as service: + blob = service.get_blob_client(self._container, key) + return await blob.exists() diff --git a/surfsense_backend/app/file_storage/backends/base.py b/surfsense_backend/app/file_storage/backends/base.py new file mode 100644 index 000000000..c65375e9e --- /dev/null +++ b/surfsense_backend/app/file_storage/backends/base.py @@ -0,0 +1,31 @@ +"""The storage backend contract: the minimal object-store surface we depend on.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator + + +class StorageBackend(ABC): + """Maps an opaque object key to durable bytes.""" + + #: Identifier stored on each row to record which backend holds the bytes. + backend_name: str + + @abstractmethod + async def put( + self, key: str, data: bytes, *, content_type: str | None = None + ) -> None: + """Store ``data`` at ``key``, overwriting any existing object.""" + + @abstractmethod + def open_stream(self, key: str) -> AsyncIterator[bytes]: + """Yield the object's bytes in chunks. Raises if the key is absent.""" + + @abstractmethod + async def delete(self, key: str) -> None: + """Remove the object at ``key``; a missing key is not an error.""" + + @abstractmethod + async def exists(self, key: str) -> bool: + """Return whether an object is stored at ``key``.""" diff --git a/surfsense_backend/app/file_storage/backends/local.py b/surfsense_backend/app/file_storage/backends/local.py new file mode 100644 index 000000000..68bb8facd --- /dev/null +++ b/surfsense_backend/app/file_storage/backends/local.py @@ -0,0 +1,63 @@ +"""Local filesystem backend for development (no cloud credentials required).""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator +from pathlib import Path + +from app.file_storage.backends.base import StorageBackend + +_CHUNK_SIZE = 1024 * 1024 + + +class LocalFileBackend(StorageBackend): + """Stores objects as files under a single root directory.""" + + backend_name = "local" + + def __init__(self, root: str) -> None: + self._root = Path(root).resolve() + + def _path_for(self, key: str) -> Path: + # Resolve and confirm the key stays inside the root to block traversal. + target = (self._root / key).resolve() + if self._root not in target.parents and target != self._root: + raise ValueError("Resolved storage key escapes the storage root") + return target + + async def put( + self, key: str, data: bytes, *, content_type: str | None = None + ) -> None: + path = self._path_for(key) + + def _write() -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + await asyncio.to_thread(_write) + + async def open_stream(self, key: str) -> AsyncIterator[bytes]: + path = self._path_for(key) + handle = await asyncio.to_thread(path.open, "rb") + try: + while True: + chunk = await asyncio.to_thread(handle.read, _CHUNK_SIZE) + if not chunk: + break + yield chunk + finally: + await asyncio.to_thread(handle.close) + + async def delete(self, key: str) -> None: + path = self._path_for(key) + + def _unlink() -> None: + with contextlib.suppress(FileNotFoundError): + path.unlink() + + await asyncio.to_thread(_unlink) + + async def exists(self, key: str) -> bool: + return await asyncio.to_thread(self._path_for(key).exists) diff --git a/surfsense_backend/app/file_storage/factory.py b/surfsense_backend/app/file_storage/factory.py new file mode 100644 index 000000000..e30f6557b --- /dev/null +++ b/surfsense_backend/app/file_storage/factory.py @@ -0,0 +1,38 @@ +"""Resolve the configured :class:`StorageBackend` as a process-wide singleton.""" + +from __future__ import annotations + +from functools import lru_cache + +from app.file_storage.backends.base import StorageBackend +from app.file_storage.settings import ( + AZURE_BACKEND, + LOCAL_BACKEND, + load_storage_settings, +) + + +@lru_cache(maxsize=1) +def get_storage_backend() -> StorageBackend: + """Build the backend selected by ``FILE_STORAGE_BACKEND`` (lazy-imported).""" + settings = load_storage_settings() + + if settings.backend == AZURE_BACKEND: + if not settings.azure_connection_string or not settings.azure_container: + raise ValueError( + "Azure storage requires AZURE_STORAGE_CONNECTION_STRING and " + "AZURE_STORAGE_CONTAINER." + ) + from app.file_storage.backends.azure import AzureBlobBackend + + return AzureBlobBackend( + connection_string=settings.azure_connection_string, + container=settings.azure_container, + ) + + if settings.backend == LOCAL_BACKEND: + from app.file_storage.backends.local import LocalFileBackend + + return LocalFileBackend(settings.local_root) + + raise ValueError(f"Unknown FILE_STORAGE_BACKEND: {settings.backend!r}") diff --git a/surfsense_backend/app/file_storage/keys.py b/surfsense_backend/app/file_storage/keys.py new file mode 100644 index 000000000..22eaa9473 --- /dev/null +++ b/surfsense_backend/app/file_storage/keys.py @@ -0,0 +1,27 @@ +"""Object-key construction for stored document files.""" + +from __future__ import annotations + +import os +import uuid + +from app.file_storage.persistence.enums import DocumentFileKind + + +def build_document_file_key( + *, + search_space_id: int, + document_id: int, + kind: DocumentFileKind, + filename: str, +) -> str: + """Build the storage key for one document file. + + Shape: ``documents/{search_space_id}/{document_id}/{kind}/{uuid}{ext}``. + """ + extension = os.path.splitext(filename)[1].lower() + unique = uuid.uuid4().hex + return ( + f"documents/{search_space_id}/{document_id}/" + f"{kind.value.lower()}/{unique}{extension}" + ) diff --git a/surfsense_backend/app/file_storage/persistence/__init__.py b/surfsense_backend/app/file_storage/persistence/__init__.py new file mode 100644 index 000000000..4664da737 --- /dev/null +++ b/surfsense_backend/app/file_storage/persistence/__init__.py @@ -0,0 +1,11 @@ +"""Models and enums for the document file-storage tables.""" + +from __future__ import annotations + +from .enums import DocumentFileKind +from .models import DocumentFile + +__all__ = [ + "DocumentFile", + "DocumentFileKind", +] diff --git a/surfsense_backend/app/file_storage/persistence/enums.py b/surfsense_backend/app/file_storage/persistence/enums.py new file mode 100644 index 000000000..368dc8a9e --- /dev/null +++ b/surfsense_backend/app/file_storage/persistence/enums.py @@ -0,0 +1,11 @@ +"""DocumentFile kinds: the original upload plus future derived artifacts.""" + +from __future__ import annotations + +from enum import StrEnum + + +class DocumentFileKind(StrEnum): + ORIGINAL = "ORIGINAL" + REDACTED = "REDACTED" + FILLED_FORM = "FILLED_FORM" diff --git a/surfsense_backend/app/file_storage/persistence/models.py b/surfsense_backend/app/file_storage/persistence/models.py new file mode 100644 index 000000000..7433f35ed --- /dev/null +++ b/surfsense_backend/app/file_storage/persistence/models.py @@ -0,0 +1,66 @@ +"""``document_files`` table: durable blobs associated with a document.""" + +from __future__ import annotations + +from sqlalchemy import ( + BigInteger, + Column, + Enum as SQLAlchemyEnum, + ForeignKey, + Integer, + String, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship + +from app.db import BaseModel, TimestampMixin + +from .enums import DocumentFileKind + + +class DocumentFile(BaseModel, TimestampMixin): + """One stored file for a document (its original upload, or a derived copy).""" + + __tablename__ = "document_files" + + document_id = Column( + Integer, + ForeignKey("documents.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + search_space_id = Column( + Integer, + ForeignKey("searchspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + kind = Column( + SQLAlchemyEnum( + DocumentFileKind, + name="document_file_kind", + values_callable=lambda x: [e.value for e in x], + ), + nullable=False, + default=DocumentFileKind.ORIGINAL, + server_default=DocumentFileKind.ORIGINAL.value, + index=True, + ) + + # Where the bytes live: the backend that stored them and its object key. + storage_backend = Column(String(32), nullable=False) + storage_key = Column(String, nullable=False) + + original_filename = Column(String, nullable=False) + mime_type = Column(String, nullable=True) + size_bytes = Column(BigInteger, nullable=False) + checksum_sha256 = Column(String(64), nullable=True) + + created_by_id = Column( + UUID(as_uuid=True), + ForeignKey("user.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + document = relationship("Document", back_populates="files") diff --git a/surfsense_backend/app/file_storage/schemas.py b/surfsense_backend/app/file_storage/schemas.py new file mode 100644 index 000000000..21fd1a3f6 --- /dev/null +++ b/surfsense_backend/app/file_storage/schemas.py @@ -0,0 +1,23 @@ +"""API shapes for document file metadata.""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, ConfigDict + +from app.file_storage.persistence.enums import DocumentFileKind + + +class DocumentFileRead(BaseModel): + """Lightweight metadata for one stored document file (no bytes).""" + + id: int + document_id: int + kind: DocumentFileKind + original_filename: str + mime_type: str | None = None + size_bytes: int + created_at: datetime + + model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/file_storage/service.py b/surfsense_backend/app/file_storage/service.py new file mode 100644 index 000000000..bdadcfca3 --- /dev/null +++ b/surfsense_backend/app/file_storage/service.py @@ -0,0 +1,129 @@ +"""Application service: persist, locate, and remove a document's stored files. + +Coordinates the storage backend (bytes) with the ``document_files`` table +(metadata). Callers own the surrounding DB transaction/commit. +""" + +from __future__ import annotations + +import hashlib +import logging +from collections.abc import AsyncIterator, Sequence +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.file_storage.backends.base import StorageBackend +from app.file_storage.factory import get_storage_backend +from app.file_storage.keys import build_document_file_key +from app.file_storage.persistence.enums import DocumentFileKind +from app.file_storage.persistence.models import DocumentFile + +logger = logging.getLogger(__name__) + + +async def store_document_file( + session: AsyncSession, + *, + document_id: int, + search_space_id: int, + data: bytes, + filename: str, + mime_type: str | None = None, + kind: DocumentFileKind = DocumentFileKind.ORIGINAL, + created_by_id: str | UUID | None = None, + backend: StorageBackend | None = None, +) -> DocumentFile: + """Write bytes to storage and add a ``DocumentFile`` row to the session.""" + backend = backend or get_storage_backend() + key = build_document_file_key( + search_space_id=search_space_id, + document_id=document_id, + kind=kind, + filename=filename, + ) + await backend.put(key, data, content_type=mime_type) + + record = DocumentFile( + document_id=document_id, + search_space_id=search_space_id, + kind=kind, + storage_backend=backend.backend_name, + storage_key=key, + original_filename=filename, + mime_type=mime_type, + size_bytes=len(data), + checksum_sha256=hashlib.sha256(data).hexdigest(), + created_by_id=created_by_id, + ) + session.add(record) + return record + + +async def list_document_files( + session: AsyncSession, *, document_id: int +) -> list[DocumentFile]: + """Return all stored files for a document, newest first.""" + result = await session.execute( + select(DocumentFile) + .where(DocumentFile.document_id == document_id) + .order_by(DocumentFile.created_at.desc()) + ) + return list(result.scalars().all()) + + +async def get_document_file( + session: AsyncSession, + *, + document_id: int, + kind: DocumentFileKind = DocumentFileKind.ORIGINAL, +) -> DocumentFile | None: + """Return the most recent stored file of ``kind`` for a document.""" + result = await session.execute( + select(DocumentFile) + .where( + DocumentFile.document_id == document_id, + DocumentFile.kind == kind, + ) + .order_by(DocumentFile.created_at.desc()) + ) + return result.scalars().first() + + +def open_document_file_stream( + record: DocumentFile, *, backend: StorageBackend | None = None +) -> AsyncIterator[bytes]: + """Open a chunked byte stream for a stored file.""" + backend = backend or get_storage_backend() + return backend.open_stream(record.storage_key) + + +async def purge_document_blobs( + session: AsyncSession, + *, + document_ids: Sequence[int], + backend: StorageBackend | None = None, +) -> None: + """Delete stored blobs for the given documents. + + Call this before the ``document_files`` rows are removed (they cascade with + the document). Best-effort: a failed blob delete is logged, not raised, so + document deletion is never blocked by an orphaned blob. + """ + if not document_ids: + return + + backend = backend or get_storage_backend() + result = await session.execute( + select(DocumentFile.storage_key).where( + DocumentFile.document_id.in_(document_ids) + ) + ) + for storage_key in result.scalars().all(): + try: + await backend.delete(storage_key) + except Exception as delete_error: + logger.warning( + "Failed to delete stored blob %s: %s", storage_key, delete_error + ) diff --git a/surfsense_backend/app/file_storage/settings.py b/surfsense_backend/app/file_storage/settings.py new file mode 100644 index 000000000..2e0b08268 --- /dev/null +++ b/surfsense_backend/app/file_storage/settings.py @@ -0,0 +1,33 @@ +"""Configuration for the file-storage module, sourced from the central Config.""" + +from __future__ import annotations + +from dataclasses import dataclass + +LOCAL_BACKEND = "local" +AZURE_BACKEND = "azure" + + +@dataclass(frozen=True) +class StorageSettings: + """Resolved storage configuration for the current process.""" + + backend: str + azure_connection_string: str | None + azure_container: str | None + local_root: str + + +def load_storage_settings() -> StorageSettings: + """Resolve storage settings from the central ``Config`` singleton. + + Defaults to the ``local`` backend so development needs no cloud creds. + """ + from app.config import config + + return StorageSettings( + backend=config.FILE_STORAGE_BACKEND, + azure_connection_string=config.AZURE_STORAGE_CONNECTION_STRING, + azure_container=config.AZURE_STORAGE_CONTAINER, + local_root=config.FILE_STORAGE_LOCAL_PATH, + ) diff --git a/surfsense_backend/app/gateway/__init__.py b/surfsense_backend/app/gateway/__init__.py new file mode 100644 index 000000000..8b79b3160 --- /dev/null +++ b/surfsense_backend/app/gateway/__init__.py @@ -0,0 +1,19 @@ +"""Messaging gateway infrastructure for external chat channels.""" + +from __future__ import annotations + +from fastapi import HTTPException, status + +from app.config import config + + +def require_gateway_enabled() -> None: + """FastAPI dependency that gates all gateway HTTP routes on the global flag. + + Returns 404 (rather than 503) when ``GATEWAY_ENABLED`` is FALSE so that + disabling the gateway makes its webhook/OAuth/pairing surface indistinguishable + from a route that does not exist. + """ + + if not config.GATEWAY_ENABLED: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not Found") diff --git a/surfsense_backend/app/gateway/accounts.py b/surfsense_backend/app/gateway/accounts.py new file mode 100644 index 000000000..8cbf748b6 --- /dev/null +++ b/surfsense_backend/app/gateway/accounts.py @@ -0,0 +1,141 @@ +"""External chat account helpers.""" + +from __future__ import annotations + +import json + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatHealthStatus, + ExternalChatPlatform, +) +from app.utils.oauth_security import TokenEncryption + + +def account_token(account: ExternalChatAccount) -> str | None: + if account.is_system_account and account.platform == ExternalChatPlatform.TELEGRAM: + return config.TELEGRAM_SHARED_BOT_TOKEN + if not account.encrypted_credentials: + return None + return TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) + + +def slack_account_credentials(account: ExternalChatAccount) -> dict: + """Decrypt Slack gateway credentials stored as encrypted JSON.""" + if not account.encrypted_credentials: + return {} + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Backward-compatible fallback if a token string was stored directly. + return {"bot_token": raw} + return data if isinstance(data, dict) else {} + + +def discord_account_credentials(account: ExternalChatAccount) -> dict: + """Decrypt Discord gateway credentials stored as encrypted JSON.""" + if not account.encrypted_credentials: + return {} + raw = TokenEncryption(config.SECRET_KEY or "").decrypt_token( + account.encrypted_credentials + ) + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Backward-compatible fallback if a token string was stored directly. + return {"bot_token": raw} + return data if isinstance(data, dict) else {} + + +async def get_or_create_system_telegram_account( + session: AsyncSession, +) -> ExternalChatAccount: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, + ExternalChatAccount.is_system_account.is_(True), + ) + ) + account = result.scalars().first() + if account is not None: + return account + account = ExternalChatAccount( + platform=ExternalChatPlatform.TELEGRAM, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + bot_username=config.TELEGRAM_SHARED_BOT_USERNAME, + webhook_secret=config.TELEGRAM_WEBHOOK_SECRET, + cursor_state={}, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + return account + + +async def get_or_create_system_whatsapp_account( + session: AsyncSession, +) -> ExternalChatAccount: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.is_system_account.is_(True), + ) + ) + account = result.scalars().first() + if account is not None: + return account + account = ExternalChatAccount( + platform=ExternalChatPlatform.WHATSAPP, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + cursor_state={ + "phone_number_id": config.WHATSAPP_SHARED_PHONE_NUMBER_ID, + "display_phone_number": config.WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER, + "waba_id": config.WHATSAPP_SHARED_WABA_ID, + }, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + return account + + +async def get_slack_account_by_team( + session: AsyncSession, + *, + team_id: str, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.SLACK, + ExternalChatAccount.is_system_account.is_(True), + ExternalChatAccount.cursor_state["team_id"].astext == team_id, + ) + ) + return result.scalars().first() + + +async def get_discord_account_by_guild( + session: AsyncSession, + *, + guild_id: str, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.DISCORD, + ExternalChatAccount.is_system_account.is_(True), + ExternalChatAccount.cursor_state["guild_id"].astext == guild_id, + ) + ) + return result.scalars().first() diff --git a/surfsense_backend/app/gateway/agent_invoke.py b/surfsense_backend/app/gateway/agent_invoke.py new file mode 100644 index 000000000..8701ccc55 --- /dev/null +++ b/surfsense_backend/app/gateway/agent_invoke.py @@ -0,0 +1,102 @@ +"""Invoke SurfSense chat agent for external chat surfaces.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import AsyncIterator + +from sqlalchemy import update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ExternalChatBinding, NewChatMessage +from app.gateway.auth_invariant import assert_authorization_invariant +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.bindings import get_or_create_thread_for_binding +from app.gateway.hitl_filter import DEFAULT_HITL_TOOL_NAMES +from app.gateway.thread_lock import acquire_thread_lock, release_thread_lock +from app.observability.metrics import record_gateway_turn_latency +from app.tasks.chat.streaming.flows import stream_new_chat + +logger = logging.getLogger(__name__) + + +async def _events_from_sse( + chunks: AsyncIterator[str], +) -> AsyncIterator[GatewayStreamEvent]: + saw_text = False + async for chunk in chunks: + for raw_line in chunk.splitlines(): + line = raw_line.strip() + if not line.startswith("data:"): + continue + payload = line.removeprefix("data:").strip() + if payload == "[DONE]": + logger.info("Gateway SSE normalized: done") + yield GatewayStreamEvent(type="done") + continue + try: + data = json.loads(payload) + except json.JSONDecodeError: + continue + event_type = str(data.get("type") or "") + if event_type == "text-delta": + delta = data.get("delta", "") + if delta and not saw_text: + logger.info("Gateway SSE normalized: text stream started") + saw_text = True + yield GatewayStreamEvent(type="text-delta", data={"delta": delta}) + elif event_type in {"finish", "done"}: + logger.info("Gateway SSE normalized: %s", event_type) + yield GatewayStreamEvent(type="finish", data=data) + elif event_type == "data-interrupt-request": + logger.info("Gateway SSE normalized: interrupt request") + yield GatewayStreamEvent(type="data-interrupt-request", data=data) + + +async def call_agent_for_gateway( + *, + session: AsyncSession, + binding: ExternalChatBinding, + user_text: str, + translator: BaseStreamTranslator, + platform_label: str = "telegram", + request_id: str | None = None, +) -> None: + user = await assert_authorization_invariant(session, binding) + thread = await get_or_create_thread_for_binding(session, binding) + await session.commit() + + if not acquire_thread_lock(thread.id): + raise RuntimeError("gateway_thread_busy") + + try: + stream = stream_new_chat( + user_query=user_text, + search_space_id=binding.search_space_id, + chat_id=thread.id, + user_id=str(user.id), + needs_history_bootstrap=thread.needs_history_bootstrap, + thread_visibility=thread.visibility, + current_user_display_name=user.display_name or "A team member", + disabled_tools=sorted(DEFAULT_HITL_TOOL_NAMES), + request_id=request_id or "gateway", + ) + events = _events_from_sse(stream) + try: + await translator.translate(events) + finally: + await events.aclose() + await stream.aclose() + await session.execute( + update(NewChatMessage) + .where( + NewChatMessage.thread_id == thread.id, + NewChatMessage.source == "surfsense", + ) + .values(source=platform_label) + ) + await session.commit() + record_gateway_turn_latency(0, platform=platform_label) + finally: + release_thread_lock(thread.id) diff --git a/surfsense_backend/app/gateway/auth_invariant.py b/surfsense_backend/app/gateway/auth_invariant.py new file mode 100644 index 000000000..e72023ce1 --- /dev/null +++ b/surfsense_backend/app/gateway/auth_invariant.py @@ -0,0 +1,54 @@ +"""Authorization invariants for external-chat-routed turns.""" + +from __future__ import annotations + +from fastapi import HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ExternalChatBinding, Permission, User +from app.gateway.bindings import suspend_binding +from app.observability.metrics import record_gateway_auth_invariant_failure +from app.utils.rbac import check_permission, check_search_space_access + + +class GatewaySuspendedError(RuntimeError): + def __init__(self, reason: str) -> None: + self.reason = reason + super().__init__(reason) + + +async def _fail( + session: AsyncSession, + binding: ExternalChatBinding, + reason: str, +) -> None: + suspend_binding(binding, reason) + record_gateway_auth_invariant_failure(cause=reason) + await session.flush() + raise GatewaySuspendedError(reason) + + +async def assert_authorization_invariant( + session: AsyncSession, + binding: ExternalChatBinding, +) -> User: + if binding.state != "bound": + await _fail(session, binding, "binding_not_bound") + + user = await session.get(User, binding.user_id) + if user is None: + await _fail(session, binding, "owner_missing") + + try: + await check_search_space_access(session, user, binding.search_space_id) + await check_permission( + session, + user, + binding.search_space_id, + Permission.CHATS_CREATE.value, + "External chat owner no longer has permission to chat in this search space", + ) + except HTTPException as exc: + await _fail(session, binding, f"rbac_{exc.status_code}") + + return user diff --git a/surfsense_backend/app/gateway/base/__init__.py b/surfsense_backend/app/gateway/base/__init__.py new file mode 100644 index 000000000..f27e3e087 --- /dev/null +++ b/surfsense_backend/app/gateway/base/__init__.py @@ -0,0 +1 @@ +"""Base gateway interfaces.""" diff --git a/surfsense_backend/app/gateway/base/adapter.py b/surfsense_backend/app/gateway/base/adapter.py new file mode 100644 index 000000000..dfe896b4a --- /dev/null +++ b/surfsense_backend/app/gateway/base/adapter.py @@ -0,0 +1,71 @@ +"""Platform adapter interfaces for messaging gateways.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class ParsedInboundEvent: + platform: str + event_kind: str + external_peer_id: str | None + external_peer_kind: str + external_message_id: str | None + external_user_id: str | None + text: str | None + raw_payload: dict[str, Any] + display_name: str | None = None + username: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class PlatformSendResult: + external_message_id: str + raw_response: dict[str, Any] = field(default_factory=dict) + + +class BasePlatformAdapter(ABC): + platform: str + + @abstractmethod + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + """Parse a provider webhook/update into the gateway's normalized shape.""" + + @abstractmethod + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + """Send a new platform message.""" + + @abstractmethod + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + """Edit an existing platform message.""" + + @abstractmethod + async def validate_credentials(self) -> dict[str, Any]: + """Validate configured credentials and return account metadata.""" + + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: + """Yield provider updates for long-polling adapters.""" + if False: + yield {} # pragma: no cover + raise NotImplementedError("This adapter does not support long-polling") diff --git a/surfsense_backend/app/gateway/base/commands.py b/surfsense_backend/app/gateway/base/commands.py new file mode 100644 index 000000000..ea5d09e20 --- /dev/null +++ b/surfsense_backend/app/gateway/base/commands.py @@ -0,0 +1,41 @@ +"""Provider-neutral command hooks for external chat gateways.""" + +from __future__ import annotations + +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent + + +def command_name(text: str | None) -> str | None: + if not text or not text.startswith("/"): + return None + return text.split(maxsplit=1)[0].split("@", 1)[0].lower() + + +class BaseGatewayCommands: + """Default command behavior for platforms without slash-command onboarding.""" + + async def handle_start_command( + self, + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return False + + async def handle_help_command( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return False + + async def send_unbound_onboarding( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + return None diff --git a/surfsense_backend/app/gateway/base/formatting.py b/surfsense_backend/app/gateway/base/formatting.py new file mode 100644 index 000000000..d0ea6a52d --- /dev/null +++ b/surfsense_backend/app/gateway/base/formatting.py @@ -0,0 +1,38 @@ +"""Provider-neutral message formatting helpers.""" + +from __future__ import annotations + +MAX_GATEWAY_TEXT_CHARS = 4096 + + +def split_text_message( + text: str, + *, + max_chars: int = MAX_GATEWAY_TEXT_CHARS, +) -> list[str]: + """Split outbound text at readable boundaries without exceeding platform caps.""" + if not text: + return [""] + + chunks: list[str] = [] + remaining = text + while remaining: + if len(remaining) <= max_chars: + chunks.append(remaining) + break + + candidate = remaining[:max_chars] + boundary = max( + candidate.rfind("\n\n"), + candidate.rfind("\n"), + candidate.rfind(". "), + candidate.rfind(" "), + ) + if boundary <= max(200, max_chars // 2): + boundary = max_chars + split_at = boundary + (2 if candidate[boundary : boundary + 2] == ". " else 1) + chunk = remaining[:split_at].rstrip() + chunks.append(chunk or remaining[:max_chars]) + remaining = remaining[split_at:].lstrip() + + return chunks diff --git a/surfsense_backend/app/gateway/base/identity.py b/surfsense_backend/app/gateway/base/identity.py new file mode 100644 index 000000000..e445bcbbb --- /dev/null +++ b/surfsense_backend/app/gateway/base/identity.py @@ -0,0 +1,18 @@ +"""Gateway identity helpers.""" + +from __future__ import annotations + +import hashlib + + +def normalize_external_peer_id(value: str | int | None) -> str | None: + if value is None: + return None + return str(value).strip() + + +def hash_external_id(value: str | int | None) -> str | None: + normalized = normalize_external_peer_id(value) + if not normalized: + return None + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() diff --git a/surfsense_backend/app/gateway/base/translator.py b/surfsense_backend/app/gateway/base/translator.py new file mode 100644 index 000000000..2476adb2c --- /dev/null +++ b/surfsense_backend/app/gateway/base/translator.py @@ -0,0 +1,27 @@ +"""Base stream translator for platform-specific outbound UX.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class GatewayStreamEvent: + """Small provider-neutral event shape consumed by translators. + + The existing chat stack emits Vercel/assistant-ui events. Gateway code + normalizes the subset it needs into this shape before handing it to the + platform translator. + """ + + type: str + data: dict[str, Any] = field(default_factory=dict) + + +class BaseStreamTranslator(ABC): + @abstractmethod + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + """Consume agent stream events and emit platform messages.""" diff --git a/surfsense_backend/app/gateway/bindings.py b/surfsense_backend/app/gateway/bindings.py new file mode 100644 index 000000000..821dd21ca --- /dev/null +++ b/surfsense_backend/app/gateway/bindings.py @@ -0,0 +1,66 @@ +"""External chat binding helpers.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ( + ChatVisibility, + ExternalChatBinding, + ExternalChatBindingState, + NewChatThread, +) + + +async def get_or_create_thread_for_binding( + session: AsyncSession, + binding: ExternalChatBinding, +) -> NewChatThread: + if binding.new_chat_thread_id is not None: + result = await session.execute( + select(NewChatThread).where(NewChatThread.id == binding.new_chat_thread_id) + ) + thread = result.scalars().first() + if thread is not None and not thread.archived: + return thread + + source = str((binding.external_metadata or {}).get("platform") or "").strip() + if not source: + kind = str((binding.external_metadata or {}).get("kind") or "") + source = "slack" if kind.startswith("slack_") else "telegram" + + thread = NewChatThread( + title=f"{source.title()} chat", + search_space_id=binding.search_space_id, + created_by_id=binding.user_id, + visibility=ChatVisibility.PRIVATE, + source=source, + external_chat_binding_id=binding.id, + ) + session.add(thread) + await session.flush() + binding.new_chat_thread_id = thread.id + return thread + + +def suspend_binding(binding: ExternalChatBinding, reason: str) -> None: + now = datetime.now(UTC) + binding.state = ExternalChatBindingState.SUSPENDED + binding.suspended_at = now + binding.suspended_reason = reason + + +def revoke_binding(binding: ExternalChatBinding) -> None: + now = datetime.now(UTC) + binding.state = ExternalChatBindingState.REVOKED + binding.revoked_at = now + binding.new_chat_thread_id = None + + +def resume_binding(binding: ExternalChatBinding) -> None: + binding.state = ExternalChatBindingState.BOUND + binding.suspended_at = None + binding.suspended_reason = None diff --git a/surfsense_backend/app/gateway/byo_long_poll.py b/surfsense_backend/app/gateway/byo_long_poll.py new file mode 100644 index 000000000..e3f3ec093 --- /dev/null +++ b/surfsense_backend/app/gateway/byo_long_poll.py @@ -0,0 +1,166 @@ +"""FastAPI lifespan integration for self-hosted BYO Telegram long-polling.""" + +from __future__ import annotations + +import asyncio +import logging + +from sqlalchemy import select + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatPlatform, + async_session_maker, +) +from app.gateway.accounts import account_token +from app.gateway.inbox import persist_inbound_event +from app.gateway.runner import _run_telegram_account +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.observability.metrics import record_gateway_inbox_write + +logger = logging.getLogger(__name__) + +_tasks: set[asyncio.Task[None]] = set() +_shutdown_event: asyncio.Event | None = None + + +async def _sleep_or_shutdown(seconds: float) -> None: + if _shutdown_event is None: + await asyncio.sleep(seconds) + return + try: + await asyncio.wait_for(_shutdown_event.wait(), timeout=seconds) + except TimeoutError: + return + + +async def _byo_account_supervisor(account_id: int, token: str) -> None: + while _shutdown_event is None or not _shutdown_event.is_set(): + try: + await _run_telegram_account(account_id, token) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "BYO Telegram long-poll failed account_id=%s; retrying in 30s", + account_id, + ) + await _sleep_or_shutdown(30) + + +async def _whatsapp_baileys_supervisor() -> None: + adapter = WhatsAppBaileysAdapter() + while _shutdown_event is None or not _shutdown_event.is_set(): + try: + async for raw_event in adapter.fetch_updates(offset=None): + async with async_session_maker() as session: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform + == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode + == ExternalChatAccountMode.SELF_HOST_BYO, + ExternalChatAccount.is_system_account.is_(False), + ExternalChatAccount.suspended_at.is_(None), + ) + ) + account = result.scalars().first() + if account is None: + continue + message_id = str(raw_event.get("messageId") or "") + if not message_id: + continue + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.WHATSAPP, + event_dedupe_key=f"baileys:{message_id}", + external_event_id=message_id, + external_message_id=message_id, + event_kind="message", + raw_payload=raw_event, + ) + await session.commit() + record_gateway_inbox_write( + platform="whatsapp", + dedup_skipped=inbox_id is None, + ) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("WhatsApp Baileys intake failed; retrying in 10s") + await _sleep_or_shutdown(10) + + +async def start_byo_long_poll_supervisors() -> None: + """Start one BYO long-poll supervisor per active non-system Telegram account.""" + + global _shutdown_event + if not config.GATEWAY_ENABLED: + return + if ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "longpoll" + and config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys" + ): + return + if _tasks: + return + + _shutdown_event = asyncio.Event() + if config.GATEWAY_TELEGRAM_INTAKE_MODE == "longpoll": + async with async_session_maker() as session: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.platform == ExternalChatPlatform.TELEGRAM, + ExternalChatAccount.is_system_account.is_(False), + ExternalChatAccount.suspended_at.is_(None), + ) + ) + accounts = list(result.scalars()) + + for account in accounts: + token = account_token(account) + if not token: + continue + task = asyncio.create_task( + _byo_account_supervisor(int(account.id), token), + name=f"gateway-byo-telegram-{account.id}", + ) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + logger.info( + "Started BYO Telegram long-poll supervisor account_id=%s", account.id + ) + + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": + task = asyncio.create_task( + _whatsapp_baileys_supervisor(), + name="gateway-byo-whatsapp-baileys", + ) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + logger.info("Started WhatsApp Baileys bridge intake supervisor") + + +async def stop_byo_long_poll_supervisors() -> None: + """Cancel and await all BYO long-poll supervisors.""" + + global _shutdown_event + if _shutdown_event is not None: + _shutdown_event.set() + tasks = list(_tasks) + for task in tasks: + task.cancel() + if tasks: + try: + await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), timeout=10 + ) + except TimeoutError: + logger.warning( + "Timed out waiting for BYO Telegram long-poll supervisors to stop" + ) + _tasks.clear() + _shutdown_event = None diff --git a/surfsense_backend/app/gateway/discord/__init__.py b/surfsense_backend/app/gateway/discord/__init__.py new file mode 100644 index 000000000..1dd0edc96 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/__init__.py @@ -0,0 +1 @@ +"""Discord gateway platform integration.""" diff --git a/surfsense_backend/app/gateway/discord/adapter.py b/surfsense_backend/app/gateway/discord/adapter.py new file mode 100644 index 000000000..60db895fe --- /dev/null +++ b/surfsense_backend/app/gateway/discord/adapter.py @@ -0,0 +1,135 @@ +"""Discord platform adapter for bot mentions and replies.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.discord.client import DiscordGatewayClient + +MENTION_RE = re.compile(r"<@!?\d+>\s*") + + +def discord_user_peer_id(guild_id: str, discord_user_id: str) -> str: + return f"discord_user:{guild_id}:{discord_user_id}" + + +def discord_thread_peer_id(guild_id: str, channel_id: str, thread_key: str) -> str: + return f"discord_thread:{guild_id}:{channel_id}:{thread_key}" + + +class DiscordAdapter(BasePlatformAdapter): + platform = "discord" + + def __init__(self, bot_token: str, *, bot_user_id: str | None = None) -> None: + self.bot_user_id = bot_user_id + self.client = DiscordGatewayClient(bot_token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event = raw_payload.get("event") or raw_payload + event_kind = str(raw_payload.get("type") or event.get("type") or "message") + guild_id = str(event.get("guild_id") or "") + channel_id = str(event.get("channel_id") or "") + author = event.get("author") or {} + discord_user_id = str(author.get("id") or event.get("author_id") or "") + message_id = str(event.get("id") or event.get("message_id") or "") + bot_user_id = self.bot_user_id or str(raw_payload.get("bot_user_id") or "") + + if not guild_id or not channel_id or not discord_user_id or not message_id: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=message_id or None, + external_user_id=discord_user_id or None, + text=None, + raw_payload=raw_payload, + metadata={ + "guild_id": guild_id, + "channel_id": channel_id, + "bot_user_id": bot_user_id, + }, + ) + + text = str(event.get("content") or "") + if bot_user_id: + text = text.replace(f"<@{bot_user_id}>", "") + text = text.replace(f"<@!{bot_user_id}>", "") + text = MENTION_RE.sub("", text).strip() + + thread_key = str( + event.get("thread_id") + or (event.get("message_reference") or {}).get("message_id") + or message_id + ) + thread_peer_id = discord_thread_peer_id(guild_id, channel_id, thread_key) + user_peer_id = discord_user_peer_id(guild_id, discord_user_id) + mentions = event.get("mentions") or [] + mentions_bot = bool( + bot_user_id + and any(str(mention.get("id")) == bot_user_id for mention in mentions) + ) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=thread_peer_id, + external_peer_kind="channel", + external_message_id=message_id, + external_user_id=discord_user_id, + text=text, + raw_payload=raw_payload, + display_name=event.get("channel_name"), + username=author.get("username") or discord_user_id, + metadata={ + "guild_id": guild_id, + "channel_id": channel_id, + "discord_user_id": discord_user_id, + "message_id": message_id, + "thread_key": thread_key, + "bot_user_id": bot_user_id, + "discord_user_peer_id": user_peer_id, + "discord_thread_peer_id": thread_peer_id, + "mentions_bot": mentions_bot, + "is_dm": False, + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.send_message( + channel_id=external_peer_id, + content=text, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.update_message( + channel_id=external_peer_id, + message_id=external_message_id, + content=text, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() diff --git a/surfsense_backend/app/gateway/discord/client.py b/surfsense_backend/app/gateway/discord/client.py new file mode 100644 index 000000000..206abaa5f --- /dev/null +++ b/surfsense_backend/app/gateway/discord/client.py @@ -0,0 +1,109 @@ +"""Discord REST API client for gateway bot operations.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx + +from app.gateway.base.adapter import PlatformSendResult + +DISCORD_API = "https://discord.com/api/v10" + + +class DiscordGatewayClient: + def __init__(self, bot_token: str) -> None: + self.bot_token = bot_token + + async def api_call( + self, + method: str, + path: str, + *, + payload: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + retry_rate_limit: bool = True, + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.request( + method, + f"{DISCORD_API}{path}", + json=payload, + params=params, + headers={ + "Authorization": f"Bot {self.bot_token}", + "Content-Type": "application/json", + }, + ) + if response.status_code == 429 and retry_rate_limit: + data = response.json() + retry_after = float(data.get("retry_after") or 1.0) + await asyncio.sleep(min(retry_after, 5.0)) + return await self.api_call( + method, + path, + payload=payload, + params=params, + retry_rate_limit=False, + ) + response.raise_for_status() + if not response.content: + return {} + return response.json() + + async def send_message( + self, + *, + channel_id: str, + content: str, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = { + "content": content, + "allowed_mentions": {"parse": []}, + } + if reply_to_message_id: + payload["message_reference"] = { + "message_id": reply_to_message_id, + "channel_id": channel_id, + "fail_if_not_exists": False, + } + data = await self.api_call( + "POST", + f"/channels/{channel_id}/messages", + payload=payload, + ) + return PlatformSendResult( + external_message_id=str(data.get("id", "")), + raw_response=data, + ) + + async def update_message( + self, + *, + channel_id: str, + message_id: str, + content: str, + ) -> PlatformSendResult: + data = await self.api_call( + "PATCH", + f"/channels/{channel_id}/messages/{message_id}", + payload={"content": content, "allowed_mentions": {"parse": []}}, + ) + return PlatformSendResult( + external_message_id=str(data.get("id") or message_id), + raw_response=data, + ) + + async def validate(self) -> dict[str, Any]: + data = await self.api_call("GET", "/users/@me") + return { + "ok": True, + "bot_user_id": data.get("id"), + "bot_username": data.get("username"), + "global_name": data.get("global_name"), + } + + async def get_guild(self, guild_id: str) -> dict[str, Any]: + return await self.api_call("GET", f"/guilds/{guild_id}") diff --git a/surfsense_backend/app/gateway/discord/commands.py b/surfsense_backend/app/gateway/discord/commands.py new file mode 100644 index 000000000..2152e75c5 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/commands.py @@ -0,0 +1,66 @@ +"""Discord command/onboarding handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.discord.adapter import DiscordAdapter +from app.gateway.ratelimit import acquire_token + +HELP_TEXT = ( + "SurfSense Discord commands:\n" + "`/new` - start a fresh SurfSense conversation for this Discord thread\n" + "`/help` - show this help\n\n" + "Mention the SurfSense bot in a Discord channel to ask your agent a question. " + "Discord search remains controlled by the Discord connector in SurfSense." +) + + +class DiscordGatewayCommands(BaseGatewayCommands): + async def handle_help_command( + self, + *, + adapter: DiscordAdapter, + event: ParsedInboundEvent, + ) -> bool: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + if not channel_id: + return True + await adapter.send_message( + external_peer_id=channel_id, + text=HELP_TEXT, + reply_to_message_id=message_id, + ) + return True + + async def send_unbound_onboarding( + self, + *, + adapter: DiscordAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + guild_id = event.metadata.get("guild_id") + discord_user_id = event.metadata.get("discord_user_id") + if not channel_id or not message_id: + return + + wait_ms = await acquire_token( + f"discord:onboarded:{guild_id}:{discord_user_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + + await adapter.send_message( + external_peer_id=channel_id, + reply_to_message_id=message_id, + text=( + "Hi! Connect your Discord user to SurfSense before using the bot here: " + f"{dashboard_url}" + ), + ) diff --git a/surfsense_backend/app/gateway/discord/intake.py b/surfsense_backend/app/gateway/discord/intake.py new file mode 100644 index 000000000..fa0c26780 --- /dev/null +++ b/surfsense_backend/app/gateway/discord/intake.py @@ -0,0 +1,210 @@ +"""FastAPI lifespan supervisor for Discord Gateway WebSocket intake.""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from contextlib import suppress +from typing import Any + +import discord + +from app.config import config +from app.db import ExternalChatPlatform, async_session_maker +from app.gateway.accounts import get_discord_account_by_guild +from app.gateway.inbox import discord_message_dedupe_key, persist_inbound_event +from app.observability.metrics import record_gateway_inbox_write + +logger = logging.getLogger(__name__) + +_task: asyncio.Task[None] | None = None +_client: discord.Client | None = None +_shutdown_event: asyncio.Event | None = None + + +def _message_reference_payload(message: discord.Message) -> dict[str, Any] | None: + if message.reference is None: + return None + return { + "message_id": str(message.reference.message_id) + if message.reference.message_id + else None, + "channel_id": str(message.reference.channel_id) + if message.reference.channel_id + else None, + "guild_id": str(message.reference.guild_id) + if message.reference.guild_id + else None, + } + + +def _serialize_message( + message: discord.Message, *, bot_user_id: str | None +) -> dict[str, Any]: + guild = message.guild + channel = message.channel + thread_id = str(channel.id) if isinstance(channel, discord.Thread) else None + parent_id = str(channel.parent_id) if isinstance(channel, discord.Thread) else None + return { + "type": "message", + "bot_user_id": bot_user_id, + "event": { + "type": "message", + "id": str(message.id), + "guild_id": str(guild.id) if guild else None, + "guild_name": guild.name if guild else None, + "channel_id": parent_id or str(message.channel.id), + "thread_id": thread_id, + "channel_name": getattr(channel, "name", None), + "content": message.content, + "author": { + "id": str(message.author.id), + "username": message.author.name, + "bot": message.author.bot, + }, + "mentions": [ + {"id": str(user.id), "username": user.name} for user in message.mentions + ], + "message_reference": _message_reference_payload(message), + "created_at": message.created_at.isoformat() + if message.created_at + else None, + }, + } + + +async def _persist_message( + message: discord.Message, *, bot_user_id: str | None +) -> None: + if message.guild is None: + return + guild_id = str(message.guild.id) + raw_payload = _serialize_message(message, bot_user_id=bot_user_id) + + async with async_session_maker() as session: + account = await get_discord_account_by_guild(session, guild_id=guild_id) + if account is None: + logger.info( + "Ignoring Discord message for uninstalled guild_id=%s", guild_id + ) + return + + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.DISCORD, + event_dedupe_key=discord_message_dedupe_key(message.id), + external_event_id=str(message.id), + external_message_id=str(message.id), + event_kind="message", + raw_payload=raw_payload, + request_id=f"gateway_{uuid.uuid4().hex[:16]}", + ) + await session.commit() + record_gateway_inbox_write(platform="discord", dedup_skipped=inbox_id is None) + logger.info( + "Persisted Discord gateway message_id=%s guild_id=%s inbox_id=%s", + message.id, + guild_id, + inbox_id, + ) + + +def _build_client() -> discord.Client: + intents = discord.Intents.default() + intents.guilds = True + intents.messages = True + intents.message_content = True + client = discord.Client(intents=intents) + + @client.event + async def on_ready() -> None: + logger.info( + "Discord gateway connected as %s (%s)", + client.user, + getattr(client.user, "id", None), + ) + + @client.event + async def on_message(message: discord.Message) -> None: + if message.author.bot: + return + bot_user = client.user + if bot_user is None: + return + if message.author.id == bot_user.id: + return + bot_user_id = str(bot_user.id) + mention_ids = {str(user.id) for user in message.mentions} + if bot_user_id not in mention_ids: + return + logger.info( + "Received Discord gateway mention message_id=%s guild_id=%s channel_id=%s content_present=%s", + message.id, + getattr(message.guild, "id", None), + getattr(message.channel, "id", None), + bool(message.content), + ) + try: + await _persist_message(message, bot_user_id=bot_user_id) + except Exception: + logger.exception( + "Discord gateway failed to persist message_id=%s", message.id + ) + + return client + + +async def _run_discord_gateway() -> None: + global _client + token = config.DISCORD_BOT_TOKEN + if not token: + logger.warning("Discord gateway enabled but DISCORD_BOT_TOKEN is not set") + return + + while _shutdown_event is None or not _shutdown_event.is_set(): + _client = _build_client() + try: + await _client.start(token) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Discord gateway WebSocket failed; retrying in 30s") + finally: + if _client is not None and not _client.is_closed(): + await _client.close() + if _shutdown_event is not None and _shutdown_event.is_set(): + break + try: + await asyncio.wait_for(_shutdown_event.wait(), timeout=30.0) + except (TimeoutError, AttributeError): + continue + + +async def start_discord_gateway_supervisor() -> None: + global _shutdown_event, _task + if not config.GATEWAY_ENABLED: + return + if not config.GATEWAY_DISCORD_ENABLED: + return + if _task is not None and not _task.done(): + return + _shutdown_event = asyncio.Event() + _task = asyncio.create_task(_run_discord_gateway(), name="gateway-discord-intake") + logger.info("Started Discord gateway intake supervisor") + + +async def stop_discord_gateway_supervisor() -> None: + global _client, _shutdown_event, _task + if _shutdown_event is not None: + _shutdown_event.set() + if _client is not None and not _client.is_closed(): + await _client.close() + if _task is not None: + _task.cancel() + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(_task, timeout=10) + _client = None + _task = None + _shutdown_event = None diff --git a/surfsense_backend/app/gateway/discord/translator.py b/surfsense_backend/app/gateway/discord/translator.py new file mode 100644 index 000000000..c09b012cf --- /dev/null +++ b/surfsense_backend/app/gateway/discord/translator.py @@ -0,0 +1,90 @@ +"""Translate agent stream events into Discord replies.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.discord.adapter import DiscordAdapter +from app.gateway.ratelimit import wait_for_token +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +DISCORD_MAX_MESSAGE_CHARS = 1900 +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Discord. " + "Try again with a different request." +) + + +class DiscordStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: DiscordAdapter, + channel_id: str, + reply_to_message_id: str | None, + ) -> None: + self.adapter = adapter + self.channel_id = channel_id + self.reply_to_message_id = reply_to_message_id + self._buffer = "" + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message( + self._buffer, max_chars=DISCORD_MAX_MESSAGE_CHARS + ): + await self._send_text(chunk) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + try: + result = await self.adapter.send_message( + external_peer_id=self.channel_id, + text=text, + reply_to_message_id=self.reply_to_message_id, + ) + except Exception: + record_gateway_outbound(platform="discord", kind="send", status="failed") + raise + record_gateway_outbound(platform="discord", kind="send", status="sent") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"discord:channel:{self.channel_id}", + capacity=5, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="discord:channel") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="discord") diff --git a/surfsense_backend/app/gateway/hitl_filter.py b/surfsense_backend/app/gateway/hitl_filter.py new file mode 100644 index 000000000..4f0422f44 --- /dev/null +++ b/surfsense_backend/app/gateway/hitl_filter.py @@ -0,0 +1,34 @@ +"""Filter approval-required tools from gateway agent invocations.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +DEFAULT_HITL_TOOL_NAMES = { + "delete_document", + "delete_folder", + "delete_note", + "delete_report", + "delete_connector", + "send_email", + "share_chat", +} + + +def _tool_name(tool: Any) -> str | None: + if isinstance(tool, str): + return tool + return getattr(tool, "name", None) or getattr(tool, "__name__", None) + + +def filter_hitl_tools( + toolkit: Iterable[Any] | None, + *, + blocked_names: set[str] | None = None, +) -> list[Any] | None: + """Return a toolkit with known approval-required tools removed.""" + if toolkit is None: + return None + blocked = blocked_names or DEFAULT_HITL_TOOL_NAMES + return [tool for tool in toolkit if (_tool_name(tool) or "") not in blocked] diff --git a/surfsense_backend/app/gateway/inbox.py b/surfsense_backend/app/gateway/inbox.py new file mode 100644 index 000000000..7dcade65e --- /dev/null +++ b/surfsense_backend/app/gateway/inbox.py @@ -0,0 +1,53 @@ +"""Durable gateway inbox helpers.""" + +from __future__ import annotations + +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ExternalChatInboundEvent, ExternalChatPlatform + + +def telegram_event_dedupe_key(update_id: int | str) -> str: + return f"update:{update_id}" + + +def slack_event_dedupe_key(event_id: int | str) -> str: + return f"slack_event:{event_id}" + + +def discord_message_dedupe_key(message_id: int | str) -> str: + return f"discord_message:{message_id}" + + +async def persist_inbound_event( + session: AsyncSession, + *, + account_id: int, + platform: ExternalChatPlatform, + event_dedupe_key: str, + event_kind: str, + raw_payload: dict, + external_event_id: str | None = None, + external_message_id: str | None = None, + request_id: str | None = None, +) -> int | None: + stmt = ( + insert(ExternalChatInboundEvent) + .values( + account_id=account_id, + platform=platform, + event_dedupe_key=event_dedupe_key, + external_event_id=external_event_id, + external_message_id=external_message_id, + event_kind=event_kind, + raw_payload=raw_payload, + request_id=request_id, + ) + .on_conflict_do_nothing( + index_elements=["account_id", "event_dedupe_key"], + ) + .returning(ExternalChatInboundEvent.id) + ) + result = await session.execute(stmt) + return result.scalar_one_or_none() diff --git a/surfsense_backend/app/gateway/inbox_processor.py b/surfsense_backend/app/gateway/inbox_processor.py new file mode 100644 index 000000000..7343b3a03 --- /dev/null +++ b/surfsense_backend/app/gateway/inbox_processor.py @@ -0,0 +1,452 @@ +"""Long-lived external chat inbox processing. + +This module owns the agent-turn execution path for external chat surfaces. +FastAPI calls into it after webhook and BYO long-poll intake persist inbox rows. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from datetime import UTC, datetime + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatBinding, + ExternalChatBindingState, + ExternalChatEventStatus, + ExternalChatInboundEvent, + ExternalChatPeerKind, + ExternalChatPlatform, + NewChatThread, + async_session_maker, +) +from app.gateway.agent_invoke import call_agent_for_gateway +from app.gateway.base.commands import command_name +from app.gateway.bindings import get_or_create_thread_for_binding +from app.gateway.registry import resolve_platform_bundle +from app.observability.metrics import record_gateway_inbox_processed + +logger = logging.getLogger(__name__) + +SessionMaker = async_sessionmaker[AsyncSession] | Callable[[], AsyncSession] + + +def _dashboard_url() -> str: + return config.NEXT_FRONTEND_URL or "/dashboard" + + +def _active_whatsapp_account_mode() -> ExternalChatAccountMode | None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "cloud": + return ExternalChatAccountMode.CLOUD_SHARED + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": + return ExternalChatAccountMode.SELF_HOST_BYO + return None + + +def _is_inactive_whatsapp_account(account: ExternalChatAccount) -> bool: + return ( + account.platform == ExternalChatPlatform.WHATSAPP + and account.mode != _active_whatsapp_account_mode() + ) + + +async def claim_next_inbound_event( + session_maker: SessionMaker = async_session_maker, +) -> int | None: + """Claim the oldest received inbox event for processing.""" + + async with session_maker() as session: + result = await session.execute( + select(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.status == ExternalChatEventStatus.RECEIVED) + .order_by(ExternalChatInboundEvent.received_at.asc()) + .with_for_update(skip_locked=True) + .limit(1) + ) + event = result.scalars().first() + if event is None: + return None + event.status = ExternalChatEventStatus.PROCESSING + event.attempt_count += 1 + await session.commit() + return int(event.id) + + +async def process_inbound_event( + inbox_id: int, + session_maker: SessionMaker = async_session_maker, +) -> None: + """Process one external chat inbox row and mark its terminal status.""" + + async with session_maker() as session: + result = await session.execute( + select(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) + .with_for_update(skip_locked=True) + ) + event = result.scalars().first() + if event is None or event.status in { + ExternalChatEventStatus.PROCESSED, + ExternalChatEventStatus.IGNORED, + }: + return + if event.status == ExternalChatEventStatus.RECEIVED: + event.status = ExternalChatEventStatus.PROCESSING + event.attempt_count += 1 + await session.commit() + + try: + await _dispatch_inbound_event(inbox_id, session_maker) + except RuntimeError as exc: + if str(exc) == "gateway_thread_busy": + async with session_maker() as session: + await session.execute( + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) + .values( + status=ExternalChatEventStatus.RECEIVED, + last_error="gateway_thread_busy", + ) + ) + await session.commit() + raise + await _mark_failed(inbox_id, str(exc), session_maker) + raise + except Exception as exc: + await _mark_failed(inbox_id, str(exc), session_maker) + raise + + async with session_maker() as session: + event = await session.get(ExternalChatInboundEvent, inbox_id) + if event is not None and event.status == ExternalChatEventStatus.PROCESSING: + event.status = ExternalChatEventStatus.PROCESSED + event.processed_at = datetime.now(UTC) + await session.commit() + record_gateway_inbox_processed( + platform=event.platform.value, status="processed" + ) + + +async def _mark_failed( + inbox_id: int, + error: str, + session_maker: SessionMaker, +) -> None: + async with session_maker() as session: + await session.execute( + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.id == inbox_id) + .values(status=ExternalChatEventStatus.FAILED, last_error=error) + ) + await session.commit() + + +async def _resolve_binding_for_event( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + if account.platform == ExternalChatPlatform.SLACK: + return await _resolve_slack_thread_binding(session, account, parsed) + if account.platform == ExternalChatPlatform.DISCORD: + return await _resolve_discord_thread_binding(session, account, parsed) + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == parsed.external_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + return result.scalars().first() + + +async def _resolve_slack_thread_binding( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + user_peer_id = parsed.metadata.get("slack_user_peer_id") + thread_peer_id = ( + parsed.metadata.get("slack_thread_peer_id") or parsed.external_peer_id + ) + if not user_peer_id or not thread_peer_id: + return None + + user_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == user_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + user_binding = user_result.scalars().first() + if user_binding is None: + return None + + thread_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == thread_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + thread_binding = thread_result.scalars().first() + if thread_binding is not None: + return thread_binding + + thread_binding = ExternalChatBinding( + account_id=account.id, + user_id=user_binding.user_id, + search_space_id=user_binding.search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=thread_peer_id, + external_peer_kind=ExternalChatPeerKind.CHANNEL, + external_thread_id=parsed.metadata.get("thread_ts"), + external_display_name=parsed.metadata.get("channel_id"), + external_username=parsed.external_user_id, + external_metadata={ + "kind": "slack_thread", + "team_id": parsed.metadata.get("team_id"), + "channel_id": parsed.metadata.get("channel_id"), + "thread_ts": parsed.metadata.get("thread_ts"), + "slack_user_id": parsed.metadata.get("slack_user_id"), + "user_binding_id": user_binding.id, + }, + ) + session.add(thread_binding) + await session.flush() + return thread_binding + + +async def _resolve_discord_thread_binding( + session: AsyncSession, + account: ExternalChatAccount, + parsed, +) -> ExternalChatBinding | None: + user_peer_id = parsed.metadata.get("discord_user_peer_id") + thread_peer_id = ( + parsed.metadata.get("discord_thread_peer_id") or parsed.external_peer_id + ) + if not user_peer_id or not thread_peer_id: + return None + + user_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == user_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + user_binding = user_result.scalars().first() + if user_binding is None: + return None + + thread_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == thread_peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + thread_binding = thread_result.scalars().first() + if thread_binding is not None: + return thread_binding + + thread_binding = ExternalChatBinding( + account_id=account.id, + user_id=user_binding.user_id, + search_space_id=user_binding.search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=thread_peer_id, + external_peer_kind=ExternalChatPeerKind.CHANNEL, + external_thread_id=parsed.metadata.get("thread_key"), + external_display_name=parsed.metadata.get("channel_id"), + external_username=parsed.external_user_id, + external_metadata={ + "kind": "discord_thread", + "guild_id": parsed.metadata.get("guild_id"), + "channel_id": parsed.metadata.get("channel_id"), + "thread_key": parsed.metadata.get("thread_key"), + "discord_user_id": parsed.metadata.get("discord_user_id"), + "user_binding_id": user_binding.id, + }, + ) + session.add(thread_binding) + await session.flush() + return thread_binding + + +def _reply_target(parsed) -> tuple[str | None, str | None]: + if parsed.platform == "slack": + return parsed.metadata.get("channel_id"), parsed.metadata.get("thread_ts") + if parsed.platform == "discord": + return parsed.metadata.get("channel_id"), parsed.metadata.get("message_id") + return parsed.external_peer_id, None + + +async def _dispatch_inbound_event( + inbox_id: int, + session_maker: SessionMaker, +) -> None: + async with session_maker() as session: + event = await session.get(ExternalChatInboundEvent, inbox_id) + if event is None: + return + account = await session.get(ExternalChatAccount, event.account_id) + if account is None: + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "account_missing" + await session.commit() + return + if _is_inactive_whatsapp_account(account): + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "inactive_whatsapp_mode" + await session.commit() + return + + try: + bundle = resolve_platform_bundle(account) + except RuntimeError as exc: + event.status = ExternalChatEventStatus.FAILED + event.last_error = str(exc) + await session.commit() + return + + adapter = bundle.adapter + parsed = adapter.parse_inbound(event.raw_payload or {}) + if parsed.external_peer_id is None: + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "missing_external_peer_id" + await session.commit() + return + + _update_account_cursor(account, parsed.metadata.get("update_id")) + + binding = await _resolve_binding_for_event(session, account, parsed) + + if ( + account.platform + not in {ExternalChatPlatform.SLACK, ExternalChatPlatform.DISCORD} + and parsed.external_peer_kind != ExternalChatPeerKind.DIRECT.value + ): + if hasattr(adapter, "leave_chat"): + await adapter.leave_chat(external_peer_id=parsed.external_peer_id) + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "group_rejected" + await session.commit() + return + + cmd = command_name(parsed.text) + if cmd == "/start": + handled = await bundle.commands.handle_start_command( + session=session, adapter=adapter, event=parsed + ) + await session.commit() + if handled: + return + + if binding is None: + if ( + bundle.auto_bind_owner + and account.owner_user_id + and account.owner_search_space_id + ): + binding = ExternalChatBinding( + account_id=account.id, + user_id=account.owner_user_id, + search_space_id=account.owner_search_space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=parsed.external_peer_id, + external_peer_kind=parsed.external_peer_kind, + external_display_name=parsed.display_name, + external_username=parsed.username, + external_metadata=parsed.metadata, + ) + session.add(binding) + await session.flush() + else: + await bundle.commands.send_unbound_onboarding( + adapter=adapter, + event=parsed, + dashboard_url=_dashboard_url(), + ) + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "unbound_chat" + await session.commit() + return + + event.external_chat_binding_id = binding.id + + if cmd == "/help": + handled = await bundle.commands.handle_help_command( + adapter=adapter, event=parsed + ) + if handled: + event.status = ExternalChatEventStatus.PROCESSED + await session.commit() + return + if cmd == "/new": + binding.new_chat_thread_id = None + reply_peer_id, reply_message_id = _reply_target(parsed) + if reply_peer_id: + await adapter.send_message( + external_peer_id=reply_peer_id, + text="Started a new SurfSense conversation.", + reply_to_message_id=reply_message_id, + ) + event.status = ExternalChatEventStatus.PROCESSED + await session.commit() + return + + if not parsed.text: + event.status = ExternalChatEventStatus.IGNORED + event.last_error = "empty_message" + await session.commit() + return + + thread = await get_or_create_thread_for_binding(session, binding) + await session.commit() + + translator = bundle.translator_factory(adapter, parsed) + await call_agent_for_gateway( + session=session, + binding=binding, + user_text=parsed.text, + translator=translator, + platform_label=bundle.platform_label, + request_id=event.request_id or f"gateway:{inbox_id}", + ) + + thread = await session.get(NewChatThread, thread.id) + if thread is not None: + thread.source = bundle.platform_label + await session.commit() + + +def _update_account_cursor(account: ExternalChatAccount, update_id: object) -> None: + if update_id is None: + return + account.cursor_state = { + **(account.cursor_state or {}), + "last_update_id": max( + int((account.cursor_state or {}).get("last_update_id", 0)), + int(update_id), + ), + } diff --git a/surfsense_backend/app/gateway/inbox_worker.py b/surfsense_backend/app/gateway/inbox_worker.py new file mode 100644 index 000000000..2170b9b1d --- /dev/null +++ b/surfsense_backend/app/gateway/inbox_worker.py @@ -0,0 +1,57 @@ +"""FastAPI lifespan worker for gateway inbox processing.""" + +from __future__ import annotations + +import asyncio +import logging +from contextlib import suppress + +from app.config import config +from app.gateway.inbox_processor import claim_next_inbound_event, process_inbound_event + +logger = logging.getLogger(__name__) + +_task: asyncio.Task[None] | None = None + + +async def _process_inbox_forever() -> None: + logger.info("Gateway inbox processor started in FastAPI process") + while True: + try: + inbox_id = await claim_next_inbound_event() + if inbox_id is None: + await asyncio.sleep(0.5) + continue + logger.info("Gateway processing inbox_id=%s", inbox_id) + await process_inbound_event(inbox_id) + logger.info("Gateway processed inbox_id=%s", inbox_id) + except asyncio.CancelledError: + raise + except RuntimeError as exc: + if str(exc) == "gateway_thread_busy": + logger.info("Gateway inbox_id busy; will retry from RECEIVED state") + else: + logger.exception("Gateway inbox processor failed one iteration") + await asyncio.sleep(1) + except Exception: + logger.exception("Gateway inbox processor failed one iteration") + await asyncio.sleep(1) + + +async def start_gateway_inbox_worker() -> None: + global _task + if not config.GATEWAY_ENABLED: + return + if _task is not None and not _task.done(): + return + _task = asyncio.create_task(_process_inbox_forever(), name="gateway-inbox-worker") + + +async def stop_gateway_inbox_worker() -> None: + global _task + if _task is None: + return + _task.cancel() + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for(_task, timeout=10) + _task = None diff --git a/surfsense_backend/app/gateway/pairing.py b/surfsense_backend/app/gateway/pairing.py new file mode 100644 index 000000000..bafb6df05 --- /dev/null +++ b/surfsense_backend/app/gateway/pairing.py @@ -0,0 +1,53 @@ +"""Pairing code lifecycle for external chat bindings.""" + +from __future__ import annotations + +import secrets +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ExternalChatBinding, ExternalChatBindingState + +PAIRING_CODE_TTL = timedelta(minutes=10) + + +def generate_pairing_code() -> str: + return secrets.token_urlsafe(6) + + +def pairing_expires_at() -> datetime: + return datetime.now(UTC) + PAIRING_CODE_TTL + + +async def redeem_pairing_code( + session: AsyncSession, + *, + code: str, + external_peer_id: str, + external_peer_kind: str, + external_display_name: str | None, + external_username: str | None, + external_metadata: dict | None = None, +) -> ExternalChatBinding | None: + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.pairing_code == code, + ExternalChatBinding.state == ExternalChatBindingState.PENDING, + ExternalChatBinding.pairing_code_expires_at > datetime.now(UTC), + ) + ) + binding = result.scalars().first() + if binding is None: + return None + + binding.state = ExternalChatBindingState.BOUND + binding.pairing_code = None + binding.pairing_code_expires_at = None + binding.external_peer_id = external_peer_id + binding.external_peer_kind = external_peer_kind + binding.external_display_name = external_display_name + binding.external_username = external_username + binding.external_metadata = external_metadata or {} + return binding diff --git a/surfsense_backend/app/gateway/ratelimit.py b/surfsense_backend/app/gateway/ratelimit.py new file mode 100644 index 000000000..1afc839d5 --- /dev/null +++ b/surfsense_backend/app/gateway/ratelimit.py @@ -0,0 +1,135 @@ +"""Redis token-bucket rate limiter for gateway outbound traffic.""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass + +import redis.asyncio as aioredis + +from app.config import config +from app.observability.metrics import record_gateway_redis_fallback + +logger = logging.getLogger(__name__) + +_TOKEN_BUCKET_LUA = """ +local capacity = tonumber(ARGV[1]) +local refill_rate = tonumber(ARGV[2]) +local now = tonumber(ARGV[3]) +local consume = tonumber(ARGV[4]) + +local bucket = redis.call('HMGET', KEYS[1], 'tokens', 'last_refill') +local tokens = tonumber(bucket[1]) or capacity +local last_refill = tonumber(bucket[2]) or now + +local elapsed = math.max(0, now - last_refill) +tokens = math.min(capacity, tokens + (elapsed * refill_rate)) + +if tokens >= consume then + tokens = tokens - consume + redis.call('HMSET', KEYS[1], 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', KEYS[1], 3600) + return 0 +else + redis.call('HMSET', KEYS[1], 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', KEYS[1], 3600) + local needed = consume - tokens + return math.ceil((needed / refill_rate) * 1000) +end +""" + +_redis_client: aioredis.Redis | None = None + + +@dataclass +class _MemoryBucket: + tokens: float + last_refill: float + + +_memory_buckets: dict[str, _MemoryBucket] = {} +_memory_lock = asyncio.Lock() + + +def _redis() -> aioredis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = aioredis.from_url(config.REDIS_APP_URL, decode_responses=True) + return _redis_client + + +async def _memory_fallback_acquire( + scope: str, + capacity: int, + refill_per_sec: float, + consume: float, +) -> int: + now = time.time() + async with _memory_lock: + bucket = _memory_buckets.get(scope) + if bucket is None: + bucket = _MemoryBucket(tokens=float(capacity), last_refill=now) + _memory_buckets[scope] = bucket + + elapsed = max(0.0, now - bucket.last_refill) + bucket.tokens = min(float(capacity), bucket.tokens + elapsed * refill_per_sec) + bucket.last_refill = now + + if bucket.tokens >= consume: + bucket.tokens -= consume + return 0 + + needed = consume - bucket.tokens + return int((needed / refill_per_sec) * 1000) if refill_per_sec > 0 else 1000 + + +async def acquire_token( + scope: str, + *, + capacity: int, + refill_per_sec: float, + consume: float = 1.0, +) -> int: + """Return 0 if allowed, otherwise milliseconds to wait. + + Redis is the primary coordination mechanism. If Redis is unavailable, + fall back to per-process memory so the gateway degrades instead of failing + closed during a short Redis outage. + """ + + redis_key = f"gateway:bucket:{scope}" + try: + wait_ms = await _redis().eval( + _TOKEN_BUCKET_LUA, + 1, + redis_key, + capacity, + refill_per_sec, + time.time(), + consume, + ) + return int(wait_ms) + except (aioredis.RedisError, OSError) as exc: + logger.warning("Redis rate limiter unavailable; using memory fallback: %s", exc) + record_gateway_redis_fallback() + return await _memory_fallback_acquire(scope, capacity, refill_per_sec, consume) + + +async def wait_for_token( + scope: str, + *, + capacity: int, + refill_per_sec: float, + consume: float = 1.0, +) -> int: + wait_ms = await acquire_token( + scope, + capacity=capacity, + refill_per_sec=refill_per_sec, + consume=consume, + ) + if wait_ms > 0: + await asyncio.sleep(wait_ms / 1000) + return wait_ms diff --git a/surfsense_backend/app/gateway/registry.py b/surfsense_backend/app/gateway/registry.py new file mode 100644 index 000000000..fb2aeefc0 --- /dev/null +++ b/surfsense_backend/app/gateway/registry.py @@ -0,0 +1,191 @@ +"""Resolve gateway platform implementations from account rows.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform +from app.gateway.accounts import ( + account_token, + discord_account_credentials, + slack_account_credentials, +) +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.base.translator import BaseStreamTranslator +from app.gateway.telegram.adapter import TelegramAdapter +from app.gateway.telegram.commands import TelegramGatewayCommands +from app.gateway.telegram.translator import TelegramStreamTranslator + +TranslatorFactory = Callable[ + [BasePlatformAdapter, ParsedInboundEvent], + BaseStreamTranslator, +] + + +@dataclass(frozen=True) +class PlatformBundle: + adapter: BasePlatformAdapter + translator_factory: TranslatorFactory + platform_label: str + commands: BaseGatewayCommands + auto_bind_owner: bool = False + + +def _telegram_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + return TelegramStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + external_peer_id=event.external_peer_id, + ) + + +def _whatsapp_cloud_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + from app.gateway.whatsapp.translator import WhatsAppCloudStreamTranslator + + return WhatsAppCloudStreamTranslator( + adapter=adapter, + external_peer_id=event.external_peer_id, + inbound_message_id=event.external_message_id, + ) + + +def _whatsapp_baileys_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + if event.external_peer_id is None: + raise RuntimeError("missing_external_peer_id") + from app.gateway.whatsapp.translator_baileys import WhatsAppBaileysStreamTranslator + + return WhatsAppBaileysStreamTranslator( + adapter=adapter, + external_peer_id=event.external_peer_id, + ) + + +def _slack_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + if not channel_id or not thread_ts: + raise RuntimeError("missing_slack_thread_metadata") + from app.gateway.slack.translator import SlackStreamTranslator + + return SlackStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + channel_id=channel_id, + thread_ts=thread_ts, + ) + + +def _discord_translator_factory( + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> BaseStreamTranslator: + channel_id = event.metadata.get("channel_id") + message_id = event.metadata.get("message_id") + if not channel_id: + raise RuntimeError("missing_discord_channel_metadata") + from app.gateway.discord.translator import DiscordStreamTranslator + + return DiscordStreamTranslator( + adapter=adapter, # type: ignore[arg-type] + channel_id=channel_id, + reply_to_message_id=message_id, + ) + + +def resolve_platform_bundle(account: ExternalChatAccount) -> PlatformBundle: + if account.platform == ExternalChatPlatform.TELEGRAM: + token = account_token(account) + if not token: + raise RuntimeError("missing_telegram_token") + return PlatformBundle( + adapter=TelegramAdapter(token), + translator_factory=_telegram_translator_factory, + platform_label="telegram", + commands=TelegramGatewayCommands(), + ) + + if account.platform == ExternalChatPlatform.WHATSAPP: + if account.mode == ExternalChatAccountMode.CLOUD_SHARED: + from app.gateway.whatsapp.adapter_cloud import WhatsAppCloudAdapter + from app.gateway.whatsapp.commands import WhatsAppGatewayCommands + from app.gateway.whatsapp.credentials import ( + load_system_whatsapp_credentials, + ) + + return PlatformBundle( + adapter=WhatsAppCloudAdapter(load_system_whatsapp_credentials()), + translator_factory=_whatsapp_cloud_translator_factory, + platform_label="whatsapp", + commands=WhatsAppGatewayCommands(), + auto_bind_owner=False, + ) + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO: + from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter + + return PlatformBundle( + adapter=WhatsAppBaileysAdapter(), + translator_factory=_whatsapp_baileys_translator_factory, + platform_label="whatsapp", + commands=BaseGatewayCommands(), + auto_bind_owner=True, + ) + + if account.platform == ExternalChatPlatform.SLACK: + from app.gateway.slack.adapter import SlackAdapter + from app.gateway.slack.commands import SlackGatewayCommands + + credentials = slack_account_credentials(account) + bot_token = credentials.get("bot_token") + if not bot_token: + raise RuntimeError("missing_slack_bot_token") + cursor_state = account.cursor_state or {} + return PlatformBundle( + adapter=SlackAdapter( + bot_token, + bot_user_id=cursor_state.get("bot_user_id"), + ), + translator_factory=_slack_translator_factory, + platform_label="slack", + commands=SlackGatewayCommands(), + auto_bind_owner=False, + ) + + if account.platform == ExternalChatPlatform.DISCORD: + from app.gateway.discord.adapter import DiscordAdapter + from app.gateway.discord.commands import DiscordGatewayCommands + + credentials = discord_account_credentials(account) + bot_token = credentials.get("bot_token") + if not bot_token: + raise RuntimeError("missing_discord_bot_token") + cursor_state = account.cursor_state or {} + return PlatformBundle( + adapter=DiscordAdapter( + bot_token, + bot_user_id=cursor_state.get("bot_user_id"), + ), + translator_factory=_discord_translator_factory, + platform_label="discord", + commands=DiscordGatewayCommands(), + auto_bind_owner=False, + ) + + raise RuntimeError( + f"unsupported_gateway_platform:{account.platform.value}:{account.mode.value}" + ) diff --git a/surfsense_backend/app/gateway/runner.py b/surfsense_backend/app/gateway/runner.py new file mode 100644 index 000000000..64e0bb2e2 --- /dev/null +++ b/surfsense_backend/app/gateway/runner.py @@ -0,0 +1,75 @@ +"""Telegram BYO long-poll helper for FastAPI lifespan.""" + +from __future__ import annotations + +import hashlib +import logging +import uuid + +from sqlalchemy import text + +from app.db import ( + ExternalChatAccount, + ExternalChatPlatform, + async_session_maker, + engine, +) +from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.telegram.adapter import TelegramAdapter +from app.observability.metrics import record_gateway_byo_longpoll_running_delta + +logger = logging.getLogger(__name__) + + +def _lock_key(token: str) -> int: + digest = hashlib.sha256(f"gateway:telegram:{token}".encode()).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +async def _run_telegram_account(account_id: int, token: str) -> None: + async with engine.connect() as conn: + lock_key = _lock_key(token) + got_lock = await conn.scalar( + text("SELECT pg_try_advisory_lock(:key)"), + {"key": lock_key}, + ) + if not got_lock: + logger.warning("Another Telegram gateway runner is active; exiting") + return + + record_gateway_byo_longpoll_running_delta(1, account_id=account_id) + try: + adapter = TelegramAdapter(token) + async with async_session_maker() as session: + account = await session.get(ExternalChatAccount, account_id) + offset = None + if account is not None: + offset = ( + int((account.cursor_state or {}).get("last_update_id", 0)) + 1 + ) + + async for update in adapter.fetch_updates(offset=offset): + request_id = f"gateway_{uuid.uuid4().hex[:16]}" + async with async_session_maker() as session: + parsed = adapter.parse_inbound(update) + inbox_id = await persist_inbound_event( + session, + account_id=account_id, + platform=ExternalChatPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(update["update_id"]), + external_event_id=str(update["update_id"]), + external_message_id=parsed.external_message_id, + event_kind=parsed.event_kind, + raw_payload=update, + request_id=request_id, + ) + await session.commit() + if inbox_id is not None: + logger.debug( + "Persisted Telegram polling update inbox_id=%s", inbox_id + ) + finally: + record_gateway_byo_longpoll_running_delta(-1, account_id=account_id) + await conn.execute( + text("SELECT pg_advisory_unlock(:key)"), {"key": lock_key} + ) diff --git a/surfsense_backend/app/gateway/slack/__init__.py b/surfsense_backend/app/gateway/slack/__init__.py new file mode 100644 index 000000000..7f7aaf2fc --- /dev/null +++ b/surfsense_backend/app/gateway/slack/__init__.py @@ -0,0 +1 @@ +"""Slack gateway integration.""" diff --git a/surfsense_backend/app/gateway/slack/adapter.py b/surfsense_backend/app/gateway/slack/adapter.py new file mode 100644 index 000000000..9890261bd --- /dev/null +++ b/surfsense_backend/app/gateway/slack/adapter.py @@ -0,0 +1,122 @@ +"""Slack platform adapter for app mentions and threaded replies.""" + +from __future__ import annotations + +import re +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.slack.client import SlackGatewayClient + +MENTION_RE = re.compile(r"<@[^>]+>\s*") + + +def slack_user_peer_id(team_id: str, slack_user_id: str) -> str: + return f"slack_user:{team_id}:{slack_user_id}" + + +def slack_thread_peer_id(team_id: str, channel_id: str, thread_ts: str) -> str: + return f"slack_thread:{team_id}:{channel_id}:{thread_ts}" + + +class SlackAdapter(BasePlatformAdapter): + platform = "slack" + + def __init__(self, bot_token: str, *, bot_user_id: str | None = None) -> None: + self.bot_user_id = bot_user_id + self.client = SlackGatewayClient(bot_token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event = raw_payload.get("event") or {} + event_type = str(event.get("type") or "other") + team_id = str(raw_payload.get("team_id") or event.get("team") or "") + channel_id = str(event.get("channel") or "") + slack_user_id = str(event.get("user") or "") + message_ts = str(event.get("ts") or "") + thread_ts = str(event.get("thread_ts") or message_ts) + bot_user_id = self.bot_user_id or str( + raw_payload.get("authorizations", [{}])[0].get("user_id") or "" + ) + + if not channel_id or not slack_user_id or not message_ts: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_type, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=message_ts or None, + external_user_id=slack_user_id or None, + text=None, + raw_payload=raw_payload, + metadata={"team_id": team_id, "bot_user_id": bot_user_id}, + ) + + text = str(event.get("text") or "") + if bot_user_id: + text = text.replace(f"<@{bot_user_id}>", "") + text = MENTION_RE.sub("", text).strip() + + peer_kind = "direct" if str(event.get("channel_type")) == "im" else "channel" + thread_key = slack_thread_peer_id(team_id, channel_id, thread_ts) + user_key = slack_user_peer_id(team_id, slack_user_id) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_type, + external_peer_id=thread_key, + external_peer_kind=peer_kind, + external_message_id=message_ts, + external_user_id=slack_user_id, + text=text, + raw_payload=raw_payload, + display_name=None, + username=slack_user_id, + metadata={ + "team_id": team_id, + "channel_id": channel_id, + "slack_user_id": slack_user_id, + "message_ts": message_ts, + "thread_ts": thread_ts, + "bot_user_id": bot_user_id, + "slack_user_peer_id": user_key, + "slack_thread_peer_id": thread_key, + "channel_type": event.get("channel_type"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.send_message( + channel=external_peer_id, + text=text, + thread_ts=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + del parse_mode + return await self.client.update_message( + channel=external_peer_id, + ts=external_message_id, + text=text, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() diff --git a/surfsense_backend/app/gateway/slack/client.py b/surfsense_backend/app/gateway/slack/client.py new file mode 100644 index 000000000..8f2f16cec --- /dev/null +++ b/surfsense_backend/app/gateway/slack/client.py @@ -0,0 +1,76 @@ +"""Slack Web API client for gateway bot operations.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from app.gateway.base.adapter import PlatformSendResult + +SLACK_API = "https://slack.com/api" + + +class SlackGatewayClient: + def __init__(self, bot_token: str) -> None: + self.bot_token = bot_token + + async def api_call( + self, method: str, payload: dict[str, Any] | None = None + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=20.0) as client: + response = await client.post( + f"{SLACK_API}/{method}", + json=payload or {}, + headers={ + "Authorization": f"Bearer {self.bot_token}", + "Content-Type": "application/json; charset=utf-8", + }, + ) + response.raise_for_status() + data = response.json() + if not data.get("ok", False): + error = data.get("error", "unknown_error") + raise RuntimeError(f"Slack API {method} failed: {error}") + return data + + async def send_message( + self, + *, + channel: str, + text: str, + thread_ts: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = {"channel": channel, "text": text} + if thread_ts: + payload["thread_ts"] = thread_ts + data = await self.api_call("chat.postMessage", payload) + return PlatformSendResult( + external_message_id=str(data.get("ts", "")), + raw_response=data, + ) + + async def update_message( + self, + *, + channel: str, + ts: str, + text: str, + ) -> PlatformSendResult: + data = await self.api_call( + "chat.update", {"channel": channel, "ts": ts, "text": text} + ) + return PlatformSendResult( + external_message_id=str(data.get("ts") or ts), + raw_response=data, + ) + + async def validate(self) -> dict[str, Any]: + data = await self.api_call("auth.test") + return { + "ok": True, + "team_id": data.get("team_id"), + "team": data.get("team"), + "bot_user_id": data.get("user_id"), + "bot_username": data.get("user"), + } diff --git a/surfsense_backend/app/gateway/slack/commands.py b/surfsense_backend/app/gateway/slack/commands.py new file mode 100644 index 000000000..ffbd5863b --- /dev/null +++ b/surfsense_backend/app/gateway/slack/commands.py @@ -0,0 +1,64 @@ +"""Slack command/onboarding handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.ratelimit import acquire_token +from app.gateway.slack.adapter import SlackAdapter + +HELP_TEXT = ( + "SurfSense Slack commands:\n" + "`/new` - start a fresh SurfSense conversation in this thread\n" + "`/help` - show this help\n\n" + "Mention the SurfSense bot in a channel thread to ask your agent a question." +) + + +class SlackGatewayCommands(BaseGatewayCommands): + async def handle_help_command( + self, + *, + adapter: SlackAdapter, + event: ParsedInboundEvent, + ) -> bool: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + if not channel_id or not thread_ts: + return True + await adapter.send_message( + external_peer_id=channel_id, + text=HELP_TEXT, + reply_to_message_id=thread_ts, + ) + return True + + async def send_unbound_onboarding( + self, + *, + adapter: SlackAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + channel_id = event.metadata.get("channel_id") + thread_ts = event.metadata.get("thread_ts") + slack_user_id = event.metadata.get("slack_user_id") + if not channel_id or not thread_ts: + return + + wait_ms = await acquire_token( + f"slack:onboarded:{event.metadata.get('team_id')}:{slack_user_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + + await adapter.send_message( + external_peer_id=channel_id, + reply_to_message_id=thread_ts, + text=( + "Hi! Connect your Slack user to SurfSense before using the bot here: " + f"{dashboard_url}" + ), + ) diff --git a/surfsense_backend/app/gateway/slack/translator.py b/surfsense_backend/app/gateway/slack/translator.py new file mode 100644 index 000000000..a591474e5 --- /dev/null +++ b/surfsense_backend/app/gateway/slack/translator.py @@ -0,0 +1,90 @@ +"""Translate agent stream events into Slack thread replies.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.ratelimit import wait_for_token +from app.gateway.slack.adapter import SlackAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +SLACK_MAX_MESSAGE_CHARS = 35000 +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Slack. " + "Try again with a different request." +) + + +class SlackStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: SlackAdapter, + channel_id: str, + thread_ts: str, + ) -> None: + self.adapter = adapter + self.channel_id = channel_id + self.thread_ts = thread_ts + self._buffer = "" + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message( + self._buffer, max_chars=SLACK_MAX_MESSAGE_CHARS + ): + await self._send_text(chunk) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + try: + result = await self.adapter.send_message( + external_peer_id=self.channel_id, + text=text, + reply_to_message_id=self.thread_ts, + ) + except Exception: + record_gateway_outbound(platform="slack", kind="send", status="failed") + raise + record_gateway_outbound(platform="slack", kind="send", status="sent") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"slack:channel:{self.channel_id}", + capacity=1, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="slack:channel") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="slack") diff --git a/surfsense_backend/app/gateway/telegram/__init__.py b/surfsense_backend/app/gateway/telegram/__init__.py new file mode 100644 index 000000000..a4e252642 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/__init__.py @@ -0,0 +1 @@ +"""Telegram gateway adapter.""" diff --git a/surfsense_backend/app/gateway/telegram/adapter.py b/surfsense_backend/app/gateway/telegram/adapter.py new file mode 100644 index 000000000..dc4266d42 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/adapter.py @@ -0,0 +1,120 @@ +"""Telegram platform adapter.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.telegram.client import TelegramClient + + +class TelegramAdapter(BasePlatformAdapter): + platform = "telegram" + + def __init__(self, token: str) -> None: + self.client = TelegramClient(token) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + event_kind = "other" + message = raw_payload.get("message") + if message is not None: + event_kind = "message" + else: + message = raw_payload.get("edited_message") + if message is not None: + event_kind = "edited_message" + + if message is None: + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=None, + external_user_id=None, + text=None, + raw_payload=raw_payload, + ) + + chat = message.get("chat") or {} + sender = message.get("from") or {} + chat_type = str(chat.get("type") or "unknown") + peer_kind = { + "private": "direct", + "group": "group", + "supergroup": "group", + "channel": "channel", + }.get(chat_type, "unknown") + display_name = chat.get("title") or " ".join( + part for part in (sender.get("first_name"), sender.get("last_name")) if part + ) + + return ParsedInboundEvent( + platform=self.platform, + event_kind=event_kind, + external_peer_id=str(chat["id"]) if chat.get("id") is not None else None, + external_peer_kind=peer_kind, + external_message_id=( + str(message["message_id"]) + if message.get("message_id") is not None + else None + ), + external_user_id=str(sender["id"]) + if sender.get("id") is not None + else None, + text=message.get("text") or message.get("caption"), + raw_payload=raw_payload, + display_name=display_name or None, + username=sender.get("username") or chat.get("username"), + metadata={ + "chat_type": chat_type, + "update_id": raw_payload.get("update_id"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + return await self.client.send_message( + chat_id=external_peer_id, + text=text, + parse_mode=parse_mode, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + return await self.client.edit_message( + chat_id=external_peer_id, + message_id=external_message_id, + text=text, + parse_mode=parse_mode, + ) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() + + async def leave_chat(self, *, external_peer_id: str) -> None: + await self.client.leave_chat(chat_id=external_peer_id) + + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: + async for update in self.client.get_updates(offset=offset): + yield update diff --git a/surfsense_backend/app/gateway/telegram/client.py b/surfsense_backend/app/gateway/telegram/client.py new file mode 100644 index 000000000..d3b054451 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/client.py @@ -0,0 +1,108 @@ +"""Thin async Telegram Bot API client.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any + +from telegram import Bot +from telegram.error import BadRequest, RetryAfter + +from app.gateway.base.adapter import PlatformSendResult + + +def retry_after_seconds(value: int | timedelta) -> float: + if isinstance(value, timedelta): + return value.total_seconds() + return float(value) + + +class TelegramClient: + def __init__(self, token: str) -> None: + self.token = token + self.bot = Bot(token=token) + + async def send_message( + self, + *, + chat_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + kwargs: dict[str, Any] = {} + if parse_mode: + kwargs["parse_mode"] = parse_mode + if reply_to_message_id: + kwargs["reply_to_message_id"] = int(reply_to_message_id) + try: + msg = await self.bot.send_message(chat_id=chat_id, text=text, **kwargs) + except RetryAfter as exc: + await asyncio.sleep(retry_after_seconds(exc.retry_after)) + msg = await self.bot.send_message(chat_id=chat_id, text=text, **kwargs) + return PlatformSendResult( + external_message_id=str(msg.message_id), + raw_response=msg.to_dict(), + ) + + async def edit_message( + self, + *, + chat_id: str, + message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + kwargs: dict[str, Any] = {} + if parse_mode: + kwargs["parse_mode"] = parse_mode + try: + msg = await self.bot.edit_message_text( + chat_id=chat_id, + message_id=int(message_id), + text=text, + **kwargs, + ) + except RetryAfter as exc: + await asyncio.sleep(retry_after_seconds(exc.retry_after)) + msg = await self.bot.edit_message_text( + chat_id=chat_id, + message_id=int(message_id), + text=text, + **kwargs, + ) + return PlatformSendResult( + external_message_id=str(msg.message_id), + raw_response=msg.to_dict(), + ) + + async def validate(self) -> dict[str, Any]: + me = await self.bot.get_me() + return me.to_dict() + + async def leave_chat(self, *, chat_id: str) -> None: + await self.bot.leave_chat(chat_id=chat_id) + + async def get_updates(self, *, offset: int | None) -> AsyncIterator[dict[str, Any]]: + next_offset = offset + while True: + updates = await self.bot.get_updates( + offset=next_offset, + timeout=30, + allowed_updates=["message", "edited_message"], + ) + for update in updates: + next_offset = update.update_id + 1 + yield update.to_dict() + + +async def retry_plaintext_on_bad_markdown(call, *args, **kwargs) -> PlatformSendResult: + try: + return await call(*args, **kwargs) + except BadRequest as exc: + if "can't parse entities" not in str(exc).lower(): + raise + kwargs["parse_mode"] = None + return await call(*args, **kwargs) diff --git a/surfsense_backend/app/gateway/telegram/commands.py b/surfsense_backend/app/gateway/telegram/commands.py new file mode 100644 index 000000000..c9965fd90 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/commands.py @@ -0,0 +1,119 @@ +"""Telegram command handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.pairing import redeem_pairing_code +from app.gateway.ratelimit import acquire_token +from app.gateway.telegram.adapter import TelegramAdapter + +HELP_TEXT = ( + "SurfSense Telegram commands:\n" + "/start - pair this chat\n" + "/new - start a fresh conversation\n" + "/help - show this help" +) + + +async def handle_start_command( + *, + session, + adapter: TelegramAdapter, + event: ParsedInboundEvent, +) -> bool: + text = event.text or "" + parts = text.split(maxsplit=1) + if len(parts) != 2 or not event.external_peer_id: + await adapter.send_message( + external_peer_id=event.external_peer_id or "", + text="Generate a pairing code in SurfSense Settings > Messaging Channels, then send /start CODE here.", + ) + return True + + binding = await redeem_pairing_code( + session, + code=parts[1].strip(), + external_peer_id=event.external_peer_id, + external_peer_kind=event.external_peer_kind, + external_display_name=event.display_name, + external_username=event.username, + external_metadata=event.metadata, + ) + if binding is None: + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="That pairing code is invalid or expired. Generate a new code in SurfSense.", + ) + return True + + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="SurfSense is connected. Send a message here to chat with your agent.", + ) + return True + + +async def handle_help_command( + *, adapter: TelegramAdapter, event: ParsedInboundEvent +) -> bool: + if not event.external_peer_id: + return True + await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT) + return True + + +async def send_unbound_onboarding( + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + dashboard_url: str, +) -> None: + if not event.external_peer_id: + return + wait_ms = await acquire_token( + f"tg:onboarded:{event.external_peer_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + await adapter.send_message( + external_peer_id=event.external_peer_id, + text=( + "Hi! To use SurfSense via Telegram, generate a pairing code at " + f"{dashboard_url} and send /start CODE here." + ), + ) + + +class TelegramGatewayCommands(BaseGatewayCommands): + async def handle_start_command( + self, + *, + session, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_start_command(session=session, adapter=adapter, event=event) + + async def handle_help_command( + self, + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_help_command(adapter=adapter, event=event) + + async def send_unbound_onboarding( + self, + *, + adapter: TelegramAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + await send_unbound_onboarding( + adapter=adapter, + event=event, + dashboard_url=dashboard_url, + ) diff --git a/surfsense_backend/app/gateway/telegram/formatting.py b/surfsense_backend/app/gateway/telegram/formatting.py new file mode 100644 index 000000000..668a6c7ed --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/formatting.py @@ -0,0 +1,62 @@ +"""Telegram formatting helpers.""" + +from __future__ import annotations + +import re + +from app.gateway.base.formatting import split_text_message + +MARKDOWN_V2_RESERVED = r"_*[]()~`>#+-=|{}.!" +MAX_TELEGRAM_MESSAGE_UNITS = 4096 + +_RESERVED_RE = re.compile(r"([_\*\[\]\(\)~`>#+\-=|{}\.!])") + + +def escape_markdown_v2(text: str) -> str: + """Escape all Telegram MarkdownV2 reserved characters.""" + return _RESERVED_RE.sub(r"\\\1", text) + + +def _utf16_len(text: str) -> int: + return len(text.encode("utf-16-le")) // 2 + + +def _split_at_boundary(text: str, max_units: int) -> tuple[str, str]: + if _utf16_len(text) <= max_units: + return text, "" + + # Build a hard upper bound by code point, then walk back to natural + # boundaries. Telegram's limit is UTF-16 code units, so verify candidates. + end = min(len(text), max_units) + while end > 0 and _utf16_len(text[:end]) > max_units: + end -= 1 + + candidate = text[:end] + boundary = max( + candidate.rfind("\n\n"), candidate.rfind(". "), candidate.rfind("\n") + ) + if boundary > max(200, end // 2): + end = boundary + ( + 2 if candidate[boundary : boundary + 2] in {"\n\n", ". "} else 1 + ) + + return text[:end], text[end:] + + +def chunk_message( + text: str, + *, + max_units: int = MAX_TELEGRAM_MESSAGE_UNITS, +) -> list[str]: + """Split a Telegram message at paragraph/sentence boundaries.""" + if max_units == MAX_TELEGRAM_MESSAGE_UNITS: + if not text: + return [""] + + chunks: list[str] = [] + remaining = text + while remaining: + chunk, remaining = _split_at_boundary(remaining, max_units) + chunks.append(chunk) + return chunks + return split_text_message(text, max_chars=max_units) diff --git a/surfsense_backend/app/gateway/telegram/translator.py b/surfsense_backend/app/gateway/telegram/translator.py new file mode 100644 index 000000000..a1600b0e7 --- /dev/null +++ b/surfsense_backend/app/gateway/telegram/translator.py @@ -0,0 +1,174 @@ +"""Translate agent stream events into Telegram messages.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator + +from telegram.constants import ParseMode + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.ratelimit import wait_for_token +from app.gateway.telegram.adapter import TelegramAdapter +from app.gateway.telegram.client import retry_plaintext_on_bad_markdown +from app.gateway.telegram.formatting import chunk_message, escape_markdown_v2 +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, + record_gateway_rate_limit_hit, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from Telegram. " + "Try again with a different request." +) + + +class TelegramStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: TelegramAdapter, + external_peer_id: str, + assistant_message_id: int | None = None, + debounce_seconds: float = 1.5, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.assistant_message_id = assistant_message_id + self.debounce_seconds = debounce_seconds + self._buffer = "" + self._last_flush_at = 0.0 + self._external_message_ids: list[str] = [] + self._plaintext_mode = False + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) + await self._maybe_flush() + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush(final=True) + + async def _maybe_flush(self) -> None: + now = asyncio.get_running_loop().time() + if now - self._last_flush_at < self.debounce_seconds: + return + await self._flush(final=False) + self._last_flush_at = now + + async def _flush(self, *, final: bool) -> None: + if not self._buffer: + return + + chunks = chunk_message(self._buffer) + # During streaming, keep edits on the last chunk only. At final flush, + # send any additional chunks and mark the message as finalized by the + # persistence layer (wired through agent/task code). + if len(chunks) > 1: + for chunk in chunks[:-1]: + result = await self._send_text(chunk) + self._external_message_ids.append(result.external_message_id) + self._buffer = chunks[-1] + + text = self._format_text(self._buffer) + if self._external_message_ids: + await self._edit_text(self._external_message_ids[-1], text) + else: + result = await self._send_text(self._buffer) + self._external_message_ids.append(result.external_message_id) + + if final: + logger.debug( + "Telegram gateway finalized assistant message id=%s external_ids=%s", + self.assistant_message_id, + self._external_message_ids, + ) + + def _format_text(self, text: str) -> str: + return text if self._plaintext_mode else escape_markdown_v2(text) + + async def _send_text(self, text: str) -> PlatformSendResult: + await self._throttle() + parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + logger.info( + "Telegram gateway sending message peer=%s chars=%d", + self.external_peer_id, + len(text), + ) + try: + result = await retry_plaintext_on_bad_markdown( + self.adapter.send_message, + external_peer_id=self.external_peer_id, + text=self._format_text(text), + parse_mode=parse_mode, + ) + except Exception: + record_gateway_outbound(platform="telegram", kind="send", status="failed") + raise + logger.info( + "Telegram gateway sent message peer=%s message_id=%s", + self.external_peer_id, + result.external_message_id, + ) + record_gateway_outbound(platform="telegram", kind="send", status="sent") + return result + + async def _edit_text(self, message_id: str, text: str) -> PlatformSendResult: + await self._throttle() + parse_mode = None if self._plaintext_mode else ParseMode.MARKDOWN_V2 + logger.info( + "Telegram gateway editing message peer=%s message_id=%s chars=%d", + self.external_peer_id, + message_id, + len(text), + ) + try: + result = await retry_plaintext_on_bad_markdown( + self.adapter.edit_message, + external_peer_id=self.external_peer_id, + external_message_id=message_id, + text=text, + parse_mode=parse_mode, + ) + except Exception: + record_gateway_outbound(platform="telegram", kind="edit", status="failed") + raise + logger.info( + "Telegram gateway edited message peer=%s message_id=%s", + self.external_peer_id, + result.external_message_id, + ) + record_gateway_outbound(platform="telegram", kind="edit", status="edited") + return result + + async def _throttle(self) -> None: + chat_wait = await wait_for_token( + f"tg:chat:{self.external_peer_id}", + capacity=1, + refill_per_sec=1.0, + ) + if chat_wait: + record_gateway_rate_limit_hit(bucket="tg:chat") + global_wait = await wait_for_token( + "tg:global", capacity=25, refill_per_sec=25.0 + ) + if global_wait: + record_gateway_rate_limit_hit(bucket="tg:global") + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush(final=False) + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="telegram") diff --git a/surfsense_backend/app/gateway/thread_lock.py b/surfsense_backend/app/gateway/thread_lock.py new file mode 100644 index 000000000..208b2898d --- /dev/null +++ b/surfsense_backend/app/gateway/thread_lock.py @@ -0,0 +1,41 @@ +"""Redis-backed distributed locks for gateway conversation turns.""" + +from __future__ import annotations + +import logging + +import redis + +from app.config import config +from app.observability.metrics import record_gateway_thread_lock_contention + +logger = logging.getLogger(__name__) + +_redis_client: redis.Redis | None = None + + +def _redis() -> redis.Redis: + global _redis_client + if _redis_client is None: + _redis_client = redis.from_url(config.REDIS_APP_URL, decode_responses=True) + return _redis_client + + +def _lock_key(thread_id: int) -> str: + return f"gateway:thread_lock:{thread_id}" + + +def acquire_thread_lock(thread_id: int, ttl: int = 60) -> bool: + acquired = bool(_redis().set(_lock_key(thread_id), "1", nx=True, ex=ttl)) + if not acquired: + record_gateway_thread_lock_contention() + return acquired + + +def release_thread_lock(thread_id: int) -> None: + try: + _redis().delete(_lock_key(thread_id)) + except redis.RedisError as exc: + logger.warning( + "Failed to release gateway thread lock for %s: %s", thread_id, exc + ) diff --git a/surfsense_backend/app/gateway/whatsapp/__init__.py b/surfsense_backend/app/gateway/whatsapp/__init__.py new file mode 100644 index 000000000..5c54d2caf --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/__init__.py @@ -0,0 +1 @@ +"""WhatsApp gateway implementations.""" diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py new file mode 100644 index 000000000..330ef3bb9 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/adapter_baileys.py @@ -0,0 +1,121 @@ +"""Baileys bridge platform adapter.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any + +import httpx + +from app.config import config +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) + + +class WhatsAppBaileysAdapter(BasePlatformAdapter): + platform = "whatsapp" + + def __init__(self, bridge_url: str | None = None) -> None: + self.bridge_url = (bridge_url or config.WHATSAPP_BRIDGE_URL).rstrip("/") + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + chat_id = str(raw_payload.get("chatId") or "") + sender_id = str(raw_payload.get("senderId") or chat_id) + message_id = str(raw_payload.get("messageId") or "") + body = raw_payload.get("body") + is_group = bool(raw_payload.get("isGroup")) + return ParsedInboundEvent( + platform=self.platform, + event_kind="message", + external_peer_id=chat_id or None, + external_peer_kind="group" if is_group else "direct", + external_message_id=message_id or None, + external_user_id=sender_id or None, + text=str(body) if body is not None else None, + raw_payload=raw_payload, + display_name=str(raw_payload.get("chatName") or sender_id or chat_id) + or None, + username=None, + metadata={ + "sender_id": sender_id, + "from_me": bool(raw_payload.get("fromMe")), + "timestamp": raw_payload.get("timestamp"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = {"chatId": external_peer_id, "message": text} + if reply_to_message_id: + payload["replyTo"] = reply_to_message_id + data = await self._post("/send", payload) + return PlatformSendResult( + external_message_id=str(data.get("messageId") or ""), + raw_response=data, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + data = await self._post( + "/edit", + { + "chatId": external_peer_id, + "messageId": external_message_id, + "message": text, + }, + ) + return PlatformSendResult( + external_message_id=str(data.get("messageId") or external_message_id), + raw_response=data, + ) + + async def send_typing_indicator(self, *, external_peer_id: str) -> None: + await self._post("/typing", {"chatId": external_peer_id}, expect_json=False) + + async def validate_credentials(self) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=10) as client: + response = await client.get(f"{self.bridge_url}/health") + response.raise_for_status() + return response.json() + + async def fetch_updates( + self, *, offset: int | None + ) -> AsyncIterator[dict[str, Any]]: + async with httpx.AsyncClient(timeout=35) as client: + response = await client.get(f"{self.bridge_url}/messages") + response.raise_for_status() + for message in response.json(): + if isinstance(message, dict): + yield message + + async def request_pairing_code(self, *, phone_number: str) -> dict[str, Any]: + return await self._post("/pair", {"phoneNumber": phone_number}) + + async def _post( + self, + path: str, + payload: dict[str, Any], + *, + expect_json: bool = True, + ) -> dict[str, Any]: + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post(f"{self.bridge_url}{path}", json=payload) + response.raise_for_status() + if not expect_json or response.status_code == 204: + return {} + return response.json() diff --git a/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py new file mode 100644 index 000000000..58d13e83e --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/adapter_cloud.py @@ -0,0 +1,153 @@ +"""WhatsApp Cloud API platform adapter.""" + +from __future__ import annotations + +from typing import Any + +from app.gateway.base.adapter import ( + BasePlatformAdapter, + ParsedInboundEvent, + PlatformSendResult, +) +from app.gateway.whatsapp.client_cloud import WhatsAppCloudClient +from app.gateway.whatsapp.credentials import WhatsAppCredentials + + +class WhatsAppCloudAdapter(BasePlatformAdapter): + platform = "whatsapp" + + def __init__(self, credentials: WhatsAppCredentials) -> None: + self.credentials = credentials + self.client = WhatsAppCloudClient( + business_token=credentials["business_token"], + phone_number_id=credentials["phone_number_id"], + api_version=credentials.get("api_version"), + ) + + def parse_inbound(self, raw_payload: dict[str, Any]) -> ParsedInboundEvent: + message = _first_message(raw_payload) + if message is None: + return ParsedInboundEvent( + platform=self.platform, + event_kind="other", + external_peer_id=None, + external_peer_kind="unknown", + external_message_id=None, + external_user_id=None, + text=None, + raw_payload=raw_payload, + ) + + contact = _first_contact(raw_payload, message.get("from")) + text = _message_text(message) + wa_id = str(message.get("from") or "") + return ParsedInboundEvent( + platform=self.platform, + event_kind=str(message.get("type") or "message"), + external_peer_id=wa_id or None, + external_peer_kind="direct", + external_message_id=str(message.get("id")) if message.get("id") else None, + external_user_id=wa_id or None, + text=text, + raw_payload=raw_payload, + display_name=(contact.get("profile") or {}).get("name"), + username=None, + metadata={ + "phone_number_id": _metadata(raw_payload).get("phone_number_id"), + "display_phone_number": _metadata(raw_payload).get( + "display_phone_number" + ), + "timestamp": message.get("timestamp"), + "message_type": message.get("type"), + }, + ) + + async def send_message( + self, + *, + external_peer_id: str, + text: str, + parse_mode: str | None = None, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + return await self.client.send_text( + to=external_peer_id, + text=text, + reply_to_message_id=reply_to_message_id, + ) + + async def edit_message( + self, + *, + external_peer_id: str, + external_message_id: str, + text: str, + parse_mode: str | None = None, + ) -> PlatformSendResult: + raise NotImplementedError("WhatsApp Cloud API does not support message edits") + + async def send_typing_indicator(self, *, inbound_message_id: str) -> None: + await self.client.send_typing_indicator(message_id=inbound_message_id) + + async def validate_credentials(self) -> dict[str, Any]: + return await self.client.validate() + + +def _changes(raw_payload: dict[str, Any]) -> list[dict[str, Any]]: + changes: list[dict[str, Any]] = [] + for entry in raw_payload.get("entry") or []: + if isinstance(entry, dict): + changes.extend( + change + for change in (entry.get("changes") or []) + if isinstance(change, dict) + ) + return changes + + +def _first_message(raw_payload: dict[str, Any]) -> dict[str, Any] | None: + for change in _changes(raw_payload): + value = change.get("value") or {} + messages = value.get("messages") or [] + if messages and isinstance(messages[0], dict): + return messages[0] + if "message" in raw_payload and isinstance(raw_payload["message"], dict): + return raw_payload["message"] + return None + + +def _first_contact( + raw_payload: dict[str, Any], + wa_id: object, +) -> dict[str, Any]: + for change in _changes(raw_payload): + value = change.get("value") or {} + for contact in value.get("contacts") or []: + if isinstance(contact, dict) and ( + wa_id is None or str(contact.get("wa_id")) == str(wa_id) + ): + return contact + return {} + + +def _metadata(raw_payload: dict[str, Any]) -> dict[str, Any]: + for change in _changes(raw_payload): + value = change.get("value") or {} + metadata = value.get("metadata") + if isinstance(metadata, dict): + return metadata + return {} + + +def _message_text(message: dict[str, Any]) -> str | None: + message_type = message.get("type") + if message_type == "text": + return (message.get("text") or {}).get("body") + if message_type == "button": + return (message.get("button") or {}).get("text") + if message_type == "interactive": + interactive = message.get("interactive") or {} + button_reply = interactive.get("button_reply") or {} + list_reply = interactive.get("list_reply") or {} + return button_reply.get("title") or list_reply.get("title") + return None diff --git a/surfsense_backend/app/gateway/whatsapp/client_cloud.py b/surfsense_backend/app/gateway/whatsapp/client_cloud.py new file mode 100644 index 000000000..e39e022aa --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/client_cloud.py @@ -0,0 +1,99 @@ +"""Small httpx wrapper for the WhatsApp Cloud API.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from app.config import config +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.ratelimit import wait_for_token +from app.observability.metrics import record_gateway_rate_limit_hit + + +class WhatsAppCloudClient: + def __init__( + self, + *, + business_token: str, + phone_number_id: str, + api_version: str | None = None, + ) -> None: + self.business_token = business_token + self.phone_number_id = phone_number_id + self.api_version = api_version or config.WHATSAPP_GRAPH_API_VERSION + self.base_url = f"https://graph.facebook.com/{self.api_version}" + + async def send_text( + self, + *, + to: str, + text: str, + reply_to_message_id: str | None = None, + ) -> PlatformSendResult: + payload: dict[str, Any] = { + "messaging_product": "whatsapp", + "recipient_type": "individual", + "to": to, + "type": "text", + "text": {"preview_url": True, "body": text}, + } + if reply_to_message_id: + payload["context"] = {"message_id": reply_to_message_id} + data = await self._post(f"/{self.phone_number_id}/messages", json=payload) + message_id = str((data.get("messages") or [{}])[0].get("id") or "") + return PlatformSendResult(external_message_id=message_id, raw_response=data) + + async def send_typing_indicator(self, *, message_id: str) -> dict[str, Any]: + payload = { + "messaging_product": "whatsapp", + "status": "read", + "message_id": message_id, + "typing_indicator": {"type": "text"}, + } + return await self._post(f"/{self.phone_number_id}/messages", json=payload) + + async def validate(self) -> dict[str, Any]: + return await self._get( + f"/{self.phone_number_id}", + params={ + "fields": "verified_name,quality_rating,account_review_status,display_phone_number" + }, + ) + + async def _post(self, path: str, *, json: dict[str, Any]) -> dict[str, Any]: + await self._throttle() + async with httpx.AsyncClient(timeout=20) as client: + response = await client.post( + f"{self.base_url}{path}", + headers={"Authorization": f"Bearer {self.business_token}"}, + json=json, + ) + response.raise_for_status() + return response.json() + + async def _get( + self, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + await self._throttle() + async with httpx.AsyncClient(timeout=20) as client: + response = await client.get( + f"{self.base_url}{path}", + headers={"Authorization": f"Bearer {self.business_token}"}, + params=params, + ) + response.raise_for_status() + return response.json() + + async def _throttle(self) -> None: + wait_ms = await wait_for_token( + f"wa:phone:{self.phone_number_id}", + capacity=10, + refill_per_sec=10.0, + ) + if wait_ms: + record_gateway_rate_limit_hit(bucket="wa:phone") diff --git a/surfsense_backend/app/gateway/whatsapp/commands.py b/surfsense_backend/app/gateway/whatsapp/commands.py new file mode 100644 index 000000000..28b765347 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/commands.py @@ -0,0 +1,123 @@ +"""WhatsApp command handlers.""" + +from __future__ import annotations + +from app.gateway.base.adapter import BasePlatformAdapter, ParsedInboundEvent +from app.gateway.base.commands import BaseGatewayCommands +from app.gateway.pairing import redeem_pairing_code +from app.gateway.ratelimit import acquire_token + +HELP_TEXT = ( + "SurfSense WhatsApp commands:\n" + "/start - pair this chat\n" + "/new - start a fresh conversation\n" + "/help - show this help" +) + + +async def handle_start_command( + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> bool: + text = event.text or "" + parts = text.split(maxsplit=1) + if len(parts) != 2 or not event.external_peer_id: + await adapter.send_message( + external_peer_id=event.external_peer_id or "", + text=( + "Generate a pairing code in SurfSense Settings > Messaging Channels, " + "then send /start CODE here." + ), + ) + return True + + binding = await redeem_pairing_code( + session, + code=parts[1].strip(), + external_peer_id=event.external_peer_id, + external_peer_kind=event.external_peer_kind, + external_display_name=event.display_name, + external_username=event.username, + external_metadata=event.metadata, + ) + if binding is None: + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="That pairing code is invalid or expired. Generate a new code in SurfSense.", + ) + return True + + await adapter.send_message( + external_peer_id=event.external_peer_id, + text="SurfSense is connected. Send a message here to chat with your agent.", + ) + return True + + +async def handle_help_command( + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, +) -> bool: + if not event.external_peer_id: + return True + await adapter.send_message(external_peer_id=event.external_peer_id, text=HELP_TEXT) + return True + + +async def send_unbound_onboarding( + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, +) -> None: + if not event.external_peer_id: + return + wait_ms = await acquire_token( + f"wa:onboarded:{event.external_peer_id}", + capacity=1, + refill_per_sec=1 / 3600, + ) + if wait_ms > 0: + return + await adapter.send_message( + external_peer_id=event.external_peer_id, + text=( + "Hi! To use SurfSense via WhatsApp, generate a pairing code at " + f"{dashboard_url} and send /start CODE here." + ), + ) + + +class WhatsAppGatewayCommands(BaseGatewayCommands): + async def handle_start_command( + self, + *, + session, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_start_command(session=session, adapter=adapter, event=event) + + async def handle_help_command( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + ) -> bool: + return await handle_help_command(adapter=adapter, event=event) + + async def send_unbound_onboarding( + self, + *, + adapter: BasePlatformAdapter, + event: ParsedInboundEvent, + dashboard_url: str, + ) -> None: + await send_unbound_onboarding( + adapter=adapter, + event=event, + dashboard_url=dashboard_url, + ) diff --git a/surfsense_backend/app/gateway/whatsapp/credentials.py b/surfsense_backend/app/gateway/whatsapp/credentials.py new file mode 100644 index 000000000..ed725944a --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/credentials.py @@ -0,0 +1,30 @@ +"""Credential helpers for WhatsApp gateway accounts.""" + +from __future__ import annotations + +from typing import TypedDict + +from app.config import config + + +class WhatsAppCredentials(TypedDict, total=False): + business_token: str + waba_id: str + phone_number_id: str + business_id: str + registration_pin: str + api_version: str + + +def load_system_whatsapp_credentials() -> WhatsAppCredentials: + if not ( + config.WHATSAPP_SHARED_BUSINESS_TOKEN and config.WHATSAPP_SHARED_PHONE_NUMBER_ID + ): + raise RuntimeError("whatsapp_system_credentials_not_configured") + + return { + "business_token": config.WHATSAPP_SHARED_BUSINESS_TOKEN, + "phone_number_id": config.WHATSAPP_SHARED_PHONE_NUMBER_ID, + "waba_id": config.WHATSAPP_SHARED_WABA_ID, + "api_version": config.WHATSAPP_GRAPH_API_VERSION, + } diff --git a/surfsense_backend/app/gateway/whatsapp/translator.py b/surfsense_backend/app/gateway/whatsapp/translator.py new file mode 100644 index 000000000..4673a51ca --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/translator.py @@ -0,0 +1,92 @@ +"""Translate agent stream events into WhatsApp Cloud API messages.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import BasePlatformAdapter, PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.whatsapp.adapter_cloud import WhatsAppCloudAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from WhatsApp. " + "Try again with a different request." +) + + +class WhatsAppCloudStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: BasePlatformAdapter, + external_peer_id: str, + inbound_message_id: str | None = None, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.inbound_message_id = inbound_message_id + self._buffer = "" + self._typing_sent = False + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + if not self._typing_sent: + await self._send_typing_indicator() + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush_final() + + async def _flush_final(self) -> None: + if not self._buffer: + return + for chunk in split_text_message(self._buffer): + await self._send_text(chunk) + + async def _send_typing_indicator(self) -> None: + self._typing_sent = True + if not self.inbound_message_id: + return + if not isinstance(self.adapter, WhatsAppCloudAdapter): + return + try: + await self.adapter.send_typing_indicator( + inbound_message_id=self.inbound_message_id + ) + record_gateway_outbound(platform="whatsapp", kind="typing", status="sent") + except Exception: + logger.debug("WhatsApp typing indicator failed", exc_info=True) + record_gateway_outbound(platform="whatsapp", kind="typing", status="failed") + + async def _send_text(self, text: str) -> PlatformSendResult: + try: + result = await self.adapter.send_message( + external_peer_id=self.external_peer_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="send", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="send", status="sent") + return result + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush_final() + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="whatsapp") diff --git a/surfsense_backend/app/gateway/whatsapp/translator_baileys.py b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py new file mode 100644 index 000000000..ab2afb9d7 --- /dev/null +++ b/surfsense_backend/app/gateway/whatsapp/translator_baileys.py @@ -0,0 +1,127 @@ +"""Translate agent stream events into Baileys bridge messages.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator + +from app.gateway.base.adapter import BasePlatformAdapter, PlatformSendResult +from app.gateway.base.formatting import split_text_message +from app.gateway.base.translator import BaseStreamTranslator, GatewayStreamEvent +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.observability.metrics import ( + record_gateway_hitl_aborted, + record_gateway_outbound, +) + +logger = logging.getLogger(__name__) + +HITL_UNSUPPORTED_MESSAGE = ( + "This action requires approval and is not yet supported from WhatsApp. " + "Try again with a different request." +) + + +class WhatsAppBaileysStreamTranslator(BaseStreamTranslator): + def __init__( + self, + *, + adapter: BasePlatformAdapter, + external_peer_id: str, + debounce_seconds: float = 1.5, + ) -> None: + self.adapter = adapter + self.external_peer_id = external_peer_id + self.debounce_seconds = debounce_seconds + self._buffer = "" + self._last_flush_at = 0.0 + self._external_message_ids: list[str] = [] + + async def translate(self, events: AsyncIterator[GatewayStreamEvent]) -> None: + await self._send_typing_indicator() + async for event in events: + if event.type in {"text-delta", "text_delta", "text"}: + self._buffer += str( + event.data.get("text") or event.data.get("delta") or "" + ) + await self._maybe_flush() + elif event.type in {"data-interrupt-request", "interrupt"}: + await self._handle_hitl_interrupt() + return + elif event.type in {"finish", "done"}: + break + + await self._flush(final=True) + + async def _maybe_flush(self) -> None: + now = asyncio.get_running_loop().time() + if now - self._last_flush_at < self.debounce_seconds: + return + await self._flush(final=False) + self._last_flush_at = now + + async def _flush(self, *, final: bool) -> None: + if not self._buffer: + return + + chunks = split_text_message(self._buffer) + if len(chunks) > 1: + for chunk in chunks[:-1]: + result = await self._send_text(chunk) + self._external_message_ids.append(result.external_message_id) + self._buffer = chunks[-1] + + if self._external_message_ids: + await self._edit_text(self._external_message_ids[-1], self._buffer) + else: + result = await self._send_text(self._buffer) + self._external_message_ids.append(result.external_message_id) + + if final: + logger.debug( + "WhatsApp Baileys finalized external_ids=%s", + self._external_message_ids, + ) + + async def _send_typing_indicator(self) -> None: + if not isinstance(self.adapter, WhatsAppBaileysAdapter): + return + try: + await self.adapter.send_typing_indicator( + external_peer_id=self.external_peer_id + ) + record_gateway_outbound(platform="whatsapp", kind="typing", status="sent") + except Exception: + logger.debug("WhatsApp Baileys typing indicator failed", exc_info=True) + + async def _send_text(self, text: str) -> PlatformSendResult: + try: + result = await self.adapter.send_message( + external_peer_id=self.external_peer_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="send", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="send", status="sent") + return result + + async def _edit_text(self, message_id: str, text: str) -> PlatformSendResult: + try: + result = await self.adapter.edit_message( + external_peer_id=self.external_peer_id, + external_message_id=message_id, + text=text, + ) + except Exception: + record_gateway_outbound(platform="whatsapp", kind="edit", status="failed") + raise + record_gateway_outbound(platform="whatsapp", kind="edit", status="edited") + return result + + async def _handle_hitl_interrupt(self) -> None: + if self._buffer: + await self._flush(final=False) + await self._send_text(HITL_UNSUPPORTED_MESSAGE) + record_gateway_hitl_aborted(platform="whatsapp") diff --git a/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py b/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py index 0bbb67105..9a9e4e4d6 100644 --- a/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py +++ b/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py @@ -18,8 +18,6 @@ class UploadDocumentAdapter: etl_service: str, search_space_id: int, user_id: str, - llm, - should_summarize: bool = False, ) -> None: connector_doc = ConnectorDocument( title=filename, @@ -29,9 +27,7 @@ class UploadDocumentAdapter: search_space_id=search_space_id, created_by_id=user_id, connector_id=None, - should_summarize=should_summarize, should_use_code_chunker=False, - fallback_summary=markdown_content[:4000], metadata={ "FILE_NAME": filename, "ETL_SERVICE": etl_service, @@ -43,7 +39,7 @@ class UploadDocumentAdapter: if not documents: raise RuntimeError("prepare_for_indexing returned no documents") - indexed = await self._service.index(documents[0], connector_doc, llm) + indexed = await self._service.index(documents[0], connector_doc) if not DocumentStatus.is_state(indexed.status, DocumentStatus.READY): raise RuntimeError(indexed.status.get("reason", "Indexing failed")) @@ -51,7 +47,7 @@ class UploadDocumentAdapter: indexed.content_needs_reindexing = False await self._session.commit() - async def reindex(self, document: Document, llm) -> None: + async def reindex(self, document: Document) -> None: """Re-index an existing document after its source_markdown has been updated.""" if not document.source_markdown: raise RuntimeError("Document has no source_markdown to reindex") @@ -66,15 +62,13 @@ class UploadDocumentAdapter: search_space_id=document.search_space_id, created_by_id=str(document.created_by_id), connector_id=document.connector_id, - should_summarize=True, should_use_code_chunker=False, - fallback_summary=document.source_markdown[:4000], metadata=metadata, ) document.content_hash = compute_content_hash(connector_doc) - indexed = await self._service.index(document, connector_doc, llm) + indexed = await self._service.index(document, connector_doc) if not DocumentStatus.is_state(indexed.status, DocumentStatus.READY): raise RuntimeError(indexed.status.get("reason", "Reindexing failed")) diff --git a/surfsense_backend/app/indexing_pipeline/connector_document.py b/surfsense_backend/app/indexing_pipeline/connector_document.py index 4f5d6e2e0..1297a6b46 100644 --- a/surfsense_backend/app/indexing_pipeline/connector_document.py +++ b/surfsense_backend/app/indexing_pipeline/connector_document.py @@ -11,9 +11,7 @@ class ConnectorDocument(BaseModel): unique_id: str document_type: DocumentType search_space_id: int = Field(gt=0) - should_summarize: bool = True should_use_code_chunker: bool = False - fallback_summary: str | None = None metadata: dict = {} connector_id: int | None = None created_by_id: str diff --git a/surfsense_backend/app/indexing_pipeline/document_persistence.py b/surfsense_backend/app/indexing_pipeline/document_persistence.py index d7810e516..9fd8867e2 100644 --- a/surfsense_backend/app/indexing_pipeline/document_persistence.py +++ b/surfsense_backend/app/indexing_pipeline/document_persistence.py @@ -1,3 +1,5 @@ +import contextlib +import logging from datetime import UTC, datetime from sqlalchemy.ext.asyncio import AsyncSession @@ -6,6 +8,8 @@ from sqlalchemy.orm.attributes import set_committed_value from app.db import Document, DocumentStatus +logger = logging.getLogger(__name__) + async def rollback_and_persist_failure( session: AsyncSession, document: Document, message: str @@ -18,14 +22,28 @@ async def rollback_and_persist_failure( try: await session.rollback() except Exception: - return # Session is completely dead; nothing further we can do. + # Session is completely dead; surface it but never raise. + logger.warning( + "Rollback failed; cannot persist failed status for document %s", + getattr(document, "id", "unknown"), + exc_info=True, + ) + return try: await session.refresh(document) document.updated_at = datetime.now(UTC) document.status = DocumentStatus.failed(message) await session.commit() except Exception: - pass # Best-effort; document will be retried on the next sync. + # Best-effort: the document stays non-ready and is retried next sync. + # Log it so a permanently-stuck document is at least traceable. + logger.warning( + "Could not persist failed status for document %s; will retry next sync", + getattr(document, "id", "unknown"), + exc_info=True, + ) + with contextlib.suppress(Exception): + await session.rollback() def attach_chunks_to_document(document: Document, chunks: list) -> None: diff --git a/surfsense_backend/app/indexing_pipeline/document_summarizer.py b/surfsense_backend/app/indexing_pipeline/document_summarizer.py deleted file mode 100644 index 76cc77377..000000000 --- a/surfsense_backend/app/indexing_pipeline/document_summarizer.py +++ /dev/null @@ -1,30 +0,0 @@ -from app.prompts import SUMMARY_PROMPT_TEMPLATE -from app.utils.document_converters import optimize_content_for_context_window - - -async def summarize_document( - source_markdown: str, llm, metadata: dict | None = None -) -> str: - """Generate a text summary of a document using an LLM, prefixed with metadata when provided.""" - model_name = getattr(llm, "model", "gpt-3.5-turbo") - optimized_content = optimize_content_for_context_window( - source_markdown, metadata, model_name - ) - - summary_chain = SUMMARY_PROMPT_TEMPLATE | llm - content_with_metadata = ( - f"\n\n{metadata}\n\n" - f"\n\n\n\n{optimized_content}\n\n" - ) - summary_result = await summary_chain.ainvoke({"document": content_with_metadata}) - summary_content = summary_result.content - - if metadata: - metadata_parts = ["# DOCUMENT METADATA"] - for key, value in metadata.items(): - if value: - metadata_parts.append(f"**{key.replace('_', ' ').title()}:** {value}") - metadata_section = "\n".join(metadata_parts) - return f"{metadata_section}\n\n# DOCUMENT SUMMARY\n\n{summary_content}" - - return summary_content diff --git a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py index 282bd6034..67a6778e0 100644 --- a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py +++ b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py @@ -31,7 +31,6 @@ from app.indexing_pipeline.document_persistence import ( attach_chunks_to_document, rollback_and_persist_failure, ) -from app.indexing_pipeline.document_summarizer import summarize_document from app.indexing_pipeline.exceptions import ( EMBEDDING_ERRORS, PERMANENT_LLM_ERRORS, @@ -204,7 +203,7 @@ class IndexingPipelineService: await self.session.commit() async def index_batch( - self, connector_docs: list[ConnectorDocument], llm + self, connector_docs: list[ConnectorDocument] ) -> list[Document]: """Convenience method: prepare_for_indexing then index each document. @@ -218,7 +217,7 @@ class IndexingPipelineService: connector_doc = doc_map.get(document.unique_identifier_hash) if connector_doc is None: continue - result = await self.index(document, connector_doc, llm) + result = await self.index(document, connector_doc) results.append(result) return results @@ -351,10 +350,10 @@ class IndexingPipelineService: return [] async def index( - self, document: Document, connector_doc: ConnectorDocument, llm + self, document: Document, connector_doc: ConnectorDocument ) -> Document: """ - Run summarization, embedding, and chunking for a document and persist the results. + Run deterministic content storage, embedding, and chunking for a document. """ ctx = PipelineLogContext( connector_id=connector_doc.connector_id, @@ -379,20 +378,7 @@ class IndexingPipelineService: document.status = DocumentStatus.processing() await self.session.commit() - t_step = time.perf_counter() - if connector_doc.should_summarize and llm is not None: - content = await summarize_document( - connector_doc.source_markdown, llm, connector_doc.metadata - ) - perf.info( - "[indexing] summarize_document doc=%d in %.3fs", - document.id, - time.perf_counter() - t_step, - ) - elif connector_doc.should_summarize and connector_doc.fallback_summary: - content = connector_doc.fallback_summary - else: - content = connector_doc.source_markdown + content = connector_doc.source_markdown await self.session.execute( delete(Chunk).where(Chunk.document_id == document.id) @@ -523,7 +509,6 @@ class IndexingPipelineService: async def index_batch_parallel( self, connector_docs: list[ConnectorDocument], - get_llm: Callable[[AsyncSession], Awaitable], *, max_concurrency: int = 4, on_heartbeat: Callable[[int], Awaitable[None]] | None = None, @@ -532,8 +517,8 @@ class IndexingPipelineService: """Index documents in parallel with bounded concurrency. Phase 1 (serial): prepare_for_indexing using self.session. - Phase 2 (parallel): index each document in an isolated session, - bounded by a semaphore to avoid overwhelming APIs/DB. + Phase 2 (parallel): index each document in an isolated session, bounded + by a semaphore to avoid overwhelming embedding APIs/DB. """ logger = logging.getLogger(__name__) perf = get_perf_logger() @@ -577,9 +562,8 @@ class IndexingPipelineService: failed_count += 1 return document - llm = await get_llm(isolated_session) iso_pipeline = IndexingPipelineService(isolated_session) - result = await iso_pipeline.index(refetched, connector_doc, llm) + result = await iso_pipeline.index(refetched, connector_doc) async with lock: if DocumentStatus.is_state( diff --git a/surfsense_backend/app/notifications/__init__.py b/surfsense_backend/app/notifications/__init__.py new file mode 100644 index 000000000..554872d85 --- /dev/null +++ b/surfsense_backend/app/notifications/__init__.py @@ -0,0 +1,18 @@ +"""User notifications: persistence, service, and HTTP API. + +Emit notifications via :class:`~app.notifications.service.NotificationService`; +the router in :mod:`app.notifications.api` exposes the inbox endpoints. +""" + +from __future__ import annotations + +# Initialize app.db first to avoid a partial-init circular import when this +# package is the entry point (e.g. Celery loading it before any ORM code). +import app.db # noqa: F401 +from app.notifications.persistence import Notification +from app.notifications.service import NotificationService + +__all__ = [ + "Notification", + "NotificationService", +] diff --git a/surfsense_backend/app/notifications/api/__init__.py b/surfsense_backend/app/notifications/api/__init__.py new file mode 100644 index 000000000..2708c8805 --- /dev/null +++ b/surfsense_backend/app/notifications/api/__init__.py @@ -0,0 +1,7 @@ +"""Notifications HTTP API.""" + +from __future__ import annotations + +from app.notifications.api.api import router + +__all__ = ["router"] diff --git a/surfsense_backend/app/routes/notifications_routes.py b/surfsense_backend/app/notifications/api/api.py similarity index 63% rename from surfsense_backend/app/routes/notifications_routes.py rename to surfsense_backend/app/notifications/api/api.py index 611227795..ddca09c66 100644 --- a/surfsense_backend/app/routes/notifications_routes.py +++ b/surfsense_backend/app/notifications/api/api.py @@ -1,124 +1,36 @@ -""" -Notifications API routes. -These endpoints allow marking notifications as read and fetching older notifications. -Zero automatically syncs the changes to all connected clients for recent items. -For older items (beyond the sync window), use the list endpoint. -""" +"""HTTP routes for the notifications inbox (list, counts, mark-read).""" + +from __future__ import annotations from datetime import UTC, datetime, timedelta -from typing import Literal from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic import BaseModel from sqlalchemy import case, desc, func, literal, literal_column, select, update from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Notification, User, get_async_session +from app.db import User, get_async_session +from app.notifications.api.schemas import ( + BatchUnreadCountResponse, + CategoryUnreadCount, + MarkAllReadResponse, + MarkReadResponse, + NotificationListResponse, + SourceTypeItem, + SourceTypesResponse, + UnreadCountResponse, +) +from app.notifications.api.transform import ( + parse_before_date, + parse_source_type, + to_response, +) +from app.notifications.constants import CATEGORY_TYPES, SYNC_WINDOW_DAYS +from app.notifications.persistence import Notification +from app.notifications.types import NotificationCategory, NotificationType from app.users import current_active_user router = APIRouter(prefix="/notifications", tags=["notifications"]) -# Must match frontend SYNC_WINDOW_DAYS in use-inbox.ts -SYNC_WINDOW_DAYS = 14 - -# Valid notification types - must match frontend InboxItemTypeEnum -NotificationType = Literal[ - "connector_indexing", - "connector_deletion", - "document_processing", - "new_mention", - "comment_reply", - "page_limit_exceeded", -] - -# Category-to-types mapping for filtering by tab -NotificationCategory = Literal["comments", "status"] -CATEGORY_TYPES: dict[str, tuple[str, ...]] = { - "comments": ("new_mention", "comment_reply"), - "status": ( - "connector_indexing", - "connector_deletion", - "document_processing", - "page_limit_exceeded", - ), -} - - -class NotificationResponse(BaseModel): - """Response model for a single notification.""" - - id: int - user_id: str - search_space_id: int | None - type: str - title: str - message: str - read: bool - metadata: dict - created_at: str - updated_at: str | None - - class Config: - from_attributes = True - - -class NotificationListResponse(BaseModel): - """Response for listing notifications with pagination.""" - - items: list[NotificationResponse] - total: int - has_more: bool - next_offset: int | None - - -class MarkReadResponse(BaseModel): - """Response for mark as read operations.""" - - success: bool - message: str - - -class MarkAllReadResponse(BaseModel): - """Response for mark all as read operation.""" - - success: bool - message: str - updated_count: int - - -class SourceTypeItem(BaseModel): - """A single source type with its category and count.""" - - key: str - type: str - category: str # "connector" or "document" - count: int - - -class SourceTypesResponse(BaseModel): - """Response for notification source types used in status tab filter.""" - - sources: list[SourceTypeItem] - - -class UnreadCountResponse(BaseModel): - """Response for unread count with split between recent and older items.""" - - total_unread: int - recent_unread: int # Within SYNC_WINDOW_DAYS - - -class CategoryUnreadCount(BaseModel): - total_unread: int - recent_unread: int - - -class BatchUnreadCountResponse(BaseModel): - """Batched unread counts for all categories in a single response.""" - - comments: CategoryUnreadCount - status: CategoryUnreadCount - @router.get("/unread-counts-batch", response_model=BatchUnreadCountResponse) async def get_unread_counts_batch( @@ -126,12 +38,7 @@ async def get_unread_counts_batch( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> BatchUnreadCountResponse: - """ - Get unread counts for all notification categories in a single DB query. - - Replaces multiple separate calls to /unread-count with different category - filters, reducing round-trips from 2+ to 1. - """ + """Unread counts for every category in a single query.""" cutoff_date = datetime.now(UTC) - timedelta(days=SYNC_WINDOW_DAYS) base_filter = [ @@ -140,6 +47,7 @@ async def get_unread_counts_batch( ] if search_space_id is not None: + # Include global (null search-space) notifications. base_filter.append( (Notification.search_space_id == search_space_id) | (Notification.search_space_id.is_(None)) @@ -181,14 +89,11 @@ async def get_notification_source_types( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> SourceTypesResponse: - """ - Get all distinct connector types and document types from the user's - status notifications. Used to populate the filter dropdown in the - inbox Status tab so that all types are shown regardless of pagination. - """ + """Distinct connector/document source types for the Status tab filter.""" base_filter = [Notification.user_id == user.id] if search_space_id is not None: + # Include global (null search-space) notifications. base_filter.append( (Notification.search_space_id == search_space_id) | (Notification.search_space_id.is_(None)) @@ -258,47 +163,35 @@ async def get_unread_count( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> UnreadCountResponse: - """ - Get the total unread notification count for the current user. + """Total and recent (within sync window) unread counts for the user. - Returns both: - - total_unread: All unread notifications (for accurate badge count) - - recent_unread: Unread notifications within the sync window (last 14 days) - - This allows the frontend to calculate: - - older_unread = total_unread - recent_unread (static until reconciliation) - - Display count = older_unread + live_recent_count (from Zero) + Returning both lets a client hold the older count static while + live-syncing the recent ones. """ - # Calculate cutoff date for sync window cutoff_date = datetime.now(UTC) - timedelta(days=SYNC_WINDOW_DAYS) - # Base filter for user's unread notifications base_filter = [ Notification.user_id == user.id, Notification.read == False, # noqa: E712 ] - # Add search space filter if provided (include null for global notifications) if search_space_id is not None: + # Include global (null search-space) notifications. base_filter.append( (Notification.search_space_id == search_space_id) | (Notification.search_space_id.is_(None)) ) - # Filter by notification type if provided if type_filter: base_filter.append(Notification.type == type_filter) - # Filter by category (maps to multiple types) if category: base_filter.append(Notification.type.in_(CATEGORY_TYPES[category])) - # Total unread count (all time) total_query = select(func.count(Notification.id)).where(*base_filter) total_result = await session.execute(total_query) total_unread = total_result.scalar() or 0 - # Recent unread count (within sync window) recent_query = select(func.count(Notification.id)).where( *base_filter, Notification.created_at > cutoff_date, @@ -340,22 +233,14 @@ async def list_notifications( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> NotificationListResponse: - """ - List notifications for the current user with pagination. - - This endpoint is used as a fallback for older notifications that are - outside the Zero sync window (2 weeks). - - Use `before_date` to paginate through older notifications efficiently. - """ - # Build base query + """Paginated inbox fallback for items outside the Zero sync window.""" query = select(Notification).where(Notification.user_id == user.id) count_query = select(func.count(Notification.id)).where( Notification.user_id == user.id ) - # Filter by search space (include null search_space_id for global notifications) if search_space_id is not None: + # Include global (null search-space) notifications. query = query.where( (Notification.search_space_id == search_space_id) | (Notification.search_space_id.is_(None)) @@ -365,39 +250,26 @@ async def list_notifications( | (Notification.search_space_id.is_(None)) ) - # Filter by type if type_filter: query = query.where(Notification.type == type_filter) count_query = count_query.where(Notification.type == type_filter) - # Filter by category (maps to multiple types) if category: cat_types = CATEGORY_TYPES[category] query = query.where(Notification.type.in_(cat_types)) count_query = count_query.where(Notification.type.in_(cat_types)) - # Filter by source type (connector or document type from JSONB metadata) + # source_type encodes the JSONB facet to match: 'connector:' or 'doctype:'. if source_type: - if source_type.startswith("connector:"): - connector_val = source_type[len("connector:") :] - source_filter = Notification.type.in_( - ("connector_indexing", "connector_deletion") - ) & ( - Notification.notification_metadata["connector_type"].astext - == connector_val - ) - query = query.where(source_filter) - count_query = count_query.where(source_filter) - elif source_type.startswith("doctype:"): - doctype_val = source_type[len("doctype:") :] - source_filter = Notification.type.in_(("document_processing",)) & ( - Notification.notification_metadata["document_type"].astext - == doctype_val + parsed_source = parse_source_type(source_type) + if parsed_source: + source_filter = Notification.type.in_(parsed_source.types) & ( + Notification.notification_metadata[parsed_source.metadata_key].astext + == parsed_source.value ) query = query.where(source_filter) count_query = count_query.where(source_filter) - # Filter by preset: 'unread' or 'errors' if filter == "unread": unread_filter = Notification.read == False # noqa: E712 query = query.where(unread_filter) @@ -409,10 +281,9 @@ async def list_notifications( query = query.where(error_filter) count_query = count_query.where(error_filter) - # Filter by date (for efficient pagination of older items) if before_date: try: - before_datetime = datetime.fromisoformat(before_date.replace("Z", "+00:00")) + before_datetime = parse_before_date(before_date) query = query.where(Notification.created_at < before_datetime) count_query = count_query.where(Notification.created_at < before_datetime) except ValueError: @@ -421,7 +292,6 @@ async def list_notifications( detail="Invalid date format. Use ISO format (e.g., 2024-01-15T00:00:00Z)", ) from None - # Filter by search query (case-insensitive title/message search) if search: search_term = f"%{search}%" search_filter = Notification.title.ilike( @@ -430,45 +300,22 @@ async def list_notifications( query = query.where(search_filter) count_query = count_query.where(search_filter) - # Get total count total_result = await session.execute(count_query) total = total_result.scalar() or 0 - # Apply ordering and pagination + # Over-fetch by one to tell whether another page exists. query = ( query.order_by(desc(Notification.created_at)).offset(offset).limit(limit + 1) ) - # Execute query result = await session.execute(query) notifications = result.scalars().all() - # Check if there are more items has_more = len(notifications) > limit if has_more: notifications = notifications[:limit] - # Convert to response format - items = [] - for notification in notifications: - items.append( - NotificationResponse( - id=notification.id, - user_id=str(notification.user_id), - search_space_id=notification.search_space_id, - type=notification.type, - title=notification.title, - message=notification.message, - read=notification.read, - metadata=notification.notification_metadata or {}, - created_at=notification.created_at.isoformat() - if notification.created_at - else "", - updated_at=notification.updated_at.isoformat() - if notification.updated_at - else None, - ) - ) + items = [to_response(notification) for notification in notifications] return NotificationListResponse( items=items, @@ -484,12 +331,8 @@ async def mark_notification_as_read( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> MarkReadResponse: - """ - Mark a single notification as read. - - Zero will automatically sync this change to all connected clients. - """ - # Verify the notification belongs to the user + """Mark one of the user's notifications read; Zero syncs the change.""" + # Scope to the caller's own notifications. result = await session.execute( select(Notification).where( Notification.id == notification_id, @@ -510,7 +353,6 @@ async def mark_notification_as_read( message="Notification already marked as read", ) - # Update the notification notification.read = True await session.commit() @@ -525,12 +367,7 @@ async def mark_all_notifications_as_read( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ) -> MarkAllReadResponse: - """ - Mark all notifications as read for the current user. - - Zero will automatically sync these changes to all connected clients. - """ - # Update all unread notifications for the user + """Mark all of the user's notifications read; Zero syncs the changes.""" result = await session.execute( update(Notification) .where( diff --git a/surfsense_backend/app/notifications/api/schemas.py b/surfsense_backend/app/notifications/api/schemas.py new file mode 100644 index 000000000..727e5485a --- /dev/null +++ b/surfsense_backend/app/notifications/api/schemas.py @@ -0,0 +1,81 @@ +"""Response shapes for the notifications API.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class NotificationResponse(BaseModel): + """A single notification.""" + + id: int + user_id: str + search_space_id: int | None + type: str + title: str + message: str + read: bool + metadata: dict + created_at: str + updated_at: str | None + + class Config: + from_attributes = True + + +class NotificationListResponse(BaseModel): + """A page of notifications.""" + + items: list[NotificationResponse] + total: int + has_more: bool + next_offset: int | None + + +class MarkReadResponse(BaseModel): + """Outcome of marking one notification read.""" + + success: bool + message: str + + +class MarkAllReadResponse(BaseModel): + """Outcome of marking every notification read.""" + + success: bool + message: str + updated_count: int + + +class SourceTypeItem(BaseModel): + """A source type with its category and count.""" + + key: str + type: str + category: str # "connector" or "document" + count: int + + +class SourceTypesResponse(BaseModel): + """Source types available for the Status tab filter.""" + + sources: list[SourceTypeItem] + + +class UnreadCountResponse(BaseModel): + """Unread totals, split by sync-window recency.""" + + total_unread: int + recent_unread: int + + +class CategoryUnreadCount(BaseModel): + total_unread: int + recent_unread: int + + +class BatchUnreadCountResponse(BaseModel): + """Per-category unread counts in one response.""" + + comments: CategoryUnreadCount + status: CategoryUnreadCount diff --git a/surfsense_backend/app/notifications/api/transform.py b/surfsense_backend/app/notifications/api/transform.py new file mode 100644 index 000000000..8970cb0b8 --- /dev/null +++ b/surfsense_backend/app/notifications/api/transform.py @@ -0,0 +1,62 @@ +"""Pure request/response helpers for the notifications API. + +No DB or framework objects, so these are unit-testable in isolation. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import NamedTuple + +from app.notifications.api.schemas import NotificationResponse +from app.notifications.persistence import Notification + + +class SourceTypeFilter(NamedTuple): + """The notification types and JSONB facet a source-type filter selects.""" + + types: tuple[str, ...] + metadata_key: str + value: str + + +def parse_source_type(source_type: str) -> SourceTypeFilter | None: + """Decode a `connector:` / `doctype:` filter, or None if unknown.""" + if source_type.startswith("connector:"): + return SourceTypeFilter( + types=("connector_indexing", "connector_deletion"), + metadata_key="connector_type", + value=source_type[len("connector:") :], + ) + if source_type.startswith("doctype:"): + return SourceTypeFilter( + types=("document_processing",), + metadata_key="document_type", + value=source_type[len("doctype:") :], + ) + return None + + +def parse_before_date(before_date: str) -> datetime: + """Parse an ISO date for pagination; raises ValueError if malformed.""" + return datetime.fromisoformat(before_date.replace("Z", "+00:00")) + + +def to_response(notification: Notification) -> NotificationResponse: + """Map a persisted notification to its API response shape.""" + return NotificationResponse( + id=notification.id, + user_id=str(notification.user_id), + search_space_id=notification.search_space_id, + type=notification.type, + title=notification.title, + message=notification.message, + read=notification.read, + metadata=notification.notification_metadata or {}, + created_at=notification.created_at.isoformat() + if notification.created_at + else "", + updated_at=notification.updated_at.isoformat() + if notification.updated_at + else None, + ) diff --git a/surfsense_backend/app/notifications/constants.py b/surfsense_backend/app/notifications/constants.py new file mode 100644 index 000000000..e8bd8391d --- /dev/null +++ b/surfsense_backend/app/notifications/constants.py @@ -0,0 +1,17 @@ +"""Notification policy constants.""" + +from __future__ import annotations + +# Notifications newer than this are live-synced; older ones load via the list endpoint. +SYNC_WINDOW_DAYS = 14 + +# Maps an inbox tab to the notification types it shows. +CATEGORY_TYPES: dict[str, tuple[str, ...]] = { + "comments": ("new_mention", "comment_reply"), + "status": ( + "connector_indexing", + "connector_deletion", + "document_processing", + "page_limit_exceeded", + ), +} diff --git a/surfsense_backend/app/notifications/persistence/__init__.py b/surfsense_backend/app/notifications/persistence/__init__.py new file mode 100644 index 000000000..82f9e6f01 --- /dev/null +++ b/surfsense_backend/app/notifications/persistence/__init__.py @@ -0,0 +1,7 @@ +"""Notification persistence models.""" + +from __future__ import annotations + +from .models import Notification + +__all__ = ["Notification"] diff --git a/surfsense_backend/app/notifications/persistence/models.py b/surfsense_backend/app/notifications/persistence/models.py new file mode 100644 index 000000000..557c4bf17 --- /dev/null +++ b/surfsense_backend/app/notifications/persistence/models.py @@ -0,0 +1,72 @@ +"""Per-user inbox notifications, synced to clients via Zero.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import ( + TIMESTAMP, + Boolean, + Column, + ForeignKey, + Index, + Integer, + String, + Text, + text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import relationship + +from app.db import BaseModel, TimestampMixin + + +class Notification(BaseModel, TimestampMixin): + __tablename__ = "notifications" + __table_args__ = ( + # Serves unread-count queries. + Index( + "ix_notifications_user_read_type_created", + "user_id", + "read", + "type", + "created_at", + ), + # Serves the paginated inbox list query. + Index( + "ix_notifications_user_space_created", + "user_id", + "search_space_id", + "created_at", + ), + ) + + user_id = Column( + UUID(as_uuid=True), + ForeignKey("user.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + search_space_id = Column( + Integer, + ForeignKey("searchspaces.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + type = Column(String(50), nullable=False, index=True) + title = Column(String(200), nullable=False) + message = Column(Text, nullable=False) + read = Column( + Boolean, nullable=False, default=False, server_default=text("false"), index=True + ) + notification_metadata = Column("metadata", JSONB, nullable=True, default={}) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=True, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + index=True, + ) + + user = relationship("User", back_populates="notifications") + search_space = relationship("SearchSpace", back_populates="notifications") diff --git a/surfsense_backend/app/notifications/service/__init__.py b/surfsense_backend/app/notifications/service/__init__.py new file mode 100644 index 000000000..8cb8491ac --- /dev/null +++ b/surfsense_backend/app/notifications/service/__init__.py @@ -0,0 +1,7 @@ +"""Notification creation/update service.""" + +from __future__ import annotations + +from app.notifications.service.facade import NotificationService + +__all__ = ["NotificationService"] diff --git a/surfsense_backend/app/notifications/service/base.py b/surfsense_backend/app/notifications/service/base.py new file mode 100644 index 000000000..31b378cda --- /dev/null +++ b/surfsense_backend/app/notifications/service/base.py @@ -0,0 +1,119 @@ +"""Shared find/upsert/update logic for a single notification type.""" + +from __future__ import annotations + +import logging +from typing import Any +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm.attributes import flag_modified + +from app.notifications.persistence import Notification +from app.notifications.service.metadata import apply_update, start_metadata + +logger = logging.getLogger(__name__) + + +class BaseNotificationHandler: + """Find, upsert, and update notifications of one ``type``.""" + + def __init__(self, notification_type: str): + self.notification_type = notification_type + + async def find_notification_by_operation( + self, + session: AsyncSession, + user_id: UUID, + operation_id: str, + search_space_id: int | None = None, + ) -> Notification | None: + """Return the notification for ``operation_id``, if one exists.""" + query = select(Notification).where( + Notification.user_id == user_id, + Notification.type == self.notification_type, + Notification.notification_metadata["operation_id"].astext == operation_id, + ) + if search_space_id is not None: + query = query.where(Notification.search_space_id == search_space_id) + + result = await session.execute(query) + return result.scalar_one_or_none() + + async def find_or_create_notification( + self, + session: AsyncSession, + user_id: UUID, + operation_id: str, + title: str, + message: str, + search_space_id: int | None = None, + initial_metadata: dict[str, Any] | None = None, + ) -> Notification: + """Upsert a notification keyed by ``operation_id``.""" + notification = await self.find_notification_by_operation( + session, user_id, operation_id, search_space_id + ) + + if notification: + notification.title = title + notification.message = message + if initial_metadata: + notification.notification_metadata = apply_update( + notification.notification_metadata, + metadata_updates=initial_metadata, + ) + # Tell SQLAlchemy the JSONB dict changed in place. + flag_modified(notification, "notification_metadata") + await session.commit() + await session.refresh(notification) + logger.info( + f"Updated notification {notification.id} for operation {operation_id}" + ) + return notification + + metadata = start_metadata(operation_id, initial_metadata) + + notification = Notification( + user_id=user_id, + search_space_id=search_space_id, + type=self.notification_type, + title=title, + message=message, + notification_metadata=metadata, + ) + session.add(notification) + await session.commit() + await session.refresh(notification) + logger.info( + f"Created notification {notification.id} for operation {operation_id}" + ) + return notification + + async def update_notification( + self, + session: AsyncSession, + notification: Notification, + title: str | None = None, + message: str | None = None, + status: str | None = None, + metadata_updates: dict[str, Any] | None = None, + ) -> Notification: + """Apply field/status/metadata changes and persist.""" + if title is not None: + notification.title = title + if message is not None: + notification.message = message + + if status is not None or metadata_updates: + notification.notification_metadata = apply_update( + notification.notification_metadata, status, metadata_updates + ) + # Tell SQLAlchemy the JSONB dict changed in place. + flag_modified(notification, "notification_metadata") + + await session.commit() + await session.refresh(notification) + logger.info(f"Updated notification {notification.id}") + return notification diff --git a/surfsense_backend/app/notifications/service/facade.py b/surfsense_backend/app/notifications/service/facade.py new file mode 100644 index 000000000..63154301c --- /dev/null +++ b/surfsense_backend/app/notifications/service/facade.py @@ -0,0 +1,55 @@ +"""Single entry point that composes the per-type notification handlers.""" + +from __future__ import annotations + +import logging +from typing import Any +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.handlers import ( + CommentReplyNotificationHandler, + ConnectorIndexingNotificationHandler, + DocumentProcessingNotificationHandler, + MentionNotificationHandler, + PageLimitNotificationHandler, +) + +logger = logging.getLogger(__name__) + + +class NotificationService: + """Facade over the per-type handlers; mutations sync via Zero.""" + + connector_indexing = ConnectorIndexingNotificationHandler() + document_processing = DocumentProcessingNotificationHandler() + mention = MentionNotificationHandler() + comment_reply = CommentReplyNotificationHandler() + page_limit = PageLimitNotificationHandler() + + @staticmethod + async def create_notification( + session: AsyncSession, + user_id: UUID, + notification_type: str, + title: str, + message: str, + search_space_id: int | None = None, + notification_metadata: dict[str, Any] | None = None, + ) -> Notification: + """Create a generic notification of any ``notification_type``.""" + notification = Notification( + user_id=user_id, + search_space_id=search_space_id, + type=notification_type, + title=title, + message=message, + notification_metadata=notification_metadata or {}, + ) + session.add(notification) + await session.commit() + await session.refresh(notification) + logger.info(f"Created notification {notification.id} for user {user_id}") + return notification diff --git a/surfsense_backend/app/notifications/service/handlers/__init__.py b/surfsense_backend/app/notifications/service/handlers/__init__.py new file mode 100644 index 000000000..8c32dea3b --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/__init__.py @@ -0,0 +1,17 @@ +"""Per-type notification handlers.""" + +from __future__ import annotations + +from .comment_reply import CommentReplyNotificationHandler +from .connector_indexing import ConnectorIndexingNotificationHandler +from .document_processing import DocumentProcessingNotificationHandler +from .mention import MentionNotificationHandler +from .page_limit import PageLimitNotificationHandler + +__all__ = [ + "CommentReplyNotificationHandler", + "ConnectorIndexingNotificationHandler", + "DocumentProcessingNotificationHandler", + "MentionNotificationHandler", + "PageLimitNotificationHandler", +] diff --git a/surfsense_backend/app/notifications/service/handlers/comment_reply.py b/surfsense_backend/app/notifications/service/handlers/comment_reply.py new file mode 100644 index 000000000..7d9a9495a --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/comment_reply.py @@ -0,0 +1,108 @@ +"""Notifications for replies to a user's comments.""" + +from __future__ import annotations + +import logging +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages.text import truncate + +logger = logging.getLogger(__name__) + + +class CommentReplyNotificationHandler(BaseNotificationHandler): + """Notifications for replies to a user's comments.""" + + def __init__(self): + super().__init__("comment_reply") + + async def find_notification_by_reply( + self, + session: AsyncSession, + reply_id: int, + user_id: UUID, + ) -> Notification | None: + query = select(Notification).where( + Notification.type == self.notification_type, + Notification.user_id == user_id, + Notification.notification_metadata["reply_id"].astext == str(reply_id), + ) + result = await session.execute(query) + return result.scalar_one_or_none() + + async def notify_comment_reply( + self, + session: AsyncSession, + user_id: UUID, + reply_id: int, + parent_comment_id: int, + message_id: int, + thread_id: int, + thread_title: str, + author_id: str, + author_name: str, + author_avatar_url: str | None, + author_email: str, + content_preview: str, + search_space_id: int, + ) -> Notification: + """Notify of a reply; idempotent on ``reply_id`` per user.""" + existing = await self.find_notification_by_reply(session, reply_id, user_id) + if existing: + logger.info( + f"Notification already exists for reply {reply_id} to user {user_id}" + ) + return existing + + title = f"{author_name} replied in a thread" + message = truncate(content_preview, 100) + + metadata = { + "reply_id": reply_id, + "parent_comment_id": parent_comment_id, + "message_id": message_id, + "thread_id": thread_id, + "thread_title": thread_title, + "author_id": author_id, + "author_name": author_name, + "author_avatar_url": author_avatar_url, + "author_email": author_email, + "content_preview": content_preview[:200], + } + + try: + notification = Notification( + user_id=user_id, + search_space_id=search_space_id, + type=self.notification_type, + title=title, + message=message, + notification_metadata=metadata, + ) + session.add(notification) + await session.commit() + await session.refresh(notification) + logger.info( + f"Created comment_reply notification {notification.id} for user {user_id}" + ) + return notification + except Exception as e: + await session.rollback() + if ( + "duplicate key" in str(e).lower() + or "unique constraint" in str(e).lower() + ): + logger.warning( + f"Duplicate notification for reply {reply_id} to user {user_id}" + ) + existing = await self.find_notification_by_reply( + session, reply_id, user_id + ) + if existing: + return existing + raise diff --git a/surfsense_backend/app/notifications/service/handlers/connector_indexing.py b/surfsense_backend/app/notifications/service/handlers/connector_indexing.py new file mode 100644 index 000000000..9ebfae2ea --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/connector_indexing.py @@ -0,0 +1,183 @@ +"""Notifications for connector indexing runs.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages import connector_indexing as msg + + +class ConnectorIndexingNotificationHandler(BaseNotificationHandler): + """Notifications for connector indexing runs.""" + + def __init__(self): + super().__init__("connector_indexing") + + async def notify_indexing_started( + self, + session: AsyncSession, + user_id: UUID, + connector_id: int, + connector_name: str, + connector_type: str, + search_space_id: int, + start_date: str | None = None, + end_date: str | None = None, + ) -> Notification: + """Open (or refresh) the notification when indexing starts.""" + operation_id = msg.operation_id(connector_id, start_date, end_date) + title = f"Syncing: {connector_name}" + message = "Connecting to your account" + + metadata = { + "connector_id": connector_id, + "connector_name": connector_name, + "connector_type": connector_type, + "start_date": start_date, + "end_date": end_date, + "indexed_count": 0, + "sync_stage": "connecting", + } + + return await self.find_or_create_notification( + session=session, + user_id=user_id, + operation_id=operation_id, + title=title, + message=message, + search_space_id=search_space_id, + initial_metadata=metadata, + ) + + async def notify_indexing_progress( + self, + session: AsyncSession, + notification: Notification, + indexed_count: int, + total_count: int | None = None, + stage: str | None = None, + stage_message: str | None = None, + ) -> Notification: + """Update the notification with indexing progress.""" + message, metadata_updates = msg.progress( + indexed_count, total_count, stage, stage_message + ) + return await self.update_notification( + session=session, + notification=notification, + message=message, + status="in_progress", + metadata_updates=metadata_updates, + ) + + async def notify_retry_progress( + self, + session: AsyncSession, + notification: Notification, + indexed_count: int, + retry_reason: str, + attempt: int, + max_attempts: int, + wait_seconds: float | None = None, + service_name: str | None = None, + ) -> Notification: + """Surface that an external service is rate-limiting/retrying.""" + connector_name = notification.notification_metadata.get( + "connector_name", "Service" + ) + message, metadata_updates = msg.retry( + connector_name, + indexed_count, + retry_reason, + attempt, + max_attempts, + wait_seconds, + service_name, + ) + return await self.update_notification( + session=session, + notification=notification, + message=message, + status="in_progress", + metadata_updates=metadata_updates, + ) + + async def notify_indexing_completed( + self, + session: AsyncSession, + notification: Notification, + indexed_count: int, + error_message: str | None = None, + is_warning: bool = False, + skipped_count: int | None = None, + unsupported_count: int | None = None, + ) -> Notification: + """Finalize the notification as ready/failed when indexing ends.""" + connector_name = notification.notification_metadata.get( + "connector_name", "Connector" + ) + title, message, status, metadata_updates = msg.completion( + connector_name, + indexed_count, + error_message, + is_warning, + skipped_count, + unsupported_count, + ) + return await self.update_notification( + session=session, + notification=notification, + title=title, + message=message, + status=status, + metadata_updates=metadata_updates, + ) + + async def notify_google_drive_indexing_started( + self, + session: AsyncSession, + user_id: UUID, + connector_id: int, + connector_name: str, + connector_type: str, + search_space_id: int, + folder_count: int, + file_count: int, + folder_names: list[str] | None = None, + file_names: list[str] | None = None, + ) -> Notification: + """Open (or refresh) the notification when Drive indexing starts.""" + operation_id = msg.google_drive_operation_id( + connector_id, folder_count, file_count + ) + title = f"Syncing: {connector_name}" + message = "Preparing your files" + + metadata = { + "connector_id": connector_id, + "connector_name": connector_name, + "connector_type": connector_type, + "folder_count": folder_count, + "file_count": file_count, + "indexed_count": 0, + "sync_stage": "connecting", + } + + if folder_names: + metadata["folder_names"] = folder_names + if file_names: + metadata["file_names"] = file_names + + return await self.find_or_create_notification( + session=session, + user_id=user_id, + operation_id=operation_id, + title=title, + message=message, + search_space_id=search_space_id, + initial_metadata=metadata, + ) diff --git a/surfsense_backend/app/notifications/service/handlers/document_processing.py b/surfsense_backend/app/notifications/service/handlers/document_processing.py new file mode 100644 index 000000000..8644df2c8 --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/document_processing.py @@ -0,0 +1,95 @@ +"""Notifications for single-document processing.""" + +from __future__ import annotations + +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages import document_processing as msg + + +class DocumentProcessingNotificationHandler(BaseNotificationHandler): + """Notifications for single-document processing.""" + + def __init__(self): + super().__init__("document_processing") + + async def notify_processing_started( + self, + session: AsyncSession, + user_id: UUID, + document_type: str, + document_name: str, + search_space_id: int, + file_size: int | None = None, + ) -> Notification: + """Open the notification when document processing is queued.""" + operation_id = msg.operation_id(document_type, document_name, search_space_id) + title = f"Processing: {document_name}" + message = "Waiting in queue" + + metadata = { + "document_type": document_type, + "document_name": document_name, + "processing_stage": "queued", + } + + if file_size is not None: + metadata["file_size"] = file_size + + return await self.find_or_create_notification( + session=session, + user_id=user_id, + operation_id=operation_id, + title=title, + message=message, + search_space_id=search_space_id, + initial_metadata=metadata, + ) + + async def notify_processing_progress( + self, + session: AsyncSession, + notification: Notification, + stage: str, + stage_message: str | None = None, + chunks_count: int | None = None, + ) -> Notification: + """Update the notification with the current processing stage.""" + message, metadata_updates = msg.progress(stage, stage_message, chunks_count) + + return await self.update_notification( + session=session, + notification=notification, + message=message, + status="in_progress", + metadata_updates=metadata_updates, + ) + + async def notify_processing_completed( + self, + session: AsyncSession, + notification: Notification, + document_id: int | None = None, + chunks_count: int | None = None, + error_message: str | None = None, + ) -> Notification: + """Finalize the notification as ready/failed when processing ends.""" + document_name = notification.notification_metadata.get( + "document_name", "Document" + ) + title, message, status, metadata_updates = msg.completion( + document_name, error_message, document_id, chunks_count + ) + + return await self.update_notification( + session=session, + notification=notification, + title=title, + message=message, + status=status, + metadata_updates=metadata_updates, + ) diff --git a/surfsense_backend/app/notifications/service/handlers/mention.py b/surfsense_backend/app/notifications/service/handlers/mention.py new file mode 100644 index 000000000..568dc01de --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/mention.py @@ -0,0 +1,106 @@ +"""Notifications for @mentions in comments.""" + +from __future__ import annotations + +import logging +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages.text import truncate + +logger = logging.getLogger(__name__) + + +class MentionNotificationHandler(BaseNotificationHandler): + """Notifications for @mentions in comments.""" + + def __init__(self): + super().__init__("new_mention") + + async def find_notification_by_mention( + self, + session: AsyncSession, + mention_id: int, + ) -> Notification | None: + """Return the notification for ``mention_id``, if one exists.""" + query = select(Notification).where( + Notification.type == self.notification_type, + Notification.notification_metadata["mention_id"].astext == str(mention_id), + ) + result = await session.execute(query) + return result.scalar_one_or_none() + + async def notify_new_mention( + self, + session: AsyncSession, + mentioned_user_id: UUID, + mention_id: int, + comment_id: int, + message_id: int, + thread_id: int, + thread_title: str, + author_id: str, + author_name: str, + author_avatar_url: str | None, + author_email: str, + content_preview: str, + search_space_id: int, + ) -> Notification: + """Notify a mentioned user; idempotent on ``mention_id``.""" + existing = await self.find_notification_by_mention(session, mention_id) + if existing: + logger.info( + f"Notification already exists for mention {mention_id}, returning existing" + ) + return existing + + title = f"{author_name} mentioned you" + message = truncate(content_preview, 100) + + metadata = { + "mention_id": mention_id, + "comment_id": comment_id, + "message_id": message_id, + "thread_id": thread_id, + "thread_title": thread_title, + "author_id": author_id, + "author_name": author_name, + "author_avatar_url": author_avatar_url, + "author_email": author_email, + "content_preview": content_preview[:200], + } + + try: + notification = Notification( + user_id=mentioned_user_id, + search_space_id=search_space_id, + type=self.notification_type, + title=title, + message=message, + notification_metadata=metadata, + ) + session.add(notification) + await session.commit() + await session.refresh(notification) + logger.info( + f"Created new_mention notification {notification.id} for user {mentioned_user_id}" + ) + return notification + except Exception as e: + # Race: a concurrent insert won; fetch the existing row instead. + await session.rollback() + if ( + "duplicate key" in str(e).lower() + or "unique constraint" in str(e).lower() + ): + logger.warning( + f"Duplicate notification detected for mention {mention_id}, fetching existing" + ) + existing = await self.find_notification_by_mention(session, mention_id) + if existing: + return existing + raise diff --git a/surfsense_backend/app/notifications/service/handlers/page_limit.py b/surfsense_backend/app/notifications/service/handlers/page_limit.py new file mode 100644 index 000000000..90722dc62 --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/page_limit.py @@ -0,0 +1,68 @@ +"""Notifications for exceeding the page limit.""" + +from __future__ import annotations + +import logging +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages import page_limit as msg + +logger = logging.getLogger(__name__) + + +class PageLimitNotificationHandler(BaseNotificationHandler): + """Notifications for exceeding the page limit.""" + + def __init__(self): + super().__init__("page_limit_exceeded") + + async def notify_page_limit_exceeded( + self, + session: AsyncSession, + user_id: UUID, + document_name: str, + document_type: str, + search_space_id: int, + pages_used: int, + pages_limit: int, + pages_to_add: int, + ) -> Notification: + """Notify that a document was blocked by the page limit.""" + operation_id = msg.operation_id(document_name, search_space_id) + title, message = msg.summary( + document_name, pages_used, pages_limit, pages_to_add + ) + + metadata = { + "operation_id": operation_id, + "document_name": document_name, + "document_type": document_type, + "pages_used": pages_used, + "pages_limit": pages_limit, + "pages_to_add": pages_to_add, + "status": "failed", + "error_type": "page_limit_exceeded", + # Where the inbox item links to. + "action_url": f"/dashboard/{search_space_id}/more-pages", + "action_label": "Upgrade Plan", + } + + notification = Notification( + user_id=user_id, + search_space_id=search_space_id, + type=self.notification_type, + title=title, + message=message, + notification_metadata=metadata, + ) + session.add(notification) + await session.commit() + await session.refresh(notification) + logger.info( + f"Created page_limit_exceeded notification {notification.id} for user {user_id}" + ) + return notification diff --git a/surfsense_backend/app/notifications/service/messages/__init__.py b/surfsense_backend/app/notifications/service/messages/__init__.py new file mode 100644 index 000000000..95373537d --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/__init__.py @@ -0,0 +1,6 @@ +"""Pure, side-effect-free presentation logic for notifications. + +Handlers compute their user-facing title/message/status/metadata here, then +persist the result. Keeping this layer free of I/O makes it unit-testable +without a database. +""" diff --git a/surfsense_backend/app/notifications/service/messages/connector_indexing.py b/surfsense_backend/app/notifications/service/messages/connector_indexing.py new file mode 100644 index 000000000..8a2926211 --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/connector_indexing.py @@ -0,0 +1,164 @@ +"""Pure presentation logic for connector-indexing notifications.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + + +def operation_id( + connector_id: int, + start_date: str | None = None, + end_date: str | None = None, +) -> str: + """Build a unique id for a connector indexing run.""" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + date_range = "" + if start_date or end_date: + date_range = f"_{start_date or 'none'}_{end_date or 'none'}" + return f"connector_{connector_id}_{timestamp}{date_range}" + + +def google_drive_operation_id( + connector_id: int, folder_count: int, file_count: int +) -> str: + """Build a unique id for a Google Drive indexing run.""" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + items_info = f"_{folder_count}f_{file_count}files" + return f"drive_{connector_id}_{timestamp}{items_info}" + + +def progress( + indexed_count: int, + total_count: int | None = None, + stage: str | None = None, + stage_message: str | None = None, +) -> tuple[str, dict[str, Any]]: + """Compute the progress message and metadata updates for an indexing run.""" + stage_messages = { + "connecting": "Connecting to your account", + "fetching": "Fetching your content", + "processing": "Preparing for search", + "storing": "Almost done", + } + + if stage or stage_message: + progress_msg = stage_message or stage_messages.get(stage, "Processing") + else: + # Legacy callers that pass neither stage nor message. + progress_msg = "Fetching your content" + + metadata_updates: dict[str, Any] = {"indexed_count": indexed_count} + if total_count is not None: + metadata_updates["total_count"] = total_count + progress_percent = int((indexed_count / total_count) * 100) + metadata_updates["progress_percent"] = progress_percent + if stage: + metadata_updates["sync_stage"] = stage + + return progress_msg, metadata_updates + + +def retry( + connector_name: str, + indexed_count: int, + retry_reason: str, + attempt: int, + max_attempts: int, + wait_seconds: float | None = None, + service_name: str | None = None, +) -> tuple[str, dict[str, Any]]: + """Compute the retry message and metadata, framing the delay as the provider's.""" + if not service_name: + service_name = connector_name + # Strip the workspace suffix, e.g. "Notion - My Workspace" -> "Notion". + if " - " in service_name: + service_name = service_name.split(" - ")[0] + + # Worded so the delay reads as the provider's, not ours. + retry_messages = { + "rate_limit": f"{service_name} rate limit reached", + "server_error": f"{service_name} is slow to respond", + "timeout": f"{service_name} took too long", + "temporary_error": f"{service_name} temporarily unavailable", + } + + base_message = retry_messages.get(retry_reason, f"Waiting for {service_name}") + + # Only surface a wait time when it's long enough to be worth showing. + if wait_seconds and wait_seconds > 5: + message = f"{base_message}. Retrying in {int(wait_seconds)}s..." + else: + message = f"{base_message}. Retrying..." + + if indexed_count > 0: + item_text = "item" if indexed_count == 1 else "items" + message = f"{message} ({indexed_count} {item_text} synced so far)" + + metadata_updates = { + "indexed_count": indexed_count, + "sync_stage": "waiting_retry", + "retry_attempt": attempt, + "retry_max_attempts": max_attempts, + "retry_reason": retry_reason, + "retry_wait_seconds": wait_seconds, + } + + return message, metadata_updates + + +def completion( + connector_name: str, + indexed_count: int, + error_message: str | None = None, + is_warning: bool = False, + skipped_count: int | None = None, + unsupported_count: int | None = None, +) -> tuple[str, str, str, dict[str, Any]]: + """Compute the final title, message, status, and metadata for a finished run.""" + unsupported_text = "" + if unsupported_count and unsupported_count > 0: + file_word = "file was" if unsupported_count == 1 else "files were" + unsupported_text = f" {unsupported_count} {file_word} not supported." + + if error_message: + if indexed_count > 0: + title = f"Ready: {connector_name}" + file_text = "file" if indexed_count == 1 else "files" + message = f"Now searchable! {indexed_count} {file_text} synced.{unsupported_text} Note: {error_message}" + status = "completed" + elif is_warning: + title = f"Ready: {connector_name}" + message = f"Sync complete.{unsupported_text} {error_message}" + status = "completed" + else: + title = f"Failed: {connector_name}" + message = f"Sync failed: {error_message}" + if unsupported_text: + message += unsupported_text + status = "failed" + else: + title = f"Ready: {connector_name}" + if indexed_count == 0: + if unsupported_count and unsupported_count > 0: + message = f"Sync complete.{unsupported_text}" + else: + message = "Already up to date!" + else: + file_text = "file" if indexed_count == 1 else "files" + message = f"Now searchable! {indexed_count} {file_text} synced." + if unsupported_text: + message += unsupported_text + status = "completed" + + metadata_updates = { + "indexed_count": indexed_count, + "skipped_count": skipped_count or 0, + "unsupported_count": unsupported_count or 0, + "sync_stage": "completed" + if (not error_message or is_warning or indexed_count > 0) + else "failed", + "error_message": error_message, + } + + return title, message, status, metadata_updates diff --git a/surfsense_backend/app/notifications/service/messages/document_processing.py b/surfsense_backend/app/notifications/service/messages/document_processing.py new file mode 100644 index 000000000..3805c2847 --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/document_processing.py @@ -0,0 +1,64 @@ +"""Pure presentation logic for document-processing notifications.""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from typing import Any + + +def operation_id(document_type: str, filename: str, search_space_id: int) -> str: + """Build a unique id for a document processing run.""" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + filename_hash = hashlib.md5(filename.encode()).hexdigest()[:8] + return f"doc_{document_type}_{search_space_id}_{timestamp}_{filename_hash}" + + +def progress( + stage: str, + stage_message: str | None = None, + chunks_count: int | None = None, +) -> tuple[str, dict[str, Any]]: + """Compute the progress message and metadata updates for a processing run.""" + stage_messages = { + "parsing": "Reading your file", + "chunking": "Preparing for search", + "embedding": "Preparing for search", + "storing": "Finalizing", + } + + message = stage_message or stage_messages.get(stage, "Processing") + + metadata_updates: dict[str, Any] = {"processing_stage": stage} + if chunks_count is not None: + metadata_updates["chunks_count"] = chunks_count + + return message, metadata_updates + + +def completion( + document_name: str, + error_message: str | None = None, + document_id: int | None = None, + chunks_count: int | None = None, +) -> tuple[str, str, str, dict[str, Any]]: + """Compute the final title, message, status, and metadata for a finished run.""" + if error_message: + title = f"Failed: {document_name}" + message = f"Processing failed: {error_message}" + status = "failed" + else: + title = f"Ready: {document_name}" + message = "Now searchable!" + status = "completed" + + metadata_updates: dict[str, Any] = { + "processing_stage": "completed" if not error_message else "failed", + "error_message": error_message, + } + if document_id is not None: + metadata_updates["document_id"] = document_id + if chunks_count is not None: + metadata_updates["chunks_count"] = chunks_count + + return title, message, status, metadata_updates diff --git a/surfsense_backend/app/notifications/service/messages/page_limit.py b/surfsense_backend/app/notifications/service/messages/page_limit.py new file mode 100644 index 000000000..54e5cbdec --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/page_limit.py @@ -0,0 +1,25 @@ +"""Pure presentation logic for page-limit notifications.""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime + +from app.notifications.service.messages.text import truncate + + +def operation_id(document_name: str, search_space_id: int) -> str: + """Build a unique id for a page-limit notification.""" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + doc_hash = hashlib.md5(document_name.encode()).hexdigest()[:8] + return f"page_limit_{search_space_id}_{timestamp}_{doc_hash}" + + +def summary( + document_name: str, pages_used: int, pages_limit: int, pages_to_add: int +) -> tuple[str, str]: + """Compute the title and message for a blocked-by-page-limit document.""" + display_name = truncate(document_name, 40) + title = f"Page limit exceeded: {display_name}" + message = f"This document has ~{pages_to_add} page(s) but you've used {pages_used}/{pages_limit} pages. Upgrade to process more documents." + return title, message diff --git a/surfsense_backend/app/notifications/service/messages/text.py b/surfsense_backend/app/notifications/service/messages/text.py new file mode 100644 index 000000000..98d5284cb --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/text.py @@ -0,0 +1,8 @@ +"""Shared text helpers for notification copy.""" + +from __future__ import annotations + + +def truncate(text: str, limit: int) -> str: + """Return ``text`` capped at ``limit`` chars, appending an ellipsis if cut.""" + return text[:limit] + "..." if len(text) > limit else text diff --git a/surfsense_backend/app/notifications/service/metadata.py b/surfsense_backend/app/notifications/service/metadata.py new file mode 100644 index 000000000..2679893dc --- /dev/null +++ b/surfsense_backend/app/notifications/service/metadata.py @@ -0,0 +1,33 @@ +"""Pure metadata transitions for the notification lifecycle.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + + +def start_metadata( + operation_id: str, initial_metadata: dict[str, Any] | None = None +) -> dict[str, Any]: + """Seed metadata for a freshly opened, in-progress notification.""" + metadata = dict(initial_metadata or {}) + metadata["operation_id"] = operation_id + metadata["status"] = "in_progress" + metadata["started_at"] = datetime.now(UTC).isoformat() + return metadata + + +def apply_update( + current: dict[str, Any], + status: str | None = None, + metadata_updates: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return metadata with the status/timestamp stamped and updates merged in.""" + metadata = dict(current) + if status is not None: + metadata["status"] = status + if status in ("completed", "failed"): + metadata["completed_at"] = datetime.now(UTC).isoformat() + if metadata_updates: + metadata = {**metadata, **metadata_updates} + return metadata diff --git a/surfsense_backend/app/notifications/types.py b/surfsense_backend/app/notifications/types.py new file mode 100644 index 000000000..bb8bcfab1 --- /dev/null +++ b/surfsense_backend/app/notifications/types.py @@ -0,0 +1,16 @@ +"""The notification types the API recognizes.""" + +from __future__ import annotations + +from typing import Literal + +NotificationType = Literal[ + "connector_indexing", + "connector_deletion", + "document_processing", + "new_mention", + "comment_reply", + "page_limit_exceeded", +] + +NotificationCategory = Literal["comments", "status"] diff --git a/surfsense_backend/app/observability/metrics.py b/surfsense_backend/app/observability/metrics.py index 798a6e2f7..5ba3be059 100644 --- a/surfsense_backend/app/observability/metrics.py +++ b/surfsense_backend/app/observability/metrics.py @@ -314,6 +314,135 @@ def _celery_queue_latency(): ) +@lru_cache(maxsize=1) +def _gateway_redis_fallback(): + return _get_meter().create_counter( + "surfsense.gateway.redis.fallback", + description="Count of gateway Redis fallback uses.", + ) + + +@lru_cache(maxsize=1) +def _gateway_thread_lock_contention(): + return _get_meter().create_counter( + "surfsense.gateway.thread_lock.contention", + description="Count of gateway per-thread lock contention events.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_writes(): + return _get_meter().create_counter( + "surfsense.gateway.inbox.writes", + description="Count of gateway inbound event inbox writes.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_processed(): + return _get_meter().create_counter( + "surfsense.gateway.inbox.processed", + description="Count of gateway inbound event processing outcomes.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbound_reconciled(): + return _get_meter().create_counter( + "surfsense.gateway.inbound.reconciled", + description="Count of gateway inbox events re-enqueued by reconciliation.", + ) + + +@lru_cache(maxsize=1) +def _gateway_outbound(): + return _get_meter().create_counter( + "surfsense.gateway.outbound", + description="Count of gateway outbound platform operations.", + ) + + +@lru_cache(maxsize=1) +def _gateway_turn_latency(): + return _get_meter().create_histogram( + "surfsense.gateway.turn.latency", + unit="ms", + description="Latency of gateway-routed agent turns.", + ) + + +@lru_cache(maxsize=1) +def _gateway_rate_limit_hits(): + return _get_meter().create_counter( + "surfsense.gateway.rate_limit.hits", + description="Count of gateway outbound rate limit waits.", + ) + + +@lru_cache(maxsize=1) +def _gateway_health_check_failures(): + return _get_meter().create_counter( + "surfsense.gateway.health_check.failures", + description="Count of gateway account health-check failures.", + ) + + +@lru_cache(maxsize=1) +def _gateway_auth_invariant_failures(): + return _get_meter().create_counter( + "surfsense.gateway.auth_invariant.failures", + description="Count of gateway authorization invariant failures.", + ) + + +@lru_cache(maxsize=1) +def _gateway_hitl_aborted(): + return _get_meter().create_counter( + "surfsense.gateway.hitl.aborted", + description="Count of gateway turns aborted because HITL is unsupported.", + ) + + +@lru_cache(maxsize=1) +def _gateway_active_bindings(): + return _get_meter().create_up_down_counter( + "surfsense.gateway.active_bindings", + description="Current change in active gateway bindings.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_enqueued(): + return _get_meter().create_counter( + "gateway_inbox_enqueued_total", + description="Count of gateway inbox rows enqueued for worker processing.", + ) + + +@lru_cache(maxsize=1) +def _gateway_inbox_sweep_replayed(): + return _get_meter().create_counter( + "gateway_inbox_sweep_replayed_total", + description="Count of received gateway inbox rows replayed by the sweep.", + ) + + +@lru_cache(maxsize=1) +def _gateway_byo_longpoll_running(): + return _get_meter().create_up_down_counter( + "gateway_byo_longpoll_running", + description="Current change in BYO Telegram long-poll supervisors holding a poll loop.", + ) + + +@lru_cache(maxsize=1) +def _gateway_webhook_parse_errors(): + return _get_meter().create_counter( + "gateway_webhook_parse_error_total", + description="Count of malformed gateway webhook payloads.", + ) + + def record_model_call_duration( duration_ms: float, *, model: str | None, provider: str | None ) -> None: @@ -569,6 +698,78 @@ def record_celery_queue_latency( ) +def record_gateway_redis_fallback() -> None: + _add(_gateway_redis_fallback(), 1, {}) + + +def record_gateway_thread_lock_contention() -> None: + _add(_gateway_thread_lock_contention(), 1, {}) + + +def record_gateway_inbox_write(*, platform: str, dedup_skipped: bool) -> None: + _add( + _gateway_inbox_writes(), + 1, + {"platform": platform, "dedup.skipped": bool(dedup_skipped)}, + ) + + +def record_gateway_inbox_processed(*, platform: str, status: str) -> None: + _add(_gateway_inbox_processed(), 1, {"platform": platform, "status": status}) + + +def record_gateway_inbound_reconciled(*, reason: str) -> None: + _add(_gateway_inbound_reconciled(), 1, {"reason": reason}) + + +def record_gateway_outbound(*, platform: str, kind: str, status: str) -> None: + _add( + _gateway_outbound(), + 1, + {"platform": platform, "kind": kind, "status": status}, + ) + + +def record_gateway_turn_latency(duration_ms: float, *, platform: str) -> None: + _record(_gateway_turn_latency(), duration_ms, {"platform": platform}) + + +def record_gateway_rate_limit_hit(*, bucket: str) -> None: + _add(_gateway_rate_limit_hits(), 1, {"bucket": bucket}) + + +def record_gateway_health_check_failure(*, platform: str) -> None: + _add(_gateway_health_check_failures(), 1, {"platform": platform}) + + +def record_gateway_auth_invariant_failure(*, cause: str) -> None: + _add(_gateway_auth_invariant_failures(), 1, {"cause": cause}) + + +def record_gateway_hitl_aborted(*, platform: str) -> None: + _add(_gateway_hitl_aborted(), 1, {"platform": platform}) + + +def record_gateway_active_bindings_delta(delta: int, *, platform: str) -> None: + _add(_gateway_active_bindings(), delta, {"platform": platform}) + + +def record_gateway_inbox_enqueued(*, intake: str, outcome: str) -> None: + _add(_gateway_inbox_enqueued(), 1, {"intake": intake, "outcome": outcome}) + + +def record_gateway_inbox_sweep_replayed() -> None: + _add(_gateway_inbox_sweep_replayed(), 1, {}) + + +def record_gateway_byo_longpoll_running_delta(delta: int, *, account_id: int) -> None: + _add(_gateway_byo_longpoll_running(), delta, {"account_id": account_id}) + + +def record_gateway_webhook_parse_error() -> None: + _add(_gateway_webhook_parse_errors(), 1, {}) + + def _runtime_snapshot_value(key: str, transform: Any = None) -> list[Any]: from opentelemetry.metrics import Observation diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/prompts/default_system_instructions.py similarity index 87% rename from surfsense_backend/app/agents/new_chat/system_prompt.py rename to surfsense_backend/app/prompts/default_system_instructions.py index 70634c65d..fd0a8e186 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/prompts/default_system_instructions.py @@ -1,5 +1,5 @@ """ -Thin compatibility wrapper around :mod:`app.agents.new_chat.prompts.composer`. +Thin compatibility wrapper around :mod:`app.prompts.system_prompt_composer.composer`. The composer split the previous monolithic prompt string into a fragment tree under ``prompts/`` plus a model-family dispatch step (see the @@ -7,11 +7,11 @@ composer module docstring for credits). This module preserves the public function surface (``build_surfsense_system_prompt`` / ``build_configurable_system_prompt`` / ``get_default_system_instructions`` / ``SURFSENSE_SYSTEM_PROMPT``) so -that existing call sites — `chat_deepagent.py`, anonymous chat routes, -and the configurable-prompt admin path — keep working without churn. +that existing call sites — the multi-agent chat factory, anonymous chat +routes, and the configurable-prompt admin path — keep working without churn. For new call sites prefer importing ``compose_system_prompt`` directly -from :mod:`app.agents.new_chat.prompts.composer`. +from :mod:`app.prompts.system_prompt_composer.composer`. """ from __future__ import annotations @@ -20,7 +20,7 @@ from datetime import UTC, datetime from app.db import ChatVisibility -from .prompts.composer import ( +from .system_prompt_composer.composer import ( _read_fragment, compose_system_prompt, detect_provider_variant, @@ -55,7 +55,7 @@ def build_surfsense_system_prompt( ) -> str: """Build the default SurfSense system prompt (citations on, defaults). - See :func:`app.agents.new_chat.prompts.composer.compose_system_prompt` + See :func:`app.prompts.system_prompt_composer.composer.compose_system_prompt` for full parameter docs. """ return compose_system_prompt( @@ -84,7 +84,7 @@ def build_configurable_system_prompt( ) -> str: """Build a configurable SurfSense system prompt (NewLLMConfig path). - See :func:`app.agents.new_chat.prompts.composer.compose_system_prompt` + See :func:`app.prompts.system_prompt_composer.composer.compose_system_prompt` for full parameter docs. """ return compose_system_prompt( @@ -108,7 +108,9 @@ def get_default_system_instructions() -> str: The output reflects the current fragment tree, not a baked-in constant. """ resolved_today = datetime.now(UTC).date().isoformat() - from .prompts.composer import _build_system_instructions # local import + from .system_prompt_composer.composer import ( + _build_system_instructions, # local import + ) return _build_system_instructions( visibility=ChatVisibility.PRIVATE, diff --git a/surfsense_backend/app/agents/new_chat/prompts/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/base/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/base/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/agent_private.md b/surfsense_backend/app/prompts/system_prompt_composer/base/agent_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/agent_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/agent_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/agent_team.md b/surfsense_backend/app/prompts/system_prompt_composer/base/agent_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/agent_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/agent_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/citations_off.md b/surfsense_backend/app/prompts/system_prompt_composer/base/citations_off.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/citations_off.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/citations_off.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/citations_on.md b/surfsense_backend/app/prompts/system_prompt_composer/base/citations_on.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/citations_on.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/citations_on.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/kb_only_policy_private.md b/surfsense_backend/app/prompts/system_prompt_composer/base/kb_only_policy_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/kb_only_policy_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/kb_only_policy_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/kb_only_policy_team.md b/surfsense_backend/app/prompts/system_prompt_composer/base/kb_only_policy_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/kb_only_policy_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/kb_only_policy_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_private.md b/surfsense_backend/app/prompts/system_prompt_composer/base/memory_protocol_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/memory_protocol_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_team.md b/surfsense_backend/app/prompts/system_prompt_composer/base/memory_protocol_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/memory_protocol_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/memory_protocol_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/parameter_resolution.md b/surfsense_backend/app/prompts/system_prompt_composer/base/parameter_resolution.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/parameter_resolution.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/parameter_resolution.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/tool_routing_private.md b/surfsense_backend/app/prompts/system_prompt_composer/base/tool_routing_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/tool_routing_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/tool_routing_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/base/tool_routing_team.md b/surfsense_backend/app/prompts/system_prompt_composer/base/tool_routing_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/base/tool_routing_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/base/tool_routing_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/composer.py b/surfsense_backend/app/prompts/system_prompt_composer/composer.py similarity index 99% rename from surfsense_backend/app/agents/new_chat/prompts/composer.py rename to surfsense_backend/app/prompts/system_prompt_composer/composer.py index 412665813..3849af313 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/composer.py +++ b/surfsense_backend/app/prompts/system_prompt_composer/composer.py @@ -2,7 +2,7 @@ Prompt composer for the SurfSense ``new_chat`` agent. This module assembles the agent's system prompt from the markdown fragments -under :mod:`app.agents.new_chat.prompts`. It replaces the monolithic +under :mod:`app.prompts.system_prompt_composer`. It replaces the monolithic ``system_prompt.py`` with a clean, fragment-based composition: :: @@ -119,7 +119,7 @@ def detect_provider_variant(model_name: str | None) -> ProviderVariant: # ----------------------------------------------------------------------------- -_PROMPTS_PACKAGE = "app.agents.new_chat.prompts" +_PROMPTS_PACKAGE = "app.prompts.system_prompt_composer" def _read_fragment(subpath: str) -> str: diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/examples/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/examples/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/generate_image.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/generate_image.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/generate_image.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/generate_image.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/generate_podcast.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/generate_podcast.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/generate_podcast.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/generate_podcast.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/generate_report.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/generate_report.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/generate_report.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/generate_report.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/generate_resume.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/generate_resume.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/generate_resume.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/generate_resume.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/generate_video_presentation.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/generate_video_presentation.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/generate_video_presentation.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/generate_video_presentation.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/scrape_webpage.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/scrape_webpage.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/scrape_webpage.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/scrape_webpage.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_private.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/update_memory_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/update_memory_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_team.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/update_memory_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/update_memory_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/update_memory_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/examples/web_search.md b/surfsense_backend/app/prompts/system_prompt_composer/examples/web_search.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/examples/web_search.md rename to surfsense_backend/app/prompts/system_prompt_composer/examples/web_search.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/routing/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/providers/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/routing/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/providers/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/anthropic.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/anthropic.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/deepseek.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/deepseek.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/default.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/default.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/default.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/default.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/google.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/google.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/google.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/google.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/grok.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/grok.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/grok.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/grok.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/kimi.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/kimi.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/openai_classic.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/openai_classic.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/openai_codex.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/openai_codex.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md b/surfsense_backend/app/prompts/system_prompt_composer/providers/openai_reasoning.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md rename to surfsense_backend/app/prompts/system_prompt_composer/providers/openai_reasoning.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/routing/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/routing/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/routing/jira.md b/surfsense_backend/app/prompts/system_prompt_composer/routing/jira.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/routing/jira.md rename to surfsense_backend/app/prompts/system_prompt_composer/routing/jira.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/routing/linear.md b/surfsense_backend/app/prompts/system_prompt_composer/routing/linear.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/routing/linear.md rename to surfsense_backend/app/prompts/system_prompt_composer/routing/linear.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/routing/slack.md b/surfsense_backend/app/prompts/system_prompt_composer/routing/slack.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/routing/slack.md rename to surfsense_backend/app/prompts/system_prompt_composer/routing/slack.md diff --git a/surfsense_backend/app/agents/new_chat/skills/builtin/__init__.py b/surfsense_backend/app/prompts/system_prompt_composer/tools/__init__.py similarity index 100% rename from surfsense_backend/app/agents/new_chat/skills/builtin/__init__.py rename to surfsense_backend/app/prompts/system_prompt_composer/tools/__init__.py diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/_preamble.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/_preamble.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/_preamble.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/_preamble.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/generate_image.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/generate_image.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/generate_image.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/generate_image.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/generate_podcast.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/generate_podcast.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/generate_podcast.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/generate_podcast.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/generate_report.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/generate_report.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/generate_report.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/generate_report.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/generate_resume.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/generate_resume.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/generate_resume.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/generate_resume.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/generate_video_presentation.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/generate_video_presentation.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/generate_video_presentation.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/generate_video_presentation.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/scrape_webpage.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/scrape_webpage.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/scrape_webpage.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/scrape_webpage.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_private.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/update_memory_private.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_private.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/update_memory_private.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_team.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/update_memory_team.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/update_memory_team.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/update_memory_team.md diff --git a/surfsense_backend/app/agents/new_chat/prompts/tools/web_search.md b/surfsense_backend/app/prompts/system_prompt_composer/tools/web_search.md similarity index 100% rename from surfsense_backend/app/agents/new_chat/prompts/tools/web_search.md rename to surfsense_backend/app/prompts/system_prompt_composer/tools/web_search.md diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 8373f13c3..5cc029884 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -1,6 +1,9 @@ -from fastapi import APIRouter +from fastapi import APIRouter, Depends from app.automations.api import router as automations_router +from app.file_storage.api import router as file_storage_router +from app.gateway import require_gateway_enabled +from app.notifications.api import router as notifications_router from .agent_action_log_route import router as agent_action_log_router from .agent_flags_route import router as agent_flags_router @@ -20,6 +23,9 @@ from .dropbox_add_connector_route import router as dropbox_add_connector_router from .editor_routes import router as editor_router from .export_routes import router as export_router from .folders_routes import router as folders_router +from .gateway_webhook_routes import router as gateway_router +from .gateway_whatsapp_baileys_routes import router as gateway_whatsapp_baileys_router +from .gateway_whatsapp_webhook_routes import router as gateway_whatsapp_webhook_router from .google_calendar_add_connector_route import ( router as google_calendar_add_connector_router, ) @@ -41,7 +47,6 @@ from .model_list_routes import router as model_list_router from .new_chat_routes import router as new_chat_router from .new_llm_config_routes import router as new_llm_config_router from .notes_routes import router as notes_router -from .notifications_routes import router as notifications_router from .notion_add_connector_route import router as notion_add_connector_router from .obsidian_plugin_routes import router as obsidian_plugin_router from .onedrive_add_connector_route import router as onedrive_add_connector_router @@ -69,6 +74,14 @@ router.include_router(editor_router) router.include_router(export_router) router.include_router(documents_router) router.include_router(folders_router) +_gateway_enabled_dep = [Depends(require_gateway_enabled)] +router.include_router(gateway_router, dependencies=_gateway_enabled_dep) +router.include_router( + gateway_whatsapp_webhook_router, dependencies=_gateway_enabled_dep +) +router.include_router( + gateway_whatsapp_baileys_router, dependencies=_gateway_enabled_dep +) router.include_router(notes_router) router.include_router(new_chat_router) # Chat with assistant-ui persistence router.include_router(agent_revert_router) # POST /threads/{id}/revert/{action_id} @@ -120,3 +133,4 @@ router.include_router(prompts_router) router.include_router(memory_router) # User personal memory (memory.md style) router.include_router(team_memory_router) # Search-space team memory router.include_router(automations_router) # Automations CRUD + run history +router.include_router(file_storage_router) # Original file metadata + download diff --git a/surfsense_backend/app/routes/agent_action_log_route.py b/surfsense_backend/app/routes/agent_action_log_route.py index 2608aa3b1..9a55fdec3 100644 --- a/surfsense_backend/app/routes/agent_action_log_route.py +++ b/surfsense_backend/app/routes/agent_action_log_route.py @@ -28,7 +28,7 @@ from pydantic import BaseModel from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.feature_flags import get_flags +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags from app.db import ( AgentActionLog, NewChatThread, diff --git a/surfsense_backend/app/routes/agent_flags_route.py b/surfsense_backend/app/routes/agent_flags_route.py index 99388af66..e97608cbe 100644 --- a/surfsense_backend/app/routes/agent_flags_route.py +++ b/surfsense_backend/app/routes/agent_flags_route.py @@ -22,7 +22,10 @@ from dataclasses import asdict from fastapi import APIRouter, Depends from pydantic import BaseModel -from app.agents.new_chat.feature_flags import AgentFeatureFlags, get_flags +from app.agents.chat.multi_agent_chat.shared.feature_flags import ( + AgentFeatureFlags, + get_flags, +) from app.config import config from app.db import User from app.users import current_active_user diff --git a/surfsense_backend/app/routes/agent_permissions_route.py b/surfsense_backend/app/routes/agent_permissions_route.py index 1c76e00e6..0c07eeb9c 100644 --- a/surfsense_backend/app/routes/agent_permissions_route.py +++ b/surfsense_backend/app/routes/agent_permissions_route.py @@ -30,7 +30,7 @@ from sqlalchemy import select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.feature_flags import get_flags +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags from app.db import ( AgentPermissionRule, NewChatThread, diff --git a/surfsense_backend/app/routes/agent_revert_route.py b/surfsense_backend/app/routes/agent_revert_route.py index 711081b15..ce21de69d 100644 --- a/surfsense_backend/app/routes/agent_revert_route.py +++ b/surfsense_backend/app/routes/agent_revert_route.py @@ -32,7 +32,7 @@ from sqlalchemy import select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.feature_flags import get_flags +from app.agents.chat.multi_agent_chat.shared.feature_flags import get_flags from app.db import ( AgentActionLog, User, diff --git a/surfsense_backend/app/routes/anonymous_chat_routes.py b/surfsense_backend/app/routes/anonymous_chat_routes.py index eb952e684..ad3277375 100644 --- a/surfsense_backend/app/routes/anonymous_chat_routes.py +++ b/surfsense_backend/app/routes/anonymous_chat_routes.py @@ -236,7 +236,7 @@ async def stream_anonymous_chat( detail="No-login mode is not enabled.", ) - from app.agents.new_chat.llm_config import ( + from app.agents.chat.runtime.llm_config import ( AgentConfig, create_chat_litellm_from_agent_config, ) @@ -351,12 +351,13 @@ async def stream_anonymous_chat( async def _generate(): from langchain_core.messages import AIMessage, HumanMessage - from app.agents.new_chat.anonymous_agent import create_anonymous_chat_agent - from app.agents.new_chat.checkpointer import get_checkpointer + from app.agents.chat.anonymous_chat import create_anonymous_chat_agent + from app.agents.chat.runtime.checkpointer import get_checkpointer from app.db import shielded_async_session from app.services.new_streaming_service import VercelStreamingService from app.services.token_tracking_service import start_turn - from app.tasks.chat.stream_new_chat import StreamResult, _stream_agent_events + from app.tasks.chat.streaming.agent.event_loop import stream_agent_events + from app.tasks.chat.streaming.shared.stream_result import StreamResult accumulator = start_turn() streaming_service = VercelStreamingService() @@ -419,7 +420,7 @@ async def stream_anonymous_chat( stream_result = StreamResult() - async for sse in _stream_agent_events( + async for sse in stream_agent_events( agent=agent, config=langgraph_config, input_data=input_state, diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 96c5d2344..865068fba 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import selectinload -from app.agents.new_chat.path_resolver import virtual_path_to_doc +from app.agents.chat.runtime.path_resolver import virtual_path_to_doc from app.db import ( Chunk, Document, @@ -44,10 +44,12 @@ except RuntimeError as e: print("Error setting event loop policy", e) pass +import logging import os os.environ["UNSTRUCTURED_HAS_PATCHED_LOOP"] = "1" +logger = logging.getLogger(__name__) router = APIRouter() @@ -123,7 +125,6 @@ async def create_documents( async def create_documents_file_upload( files: list[UploadFile], search_space_id: int = Form(...), - should_summarize: bool = Form(False), use_vision_llm: bool = Form(False), processing_mode: str = Form("basic"), session: AsyncSession = Depends(get_async_session), @@ -142,9 +143,11 @@ async def create_documents_file_upload( import os import tempfile from datetime import datetime + from pathlib import Path from app.db import DocumentStatus from app.etl_pipeline.etl_document import ProcessingMode + from app.file_storage.service import store_document_file from app.tasks.document_processors.base import ( check_document_by_unique_identifier, get_current_timestamp, @@ -175,11 +178,12 @@ async def create_documents_file_upload( ) # ===== Read all files concurrently to avoid blocking the event loop ===== - async def _read_and_save(file: UploadFile) -> tuple[str, str, int]: + async def _read_and_save(file: UploadFile) -> tuple[str, str, int, str | None]: """Read upload content and write to temp file off the event loop.""" content = await file.read() file_size = len(content) filename = file.filename or "unknown" + content_type = file.content_type if file_size > MAX_FILE_SIZE_BYTES: raise HTTPException( @@ -196,17 +200,18 @@ async def create_documents_file_upload( return tmp.name temp_path = await asyncio.to_thread(_write_temp) - return temp_path, filename, file_size + return temp_path, filename, file_size, content_type saved_files = await asyncio.gather(*(_read_and_save(f) for f in files)) # ===== PHASE 1: Create pending documents for all files ===== created_documents: list[Document] = [] - files_to_process: list[tuple[Document, str, str]] = [] + # (document, temp_path, filename, content_type) + files_to_process: list[tuple[Document, str, str, str | None]] = [] skipped_duplicates = 0 duplicate_document_ids: list[int] = [] - for temp_path, filename, file_size in saved_files: + for temp_path, filename, file_size, content_type in saved_files: try: unique_identifier_hash = generate_unique_identifier_hash( DocumentType.FILE, filename, search_space_id @@ -231,7 +236,9 @@ async def create_documents_file_upload( } existing.updated_at = get_current_timestamp() created_documents.append(existing) - files_to_process.append((existing, temp_path, filename)) + files_to_process.append( + (existing, temp_path, filename, content_type) + ) continue document = Document( @@ -253,7 +260,7 @@ async def create_documents_file_upload( ) session.add(document) created_documents.append(document) - files_to_process.append((document, temp_path, filename)) + files_to_process.append((document, temp_path, filename, content_type)) except HTTPException: raise @@ -269,15 +276,38 @@ async def create_documents_file_upload( for doc in created_documents: await session.refresh(doc) + # ===== PHASE 1.5: Persist the original uploads to durable storage ===== + # Best-effort: a storage failure must not block parsing or the response. + for document, temp_path, filename, content_type in files_to_process: + try: + original_bytes = await asyncio.to_thread( + lambda p=temp_path: Path(p).read_bytes() + ) + await store_document_file( + session, + document_id=document.id, + search_space_id=search_space_id, + data=original_bytes, + filename=filename, + mime_type=content_type, + created_by_id=str(user.id), + ) + except Exception as storage_error: + logger.warning( + "Failed to store original upload for document %s: %s", + document.id, + storage_error, + ) + await session.commit() + # ===== PHASE 2: Dispatch tasks for each file ===== - for document, temp_path, filename in files_to_process: + for document, temp_path, filename, _content_type in files_to_process: await dispatcher.dispatch_file_processing( document_id=document.id, temp_path=temp_path, filename=filename, search_space_id=search_space_id, user_id=str(user.id), - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=validated_mode.value, ) @@ -1554,7 +1584,6 @@ async def folder_upload( search_space_id: int = Form(...), relative_paths: str = Form(...), root_folder_id: int | None = Form(None), - enable_summary: bool = Form(False), use_vision_llm: bool = Form(False), processing_mode: str = Form("basic"), session: AsyncSession = Depends(get_async_session), @@ -1687,7 +1716,6 @@ async def folder_upload( user_id=str(user.id), folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, use_vision_llm=use_vision_llm, file_mappings=list(file_mappings), processing_mode=validated_mode.value, diff --git a/surfsense_backend/app/routes/editor_routes.py b/surfsense_backend/app/routes/editor_routes.py index 0f986e416..166164c50 100644 --- a/surfsense_backend/app/routes/editor_routes.py +++ b/surfsense_backend/app/routes/editor_routes.py @@ -38,14 +38,13 @@ logger = logging.getLogger(__name__) router = APIRouter() +EDITOR_PLATE_MAX_BYTES = 5 * 1024 * 1024 + @router.get("/search-spaces/{search_space_id}/documents/{document_id}/editor-content") async def get_editor_content( search_space_id: int, document_id: int, - max_length: int | None = Query( - None, description="Truncate source_markdown to this many characters" - ), session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): @@ -84,19 +83,16 @@ async def get_editor_content( def _build_response(md: str) -> dict: size_bytes = len(md.encode("utf-8")) - truncated = False - output_md = md - if max_length is not None and size_bytes > max_length: - output_md = md[:max_length] - truncated = True + viewer_mode = "monaco" if size_bytes > EDITOR_PLATE_MAX_BYTES else "plate" return { "document_id": document.id, "title": document.title, "document_type": document.document_type.value, - "source_markdown": output_md, + "source_markdown": md, "content_size_bytes": size_bytes, "chunk_count": chunk_count, - "truncated": truncated, + "viewer_mode": viewer_mode, + "editor_plate_max_bytes": EDITOR_PLATE_MAX_BYTES, "updated_at": document.updated_at.isoformat() if document.updated_at else None, diff --git a/surfsense_backend/app/routes/gateway_webhook_routes.py b/surfsense_backend/app/routes/gateway_webhook_routes.py new file mode 100644 index 000000000..14f929567 --- /dev/null +++ b/surfsense_backend/app/routes/gateway_webhook_routes.py @@ -0,0 +1,1119 @@ +"""Messaging gateway routes.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import time +import uuid +from datetime import UTC, datetime +from typing import Any +from urllib.parse import quote, urlencode +from uuid import UUID + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel +from sqlalchemy import or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.responses import JSONResponse, RedirectResponse, Response + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatBinding, + ExternalChatBindingState, + ExternalChatHealthStatus, + ExternalChatPeerKind, + ExternalChatPlatform, + User, + get_async_session, +) +from app.gateway.accounts import ( + get_discord_account_by_guild, + get_or_create_system_telegram_account, + get_or_create_system_whatsapp_account, + get_slack_account_by_team, +) +from app.gateway.bindings import resume_binding, revoke_binding +from app.gateway.discord.adapter import discord_user_peer_id +from app.gateway.inbox import ( + persist_inbound_event, + slack_event_dedupe_key, + telegram_event_dedupe_key, +) +from app.gateway.pairing import generate_pairing_code, pairing_expires_at +from app.gateway.slack.adapter import slack_user_peer_id +from app.observability.metrics import ( + record_gateway_inbox_write, + record_gateway_webhook_parse_error, +) +from app.users import current_active_user +from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.rbac import check_search_space_access + +router = APIRouter(prefix="/gateway", tags=["gateway"]) +logger = logging.getLogger(__name__) + +SLACK_AUTHORIZATION_URL = "https://slack.com/oauth/v2/authorize" +SLACK_TOKEN_URL = "https://slack.com/api/oauth.v2.access" +DISCORD_AUTHORIZATION_URL = "https://discord.com/api/oauth2/authorize" +DISCORD_TOKEN_URL = "https://discord.com/api/oauth2/token" +DISCORD_API = "https://discord.com/api/v10" +SLACK_BOT_SCOPES = [ + "app_mentions:read", + "chat:write", + "channels:read", + "groups:read", + "im:write", + "users:read", + "team:read", +] +DISCORD_GATEWAY_SCOPES = ["identify", "guilds", "bot"] +DISCORD_VIEW_CHANNEL = 1 << 10 +DISCORD_SEND_MESSAGES = 1 << 11 +DISCORD_READ_MESSAGE_HISTORY = 1 << 16 +DISCORD_SEND_MESSAGES_IN_THREADS = 1 << 38 +DISCORD_GATEWAY_PERMISSIONS = ( + DISCORD_VIEW_CHANNEL + | DISCORD_SEND_MESSAGES + | DISCORD_READ_MESSAGE_HISTORY + | DISCORD_SEND_MESSAGES_IN_THREADS +) +_state_manager: OAuthStateManager | None = None +_token_encryption: TokenEncryption | None = None + + +def _get_state_manager() -> OAuthStateManager: + global _state_manager + if _state_manager is None: + if not config.SECRET_KEY: + raise HTTPException(status_code=500, detail="SECRET_KEY is not configured") + _state_manager = OAuthStateManager(config.SECRET_KEY) + return _state_manager + + +def _get_token_encryption() -> TokenEncryption: + global _token_encryption + if _token_encryption is None: + if not config.SECRET_KEY: + raise HTTPException(status_code=500, detail="SECRET_KEY is not configured") + _token_encryption = TokenEncryption(config.SECRET_KEY) + return _token_encryption + + +def _slack_redirect_uri() -> str: + if config.GATEWAY_SLACK_REDIRECT_URI: + return config.GATEWAY_SLACK_REDIRECT_URI + base = config.BACKEND_URL or "" + return f"{base.rstrip('/')}/api/v1/gateway/slack/callback" + + +def _discord_redirect_uri() -> str: + if config.GATEWAY_DISCORD_REDIRECT_URI: + return config.GATEWAY_DISCORD_REDIRECT_URI + base = config.BACKEND_URL or "" + return f"{base.rstrip('/')}/api/v1/gateway/discord/callback" + + +def _slack_frontend_redirect( + space_id: int, *, success: bool = False, error: str | None = None +) -> RedirectResponse: + qs = ( + "slack_gateway=connected" + if success + else f"error={error or 'slack_gateway_failed'}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" + ) + + +def _discord_frontend_redirect( + space_id: int, *, success: bool = False, error: str | None = None +) -> RedirectResponse: + qs = ( + "discord_gateway=connected" + if success + else f"error={error or 'discord_gateway_failed'}" + ) + return RedirectResponse( + url=f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/user-settings?{qs}" + ) + + +def verify_slack_signature( + *, signing_secret: str, timestamp: str | None, signature: str | None, body: bytes +) -> bool: + if not signing_secret or not timestamp or not signature: + return False + try: + ts = int(timestamp) + except ValueError: + return False + if abs(time.time() - ts) > 60 * 5: + return False + base = b"v0:" + timestamp.encode() + b":" + body + digest = hmac.new(signing_secret.encode(), base, hashlib.sha256).hexdigest() + expected = f"v0={digest}" + return hmac.compare_digest(expected, signature) + + +def _slack_event_kind(payload: dict[str, Any]) -> str: + event_type = str((payload.get("event") or {}).get("type") or "") + return "message" if event_type in {"app_mention", "message"} else "other" + + +class StartBindingRequest(BaseModel): + platform: ExternalChatPlatform = ExternalChatPlatform.TELEGRAM + search_space_id: int + + +class StartBindingResponse(BaseModel): + binding_id: int + code: str + deep_link: str + expires_at: datetime + + +class UpdateBindingSearchSpaceRequest(BaseModel): + search_space_id: int + + +class UpdateAccountSearchSpaceRequest(BaseModel): + search_space_id: int + + +def _active_whatsapp_account_mode() -> ExternalChatAccountMode | None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "cloud": + return ExternalChatAccountMode.CLOUD_SHARED + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "baileys": + return ExternalChatAccountMode.SELF_HOST_BYO + return None + + +def _is_inactive_whatsapp_account(account: ExternalChatAccount) -> bool: + return ( + account.platform == ExternalChatPlatform.WHATSAPP + and account.mode != _active_whatsapp_account_mode() + ) + + +def _telegram_gateway_enabled() -> bool: + return ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "disabled" + and bool(config.TELEGRAM_SHARED_BOT_TOKEN) + and bool(config.TELEGRAM_SHARED_BOT_USERNAME) + and ( + config.GATEWAY_TELEGRAM_INTAKE_MODE != "webhook" + or bool(config.TELEGRAM_WEBHOOK_SECRET) + ) + ) + + +def _slack_gateway_enabled() -> bool: + return bool( + config.GATEWAY_SLACK_ENABLED + and config.GATEWAY_SLACK_CLIENT_ID + and config.GATEWAY_SLACK_CLIENT_SECRET + and config.GATEWAY_SLACK_SIGNING_SECRET + ) + + +def _discord_gateway_enabled() -> bool: + return bool( + config.GATEWAY_DISCORD_ENABLED + and config.DISCORD_CLIENT_ID + and config.DISCORD_CLIENT_SECRET + and config.DISCORD_BOT_TOKEN + ) + + +def _classify_telegram_event(payload: dict[str, Any]) -> str: + if "message" in payload: + return "message" + if "edited_message" in payload: + return "edited_message" + if "callback_query" in payload: + return "callback_query" + return "other" + + +def _telegram_message(payload: dict[str, Any]) -> dict[str, Any] | None: + return payload.get("message") or payload.get("edited_message") + + +@router.get("/slack/install") +async def install_slack_gateway( + search_space_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, str]: + if not _slack_gateway_enabled(): + raise HTTPException( + status_code=500, detail="Slack gateway OAuth is not configured" + ) + await check_search_space_access(session, user, search_space_id) + state = _get_state_manager().generate_secure_state(search_space_id, user.id) + auth_params = { + "client_id": config.GATEWAY_SLACK_CLIENT_ID, + "scope": ",".join(SLACK_BOT_SCOPES), + "redirect_uri": _slack_redirect_uri(), + "state": state, + } + return {"auth_url": f"{SLACK_AUTHORIZATION_URL}?{urlencode(auth_params)}"} + + +@router.get("/slack/callback") +async def slack_gateway_callback( + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +) -> RedirectResponse: + space_id = None + if state: + try: + state_data = _get_state_manager().validate_state(state) + space_id = int(state_data["space_id"]) + except Exception: + state_data = None + else: + state_data = None + + if error: + return _slack_frontend_redirect( + space_id or 0, error="slack_gateway_oauth_denied" + ) + if not code or state_data is None: + raise HTTPException( + status_code=400, detail="Invalid Slack gateway OAuth callback" + ) + if not _slack_gateway_enabled(): + raise HTTPException( + status_code=500, detail="Slack gateway OAuth is not configured" + ) + + user_id = UUID(state_data["user_id"]) + token_payload = { + "client_id": config.GATEWAY_SLACK_CLIENT_ID, + "client_secret": config.GATEWAY_SLACK_CLIENT_SECRET, + "code": code, + "redirect_uri": _slack_redirect_uri(), + } + async with httpx.AsyncClient(timeout=30.0) as client: + token_response = await client.post( + SLACK_TOKEN_URL, + data=token_payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + token_response.raise_for_status() + token_json = token_response.json() + if not token_json.get("ok", False): + raise HTTPException( + status_code=400, + detail=f"Slack gateway OAuth failed: {token_json.get('error', 'unknown_error')}", + ) + + bot_token = token_json.get("access_token") + team = token_json.get("team") or {} + team_id = team.get("id") + if not bot_token or not team_id: + raise HTTPException( + status_code=400, detail="Slack gateway OAuth returned incomplete data" + ) + + bot_user_id = token_json.get("bot_user_id") + app_id = token_json.get("app_id") + authed_user = token_json.get("authed_user") or {} + authed_slack_user_id = authed_user.get("id") + enc = _get_token_encryption() + credentials = { + "bot_token": bot_token, + "token_type": token_json.get("token_type", "bot"), + "scope": token_json.get("scope"), + } + cursor_state = { + "team_id": team_id, + "team_name": team.get("name"), + "enterprise_id": (token_json.get("enterprise") or {}).get("id"), + "app_id": app_id, + "bot_user_id": bot_user_id, + "scope": token_json.get("scope"), + } + + account = await get_slack_account_by_team(session, team_id=team_id) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.SLACK, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + encrypted_credentials=enc.encrypt_token(json.dumps(credentials)), + bot_username="SurfSense", + cursor_state=cursor_state, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + else: + account.encrypted_credentials = enc.encrypt_token(json.dumps(credentials)) + account.cursor_state = {**(account.cursor_state or {}), **cursor_state} + account.health_status = ExternalChatHealthStatus.UNKNOWN + + if authed_slack_user_id: + peer_id = slack_user_peer_id(team_id, authed_slack_user_id) + existing_binding_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + binding = existing_binding_result.scalars().first() + if binding is None: + session.add( + ExternalChatBinding( + account_id=account.id, + user_id=user_id, + search_space_id=space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=peer_id, + external_peer_kind=ExternalChatPeerKind.DIRECT, + external_username=authed_slack_user_id, + external_metadata={ + "kind": "slack_user", + "team_id": team_id, + "slack_user_id": authed_slack_user_id, + }, + ) + ) + elif binding.user_id == user_id: + binding.search_space_id = space_id + binding.external_metadata = { + **(binding.external_metadata or {}), + "kind": "slack_user", + "team_id": team_id, + "slack_user_id": authed_slack_user_id, + } + + await session.commit() + return _slack_frontend_redirect(space_id, success=True) + + +@router.get("/discord/install") +async def install_discord_gateway( + search_space_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, str]: + if not _discord_gateway_enabled(): + raise HTTPException( + status_code=500, detail="Discord gateway OAuth is not configured" + ) + await check_search_space_access(session, user, search_space_id) + state = _get_state_manager().generate_secure_state(search_space_id, user.id) + auth_params = { + "client_id": config.DISCORD_CLIENT_ID, + "scope": " ".join(DISCORD_GATEWAY_SCOPES), + "redirect_uri": _discord_redirect_uri(), + "response_type": "code", + "state": state, + "permissions": str(DISCORD_GATEWAY_PERMISSIONS), + } + return {"auth_url": f"{DISCORD_AUTHORIZATION_URL}?{urlencode(auth_params)}"} + + +@router.get("/discord/callback") +async def discord_gateway_callback( + code: str | None = None, + error: str | None = None, + state: str | None = None, + session: AsyncSession = Depends(get_async_session), +) -> RedirectResponse: + space_id = None + if state: + try: + state_data = _get_state_manager().validate_state(state) + space_id = int(state_data["space_id"]) + except Exception: + state_data = None + else: + state_data = None + + if error: + return _discord_frontend_redirect( + space_id or 0, error="discord_gateway_oauth_denied" + ) + if not code or state_data is None: + raise HTTPException( + status_code=400, detail="Invalid Discord gateway OAuth callback" + ) + if not _discord_gateway_enabled(): + raise HTTPException( + status_code=500, detail="Discord gateway OAuth is not configured" + ) + + user_id = UUID(state_data["user_id"]) + token_payload = { + "client_id": config.DISCORD_CLIENT_ID, + "client_secret": config.DISCORD_CLIENT_SECRET, + "grant_type": "authorization_code", + "code": code, + "redirect_uri": _discord_redirect_uri(), + } + async with httpx.AsyncClient(timeout=30.0) as client: + token_response = await client.post( + DISCORD_TOKEN_URL, + data=token_payload, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + token_response.raise_for_status() + token_json = token_response.json() + + oauth_access_token = token_json.get("access_token") + guild = token_json.get("guild") or {} + guild_id = guild.get("id") + guild_name = guild.get("name") + discord_user_id = None + discord_username = None + if oauth_access_token: + async with httpx.AsyncClient(timeout=30.0) as client: + user_response = await client.get( + f"{DISCORD_API}/users/@me", + headers={"Authorization": f"Bearer {oauth_access_token}"}, + ) + user_response.raise_for_status() + user_json = user_response.json() + discord_user_id = user_json.get("id") + discord_username = user_json.get("username") + + if not guild_id: + raise HTTPException( + status_code=400, + detail=( + "Discord gateway OAuth did not return a guild. " + "Choose a server during bot installation and try again." + ), + ) + + enc = _get_token_encryption() + credentials = { + "bot_token": config.DISCORD_BOT_TOKEN, + "token_type": "bot", + "scope": token_json.get("scope"), + } + cursor_state = { + "guild_id": guild_id, + "guild_name": guild_name, + "application_id": config.DISCORD_CLIENT_ID, + "scope": token_json.get("scope"), + "permissions": str(DISCORD_GATEWAY_PERMISSIONS), + } + + account = await get_discord_account_by_guild(session, guild_id=str(guild_id)) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.DISCORD, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + encrypted_credentials=enc.encrypt_token(json.dumps(credentials)), + bot_username="SurfSense", + cursor_state=cursor_state, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + await session.flush() + else: + account.encrypted_credentials = enc.encrypt_token(json.dumps(credentials)) + account.cursor_state = {**(account.cursor_state or {}), **cursor_state} + account.health_status = ExternalChatHealthStatus.UNKNOWN + + if discord_user_id: + peer_id = discord_user_peer_id(str(guild_id), str(discord_user_id)) + existing_binding_result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.external_peer_id == peer_id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + binding = existing_binding_result.scalars().first() + metadata = { + "kind": "discord_user", + "guild_id": guild_id, + "guild_name": guild_name, + "discord_user_id": discord_user_id, + } + if binding is None: + session.add( + ExternalChatBinding( + account_id=account.id, + user_id=user_id, + search_space_id=space_id, + state=ExternalChatBindingState.BOUND, + external_peer_id=peer_id, + external_peer_kind=ExternalChatPeerKind.DIRECT, + external_username=discord_username or discord_user_id, + external_metadata=metadata, + ) + ) + elif binding.user_id == user_id: + binding.search_space_id = space_id + binding.external_username = discord_username or binding.external_username + binding.external_metadata = { + **(binding.external_metadata or {}), + **metadata, + } + + await session.commit() + return _discord_frontend_redirect(space_id, success=True) + + +@router.post("/webhooks/slack") +async def slack_webhook( + request: Request, + session: AsyncSession = Depends(get_async_session), +) -> Response: + if not _slack_gateway_enabled(): + return Response(status_code=200) + + body = await request.body() + if not verify_slack_signature( + signing_secret=config.GATEWAY_SLACK_SIGNING_SECRET or "", + timestamp=request.headers.get("X-Slack-Request-Timestamp"), + signature=request.headers.get("X-Slack-Signature"), + body=body, + ): + raise HTTPException(status_code=403, detail="Invalid Slack signature") + + try: + payload = json.loads(body.decode()) + except ValueError: + record_gateway_webhook_parse_error() + return Response(status_code=200) + + if payload.get("type") == "url_verification": + return JSONResponse({"challenge": payload.get("challenge", "")}) + if payload.get("type") != "event_callback": + return Response(status_code=200) + + event = payload.get("event") or {} + event_id = payload.get("event_id") + team_id = payload.get("team_id") or event.get("team") + if not event_id or not team_id: + return Response(status_code=200) + + account = await get_slack_account_by_team(session, team_id=str(team_id)) + if account is None: + logger.warning("Ignoring Slack event for uninstalled team_id=%s", team_id) + return Response(status_code=200) + + bot_user_id = (account.cursor_state or {}).get("bot_user_id") + if event.get("bot_id") or (bot_user_id and event.get("user") == bot_user_id): + return Response(status_code=200) + + try: + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.SLACK, + event_dedupe_key=slack_event_dedupe_key(event_id), + external_event_id=str(event_id), + external_message_id=str(event.get("ts")) if event.get("ts") else None, + event_kind=_slack_event_kind(payload), + raw_payload=payload, + request_id=f"gateway_{uuid.uuid4().hex[:16]}", + ) + await session.commit() + record_gateway_inbox_write(platform="slack", dedup_skipped=inbox_id is None) + except Exception: + await session.rollback() + logger.exception("Slack webhook persistence failed team_id=%s", team_id) + return Response(status_code=200) + + +async def _resolve_webhook_account( + session: AsyncSession, + *, + account_id: int, + header_secret: str | None, +) -> ExternalChatAccount: + account = await session.get(ExternalChatAccount, account_id) + if account is None or account.platform != ExternalChatPlatform.TELEGRAM: + raise HTTPException(status_code=404, detail="Gateway account not found") + expected_secret = account.webhook_secret or "" + if not expected_secret or not hmac.compare_digest( + header_secret or "", expected_secret + ): + raise HTTPException(status_code=403, detail="Invalid Telegram webhook secret") + return account + + +@router.post("/webhooks/telegram/{account_id}") +async def telegram_webhook( + request: Request, + account_id: int, + session: AsyncSession = Depends(get_async_session), +) -> Response: + if not _telegram_gateway_enabled(): + return Response(status_code=200) + + request_id = f"gateway_{uuid.uuid4().hex[:16]}" + try: + payload = await request.json() + except ValueError: + record_gateway_webhook_parse_error() + return Response(status_code=200) + + account = await _resolve_webhook_account( + session, + account_id=account_id, + header_secret=request.headers.get("X-Telegram-Bot-Api-Secret-Token"), + ) + + try: + update_id = payload.get("update_id") + if update_id is None: + return Response(status_code=200) + + message = _telegram_message(payload) or {} + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(update_id), + external_event_id=str(update_id), + external_message_id=( + str(message["message_id"]) + if message.get("message_id") is not None + else None + ), + event_kind=_classify_telegram_event(payload), + raw_payload=payload, + request_id=request_id, + ) + await session.commit() + record_gateway_inbox_write(platform="telegram", dedup_skipped=inbox_id is None) + return Response(status_code=200) + except Exception: + await session.rollback() + logger.exception("Telegram webhook processing failed account_id=%s", account_id) + return Response(status_code=200) + + +@router.post("/bindings/start", response_model=StartBindingResponse) +async def start_binding( + body: StartBindingRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> StartBindingResponse: + await check_search_space_access(session, user, body.search_space_id) + code = generate_pairing_code() + if body.platform == ExternalChatPlatform.TELEGRAM: + if not _telegram_gateway_enabled(): + raise HTTPException(status_code=400, detail="Telegram gateway is disabled") + account = await get_or_create_system_telegram_account(session) + username = account.bot_username or config.TELEGRAM_SHARED_BOT_USERNAME + if not username: + raise HTTPException( + status_code=500, + detail="Telegram bot username is not configured", + ) + deep_link = f"https://t.me/{username}?start={code}" + elif body.platform == ExternalChatPlatform.WHATSAPP: + if config.GATEWAY_WHATSAPP_INTAKE_MODE != "cloud": + raise HTTPException( + status_code=400, + detail="WhatsApp /start pairing requires GATEWAY_WHATSAPP_INTAKE_MODE=cloud", + ) + account = await get_or_create_system_whatsapp_account(session) + phone = config.WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER + if not phone: + raise HTTPException( + status_code=500, + detail="WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER is not configured", + ) + normalized_phone = "".join(ch for ch in phone if ch.isdigit()) + if not normalized_phone: + raise HTTPException( + status_code=500, + detail="WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER must contain digits", + ) + deep_link = f"https://wa.me/{normalized_phone}?text={quote(f'/start {code}')}" + else: + raise HTTPException(status_code=400, detail="Unsupported platform") + + expires_at = pairing_expires_at() + binding = ExternalChatBinding( + account_id=account.id, + user_id=user.id, + search_space_id=body.search_space_id, + state=ExternalChatBindingState.PENDING, + pairing_code=code, + pairing_code_expires_at=expires_at, + ) + session.add(binding) + await session.commit() + await session.refresh(binding) + + return StartBindingResponse( + binding_id=binding.id, + code=code, + deep_link=deep_link, + expires_at=expires_at, + ) + + +@router.get("/bindings") +async def list_bindings( + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + result = await session.execute( + select(ExternalChatBinding, ExternalChatAccount) + .join( + ExternalChatAccount, + ExternalChatBinding.account_id == ExternalChatAccount.id, + ) + .where(ExternalChatBinding.user_id == user.id) + ) + return [ + { + "id": binding.id, + "platform": account.platform.value, + "state": binding.state.value, + "search_space_id": binding.search_space_id, + "external_display_name": binding.external_display_name, + "external_username": binding.external_username, + "external_metadata": binding.external_metadata, + "suspended_reason": binding.suspended_reason, + } + for binding, account in result.all() + ] + + +@router.get("/connections") +async def list_connections( + platform: ExternalChatPlatform | None = None, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + active_whatsapp_mode = _active_whatsapp_account_mode() + if platform == ExternalChatPlatform.WHATSAPP and active_whatsapp_mode is None: + return [] + if platform == ExternalChatPlatform.TELEGRAM and not _telegram_gateway_enabled(): + return [] + + filters = [ + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ] + if platform is not None: + filters.append(ExternalChatAccount.platform == platform) + if ( + platform == ExternalChatPlatform.WHATSAPP + and active_whatsapp_mode is not None + ): + filters.append(ExternalChatAccount.mode == active_whatsapp_mode) + else: + if not _telegram_gateway_enabled(): + filters.append( + ExternalChatAccount.platform != ExternalChatPlatform.TELEGRAM + ) + if active_whatsapp_mode is None: + filters.append( + ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP + ) + else: + filters.append( + or_( + ExternalChatAccount.platform != ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == active_whatsapp_mode, + ) + ) + + result = await session.execute( + select(ExternalChatBinding, ExternalChatAccount) + .join( + ExternalChatAccount, + ExternalChatBinding.account_id == ExternalChatAccount.id, + ) + .where(*filters) + ) + + connections: list[dict[str, Any]] = [] + baileys_account_ids: set[int] = set() + for binding, account in result.all(): + binding_metadata = binding.external_metadata or {} + kind = str(binding_metadata.get("kind") or "") + if kind in {"slack_thread", "discord_thread"}: + continue + + account_state = account.cursor_state or {} + workspace_name = None + workspace_id = None + route_type = "binding" + connection_id = binding.id + search_space_id = binding.search_space_id + display_name = binding.external_display_name or binding.external_username + if account.platform == ExternalChatPlatform.SLACK: + workspace_name = account_state.get("team_name") + workspace_id = account_state.get("team_id") + elif account.platform == ExternalChatPlatform.DISCORD: + workspace_name = account_state.get("guild_name") + workspace_id = account_state.get("guild_id") + elif account.platform == ExternalChatPlatform.WHATSAPP: + workspace_name = account_state.get("display_phone_number") + workspace_id = account_state.get("phone_number_id") + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO: + if int(account.id) in baileys_account_ids: + continue + baileys_account_ids.add(int(account.id)) + route_type = "account" + connection_id = account.id + search_space_id = ( + account.owner_search_space_id or binding.search_space_id + ) + display_name = "WhatsApp Bridge" + + connections.append( + { + "id": connection_id, + "account_id": account.id, + "route_type": route_type, + "platform": account.platform.value, + "mode": account.mode.value, + "state": binding.state.value, + "search_space_id": search_space_id, + "display_name": display_name or workspace_name, + "external_username": ( + None + if account.mode == ExternalChatAccountMode.SELF_HOST_BYO + else binding.external_username + ), + "workspace_name": workspace_name, + "workspace_id": workspace_id, + "health_status": account.health_status.value, + "suspended_reason": binding.suspended_reason, + } + ) + + if active_whatsapp_mode == ExternalChatAccountMode.SELF_HOST_BYO and ( + platform is None or platform == ExternalChatPlatform.WHATSAPP + ): + account_result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.owner_user_id == user.id, + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.mode == ExternalChatAccountMode.SELF_HOST_BYO, + ExternalChatAccount.owner_search_space_id.is_not(None), + ) + ) + for account in account_result.scalars(): + if int(account.id) in baileys_account_ids: + continue + account_state = account.cursor_state or {} + connections.append( + { + "id": account.id, + "account_id": account.id, + "route_type": "account", + "platform": account.platform.value, + "mode": account.mode.value, + "state": "bound", + "search_space_id": account.owner_search_space_id, + "display_name": "WhatsApp Bridge", + "external_username": None, + "workspace_name": account_state.get("display_phone_number"), + "workspace_id": account_state.get("phone_number_id"), + "health_status": account.health_status.value, + "suspended_reason": account.suspended_reason, + } + ) + + return connections + + +@router.get("/platforms") +async def list_platforms( + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> list[dict[str, Any]]: + result = await session.execute( + select(ExternalChatAccount).where( + (ExternalChatAccount.owner_user_id == user.id) + | (ExternalChatAccount.is_system_account.is_(True)) + ) + ) + return [ + { + "id": account.id, + "platform": account.platform.value, + "mode": account.mode.value, + "bot_username": account.bot_username, + "health_status": account.health_status.value, + "last_health_check_at": account.last_health_check_at, + } + for account in result.scalars() + ] + + +@router.get("/config") +async def get_gateway_config( + user: User = Depends(current_active_user), +) -> dict[str, bool | str]: + return { + "telegram_enabled": _telegram_gateway_enabled(), + "whatsapp_intake_mode": config.GATEWAY_WHATSAPP_INTAKE_MODE, + "slack_enabled": _slack_gateway_enabled(), + "discord_enabled": _discord_gateway_enabled(), + } + + +@router.patch("/bindings/{binding_id}/search-space") +async def update_binding_search_space( + binding_id: int, + body: UpdateBindingSearchSpaceRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(ExternalChatBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + if binding.state not in { + ExternalChatBindingState.BOUND, + ExternalChatBindingState.SUSPENDED, + }: + raise HTTPException( + status_code=400, detail="Only active bindings can be routed" + ) + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") + + await check_search_space_access(session, user, body.search_space_id) + if binding.search_space_id != body.search_space_id: + binding.search_space_id = body.search_space_id + binding.new_chat_thread_id = None + binding.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} + + +@router.patch("/accounts/{account_id}/search-space") +async def update_gateway_account_search_space( + account_id: int, + body: UpdateAccountSearchSpaceRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + account = await session.get(ExternalChatAccount, account_id) + if ( + account is None + or account.owner_user_id != user.id + or account.platform != ExternalChatPlatform.WHATSAPP + or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + or _is_inactive_whatsapp_account(account) + ): + raise HTTPException(status_code=404, detail="Gateway account not found") + + await check_search_space_access(session, user, body.search_space_id) + account.owner_search_space_id = body.search_space_id + account.updated_at = datetime.now(UTC) + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + for binding in result.scalars(): + binding.search_space_id = body.search_space_id + binding.new_chat_thread_id = None + binding.updated_at = datetime.now(UTC) + + await session.commit() + return {"ok": True} + + +@router.delete("/bindings/{binding_id}") +async def delete_binding( + binding_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(ExternalChatBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") + revoke_binding(binding) + await session.commit() + return {"ok": True} + + +@router.delete("/accounts/{account_id}") +async def delete_gateway_account( + account_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + account = await session.get(ExternalChatAccount, account_id) + if ( + account is None + or account.owner_user_id != user.id + or account.platform != ExternalChatPlatform.WHATSAPP + or account.mode != ExternalChatAccountMode.SELF_HOST_BYO + or _is_inactive_whatsapp_account(account) + ): + raise HTTPException(status_code=404, detail="Gateway account not found") + + result = await session.execute( + select(ExternalChatBinding).where( + ExternalChatBinding.account_id == account.id, + ExternalChatBinding.user_id == user.id, + ExternalChatBinding.state.in_( + [ExternalChatBindingState.BOUND, ExternalChatBindingState.SUSPENDED] + ), + ) + ) + for binding in result.scalars(): + revoke_binding(binding) + + account.owner_search_space_id = None + account.suspended_at = datetime.now(UTC) + account.suspended_reason = "disconnected" + account.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} + + +@router.post("/bindings/{binding_id}/resume") +async def resume_external_chat_binding( + binding_id: int, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, bool]: + binding = await session.get(ExternalChatBinding, binding_id) + if binding is None or binding.user_id != user.id: + raise HTTPException(status_code=404, detail="Binding not found") + account = await session.get(ExternalChatAccount, binding.account_id) + if account is None or _is_inactive_whatsapp_account(account): + raise HTTPException(status_code=404, detail="Binding not found") + resume_binding(binding) + binding.updated_at = datetime.now(UTC) + await session.commit() + return {"ok": True} diff --git a/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py new file mode 100644 index 000000000..1fcf5c438 --- /dev/null +++ b/surfsense_backend/app/routes/gateway_whatsapp_baileys_routes.py @@ -0,0 +1,107 @@ +"""Routes for the self-hosted WhatsApp Baileys bridge.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + ExternalChatAccount, + ExternalChatAccountMode, + ExternalChatHealthStatus, + ExternalChatPlatform, + User, + get_async_session, +) +from app.gateway.whatsapp.adapter_baileys import WhatsAppBaileysAdapter +from app.users import current_active_user +from app.utils.rbac import check_search_space_access + +router = APIRouter(prefix="/gateway/whatsapp/baileys", tags=["gateway"]) + + +class BaileysPairRequest(BaseModel): + search_space_id: int + phone_number: str + + +def _ensure_baileys_enabled() -> None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE != "baileys": + raise HTTPException( + status_code=404, detail="WhatsApp Baileys gateway is disabled" + ) + if config.is_cloud(): + raise HTTPException( + status_code=403, + detail="Baileys is only available for self-hosted SurfSense installs", + ) + + +async def _get_user_whatsapp_account( + session: AsyncSession, + user: User, +) -> ExternalChatAccount | None: + result = await session.execute( + select(ExternalChatAccount).where( + ExternalChatAccount.owner_user_id == user.id, + ExternalChatAccount.platform == ExternalChatPlatform.WHATSAPP, + ExternalChatAccount.is_system_account.is_(False), + ) + ) + return result.scalars().first() + + +@router.post("/pair") +async def request_pairing_code( + body: BaileysPairRequest, + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +) -> dict[str, Any]: + _ensure_baileys_enabled() + await check_search_space_access(session, user, body.search_space_id) + adapter = WhatsAppBaileysAdapter() + try: + pairing = await adapter.request_pairing_code(phone_number=body.phone_number) + except Exception as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + + account = await _get_user_whatsapp_account(session, user) + if account is None: + account = ExternalChatAccount( + platform=ExternalChatPlatform.WHATSAPP, + mode=ExternalChatAccountMode.SELF_HOST_BYO, + owner_user_id=user.id, + owner_search_space_id=body.search_space_id, + is_system_account=False, + cursor_state={}, + health_status=ExternalChatHealthStatus.UNKNOWN, + ) + session.add(account) + else: + account.mode = ExternalChatAccountMode.SELF_HOST_BYO + account.owner_search_space_id = body.search_space_id + account.health_status = ExternalChatHealthStatus.UNKNOWN + account.suspended_at = None + account.suspended_reason = None + account.last_health_check_at = datetime.now(UTC) + await session.commit() + await session.refresh(account) + return {"account_id": account.id, **pairing} + + +@router.get("/health") +async def bridge_health( + user: User = Depends(current_active_user), +) -> dict[str, Any]: + _ensure_baileys_enabled() + adapter = WhatsAppBaileysAdapter() + try: + return await adapter.validate_credentials() + except Exception as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc diff --git a/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py new file mode 100644 index 000000000..39dc928df --- /dev/null +++ b/surfsense_backend/app/routes/gateway_whatsapp_webhook_routes.py @@ -0,0 +1,211 @@ +"""WhatsApp Cloud API webhook routes.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import uuid +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from sqlalchemy.ext.asyncio import AsyncSession +from starlette.responses import Response + +from app.config import config +from app.db import ( + ExternalChatHealthStatus, + ExternalChatPlatform, + get_async_session, +) +from app.gateway.accounts import get_or_create_system_whatsapp_account +from app.gateway.inbox import persist_inbound_event +from app.observability.metrics import ( + record_gateway_inbox_write, + record_gateway_outbound, + record_gateway_webhook_parse_error, +) + +router = APIRouter(prefix="/gateway/webhooks/whatsapp", tags=["gateway"]) +logger = logging.getLogger(__name__) + + +def _ensure_whatsapp_enabled() -> None: + if config.GATEWAY_WHATSAPP_INTAKE_MODE == "disabled": + raise HTTPException(status_code=404, detail="WhatsApp gateway is disabled") + + +@router.get("") +async def verify_whatsapp_webhook( + hub_mode: str = Query(alias="hub.mode"), + hub_verify_token: str = Query(alias="hub.verify_token"), + hub_challenge: str = Query(alias="hub.challenge"), +) -> Response: + _ensure_whatsapp_enabled() + if ( + hub_mode == "subscribe" + and config.WHATSAPP_WEBHOOK_VERIFY_TOKEN + and hmac.compare_digest(hub_verify_token, config.WHATSAPP_WEBHOOK_VERIFY_TOKEN) + ): + return Response(content=hub_challenge, media_type="text/plain") + raise HTTPException(status_code=403, detail="Invalid WhatsApp webhook token") + + +@router.post("") +async def whatsapp_webhook( + request: Request, + session: AsyncSession = Depends(get_async_session), +) -> Response: + _ensure_whatsapp_enabled() + raw_body = await request.body() + _verify_signature(raw_body, request.headers.get("X-Hub-Signature-256")) + try: + payload = json.loads(raw_body) + except ValueError: + record_gateway_webhook_parse_error() + return Response(status_code=200) + + try: + await _process_payload(session, payload) + await session.commit() + except Exception: + await session.rollback() + logger.exception("WhatsApp webhook processing failed") + return Response(status_code=200) + return Response(status_code=200) + + +def _verify_signature(raw_body: bytes, header_signature: str | None) -> None: + if not config.WHATSAPP_WEBHOOK_APP_SECRET: + raise HTTPException( + status_code=500, detail="WhatsApp app secret is not configured" + ) + received = (header_signature or "").removeprefix("sha256=") + expected = hmac.new( + config.WHATSAPP_WEBHOOK_APP_SECRET.encode(), + raw_body, + hashlib.sha256, + ).hexdigest() + if not received or not hmac.compare_digest(received, expected): + raise HTTPException( + status_code=403, detail="Invalid WhatsApp webhook signature" + ) + + +async def _process_payload(session: AsyncSession, payload: dict[str, Any]) -> None: + for entry in payload.get("entry") or []: + if not isinstance(entry, dict): + continue + for change in entry.get("changes") or []: + if not isinstance(change, dict): + continue + field = change.get("field") + value = change.get("value") or {} + if field == "messages": + await _process_messages_change(session, payload, entry, change, value) + elif field == "account_update": + await _handle_account_update(session, entry, value) + elif field == "phone_number_quality_update": + await _handle_phone_number_quality_update(session, entry, value) + + +async def _process_messages_change( + session: AsyncSession, + payload: dict[str, Any], + entry: dict[str, Any], + change: dict[str, Any], + value: dict[str, Any], +) -> None: + statuses = [ + status for status in value.get("statuses") or [] if isinstance(status, dict) + ] + for status in statuses: + record_gateway_outbound( + platform="whatsapp", + kind="status", + status=str(status.get("status") or "unknown"), + ) + + messages = [msg for msg in value.get("messages") or [] if isinstance(msg, dict)] + if not messages: + return + + account = await get_or_create_system_whatsapp_account(session) + metadata = value.get("metadata") or {} + if isinstance(metadata, dict): + cursor_state = dict(account.cursor_state or {}) + for key in ("phone_number_id", "display_phone_number"): + if metadata.get(key): + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + + for msg in messages: + message_id = str(msg.get("id") or "") + if not message_id: + continue + request_id = f"gateway_{uuid.uuid4().hex[:16]}" + inbox_id = await persist_inbound_event( + session, + account_id=account.id, + platform=ExternalChatPlatform.WHATSAPP, + event_dedupe_key=f"wamid:{message_id}", + external_event_id=message_id, + external_message_id=message_id, + event_kind="message", + raw_payload=_single_message_payload(payload, entry, change, msg), + request_id=request_id, + ) + record_gateway_inbox_write(platform="whatsapp", dedup_skipped=inbox_id is None) + + +async def _handle_account_update( + session: AsyncSession, + entry: dict[str, Any], + value: dict[str, Any], +) -> None: + account = await get_or_create_system_whatsapp_account(session) + cursor_state = dict(account.cursor_state or {}) + if entry.get("id"): + cursor_state["waba_id"] = str(entry.get("id")) + cursor_state["account_update"] = value + account.cursor_state = cursor_state + event = str(value.get("event") or value.get("type") or "").upper() + if event in {"DISABLED_UPDATE", "ACCOUNT_RESTRICTION", "PARTNER_REMOVED"}: + account.health_status = ExternalChatHealthStatus.FAILING + account.suspended_at = datetime.now(UTC) + account.suspended_reason = event.lower() + elif event in {"VERIFIED_ACCOUNT", "ACCOUNT_ENABLED", "REINSTATED"}: + account.health_status = ExternalChatHealthStatus.OK + account.suspended_at = None + account.suspended_reason = None + account.last_health_check_at = datetime.now(UTC) + + +async def _handle_phone_number_quality_update( + session: AsyncSession, + entry: dict[str, Any], + value: dict[str, Any], +) -> None: + account = await get_or_create_system_whatsapp_account(session) + cursor_state = dict(account.cursor_state or {}) + if entry.get("id"): + cursor_state["waba_id"] = str(entry.get("id")) + cursor_state["quality_update"] = value + account.cursor_state = cursor_state + account.last_health_check_at = datetime.now(UTC) + + +def _single_message_payload( + payload: dict[str, Any], + entry: dict[str, Any], + change: dict[str, Any], + message: dict[str, Any], +) -> dict[str, Any]: + value = dict(change.get("value") or {}) + value["messages"] = [message] + value.pop("statuses", None) + single_change = {**change, "value": value} + single_entry = {**entry, "changes": [single_change]} + return {"object": payload.get("object"), "entry": [single_entry]} diff --git a/surfsense_backend/app/routes/mcp_oauth_route.py b/surfsense_backend/app/routes/mcp_oauth_route.py index 57248d631..fdeb6ecfd 100644 --- a/surfsense_backend/app/routes/mcp_oauth_route.py +++ b/surfsense_backend/app/routes/mcp_oauth_route.py @@ -665,7 +665,7 @@ def _refresh_mcp_cache(connector_id: int, space_id: int) -> None: isolated from the OAuth response flow. """ try: - from app.agents.new_chat.tools.mcp_tools_cache import ( + from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( refresh_mcp_tools_cache_for_connector, ) diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 63b7732a9..0e4e557be 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -24,18 +24,18 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import selectinload -from app.agents.new_chat.filesystem_selection import ( - ClientPlatform, - FilesystemMode, - FilesystemSelection, - LocalFilesystemMount, -) -from app.agents.new_chat.middleware.busy_mutex import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import ( get_cancel_state, is_cancel_requested, manager, request_cancel, ) +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + ClientPlatform, + FilesystemMode, + FilesystemSelection, + LocalFilesystemMount, +) from app.config import config from app.db import ( ChatComment, @@ -71,7 +71,7 @@ from app.schemas.new_chat import ( TokenUsageSummary, TurnStatusResponse, ) -from app.tasks.chat.stream_new_chat import ( +from app.tasks.chat.streaming.flows import ( stream_new_chat, stream_resume_chat, ) @@ -476,7 +476,7 @@ async def _revert_turns_for_regenerate( def _try_delete_sandbox(thread_id: int) -> None: """Fire-and-forget sandbox + local file deletion so the HTTP response isn't blocked.""" - from app.agents.new_chat.sandbox import ( + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( delete_local_sandbox_files, delete_sandbox, is_sandbox_enabled, @@ -1668,7 +1668,7 @@ async def list_agent_tools( Hidden (WIP) tools are excluded from the response. """ - from app.agents.new_chat.tools.registry import BUILTIN_TOOLS + from app.agents.chat.multi_agent_chat.shared.tools.catalog import TOOL_CATALOG return [ AgentToolInfo( @@ -1676,7 +1676,7 @@ async def list_agent_tools( description=t.description, enabled_by_default=t.enabled_by_default, ) - for t in BUILTIN_TOOLS + for t in TOOL_CATALOG if not t.hidden ] @@ -1934,7 +1934,7 @@ async def regenerate_response( """ from langchain_core.messages import HumanMessage - from app.agents.new_chat.checkpointer import get_checkpointer + from app.agents.chat.runtime.checkpointer import get_checkpointer try: # Verify thread exists and user has permission diff --git a/surfsense_backend/app/routes/new_llm_config_routes.py b/surfsense_backend/app/routes/new_llm_config_routes.py index e090a1a7c..84d66bb13 100644 --- a/surfsense_backend/app/routes/new_llm_config_routes.py +++ b/surfsense_backend/app/routes/new_llm_config_routes.py @@ -13,7 +13,6 @@ from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.new_chat.system_prompt import get_default_system_instructions from app.config import config from app.db import ( NewLLMConfig, @@ -21,6 +20,7 @@ from app.db import ( User, get_async_session, ) +from app.prompts.default_system_instructions import get_default_system_instructions from app.schemas import ( DefaultSystemInstructionsResponse, GlobalNewLLMConfigRead, diff --git a/surfsense_backend/app/routes/obsidian_plugin_routes.py b/surfsense_backend/app/routes/obsidian_plugin_routes.py index 0dae7a463..bd54a4788 100644 --- a/surfsense_backend/app/routes/obsidian_plugin_routes.py +++ b/surfsense_backend/app/routes/obsidian_plugin_routes.py @@ -25,6 +25,7 @@ from app.db import ( User, get_async_session, ) +from app.notifications.service import NotificationService from app.schemas.obsidian_plugin import ( ALLOWED_ATTACHMENT_EXTENSIONS, ATTACHMENT_MIME_TYPES, @@ -43,7 +44,6 @@ from app.schemas.obsidian_plugin import ( SyncAckItem, SyncBatchRequest, ) -from app.services.notification_service import NotificationService from app.services.obsidian_plugin_indexer import ( delete_note, get_manifest, diff --git a/surfsense_backend/app/routes/sandbox_routes.py b/surfsense_backend/app/routes/sandbox_routes.py index f656e8d76..fefe51997 100644 --- a/surfsense_backend/app/routes/sandbox_routes.py +++ b/surfsense_backend/app/routes/sandbox_routes.py @@ -51,7 +51,10 @@ async def download_sandbox_file( ): """Download a file from the Daytona sandbox associated with a chat thread.""" - from app.agents.new_chat.sandbox import get_or_create_sandbox, is_sandbox_enabled + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( + get_or_create_sandbox, + is_sandbox_enabled, + ) if not is_sandbox_enabled(): raise HTTPException(status_code=404, detail="Sandbox is not enabled") @@ -71,7 +74,9 @@ async def download_sandbox_file( "You don't have permission to access files in this thread", ) - from app.agents.new_chat.sandbox import get_local_sandbox_file + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( + get_local_sandbox_file, + ) # Prefer locally-persisted copy (sandbox may already be deleted) local_content = get_local_sandbox_file(thread_id, path) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 3060fdf4a..dc26b4c02 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -43,6 +43,7 @@ from app.db import ( async_session_maker, get_async_session, ) +from app.notifications.service import NotificationService from app.observability import metrics as ot_metrics, otel as ot from app.schemas import ( GoogleDriveIndexRequest, @@ -55,7 +56,6 @@ from app.schemas import ( SearchSourceConnectorUpdate, ) from app.services.composio_service import ComposioService, get_composio_service -from app.services.notification_service import NotificationService from app.users import current_active_user # NOTE: connector indexer functions are imported lazily inside each @@ -675,7 +675,9 @@ async def delete_search_source_connector( await session.commit() if is_mcp: - from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + invalidate_mcp_tools_cache, + ) invalidate_mcp_tools_cache(search_space_id) @@ -2687,7 +2689,7 @@ async def create_mcp_connector( f"for user {user.id} in search space {search_space_id}" ) - from app.agents.new_chat.tools.mcp_tools_cache import ( + from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( refresh_mcp_tools_cache_for_connector, ) @@ -2867,7 +2869,7 @@ async def update_mcp_connector( logger.info(f"Updated MCP connector {connector_id}") - from app.agents.new_chat.tools.mcp_tools_cache import ( + from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( refresh_mcp_tools_cache_for_connector, ) @@ -2927,7 +2929,9 @@ async def delete_mcp_connector( await session.delete(connector) await session.commit() - from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + invalidate_mcp_tools_cache, + ) invalidate_mcp_tools_cache(search_space_id) @@ -2966,7 +2970,7 @@ async def test_mcp_server_connection( Connection status and list of available tools """ try: - from app.agents.new_chat.tools.mcp_client import ( + from app.agents.chat.multi_agent_chat.shared.tools.mcp.client import ( test_mcp_connection, test_mcp_http_connection, ) @@ -3157,7 +3161,9 @@ async def trust_mcp_tool( connectors (``LINEAR_CONNECTOR``, ``JIRA_CONNECTOR``, ...) — the storage primitive is the same JSON list under ``config.trusted_tools``. """ - from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + invalidate_mcp_tools_cache, + ) from app.services.user_tool_allowlist import add_user_trust try: @@ -3197,7 +3203,9 @@ async def untrust_mcp_tool( The tool will require HITL approval again on subsequent calls. """ - from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache + from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + invalidate_mcp_tools_cache, + ) from app.services.user_tool_allowlist import remove_user_trust try: diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index db230b0f5..898077b7a 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -617,9 +617,6 @@ async def get_llm_preferences( # Get full config objects for each role agent_llm = await _get_llm_config_by_id(session, search_space.agent_llm_id) - document_summary_llm = await _get_llm_config_by_id( - session, search_space.document_summary_llm_id - ) image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) @@ -629,11 +626,9 @@ async def get_llm_preferences( return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, - document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, - document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, vision_llm_config=vision_llm_config, ) @@ -707,9 +702,6 @@ async def update_llm_preferences( # Get full config objects for response agent_llm = await _get_llm_config_by_id(session, search_space.agent_llm_id) - document_summary_llm = await _get_llm_config_by_id( - session, search_space.document_summary_llm_id - ) image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) @@ -719,11 +711,9 @@ async def update_llm_preferences( return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, - document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, - document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, vision_llm_config=vision_llm_config, ) diff --git a/surfsense_backend/app/routes/youtube_routes.py b/surfsense_backend/app/routes/youtube_routes.py index cb53f86e4..9fc6d1dfc 100644 --- a/surfsense_backend/app/routes/youtube_routes.py +++ b/surfsense_backend/app/routes/youtube_routes.py @@ -3,14 +3,14 @@ import json import logging import re +import time -import aiohttp -from fake_useragent import UserAgent from fastapi import APIRouter, Depends, HTTPException, Query +from scrapling.fetchers import AsyncFetcher from app.db import User from app.users import current_active_user -from app.utils.proxy_config import get_requests_proxies +from app.utils.proxy import get_proxy_url router = APIRouter() logger = logging.getLogger(__name__) @@ -69,26 +69,30 @@ async def _fetch_playlist_via_innertube(playlist_id: str) -> list[str]: "context": {"client": _INNERTUBE_CLIENT}, "browseId": f"VL{playlist_id}", } - proxies = get_requests_proxies() try: - async with ( - aiohttp.ClientSession() as session, - session.post( - _INNERTUBE_API_URL, - json=payload, - headers={"Content-Type": "application/json"}, - proxy=proxies["http"] if proxies else None, - ) as response, - ): - if response.status != 200: - logger.warning( - "Innertube API returned %d for playlist %s", - response.status, - playlist_id, - ) - return [] - data = await response.json() + fetch_start = time.perf_counter() + page = await AsyncFetcher.post( + _INNERTUBE_API_URL, + json=payload, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + fetch_ms = (time.perf_counter() - fetch_start) * 1000 + logger.info( + "[youtube][perf] source=innertube playlist=%s status=%s fetch_ms=%.1f", + playlist_id, + page.status, + fetch_ms, + ) + if page.status != 200: + logger.warning( + "Innertube API returned %d for playlist %s", + page.status, + playlist_id, + ) + return [] + data = page.json() return _extract_playlist_video_ids(data) except Exception as e: @@ -98,35 +102,38 @@ async def _fetch_playlist_via_innertube(playlist_id: str) -> list[str]: async def _fetch_playlist_via_html(playlist_id: str) -> list[str]: """Fallback: scrape playlist page HTML with consent cookies set.""" - ua = UserAgent() - headers = { - "User-Agent": ua.random, - "Accept-Language": "en-US,en;q=0.9", - } + # Scrapling's stealthy_headers supplies a realistic User-Agent automatically. + headers = {"Accept-Language": "en-US,en;q=0.9"} cookies = { "CONSENT": "PENDING+999", "SOCS": "CAISNQgDEitib3FfaWRlbnRpdHlmcm9udGVuZHVpc2VydmVyXzIwMjMwODI5LjA3X3AxGgJlbiADGgYIgOa_pgY", } - proxies = get_requests_proxies() playlist_url = f"https://www.youtube.com/playlist?list={playlist_id}" try: - async with ( - aiohttp.ClientSession(cookies=cookies) as session, - session.get( - playlist_url, - headers=headers, - proxy=proxies["http"] if proxies else None, - ) as response, - ): - if response.status != 200: - logger.warning( - "HTML fallback returned %d for playlist %s", - response.status, - playlist_id, - ) - return [] - html = await response.text() + fetch_start = time.perf_counter() + page = await AsyncFetcher.get( + playlist_url, + headers=headers, + cookies=cookies, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + fetch_ms = (time.perf_counter() - fetch_start) * 1000 + logger.info( + "[youtube][perf] source=html-fallback playlist=%s status=%s fetch_ms=%.1f", + playlist_id, + page.status, + fetch_ms, + ) + if page.status != 200: + logger.warning( + "HTML fallback returned %d for playlist %s", + page.status, + playlist_id, + ) + return [] + html = page.html_content yt_data = _extract_yt_initial_data(html) if not yt_data: diff --git a/surfsense_backend/app/schemas/new_llm_config.py b/surfsense_backend/app/schemas/new_llm_config.py index e64478d38..716aa0457 100644 --- a/surfsense_backend/app/schemas/new_llm_config.py +++ b/surfsense_backend/app/schemas/new_llm_config.py @@ -221,9 +221,6 @@ class LLMPreferencesRead(BaseModel): agent_llm_id: int | None = Field( None, description="ID of the LLM config to use for agent/chat tasks" ) - document_summary_llm_id: int | None = Field( - None, description="ID of the LLM config to use for document summarization" - ) image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) @@ -234,9 +231,6 @@ class LLMPreferencesRead(BaseModel): agent_llm: dict[str, Any] | None = Field( None, description="Full config for agent LLM" ) - document_summary_llm: dict[str, Any] | None = Field( - None, description="Full config for document summary LLM" - ) image_generation_config: dict[str, Any] | None = Field( None, description="Full config for image generation" ) @@ -253,9 +247,6 @@ class LLMPreferencesUpdate(BaseModel): agent_llm_id: int | None = Field( None, description="ID of the LLM config to use for agent/chat tasks" ) - document_summary_llm_id: int | None = Field( - None, description="ID of the LLM config to use for document summarization" - ) image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index aac7b92d5..982931859 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -16,7 +16,6 @@ class SearchSourceConnectorBase(BaseModel): is_indexable: bool last_indexed_at: datetime | None = None config: dict[str, Any] - enable_summary: bool = False enable_vision_llm: bool = False periodic_indexing_enabled: bool = False indexing_frequency_minutes: int | None = None @@ -67,7 +66,6 @@ class SearchSourceConnectorUpdate(BaseModel): is_indexable: bool | None = None last_indexed_at: datetime | None = None config: dict[str, Any] | None = None - enable_summary: bool | None = None enable_vision_llm: bool | None = None periodic_indexing_enabled: bool | None = None indexing_frequency_minutes: int | None = None diff --git a/surfsense_backend/app/services/chat_comments_service.py b/surfsense_backend/app/services/chat_comments_service.py index 54662fe5b..905482010 100644 --- a/surfsense_backend/app/services/chat_comments_service.py +++ b/surfsense_backend/app/services/chat_comments_service.py @@ -20,6 +20,7 @@ from app.db import ( User, has_permission, ) +from app.notifications.service import NotificationService from app.schemas.chat_comments import ( AuthorResponse, CommentBatchResponse, @@ -31,7 +32,6 @@ from app.schemas.chat_comments import ( MentionListResponse, MentionResponse, ) -from app.services.notification_service import NotificationService from app.utils.chat_comments import parse_mentions, render_mentions from app.utils.rbac import check_permission, get_user_permissions diff --git a/surfsense_backend/app/services/confluence/kb_sync_service.py b/surfsense_backend/app/services/confluence/kb_sync_service.py index cae2bef88..7154637b4 100644 --- a/surfsense_backend/app/services/confluence/kb_sync_service.py +++ b/surfsense_backend/app/services/confluence/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -65,29 +64,8 @@ class ConfluenceKBSyncService: if dup: content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "page_title": page_title, - "space_id": space_id, - "document_type": "Confluence Page", - "connector_type": "Confluence", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - page_content, user_llm, doc_metadata_for_summary - ) - else: - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Confluence Page: {page_title}\n\n{page_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(page_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -185,25 +163,8 @@ class ConfluenceKBSyncService: space_id = (document.document_metadata or {}).get("space_id", "") - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True - ) - - if user_llm: - doc_meta = { - "page_title": page_title, - "space_id": space_id, - "document_type": "Confluence Page", - "connector_type": "Confluence", - } - summary_content, summary_embedding = await generate_document_summary( - page_content, user_llm, doc_meta - ) - else: - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Confluence Page: {page_title}\n\n{page_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(page_content) diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 45bcfd00f..2694a8e69 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -1,5 +1,4 @@ import asyncio -import os import time from datetime import datetime from threading import Lock @@ -12,6 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from tavily import TavilyClient +from app.config import config from app.db import ( NATIVE_TO_LEGACY_DOCTYPE, Chunk, @@ -2856,9 +2856,7 @@ class ConnectorService: # bounded and the alternative (cross-replica fanout) is not worth the # coupling here. -_DISCOVERY_TTL_SECONDS: float = float( - os.getenv("SURFSENSE_CONNECTOR_DISCOVERY_TTL_SECONDS", "30") -) +_DISCOVERY_TTL_SECONDS: float = config.CONNECTOR_DISCOVERY_TTL_SECONDS # Per-search-space caches. Keyed by ``search_space_id``; value is # ``(expires_at_monotonic, payload)``. Plain dicts protected by a lock — diff --git a/surfsense_backend/app/services/docling_service.py b/surfsense_backend/app/services/docling_service.py index cf51efb4a..dc87e75f0 100644 --- a/surfsense_backend/app/services/docling_service.py +++ b/surfsense_backend/app/services/docling_service.py @@ -191,149 +191,6 @@ class DoclingService: logger.error(f"Full traceback: {traceback.format_exc()}") raise RuntimeError(f"Docling processing failed: {e}") from e - async def process_large_document_summary( - self, content: str, llm, document_title: str = "Document" - ) -> str: - """ - Process large documents using chunked LLM summarization. - - Args: - content: The full document content - llm: The language model to use for summarization - document_title: Title of the document for context - - Returns: - Final summary of the document - """ - # Large document threshold (100K characters ≈ 25K tokens) - large_document_threshold = 100_000 - - if len(content) <= large_document_threshold: - # For smaller documents, use direct processing - logger.info( - f"📄 Document size: {len(content)} chars - using direct processing" - ) - from app.prompts import SUMMARY_PROMPT_TEMPLATE - - summary_chain = SUMMARY_PROMPT_TEMPLATE | llm - result = await summary_chain.ainvoke({"document": content}) - return result.content - - logger.info( - f"📚 Large document detected: {len(content)} chars - using chunked processing" - ) - - # Import chunker from config - # Create LLM-optimized chunks (8K tokens max for safety) - from chonkie import OverlapRefinery, RecursiveChunker - from langchain_core.prompts import PromptTemplate - - llm_chunker = RecursiveChunker( - chunk_size=8000 # Conservative for most LLMs - ) - - # Apply overlap refinery for context preservation (10% overlap = 800 tokens) - overlap_refinery = OverlapRefinery( - context_size=0.1, # 10% overlap for context preservation - method="suffix", # Add next chunk context to current chunk - ) - - # First chunk the content, then apply overlap refinery - initial_chunks = llm_chunker.chunk(content) - chunks = overlap_refinery.refine(initial_chunks) - total_chunks = len(chunks) - - logger.info(f"📄 Split into {total_chunks} chunks for LLM processing") - - # Template for chunk processing - chunk_template = PromptTemplate( - input_variables=["chunk", "chunk_number", "total_chunks"], - template=""" -You are summarizing chunk {chunk_number} of {total_chunks} from a large document. - -Create a comprehensive summary of this document chunk. Focus on: -- Key concepts, facts, and information -- Important details and context -- Main topics and themes - -Provide a clear, structured summary that captures the essential content. - -Chunk {chunk_number}/{total_chunks}: - -{chunk} - -""", - ) - - # Process each chunk individually - chunk_summaries = [] - for i, chunk in enumerate(chunks, 1): - try: - logger.info( - f"🔄 Processing chunk {i}/{total_chunks} ({len(chunk.text)} chars)" - ) - - chunk_chain = chunk_template | llm - chunk_result = await chunk_chain.ainvoke( - { - "chunk": chunk.text, - "chunk_number": i, - "total_chunks": total_chunks, - } - ) - - chunk_summary = chunk_result.content - chunk_summaries.append(f"=== Section {i} ===\n{chunk_summary}") - - logger.info(f"✅ Completed chunk {i}/{total_chunks}") - - except Exception as e: - logger.error(f"❌ Failed to process chunk {i}/{total_chunks}: {e}") - chunk_summaries.append(f"=== Section {i} ===\n[Processing failed]") - - # Combine summaries into final document summary - logger.info(f"🔄 Combining {len(chunk_summaries)} chunk summaries") - - try: - combine_template = PromptTemplate( - input_variables=["summaries", "document_title"], - template=""" -You are combining multiple section summaries into a final comprehensive document summary. - -Create a unified, coherent summary from the following section summaries of "{document_title}". -Ensure: -- Logical flow and organization -- No redundancy or repetition -- Comprehensive coverage of all key points -- Professional, objective tone - - -{summaries} - -""", - ) - - combined_summaries = "\n\n".join(chunk_summaries) - combine_chain = combine_template | llm - - final_result = await combine_chain.ainvoke( - {"summaries": combined_summaries, "document_title": document_title} - ) - - final_summary = final_result.content - logger.info( - f"✅ Large document processing complete: {len(final_summary)} chars summary" - ) - - return final_summary - - except Exception as e: - logger.error(f"❌ Failed to combine summaries: {e}") - # Fallback: return concatenated chunk summaries - fallback_summary = "\n\n".join(chunk_summaries) - logger.warning("⚠️ Using fallback combined summary") - return fallback_summary - def create_docling_service() -> DoclingService: """Create a Docling service instance.""" diff --git a/surfsense_backend/app/services/dropbox/kb_sync_service.py b/surfsense_backend/app/services/dropbox/kb_sync_service.py index 9d1951013..a25cc054d 100644 --- a/surfsense_backend/app/services/dropbox/kb_sync_service.py +++ b/surfsense_backend/app/services/dropbox/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) logger = logging.getLogger(__name__) @@ -72,29 +71,8 @@ class DropboxKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "file_name": file_name, - "document_type": "Dropbox File", - "connector_type": "Dropbox", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/gmail/kb_sync_service.py b/surfsense_backend/app/services/gmail/kb_sync_service.py index 85e25fcb6..192570339 100644 --- a/surfsense_backend/app/services/gmail/kb_sync_service.py +++ b/surfsense_backend/app/services/gmail/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -78,30 +77,8 @@ class GmailKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "subject": subject, - "sender": sender, - "document_type": "Gmail Message", - "connector_type": "Gmail", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured -- using fallback summary") - summary_content = f"Gmail Message: {subject}\n\n{indexable_content}" - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_content = f"Gmail Message: {subject}\n\n{indexable_content}" + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/google_calendar/kb_sync_service.py b/surfsense_backend/app/services/google_calendar/kb_sync_service.py index e59868aff..495720a2d 100644 --- a/surfsense_backend/app/services/google_calendar/kb_sync_service.py +++ b/surfsense_backend/app/services/google_calendar/kb_sync_service.py @@ -19,7 +19,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -90,33 +89,10 @@ class GoogleCalendarKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, + summary_content = ( + f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) - - doc_metadata_for_summary = { - "event_summary": event_summary, - "start_time": start_time, - "end_time": end_time, - "document_type": "Google Calendar Event", - "connector_type": "Google Calendar", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured -- using fallback summary") - summary_content = ( - f"Google Calendar Event: {event_summary}\n\n{indexable_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -273,29 +249,10 @@ class GoogleCalendarKBSyncService: if not indexable_content: return {"status": "error", "message": "Event produced empty content"} - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True + summary_content = ( + f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) - - doc_metadata_for_summary = { - "event_summary": event_summary, - "start_time": start_time, - "end_time": end_time, - "document_type": "Google Calendar Event", - "connector_type": "Google Calendar", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - summary_content = ( - f"Google Calendar Event: {event_summary}\n\n{indexable_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/google_drive/kb_sync_service.py b/surfsense_backend/app/services/google_drive/kb_sync_service.py index 0a8eb47a6..30fbc14f2 100644 --- a/surfsense_backend/app/services/google_drive/kb_sync_service.py +++ b/surfsense_backend/app/services/google_drive/kb_sync_service.py @@ -8,7 +8,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -74,32 +73,8 @@ class GoogleDriveKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "file_name": file_name, - "mime_type": mime_type, - "document_type": "Google Drive File", - "connector_type": "Google Drive", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = ( - f"Google Drive File: {file_name}\n\n{indexable_content}" - ) - summary_embedding = embed_text(summary_content) + summary_content = f"Google Drive File: {file_name}\n\n{indexable_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/jira/__init__.py b/surfsense_backend/app/services/jira/__init__.py deleted file mode 100644 index fad49b68d..000000000 --- a/surfsense_backend/app/services/jira/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from app.services.jira.kb_sync_service import JiraKBSyncService -from app.services.jira.tool_metadata_service import ( - JiraIssue, - JiraToolMetadataService, - JiraWorkspace, -) - -__all__ = [ - "JiraIssue", - "JiraKBSyncService", - "JiraToolMetadataService", - "JiraWorkspace", -] diff --git a/surfsense_backend/app/services/jira/kb_sync_service.py b/surfsense_backend/app/services/jira/kb_sync_service.py deleted file mode 100644 index 37001a476..000000000 --- a/surfsense_backend/app/services/jira/kb_sync_service.py +++ /dev/null @@ -1,257 +0,0 @@ -import asyncio -import logging -from datetime import datetime - -from sqlalchemy.ext.asyncio import AsyncSession - -from app.connectors.jira_history import JiraHistoryConnector -from app.db import Document, DocumentType -from app.utils.document_converters import ( - create_document_chunks, - embed_text, - generate_content_hash, - generate_document_summary, - generate_unique_identifier_hash, -) - -logger = logging.getLogger(__name__) - - -class JiraKBSyncService: - """Syncs Jira issue documents to the knowledge base after HITL actions.""" - - def __init__(self, db_session: AsyncSession): - self.db_session = db_session - - async def sync_after_create( - self, - issue_id: str, - issue_identifier: str, - issue_title: str, - description: str | None, - state: str | None, - connector_id: int, - search_space_id: int, - user_id: str, - ) -> dict: - from app.tasks.connector_indexers.base import ( - check_document_by_unique_identifier, - check_duplicate_document_by_hash, - get_current_timestamp, - safe_set_chunks, - ) - - try: - unique_hash = generate_unique_identifier_hash( - DocumentType.JIRA_CONNECTOR, issue_id, search_space_id - ) - - existing = await check_document_by_unique_identifier( - self.db_session, unique_hash - ) - if existing: - logger.info( - "Document for Jira issue %s already exists (doc_id=%s), skipping", - issue_identifier, - existing.id, - ) - return {"status": "success"} - - indexable_content = (description or "").strip() - if not indexable_content: - indexable_content = f"Jira Issue {issue_identifier}: {issue_title}" - - issue_content = ( - f"# {issue_identifier}: {issue_title}\n\n{indexable_content}" - ) - - content_hash = generate_content_hash(issue_content, search_space_id) - - with self.db_session.no_autoflush: - dup = await check_duplicate_document_by_hash( - self.db_session, content_hash - ) - if dup: - content_hash = unique_hash - - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "issue_id": issue_identifier, - "issue_title": issue_title, - "document_type": "Jira Issue", - "connector_type": "Jira", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, doc_metadata_for_summary - ) - else: - summary_content = ( - f"Jira Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) - - chunks = await create_document_chunks(issue_content) - now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - document = Document( - title=f"{issue_identifier}: {issue_title}", - document_type=DocumentType.JIRA_CONNECTOR, - document_metadata={ - "issue_id": issue_id, - "issue_identifier": issue_identifier, - "issue_title": issue_title, - "state": state or "Unknown", - "indexed_at": now_str, - "connector_id": connector_id, - }, - content=summary_content, - content_hash=content_hash, - unique_identifier_hash=unique_hash, - embedding=summary_embedding, - search_space_id=search_space_id, - connector_id=connector_id, - updated_at=get_current_timestamp(), - created_by_id=user_id, - ) - - self.db_session.add(document) - await self.db_session.flush() - await safe_set_chunks(self.db_session, document, chunks) - await self.db_session.commit() - - logger.info( - "KB sync after create succeeded: doc_id=%s, issue=%s", - document.id, - issue_identifier, - ) - return {"status": "success"} - - except Exception as e: - error_str = str(e).lower() - if ( - "duplicate key value violates unique constraint" in error_str - or "uniqueviolationerror" in error_str - ): - await self.db_session.rollback() - return {"status": "error", "message": "Duplicate document detected"} - - logger.error( - "KB sync after create failed for issue %s: %s", - issue_identifier, - e, - exc_info=True, - ) - await self.db_session.rollback() - return {"status": "error", "message": str(e)} - - async def sync_after_update( - self, - document_id: int, - issue_id: str, - user_id: str, - search_space_id: int, - ) -> dict: - from app.tasks.connector_indexers.base import ( - get_current_timestamp, - safe_set_chunks, - ) - - try: - document = await self.db_session.get(Document, document_id) - if not document: - return {"status": "not_indexed"} - - connector_id = document.connector_id - if not connector_id: - return {"status": "error", "message": "Document has no connector_id"} - - jira_history = JiraHistoryConnector( - session=self.db_session, connector_id=connector_id - ) - jira_client = await jira_history._get_jira_client() - issue_raw = await asyncio.to_thread(jira_client.get_issue, issue_id) - formatted = jira_client.format_issue(issue_raw) - issue_content = jira_client.format_issue_to_markdown(formatted) - - if not issue_content: - return {"status": "error", "message": "Issue produced empty content"} - - issue_identifier = formatted.get("key", "") - issue_title = formatted.get("title", "") - state = formatted.get("status", "Unknown") - comment_count = len(formatted.get("comments", [])) - - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True - ) - - if user_llm: - doc_meta = { - "issue_key": issue_identifier, - "issue_title": issue_title, - "status": state, - "document_type": "Jira Issue", - "connector_type": "Jira", - } - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, doc_meta - ) - else: - summary_content = ( - f"Jira Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) - - chunks = await create_document_chunks(issue_content) - - document.title = f"{issue_identifier}: {issue_title}" - document.content = summary_content - document.content_hash = generate_content_hash( - issue_content, search_space_id - ) - document.embedding = summary_embedding - - from sqlalchemy.orm.attributes import flag_modified - - document.document_metadata = { - **(document.document_metadata or {}), - "issue_id": issue_id, - "issue_identifier": issue_identifier, - "issue_title": issue_title, - "state": state, - "comment_count": comment_count, - "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - "connector_id": connector_id, - } - flag_modified(document, "document_metadata") - await safe_set_chunks(self.db_session, document, chunks) - document.updated_at = get_current_timestamp() - - await self.db_session.commit() - - logger.info( - "KB sync successful for document %s (%s: %s)", - document_id, - issue_identifier, - issue_title, - ) - return {"status": "success"} - - except Exception as e: - logger.error( - "KB sync failed for document %s: %s", document_id, e, exc_info=True - ) - await self.db_session.rollback() - return {"status": "error", "message": str(e)} diff --git a/surfsense_backend/app/services/jira/tool_metadata_service.py b/surfsense_backend/app/services/jira/tool_metadata_service.py deleted file mode 100644 index cbc89e7be..000000000 --- a/surfsense_backend/app/services/jira/tool_metadata_service.py +++ /dev/null @@ -1,332 +0,0 @@ -import asyncio -import logging -from dataclasses import dataclass - -from sqlalchemy import and_, func, or_ -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.future import select -from sqlalchemy.orm.attributes import flag_modified - -from app.connectors.jira_history import JiraHistoryConnector -from app.db import ( - Document, - DocumentType, - SearchSourceConnector, - SearchSourceConnectorType, -) - -logger = logging.getLogger(__name__) - - -@dataclass -class JiraWorkspace: - """Represents a Jira connector as a workspace for tool context.""" - - id: int - name: str - base_url: str - - @classmethod - def from_connector(cls, connector: SearchSourceConnector) -> "JiraWorkspace": - return cls( - id=connector.id, - name=connector.name, - base_url=connector.config.get("base_url", ""), - ) - - def to_dict(self) -> dict: - return { - "id": self.id, - "name": self.name, - "base_url": self.base_url, - } - - -@dataclass -class JiraIssue: - """Represents an indexed Jira issue resolved from the knowledge base.""" - - issue_id: str - issue_identifier: str - issue_title: str - state: str - connector_id: int - document_id: int - indexed_at: str | None - - @classmethod - def from_document(cls, document: Document) -> "JiraIssue": - meta = document.document_metadata or {} - return cls( - issue_id=meta.get("issue_id", ""), - issue_identifier=meta.get("issue_identifier", ""), - issue_title=meta.get("issue_title", document.title), - state=meta.get("state", ""), - connector_id=document.connector_id, - document_id=document.id, - indexed_at=meta.get("indexed_at"), - ) - - def to_dict(self) -> dict: - return { - "issue_id": self.issue_id, - "issue_identifier": self.issue_identifier, - "issue_title": self.issue_title, - "state": self.state, - "connector_id": self.connector_id, - "document_id": self.document_id, - "indexed_at": self.indexed_at, - } - - -class JiraToolMetadataService: - """Builds interrupt context for Jira HITL tools.""" - - def __init__(self, db_session: AsyncSession): - self._db_session = db_session - - async def _check_account_health(self, connector: SearchSourceConnector) -> bool: - """Check if the Jira connector auth is still valid. - - Returns True if auth is expired/invalid, False if healthy. - """ - try: - jira_history = JiraHistoryConnector( - session=self._db_session, connector_id=connector.id - ) - jira_client = await jira_history._get_jira_client() - await asyncio.to_thread(jira_client.get_myself) - return False - except Exception as e: - logger.warning("Jira connector %s health check failed: %s", connector.id, e) - try: - connector.config = {**connector.config, "auth_expired": True} - flag_modified(connector, "config") - await self._db_session.commit() - await self._db_session.refresh(connector) - except Exception: - logger.warning( - "Failed to persist auth_expired for connector %s", - connector.id, - exc_info=True, - ) - return True - - async def get_creation_context(self, search_space_id: int, user_id: str) -> dict: - """Return context needed to create a new Jira issue. - - Fetches all connected Jira accounts, and for the first healthy one - fetches projects, issue types, and priorities. - """ - connectors = await self._get_all_jira_connectors(search_space_id, user_id) - if not connectors: - return {"error": "No Jira account connected"} - - accounts = [] - projects = [] - issue_types = [] - priorities = [] - fetched_context = False - - for connector in connectors: - auth_expired = await self._check_account_health(connector) - workspace = JiraWorkspace.from_connector(connector) - account_info = { - **workspace.to_dict(), - "auth_expired": auth_expired, - } - accounts.append(account_info) - - if not auth_expired and not fetched_context: - try: - jira_history = JiraHistoryConnector( - session=self._db_session, connector_id=connector.id - ) - jira_client = await jira_history._get_jira_client() - raw_projects = await asyncio.to_thread(jira_client.get_projects) - projects = [ - {"id": p.get("id"), "key": p.get("key"), "name": p.get("name")} - for p in raw_projects - ] - raw_types = await asyncio.to_thread(jira_client.get_issue_types) - seen_type_names: set[str] = set() - issue_types = [] - for t in raw_types: - if t.get("subtask", False): - continue - name = t.get("name") - if name not in seen_type_names: - seen_type_names.add(name) - issue_types.append({"id": t.get("id"), "name": name}) - raw_priorities = await asyncio.to_thread(jira_client.get_priorities) - priorities = [ - {"id": p.get("id"), "name": p.get("name")} - for p in raw_priorities - ] - fetched_context = True - except Exception as e: - logger.warning( - "Failed to fetch Jira context for connector %s: %s", - connector.id, - e, - ) - - return { - "accounts": accounts, - "projects": projects, - "issue_types": issue_types, - "priorities": priorities, - } - - async def get_update_context( - self, search_space_id: int, user_id: str, issue_ref: str - ) -> dict: - """Return context needed to update an indexed Jira issue. - - Resolves the issue from the KB, then fetches current details from the Jira API. - """ - document = await self._resolve_issue(search_space_id, user_id, issue_ref) - if not document: - return { - "error": f"Issue '{issue_ref}' not found in your synced Jira issues. " - "Please make sure the issue is indexed in your knowledge base." - } - - connector = await self._get_connector_for_document(document, user_id) - if not connector: - return {"error": "Connector not found or access denied"} - - auth_expired = await self._check_account_health(connector) - if auth_expired: - return { - "error": "Jira authentication has expired. Please re-authenticate.", - "auth_expired": True, - "connector_id": connector.id, - } - - workspace = JiraWorkspace.from_connector(connector) - issue = JiraIssue.from_document(document) - - try: - jira_history = JiraHistoryConnector( - session=self._db_session, connector_id=connector.id - ) - jira_client = await jira_history._get_jira_client() - issue_data = await asyncio.to_thread(jira_client.get_issue, issue.issue_id) - formatted = jira_client.format_issue(issue_data) - except Exception as e: - error_str = str(e).lower() - if ( - "401" in error_str - or "403" in error_str - or "authentication" in error_str - ): - return { - "error": f"Failed to fetch Jira issue: {e!s}", - "auth_expired": True, - "connector_id": connector.id, - } - return {"error": f"Failed to fetch Jira issue: {e!s}"} - - return { - "account": {**workspace.to_dict(), "auth_expired": False}, - "issue": { - "issue_id": formatted.get("key", issue.issue_id), - "issue_identifier": formatted.get("key", issue.issue_identifier), - "issue_title": formatted.get("title", issue.issue_title), - "state": formatted.get("status", "Unknown"), - "priority": formatted.get("priority", "Unknown"), - "issue_type": formatted.get("issue_type", "Unknown"), - "assignee": formatted.get("assignee"), - "description": formatted.get("description"), - "project": formatted.get("project", ""), - "document_id": issue.document_id, - "indexed_at": issue.indexed_at, - }, - } - - async def get_deletion_context( - self, search_space_id: int, user_id: str, issue_ref: str - ) -> dict: - """Return context needed to delete a Jira issue (KB metadata only, no API call).""" - document = await self._resolve_issue(search_space_id, user_id, issue_ref) - if not document: - return { - "error": f"Issue '{issue_ref}' not found in your synced Jira issues. " - "Please make sure the issue is indexed in your knowledge base." - } - - connector = await self._get_connector_for_document(document, user_id) - if not connector: - return {"error": "Connector not found or access denied"} - - auth_expired = connector.config.get("auth_expired", False) - workspace = JiraWorkspace.from_connector(connector) - issue = JiraIssue.from_document(document) - - return { - "account": {**workspace.to_dict(), "auth_expired": auth_expired}, - "issue": issue.to_dict(), - } - - async def _resolve_issue( - self, search_space_id: int, user_id: str, issue_ref: str - ) -> Document | None: - """Resolve an issue from KB: issue_identifier -> issue_title -> document.title.""" - ref_lower = issue_ref.lower() - - result = await self._db_session.execute( - select(Document) - .join( - SearchSourceConnector, Document.connector_id == SearchSourceConnector.id - ) - .filter( - and_( - Document.search_space_id == search_space_id, - Document.document_type == DocumentType.JIRA_CONNECTOR, - SearchSourceConnector.user_id == user_id, - or_( - func.lower( - Document.document_metadata.op("->>")("issue_identifier") - ) - == ref_lower, - func.lower(Document.document_metadata.op("->>")("issue_title")) - == ref_lower, - func.lower(Document.title) == ref_lower, - ), - ) - ) - .order_by(Document.updated_at.desc().nullslast()) - .limit(1) - ) - return result.scalars().first() - - async def _get_all_jira_connectors( - self, search_space_id: int, user_id: str - ) -> list[SearchSourceConnector]: - result = await self._db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.search_space_id == search_space_id, - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.JIRA_CONNECTOR, - ) - ) - ) - return result.scalars().all() - - async def _get_connector_for_document( - self, document: Document, user_id: str - ) -> SearchSourceConnector | None: - if not document.connector_id: - return None - result = await self._db_session.execute( - select(SearchSourceConnector).filter( - and_( - SearchSourceConnector.id == document.connector_id, - SearchSourceConnector.user_id == user_id, - ) - ) - ) - return result.scalars().first() diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 471227602..3b8def6c3 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -84,32 +83,10 @@ class LinearKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, + summary_content = ( + f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) - - doc_metadata_for_summary = { - "issue_id": issue_identifier, - "issue_title": issue_title, - "document_type": "Linear Issue", - "connector_type": "Linear", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = embed_text(summary_content) + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(issue_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -227,30 +204,10 @@ class LinearKBSyncService: comment_count = len(formatted_issue.get("comments", [])) formatted_issue.get("description", "") - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True + summary_content = ( + f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) - - if user_llm: - document_metadata_for_summary = { - "issue_id": issue_identifier, - "issue_title": issue_title, - "state": state, - "priority": priority, - "comment_count": comment_count, - "document_type": "Linear Issue", - "connector_type": "Linear", - } - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, document_metadata_for_summary - ) - else: - summary_content = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = embed_text(summary_content) + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(issue_content) diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index aadb60cde..7061a826f 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -68,7 +68,6 @@ def _is_interactive_auth_provider( class LLMRole: AGENT = "agent" # For agent/chat operations - DOCUMENT_SUMMARY = "document_summary" # For document summarization def get_global_llm_config(llm_config_id: int) -> dict | None: @@ -204,7 +203,9 @@ async def validate_llm_config( if litellm_params: litellm_kwargs.update(litellm_params) - from app.agents.new_chat.llm_config import SanitizedChatLiteLLM + from app.agents.chat.runtime.llm_config import ( + SanitizedChatLiteLLM, + ) llm = SanitizedChatLiteLLM(**litellm_kwargs) @@ -266,7 +267,7 @@ async def get_search_space_llm_instance( Args: session: Database session search_space_id: Search Space ID - role: LLM role ('agent' or 'document_summary') + role: LLM role ('agent') Returns: ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found @@ -283,11 +284,8 @@ async def get_search_space_llm_instance( return None # Get the appropriate LLM config ID based on role - llm_config_id = None if role == LLMRole.AGENT: llm_config_id = search_space.agent_llm_id - elif role == LLMRole.DOCUMENT_SUMMARY: - llm_config_id = search_space.document_summary_llm_id else: logger.error(f"Invalid LLM role: {role}") return None @@ -379,7 +377,9 @@ async def get_search_space_llm_instance( if disable_streaming: litellm_kwargs["disable_streaming"] = True - from app.agents.new_chat.llm_config import SanitizedChatLiteLLM + from app.agents.chat.runtime.llm_config import ( + SanitizedChatLiteLLM, + ) return SanitizedChatLiteLLM(**litellm_kwargs) @@ -458,7 +458,9 @@ async def get_search_space_llm_instance( if disable_streaming: litellm_kwargs["disable_streaming"] = True - from app.agents.new_chat.llm_config import SanitizedChatLiteLLM + from app.agents.chat.runtime.llm_config import ( + SanitizedChatLiteLLM, + ) return SanitizedChatLiteLLM(**litellm_kwargs) @@ -470,20 +472,13 @@ async def get_search_space_llm_instance( async def get_agent_llm( - session: AsyncSession, search_space_id: int -) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's agent LLM instance for chat operations.""" - return await get_search_space_llm_instance(session, search_space_id, LLMRole.AGENT) - - -async def get_document_summary_llm( session: AsyncSession, search_space_id: int, disable_streaming: bool = False ) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's document summary LLM instance.""" + """Get the search space's agent LLM instance for chat operations.""" return await get_search_space_llm_instance( session, search_space_id, - LLMRole.DOCUMENT_SUMMARY, + LLMRole.AGENT, disable_streaming=disable_streaming, ) @@ -580,7 +575,9 @@ async def get_vision_llm( if global_cfg.get("litellm_params"): litellm_kwargs.update(global_cfg["litellm_params"]) - from app.agents.new_chat.llm_config import SanitizedChatLiteLLM + from app.agents.chat.runtime.llm_config import ( + SanitizedChatLiteLLM, + ) inner_llm = SanitizedChatLiteLLM(**litellm_kwargs) @@ -634,7 +631,9 @@ async def get_vision_llm( if vision_cfg.litellm_params: litellm_kwargs.update(vision_cfg.litellm_params) - from app.agents.new_chat.llm_config import SanitizedChatLiteLLM + from app.agents.chat.runtime.llm_config import ( + SanitizedChatLiteLLM, + ) return SanitizedChatLiteLLM(**litellm_kwargs) @@ -645,22 +644,6 @@ async def get_vision_llm( return None -# Backward-compatible alias (LLM preferences are now per-search-space, not per-user) -async def get_user_long_context_llm( - session: AsyncSession, - user_id: str, - search_space_id: int, - disable_streaming: bool = False, -) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """ - Deprecated: Use get_document_summary_llm instead. - The user_id parameter is ignored as LLM preferences are now per-search-space. - """ - return await get_document_summary_llm( - session, search_space_id, disable_streaming=disable_streaming - ) - - def get_planner_llm() -> ChatLiteLLM | None: """Return a planner LLM instance from the first global config marked ``is_planner: true``, or ``None`` if no planner config is defined. @@ -679,7 +662,9 @@ def get_planner_llm() -> ChatLiteLLM | None: Callers MUST fall back to their chat LLM when this returns ``None`` so deployments without a planner config keep working unchanged. """ - from app.agents.new_chat.llm_config import create_chat_litellm_from_config + from app.agents.chat.runtime.llm_config import ( + create_chat_litellm_from_config, + ) planner_cfg = next( (cfg for cfg in config.GLOBAL_LLM_CONFIGS if cfg.get("is_planner") is True), diff --git a/surfsense_backend/app/services/notification_service.py b/surfsense_backend/app/services/notification_service.py deleted file mode 100644 index 5ffee12d7..000000000 --- a/surfsense_backend/app/services/notification_service.py +++ /dev/null @@ -1,1089 +0,0 @@ -"""Service for creating and managing notifications with Zero sync.""" - -import logging -from datetime import UTC, datetime -from typing import Any -from uuid import UUID - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm.attributes import flag_modified - -from app.db import Notification - -logger = logging.getLogger(__name__) - - -class BaseNotificationHandler: - """Base class for notification handlers - provides common functionality.""" - - def __init__(self, notification_type: str): - """ - Initialize the notification handler. - - Args: - notification_type: Type of notification (e.g., 'connector_indexing', 'document_processing') - """ - self.notification_type = notification_type - - async def find_notification_by_operation( - self, - session: AsyncSession, - user_id: UUID, - operation_id: str, - search_space_id: int | None = None, - ) -> Notification | None: - """ - Find an existing notification by operation ID. - - Args: - session: Database session - user_id: User ID - operation_id: Unique operation identifier - search_space_id: Optional search space ID - - Returns: - Notification if found, None otherwise - """ - query = select(Notification).where( - Notification.user_id == user_id, - Notification.type == self.notification_type, - Notification.notification_metadata["operation_id"].astext == operation_id, - ) - if search_space_id is not None: - query = query.where(Notification.search_space_id == search_space_id) - - result = await session.execute(query) - return result.scalar_one_or_none() - - async def find_or_create_notification( - self, - session: AsyncSession, - user_id: UUID, - operation_id: str, - title: str, - message: str, - search_space_id: int | None = None, - initial_metadata: dict[str, Any] | None = None, - ) -> Notification: - """ - Find an existing notification or create a new one. - - Args: - session: Database session - user_id: User ID - operation_id: Unique operation identifier - title: Notification title - message: Notification message - search_space_id: Optional search space ID - initial_metadata: Initial metadata dictionary - - Returns: - Notification: The found or created notification - """ - # Try to find existing notification - notification = await self.find_notification_by_operation( - session, user_id, operation_id, search_space_id - ) - - if notification: - # Update existing notification - notification.title = title - notification.message = message - if initial_metadata: - notification.notification_metadata = { - **notification.notification_metadata, - **initial_metadata, - } - # Mark JSONB column as modified so SQLAlchemy detects the change - flag_modified(notification, "notification_metadata") - await session.commit() - await session.refresh(notification) - logger.info( - f"Updated notification {notification.id} for operation {operation_id}" - ) - return notification - - # Create new notification - metadata = initial_metadata or {} - metadata["operation_id"] = operation_id - metadata["status"] = "in_progress" - metadata["started_at"] = datetime.now(UTC).isoformat() - - notification = Notification( - user_id=user_id, - search_space_id=search_space_id, - type=self.notification_type, - title=title, - message=message, - notification_metadata=metadata, - ) - session.add(notification) - await session.commit() - await session.refresh(notification) - logger.info( - f"Created notification {notification.id} for operation {operation_id}" - ) - return notification - - async def update_notification( - self, - session: AsyncSession, - notification: Notification, - title: str | None = None, - message: str | None = None, - status: str | None = None, - metadata_updates: dict[str, Any] | None = None, - ) -> Notification: - """ - Update an existing notification. - - Args: - session: Database session - notification: Notification to update - title: New title (optional) - message: New message (optional) - status: New status (optional) - metadata_updates: Additional metadata to merge (optional) - - Returns: - Updated notification - """ - if title is not None: - notification.title = title - if message is not None: - notification.message = message - - if status is not None: - notification.notification_metadata["status"] = status - if status in ("completed", "failed"): - notification.notification_metadata["completed_at"] = datetime.now( - UTC - ).isoformat() - # Mark JSONB column as modified so SQLAlchemy detects the change - flag_modified(notification, "notification_metadata") - - if metadata_updates: - notification.notification_metadata = { - **notification.notification_metadata, - **metadata_updates, - } - # Mark JSONB column as modified - flag_modified(notification, "notification_metadata") - - await session.commit() - await session.refresh(notification) - logger.info(f"Updated notification {notification.id}") - return notification - - -class ConnectorIndexingNotificationHandler(BaseNotificationHandler): - """Handler for connector indexing notifications.""" - - def __init__(self): - super().__init__("connector_indexing") - - def _generate_operation_id( - self, - connector_id: int, - start_date: str | None = None, - end_date: str | None = None, - ) -> str: - """ - Generate a unique operation ID for a connector indexing operation. - - Args: - connector_id: Connector ID - start_date: Start date (optional) - end_date: End date (optional) - - Returns: - Unique operation ID string - """ - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") - date_range = "" - if start_date or end_date: - date_range = f"_{start_date or 'none'}_{end_date or 'none'}" - return f"connector_{connector_id}_{timestamp}{date_range}" - - def _generate_google_drive_operation_id( - self, connector_id: int, folder_count: int, file_count: int - ) -> str: - """ - Generate a unique operation ID for a Google Drive indexing operation. - - Args: - connector_id: Connector ID - folder_count: Number of folders to index - file_count: Number of files to index - - Returns: - Unique operation ID string - """ - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") - items_info = f"_{folder_count}f_{file_count}files" - return f"drive_{connector_id}_{timestamp}{items_info}" - - async def notify_indexing_started( - self, - session: AsyncSession, - user_id: UUID, - connector_id: int, - connector_name: str, - connector_type: str, - search_space_id: int, - start_date: str | None = None, - end_date: str | None = None, - ) -> Notification: - """ - Create or update notification when connector indexing starts. - - Args: - session: Database session - user_id: User ID - connector_id: Connector ID - connector_name: Connector name - connector_type: Connector type - search_space_id: Search space ID - start_date: Start date for indexing - end_date: End date for indexing - - Returns: - Notification: The created or updated notification - """ - operation_id = self._generate_operation_id(connector_id, start_date, end_date) - title = f"Syncing: {connector_name}" - message = "Connecting to your account" - - metadata = { - "connector_id": connector_id, - "connector_name": connector_name, - "connector_type": connector_type, - "start_date": start_date, - "end_date": end_date, - "indexed_count": 0, - "sync_stage": "connecting", - } - - return await self.find_or_create_notification( - session=session, - user_id=user_id, - operation_id=operation_id, - title=title, - message=message, - search_space_id=search_space_id, - initial_metadata=metadata, - ) - - async def notify_indexing_progress( - self, - session: AsyncSession, - notification: Notification, - indexed_count: int, - total_count: int | None = None, - stage: str | None = None, - stage_message: str | None = None, - ) -> Notification: - """ - Update notification with indexing progress. - - Args: - session: Database session - notification: Notification to update - indexed_count: Number of items indexed so far - total_count: Total number of items (optional) - stage: Current sync stage (fetching, processing, storing) (optional) - stage_message: Optional custom message for the stage - - Returns: - Updated notification - """ - # User-friendly stage messages (clean, no ellipsis - spinner shows activity) - stage_messages = { - "connecting": "Connecting to your account", - "fetching": "Fetching your content", - "processing": "Preparing for search", - "storing": "Almost done", - } - - # Use stage-based message if stage provided, otherwise fallback - if stage or stage_message: - progress_msg = stage_message or stage_messages.get(stage, "Processing") - else: - # Fallback for backward compatibility - progress_msg = "Fetching your content" - - metadata_updates = {"indexed_count": indexed_count} - if total_count is not None: - metadata_updates["total_count"] = total_count - progress_percent = int((indexed_count / total_count) * 100) - metadata_updates["progress_percent"] = progress_percent - if stage: - metadata_updates["sync_stage"] = stage - - return await self.update_notification( - session=session, - notification=notification, - message=progress_msg, - status="in_progress", - metadata_updates=metadata_updates, - ) - - async def notify_retry_progress( - self, - session: AsyncSession, - notification: Notification, - indexed_count: int, - retry_reason: str, - attempt: int, - max_attempts: int, - wait_seconds: float | None = None, - service_name: str | None = None, - ) -> Notification: - """ - Update notification when a connector is retrying due to rate limits or errors. - - This method provides user-friendly feedback when external service limitations - (rate limits, temporary outages) cause delays. Users see that the delay is - not our fault and the sync is still progressing. - - This method can be used by ANY connector (Notion, Slack, Airtable, etc.) - when they hit rate limits or transient errors. - - Args: - session: Database session - notification: Notification to update - indexed_count: Number of items indexed so far - retry_reason: Reason for retry ('rate_limit', 'server_error', 'timeout') - attempt: Current retry attempt number (1-based) - max_attempts: Maximum number of retry attempts - wait_seconds: Seconds to wait before retry (optional, for display) - service_name: Name of the external service (e.g., 'Notion', 'Slack') - If not provided, extracts from notification metadata - - Returns: - Updated notification - """ - # Get service name from notification if not provided - if not service_name: - service_name = notification.notification_metadata.get( - "connector_name", "Service" - ) - # Extract just the service name if it's "Notion - My Workspace" - if " - " in service_name: - service_name = service_name.split(" - ")[0] - - # User-friendly messages for different retry reasons - # These make it clear the delay is due to the external service, not SurfSense - retry_messages = { - "rate_limit": f"{service_name} rate limit reached", - "server_error": f"{service_name} is slow to respond", - "timeout": f"{service_name} took too long", - "temporary_error": f"{service_name} temporarily unavailable", - } - - base_message = retry_messages.get(retry_reason, f"Waiting for {service_name}") - - # Add wait time and progress info - if wait_seconds and wait_seconds > 5: - # Only show wait time if it's significant - message = f"{base_message}. Retrying in {int(wait_seconds)}s..." - else: - message = f"{base_message}. Retrying..." - - # Add progress count if we have any - if indexed_count > 0: - item_text = "item" if indexed_count == 1 else "items" - message = f"{message} ({indexed_count} {item_text} synced so far)" - - metadata_updates = { - "indexed_count": indexed_count, - "sync_stage": "waiting_retry", - "retry_attempt": attempt, - "retry_max_attempts": max_attempts, - "retry_reason": retry_reason, - "retry_wait_seconds": wait_seconds, - } - - return await self.update_notification( - session=session, - notification=notification, - message=message, - status="in_progress", - metadata_updates=metadata_updates, - ) - - async def notify_indexing_completed( - self, - session: AsyncSession, - notification: Notification, - indexed_count: int, - error_message: str | None = None, - is_warning: bool = False, - skipped_count: int | None = None, - unsupported_count: int | None = None, - ) -> Notification: - """ - Update notification when connector indexing completes. - - Args: - session: Database session - notification: Notification to update - indexed_count: Total number of files indexed - error_message: Error message if indexing failed, or warning message (optional) - is_warning: If True, treat error_message as a warning (success case) rather than an error - skipped_count: Number of files skipped (e.g., unchanged) - optional - unsupported_count: Number of files skipped because the ETL parser doesn't support them - - Returns: - Updated notification - """ - connector_name = notification.notification_metadata.get( - "connector_name", "Connector" - ) - - unsupported_text = "" - if unsupported_count and unsupported_count > 0: - file_word = "file was" if unsupported_count == 1 else "files were" - unsupported_text = f" {unsupported_count} {file_word} not supported." - - if error_message: - if indexed_count > 0: - title = f"Ready: {connector_name}" - file_text = "file" if indexed_count == 1 else "files" - message = f"Now searchable! {indexed_count} {file_text} synced.{unsupported_text} Note: {error_message}" - status = "completed" - elif is_warning: - title = f"Ready: {connector_name}" - message = f"Sync complete.{unsupported_text} {error_message}" - status = "completed" - else: - title = f"Failed: {connector_name}" - message = f"Sync failed: {error_message}" - if unsupported_text: - message += unsupported_text - status = "failed" - else: - title = f"Ready: {connector_name}" - if indexed_count == 0: - if unsupported_count and unsupported_count > 0: - message = f"Sync complete.{unsupported_text}" - else: - message = "Already up to date!" - else: - file_text = "file" if indexed_count == 1 else "files" - message = f"Now searchable! {indexed_count} {file_text} synced." - if unsupported_text: - message += unsupported_text - status = "completed" - - metadata_updates = { - "indexed_count": indexed_count, - "skipped_count": skipped_count or 0, - "unsupported_count": unsupported_count or 0, - "sync_stage": "completed" - if (not error_message or is_warning or indexed_count > 0) - else "failed", - "error_message": error_message, - } - - return await self.update_notification( - session=session, - notification=notification, - title=title, - message=message, - status=status, - metadata_updates=metadata_updates, - ) - - async def notify_google_drive_indexing_started( - self, - session: AsyncSession, - user_id: UUID, - connector_id: int, - connector_name: str, - connector_type: str, - search_space_id: int, - folder_count: int, - file_count: int, - folder_names: list[str] | None = None, - file_names: list[str] | None = None, - ) -> Notification: - """ - Create or update notification when Google Drive indexing starts. - - Args: - session: Database session - user_id: User ID - connector_id: Connector ID - connector_name: Connector name - connector_type: Connector type - search_space_id: Search space ID - folder_count: Number of folders to index - file_count: Number of files to index - folder_names: List of folder names (optional) - file_names: List of file names (optional) - - Returns: - Notification: The created or updated notification - """ - operation_id = self._generate_google_drive_operation_id( - connector_id, folder_count, file_count - ) - title = f"Syncing: {connector_name}" - message = "Preparing your files" - - metadata = { - "connector_id": connector_id, - "connector_name": connector_name, - "connector_type": connector_type, - "folder_count": folder_count, - "file_count": file_count, - "indexed_count": 0, - "sync_stage": "connecting", - } - - if folder_names: - metadata["folder_names"] = folder_names - if file_names: - metadata["file_names"] = file_names - - return await self.find_or_create_notification( - session=session, - user_id=user_id, - operation_id=operation_id, - title=title, - message=message, - search_space_id=search_space_id, - initial_metadata=metadata, - ) - - -class DocumentProcessingNotificationHandler(BaseNotificationHandler): - """Handler for document processing notifications.""" - - def __init__(self): - super().__init__("document_processing") - - def _generate_operation_id( - self, document_type: str, filename: str, search_space_id: int - ) -> str: - """ - Generate a unique operation ID for a document processing operation. - - Args: - document_type: Type of document (FILE, YOUTUBE_VIDEO, CRAWLED_URL, etc.) - filename: Name of the file/document - search_space_id: Search space ID - - Returns: - Unique operation ID string - """ - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") - # Create a short hash of filename to ensure uniqueness - import hashlib - - filename_hash = hashlib.md5(filename.encode()).hexdigest()[:8] - return f"doc_{document_type}_{search_space_id}_{timestamp}_{filename_hash}" - - async def notify_processing_started( - self, - session: AsyncSession, - user_id: UUID, - document_type: str, - document_name: str, - search_space_id: int, - file_size: int | None = None, - ) -> Notification: - """ - Create notification when document processing starts. - - Args: - session: Database session - user_id: User ID - document_type: Type of document (FILE, YOUTUBE_VIDEO, CRAWLED_URL, etc.) - document_name: Name/title of the document - search_space_id: Search space ID - file_size: Size of file in bytes (optional) - - Returns: - Notification: The created notification - """ - operation_id = self._generate_operation_id( - document_type, document_name, search_space_id - ) - title = f"Processing: {document_name}" - message = "Waiting in queue" - - metadata = { - "document_type": document_type, - "document_name": document_name, - "processing_stage": "queued", - } - - if file_size is not None: - metadata["file_size"] = file_size - - return await self.find_or_create_notification( - session=session, - user_id=user_id, - operation_id=operation_id, - title=title, - message=message, - search_space_id=search_space_id, - initial_metadata=metadata, - ) - - async def notify_processing_progress( - self, - session: AsyncSession, - notification: Notification, - stage: str, - stage_message: str | None = None, - chunks_count: int | None = None, - ) -> Notification: - """ - Update notification with processing progress. - - Args: - session: Database session - notification: Notification to update - stage: Current processing stage (parsing, chunking, embedding, storing) - stage_message: Optional custom message for the stage - chunks_count: Number of chunks created (optional, stored in metadata only) - - Returns: - Updated notification - """ - # User-friendly stage messages - stage_messages = { - "parsing": "Reading your file", - "chunking": "Preparing for search", - "embedding": "Preparing for search", - "storing": "Finalizing", - } - - message = stage_message or stage_messages.get(stage, "Processing") - - metadata_updates = {"processing_stage": stage} - # Store chunks_count in metadata for debugging, but don't show to user - if chunks_count is not None: - metadata_updates["chunks_count"] = chunks_count - - return await self.update_notification( - session=session, - notification=notification, - message=message, - status="in_progress", - metadata_updates=metadata_updates, - ) - - async def notify_processing_completed( - self, - session: AsyncSession, - notification: Notification, - document_id: int | None = None, - chunks_count: int | None = None, - error_message: str | None = None, - ) -> Notification: - """ - Update notification when document processing completes. - - Args: - session: Database session - notification: Notification to update - document_id: ID of the created document (optional) - chunks_count: Total number of chunks created (optional) - error_message: Error message if processing failed (optional) - - Returns: - Updated notification - """ - document_name = notification.notification_metadata.get( - "document_name", "Document" - ) - - if error_message: - title = f"Failed: {document_name}" - message = f"Processing failed: {error_message}" - status = "failed" - else: - title = f"Ready: {document_name}" - message = "Now searchable!" - status = "completed" - - metadata_updates = { - "processing_stage": "completed" if not error_message else "failed", - "error_message": error_message, - } - - if document_id is not None: - metadata_updates["document_id"] = document_id - # Store chunks_count in metadata for debugging, but don't show to user - if chunks_count is not None: - metadata_updates["chunks_count"] = chunks_count - - return await self.update_notification( - session=session, - notification=notification, - title=title, - message=message, - status=status, - metadata_updates=metadata_updates, - ) - - -class MentionNotificationHandler(BaseNotificationHandler): - """Handler for new mention notifications.""" - - def __init__(self): - super().__init__("new_mention") - - async def find_notification_by_mention( - self, - session: AsyncSession, - mention_id: int, - ) -> Notification | None: - """ - Find an existing notification by mention ID. - - Args: - session: Database session - mention_id: The mention ID to search for - - Returns: - Notification if found, None otherwise - """ - query = select(Notification).where( - Notification.type == self.notification_type, - Notification.notification_metadata["mention_id"].astext == str(mention_id), - ) - result = await session.execute(query) - return result.scalar_one_or_none() - - async def notify_new_mention( - self, - session: AsyncSession, - mentioned_user_id: UUID, - mention_id: int, - comment_id: int, - message_id: int, - thread_id: int, - thread_title: str, - author_id: str, - author_name: str, - author_avatar_url: str | None, - author_email: str, - content_preview: str, - search_space_id: int, - ) -> Notification: - """ - Create notification when a user is @mentioned in a comment. - Uses mention_id for idempotency to prevent duplicate notifications. - - Args: - session: Database session - mentioned_user_id: User who was mentioned - mention_id: ID of the mention record (used for idempotency) - comment_id: ID of the comment containing the mention - message_id: ID of the message being commented on - thread_id: ID of the chat thread - thread_title: Title of the chat thread - author_id: ID of the comment author - author_name: Display name of the comment author - author_avatar_url: Avatar URL of the comment author - author_email: Email of the comment author (for fallback initials) - content_preview: First ~100 chars of the comment - search_space_id: Search space ID - - Returns: - Notification: The created or existing notification - """ - # Check if notification already exists for this mention (idempotency) - existing = await self.find_notification_by_mention(session, mention_id) - if existing: - logger.info( - f"Notification already exists for mention {mention_id}, returning existing" - ) - return existing - - title = f"{author_name} mentioned you" - message = content_preview[:100] + ("..." if len(content_preview) > 100 else "") - - metadata = { - "mention_id": mention_id, - "comment_id": comment_id, - "message_id": message_id, - "thread_id": thread_id, - "thread_title": thread_title, - "author_id": author_id, - "author_name": author_name, - "author_avatar_url": author_avatar_url, - "author_email": author_email, - "content_preview": content_preview[:200], - } - - try: - notification = Notification( - user_id=mentioned_user_id, - search_space_id=search_space_id, - type=self.notification_type, - title=title, - message=message, - notification_metadata=metadata, - ) - session.add(notification) - await session.commit() - await session.refresh(notification) - logger.info( - f"Created new_mention notification {notification.id} for user {mentioned_user_id}" - ) - return notification - except Exception as e: - # Handle race condition - if duplicate key error, try to fetch existing - await session.rollback() - if ( - "duplicate key" in str(e).lower() - or "unique constraint" in str(e).lower() - ): - logger.warning( - f"Duplicate notification detected for mention {mention_id}, fetching existing" - ) - existing = await self.find_notification_by_mention(session, mention_id) - if existing: - return existing - # Re-raise if not a duplicate key error or couldn't find existing - raise - - -class CommentReplyNotificationHandler(BaseNotificationHandler): - """Handler for comment reply notifications.""" - - def __init__(self): - super().__init__("comment_reply") - - async def find_notification_by_reply( - self, - session: AsyncSession, - reply_id: int, - user_id: UUID, - ) -> Notification | None: - query = select(Notification).where( - Notification.type == self.notification_type, - Notification.user_id == user_id, - Notification.notification_metadata["reply_id"].astext == str(reply_id), - ) - result = await session.execute(query) - return result.scalar_one_or_none() - - async def notify_comment_reply( - self, - session: AsyncSession, - user_id: UUID, - reply_id: int, - parent_comment_id: int, - message_id: int, - thread_id: int, - thread_title: str, - author_id: str, - author_name: str, - author_avatar_url: str | None, - author_email: str, - content_preview: str, - search_space_id: int, - ) -> Notification: - existing = await self.find_notification_by_reply(session, reply_id, user_id) - if existing: - logger.info( - f"Notification already exists for reply {reply_id} to user {user_id}" - ) - return existing - - title = f"{author_name} replied in a thread" - message = content_preview[:100] + ("..." if len(content_preview) > 100 else "") - - metadata = { - "reply_id": reply_id, - "parent_comment_id": parent_comment_id, - "message_id": message_id, - "thread_id": thread_id, - "thread_title": thread_title, - "author_id": author_id, - "author_name": author_name, - "author_avatar_url": author_avatar_url, - "author_email": author_email, - "content_preview": content_preview[:200], - } - - try: - notification = Notification( - user_id=user_id, - search_space_id=search_space_id, - type=self.notification_type, - title=title, - message=message, - notification_metadata=metadata, - ) - session.add(notification) - await session.commit() - await session.refresh(notification) - logger.info( - f"Created comment_reply notification {notification.id} for user {user_id}" - ) - return notification - except Exception as e: - await session.rollback() - if ( - "duplicate key" in str(e).lower() - or "unique constraint" in str(e).lower() - ): - logger.warning( - f"Duplicate notification for reply {reply_id} to user {user_id}" - ) - existing = await self.find_notification_by_reply( - session, reply_id, user_id - ) - if existing: - return existing - raise - - -class PageLimitNotificationHandler(BaseNotificationHandler): - """Handler for page limit exceeded notifications.""" - - def __init__(self): - super().__init__("page_limit_exceeded") - - def _generate_operation_id(self, document_name: str, search_space_id: int) -> str: - """ - Generate a unique operation ID for a page limit exceeded notification. - - Args: - document_name: Name of the document that triggered the limit - search_space_id: Search space ID - - Returns: - Unique operation ID string - """ - import hashlib - - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") - # Create a short hash of document name to ensure uniqueness - doc_hash = hashlib.md5(document_name.encode()).hexdigest()[:8] - return f"page_limit_{search_space_id}_{timestamp}_{doc_hash}" - - async def notify_page_limit_exceeded( - self, - session: AsyncSession, - user_id: UUID, - document_name: str, - document_type: str, - search_space_id: int, - pages_used: int, - pages_limit: int, - pages_to_add: int, - ) -> Notification: - """ - Create notification when a document exceeds the user's page limit. - - Args: - session: Database session - user_id: User ID - document_name: Name of the document that triggered the limit - document_type: Type of document (FILE, YOUTUBE_VIDEO, etc.) - search_space_id: Search space ID - pages_used: Current number of pages used - pages_limit: User's page limit - pages_to_add: Number of pages the document would add - - Returns: - Notification: The created notification - """ - operation_id = self._generate_operation_id(document_name, search_space_id) - - # Truncate document name for title if too long - display_name = ( - document_name[:40] + "..." if len(document_name) > 40 else document_name - ) - title = f"Page limit exceeded: {display_name}" - message = f"This document has ~{pages_to_add} page(s) but you've used {pages_used}/{pages_limit} pages. Upgrade to process more documents." - - metadata = { - "operation_id": operation_id, - "document_name": document_name, - "document_type": document_type, - "pages_used": pages_used, - "pages_limit": pages_limit, - "pages_to_add": pages_to_add, - "status": "failed", - "error_type": "page_limit_exceeded", - # Navigation target for frontend - "action_url": f"/dashboard/{search_space_id}/more-pages", - "action_label": "Upgrade Plan", - } - - notification = Notification( - user_id=user_id, - search_space_id=search_space_id, - type=self.notification_type, - title=title, - message=message, - notification_metadata=metadata, - ) - session.add(notification) - await session.commit() - await session.refresh(notification) - logger.info( - f"Created page_limit_exceeded notification {notification.id} for user {user_id}" - ) - return notification - - -class NotificationService: - """Service for creating and managing notifications that sync via Zero.""" - - # Handler instances - connector_indexing = ConnectorIndexingNotificationHandler() - document_processing = DocumentProcessingNotificationHandler() - mention = MentionNotificationHandler() - comment_reply = CommentReplyNotificationHandler() - page_limit = PageLimitNotificationHandler() - - @staticmethod - async def create_notification( - session: AsyncSession, - user_id: UUID, - notification_type: str, - title: str, - message: str, - search_space_id: int | None = None, - notification_metadata: dict[str, Any] | None = None, - ) -> Notification: - """ - Create a notification - Zero will automatically sync it to frontend. - - Args: - session: Database session - user_id: User to notify - notification_type: Type of notification (e.g., 'document_processing', 'connector_indexing') - title: Notification title - message: Notification message - search_space_id: Optional search space ID - notification_metadata: Optional metadata dictionary - - Returns: - Notification: The created notification - """ - notification = Notification( - user_id=user_id, - search_space_id=search_space_id, - type=notification_type, - title=title, - message=message, - notification_metadata=notification_metadata or {}, - ) - session.add(notification) - await session.commit() - await session.refresh(notification) - logger.info(f"Created notification {notification.id} for user {user_id}") - return notification diff --git a/surfsense_backend/app/services/notion/kb_sync_service.py b/surfsense_backend/app/services/notion/kb_sync_service.py index b10d1b157..ee85daf41 100644 --- a/surfsense_backend/app/services/notion/kb_sync_service.py +++ b/surfsense_backend/app/services/notion/kb_sync_service.py @@ -8,7 +8,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -73,30 +72,8 @@ class NotionKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "page_title": page_title, - "page_id": page_id, - "document_type": "Notion Page", - "connector_type": "Notion", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - markdown_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"Notion Page: {page_title}\n\n{markdown_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Notion Page: {page_title}\n\n{markdown_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(markdown_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -245,31 +222,10 @@ class NotionKBSyncService: f"Final content length: {len(full_content)} chars, verified={content_verified}" ) - from app.services.llm_service import get_user_long_context_llm - logger.debug("Generating summary and embeddings") - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, # disable streaming to avoid leaking into the chat - ) - if user_llm: - document_metadata_for_summary = { - "page_title": document.document_metadata.get("page_title"), - "page_id": document.document_metadata.get("page_id"), - "document_type": "Notion Page", - "connector_type": "Notion", - } - summary_content, summary_embedding = await generate_document_summary( - full_content, user_llm, document_metadata_for_summary - ) - logger.debug(f"Generated summary length: {len(summary_content)} chars") - else: - logger.warning("No LLM configured - using fallback summary") - summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" + summary_embedding = embed_text(summary_content) logger.debug("Creating new chunks") chunks = await create_document_chunks(full_content) diff --git a/surfsense_backend/app/services/obsidian_plugin_indexer.py b/surfsense_backend/app/services/obsidian_plugin_indexer.py index 0fc4f30f4..13f43d1ee 100644 --- a/surfsense_backend/app/services/obsidian_plugin_indexer.py +++ b/surfsense_backend/app/services/obsidian_plugin_indexer.py @@ -233,18 +233,6 @@ async def _resolve_attachment_vision_llm( return await get_vision_llm(session, search_space_id) -async def _resolve_summary_llm( - session: AsyncSession, *, user_id: str, search_space_id: int, should_summarize: bool -): - """Fetch summary LLM only when indexing summary is enabled.""" - if not should_summarize: - return None - - from app.services.llm_service import get_user_long_context_llm - - return await get_user_long_context_llm(session, user_id, search_space_id) - - def _require_extracted_attachment_content( *, content: str, etl_meta: dict[str, Any], path: str ) -> str: @@ -349,13 +337,6 @@ async def upsert_note( path=payload.path, ) - llm = await _resolve_summary_llm( - session, - user_id=str(user_id), - search_space_id=search_space_id, - should_summarize=connector.enable_summary, - ) - document_string = _build_document_string( payload, vault_name, content_override=content_for_index ) @@ -374,8 +355,6 @@ async def upsert_note( search_space_id=search_space_id, connector_id=connector.id, created_by_id=str(user_id), - should_summarize=connector.enable_summary, - fallback_summary=f"Obsidian Note: {payload.name}\n\n{content_for_index}", metadata=metadata, ) @@ -388,7 +367,7 @@ async def upsert_note( document = prepared[0] - return await pipeline.index(document, connector_doc, llm) + return await pipeline.index(document, connector_doc) async def rename_note( diff --git a/surfsense_backend/app/services/onedrive/kb_sync_service.py b/surfsense_backend/app/services/onedrive/kb_sync_service.py index 731f081dd..2bfea6ef4 100644 --- a/surfsense_backend/app/services/onedrive/kb_sync_service.py +++ b/surfsense_backend/app/services/onedrive/kb_sync_service.py @@ -10,7 +10,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) logger = logging.getLogger(__name__) @@ -73,30 +72,8 @@ class OneDriveKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - - doc_metadata_for_summary = { - "file_name": file_name, - "mime_type": mime_type, - "document_type": "OneDrive File", - "connector_type": "OneDrive", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}" - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}" + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/provider_capabilities.py b/surfsense_backend/app/services/provider_capabilities.py index e9a1c33e1..f094c9954 100644 --- a/surfsense_backend/app/services/provider_capabilities.py +++ b/surfsense_backend/app/services/provider_capabilities.py @@ -53,10 +53,10 @@ logger = logging.getLogger(__name__) # # Owned here because ``app.services.provider_capabilities`` is the # only edge that's safe to call from ``app.config``'s YAML loader at -# class-body init time. ``app.agents.new_chat.llm_config`` re-exports +# class-body init time. ``app.agents.chat.runtime.llm_config`` re-exports # this constant under the historical ``PROVIDER_MAP`` name; placing the # map there directly would re-introduce the -# ``app.config -> ... -> app.agents.new_chat.tools.generate_image -> +# ``app.config -> ... -> deliverables/tools/generate_image -> # app.config`` cycle that prompted the move. _PROVIDER_PREFIX_MAP: dict[str, str] = { "OPENAI": "openai", diff --git a/surfsense_backend/app/services/revert_service.py b/surfsense_backend/app/services/revert_service.py index 60f6503aa..6db5e2604 100644 --- a/surfsense_backend/app/services/revert_service.py +++ b/surfsense_backend/app/services/revert_service.py @@ -38,7 +38,7 @@ from typing import Any, Literal from sqlalchemy import delete, select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, safe_filename, safe_folder_segment, diff --git a/surfsense_backend/app/services/task_dispatcher.py b/surfsense_backend/app/services/task_dispatcher.py index 210084102..43957be03 100644 --- a/surfsense_backend/app/services/task_dispatcher.py +++ b/surfsense_backend/app/services/task_dispatcher.py @@ -18,7 +18,6 @@ class TaskDispatcher(Protocol): filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: ... @@ -35,7 +34,6 @@ class CeleryTaskDispatcher: filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: @@ -49,7 +47,6 @@ class CeleryTaskDispatcher: filename=filename, search_space_id=search_space_id, user_id=user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) diff --git a/surfsense_backend/app/services/user_tool_allowlist.py b/surfsense_backend/app/services/user_tool_allowlist.py index fdfa51560..9b87fbdea 100644 --- a/surfsense_backend/app/services/user_tool_allowlist.py +++ b/surfsense_backend/app/services/user_tool_allowlist.py @@ -16,10 +16,10 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified -from app.agents.multi_agent_chat.constants import ( +from app.agents.chat.multi_agent_chat.constants import ( CONNECTOR_TYPE_TO_CONNECTOR_AGENT_MAPS, ) -from app.agents.new_chat.permissions import Rule, Ruleset +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset from app.db import SearchSourceConnector, async_session_maker logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py index 5d6bde6c1..d36a7c05f 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py @@ -9,7 +9,6 @@ from sqlalchemy.orm import selectinload from app.celery_app import celery_app from app.db import Document from app.indexing_pipeline.adapters.file_upload_adapter import UploadDocumentAdapter -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task @@ -68,12 +67,8 @@ async def _reindex_document(document_id: int, user_id: str): logger.info(f"Reindexing document {document_id} ({document.title})") - user_llm = await get_user_long_context_llm( - session, user_id, document.search_space_id - ) - adapter = UploadDocumentAdapter(session) - await adapter.reindex(document=document, llm=user_llm) + await adapter.reindex(document=document) await task_logger.log_task_success( log_entry, diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 1f9609968..d38014124 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -9,8 +9,8 @@ from uuid import UUID from app.celery_app import celery_app from app.config import config +from app.notifications.service import NotificationService from app.observability import metrics as ot_metrics -from app.services.notification_service import NotificationService from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task from app.tasks.connector_indexers.local_folder_indexer import ( @@ -118,6 +118,7 @@ async def _delete_document_background(document_id: int) -> None: from sqlalchemy import delete as sa_delete, select from app.db import Chunk, Document + from app.file_storage.service import purge_document_blobs async with get_celery_session_maker()() as session: batch_size = 500 @@ -133,6 +134,9 @@ async def _delete_document_background(document_id: int) -> None: await session.execute(sa_delete(Chunk).where(Chunk.id.in_(chunk_ids))) await session.commit() + # Remove stored blobs before the document_files rows cascade away. + await purge_document_blobs(session, document_ids=[document_id]) + doc = await session.get(Document, document_id) if doc: await session.delete(doc) @@ -166,6 +170,7 @@ async def _delete_folder_documents( from sqlalchemy import delete as sa_delete, select from app.db import Chunk, Document, Folder + from app.file_storage.service import purge_document_blobs async with get_celery_session_maker()() as session: batch_size = 500 @@ -182,6 +187,9 @@ async def _delete_folder_documents( await session.execute(sa_delete(Chunk).where(Chunk.id.in_(chunk_ids))) await session.commit() + # Remove stored blobs before the document_files rows cascade away. + await purge_document_blobs(session, document_ids=[doc_id]) + doc = await session.get(Document, doc_id) if doc: await session.delete(doc) @@ -214,6 +222,7 @@ async def _delete_search_space_background(search_space_id: int) -> None: from sqlalchemy import delete as sa_delete, select from app.db import Chunk, Document, SearchSpace + from app.file_storage.service import purge_document_blobs async with get_celery_session_maker()() as session: batch_size = 500 @@ -240,6 +249,8 @@ async def _delete_search_space_background(search_space_id: int) -> None: doc_ids = doc_ids_result.scalars().all() if not doc_ids: break + # Remove stored blobs before the document_files rows cascade away. + await purge_document_blobs(session, document_ids=list(doc_ids)) await session.execute(sa_delete(Document).where(Document.id.in_(doc_ids))) await session.commit() @@ -754,7 +765,6 @@ def process_file_upload_with_document_task( filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ): @@ -771,7 +781,6 @@ def process_file_upload_with_document_task( filename: Original filename search_space_id: ID of the search space user_id: ID of the user - should_summarize: Whether to generate an LLM summary """ import traceback @@ -803,7 +812,6 @@ def process_file_upload_with_document_task( filename, search_space_id, user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -839,7 +847,6 @@ async def _process_file_with_document( filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ): @@ -943,7 +950,6 @@ async def _process_file_with_document( task_logger=task_logger, log_entry=log_entry, notification=notification, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -1247,7 +1253,6 @@ def index_local_folder_task( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, ): """Celery task to index a local folder. Config is passed directly — no connector row.""" @@ -1260,7 +1265,6 @@ def index_local_folder_task( exclude_patterns=exclude_patterns, file_extensions=file_extensions, root_folder_id=root_folder_id, - enable_summary=enable_summary, target_file_paths=target_file_paths, ) ) @@ -1274,7 +1278,6 @@ async def _index_local_folder_async( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, ): """Run local folder indexing with notification + heartbeat.""" @@ -1332,7 +1335,6 @@ async def _index_local_folder_async( exclude_patterns=exclude_patterns, file_extensions=file_extensions, root_folder_id=root_folder_id, - enable_summary=enable_summary, target_file_paths=target_file_paths, on_heartbeat_callback=_heartbeat_progress if (is_batch or is_full_scan) @@ -1389,7 +1391,6 @@ def index_uploaded_folder_files_task( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], use_vision_llm: bool = False, processing_mode: str = "basic", @@ -1401,7 +1402,6 @@ def index_uploaded_folder_files_task( user_id=user_id, folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, file_mappings=file_mappings, use_vision_llm=use_vision_llm, processing_mode=processing_mode, @@ -1414,7 +1414,6 @@ async def _index_uploaded_folder_files_async( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], use_vision_llm: bool = False, processing_mode: str = "basic", @@ -1464,7 +1463,6 @@ async def _index_uploaded_folder_files_async( user_id=user_id, folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, file_mappings=file_mappings, on_heartbeat_callback=_heartbeat_progress, use_vision_llm=use_vision_llm, @@ -1552,12 +1550,10 @@ async def _ai_sort_search_space_async(search_space_id: int, user_id: str): t_start = time.perf_counter() try: from app.services.ai_file_sort_service import ai_sort_all_documents - from app.services.llm_service import get_document_summary_llm + from app.services.llm_service import get_agent_llm async with get_celery_session_maker()() as session: - llm = await get_document_summary_llm( - session, search_space_id, disable_streaming=True - ) + llm = await get_agent_llm(session, search_space_id, disable_streaming=True) if llm is None: logger.warning( "No LLM configured for search_space=%d, skipping AI sort", @@ -1593,7 +1589,7 @@ def ai_sort_document_task(self, search_space_id: int, user_id: str, document_id: async def _ai_sort_document_async(search_space_id: int, user_id: str, document_id: int): from app.db import Document from app.services.ai_file_sort_service import ai_sort_document - from app.services.llm_service import get_document_summary_llm + from app.services.llm_service import get_agent_llm async with get_celery_session_maker()() as session: document = await session.get(Document, document_id) @@ -1601,9 +1597,7 @@ async def _ai_sort_document_async(search_space_id: int, user_id: str, document_i logger.warning("Document %d not found, skipping AI sort", document_id) return - llm = await get_document_summary_llm( - session, search_space_id, disable_streaming=True - ) + llm = await get_agent_llm(session, search_space_id, disable_streaming=True) if llm is None: logger.warning( "No LLM for search_space=%d, skipping AI sort of doc=%d", diff --git a/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py new file mode 100644 index 000000000..7bd3d6788 --- /dev/null +++ b/surfsense_backend/app/tasks/celery_tasks/gateway_tasks.py @@ -0,0 +1,166 @@ +"""Celery maintenance tasks for external chat surfaces.""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select, update + +from app.celery_app import celery_app +from app.db import ( + ExternalChatAccount, + ExternalChatEventStatus, + ExternalChatHealthStatus, + ExternalChatInboundEvent, + ExternalChatPlatform, +) +from app.gateway.inbox import persist_inbound_event, telegram_event_dedupe_key +from app.gateway.registry import resolve_platform_bundle +from app.gateway.telegram.adapter import TelegramAdapter +from app.observability.metrics import ( + record_gateway_health_check_failure, + record_gateway_inbound_reconciled, +) +from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task + +logger = logging.getLogger(__name__) + + +@celery_app.task(name="gateway.process_inbound_event") +def process_inbound_event_task(inbox_id: int) -> None: + logger.warning( + "Ignoring gateway.process_inbound_event for inbox_id=%s; " + "FastAPI owns external chat agent turn processing.", + inbox_id, + ) + return None + + +@celery_app.task(name="gateway.reconcile_inbox") +def reconcile_inbox_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + stale_threshold = datetime.now(UTC) - timedelta(minutes=10) + result = await session.execute( + update(ExternalChatInboundEvent) + .where( + ExternalChatInboundEvent.status + == ExternalChatEventStatus.PROCESSING, + ExternalChatInboundEvent.received_at < stale_threshold, + ) + .values( + status=ExternalChatEventStatus.RECEIVED, + last_error="stale processing reset for FastAPI inbox worker", + ) + ) + for _ in range(result.rowcount or 0): + record_gateway_inbound_reconciled(reason="stale_processing_reset") + await session.commit() + + return run_async_celery_task(_run) + + +@celery_app.task(name="gateway.health_check") +def gateway_health_check_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + result = await session.execute(select(ExternalChatAccount)) + accounts = list(result.scalars()) + for account in accounts: + try: + bundle = resolve_platform_bundle(account) + metadata = await bundle.adapter.validate_credentials() + account.health_status = ExternalChatHealthStatus.OK + if account.platform == ExternalChatPlatform.TELEGRAM: + account.bot_username = metadata.get("username") + elif account.platform == ExternalChatPlatform.WHATSAPP: + cursor_state = dict(account.cursor_state or {}) + for key in ( + "quality_rating", + "account_review_status", + "status", + ): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + elif account.platform == ExternalChatPlatform.SLACK: + cursor_state = dict(account.cursor_state or {}) + for key in ("team_id", "team", "bot_user_id", "bot_username"): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + account.bot_username = metadata.get("bot_username") + elif account.platform == ExternalChatPlatform.DISCORD: + cursor_state = dict(account.cursor_state or {}) + for key in ("bot_user_id", "bot_username", "global_name"): + if key in metadata: + cursor_state[key] = metadata[key] + account.cursor_state = cursor_state + account.bot_username = metadata.get("bot_username") + except Exception: + logger.warning( + "External chat health check failed platform=%s account_id=%s", + account.platform.value, + account.id, + exc_info=True, + ) + account.health_status = ExternalChatHealthStatus.FAILING + record_gateway_health_check_failure(platform=account.platform.value) + account.last_health_check_at = datetime.now(UTC) + await session.commit() + + return run_async_celery_task(_run) + + +@celery_app.task(name="gateway.enqueue_received_sweep") +def enqueue_received_sweep_task() -> int: + logger.info( + "Skipping gateway.enqueue_received_sweep; " + "FastAPI inbox worker scans RECEIVED rows directly." + ) + return 0 + + +@celery_app.task(name="gateway.retention_sweep") +def gateway_retention_sweep_task() -> None: + async def _run() -> None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + raw_cutoff = datetime.now(UTC) - timedelta(days=30) + delete_cutoff = datetime.now(UTC) - timedelta(days=365) + await session.execute( + update(ExternalChatInboundEvent) + .where(ExternalChatInboundEvent.received_at < raw_cutoff) + .values(raw_payload=None) + ) + result = await session.execute( + select(ExternalChatInboundEvent).where( + ExternalChatInboundEvent.received_at < delete_cutoff + ) + ) + for event in result.scalars(): + await session.delete(event) + await session.commit() + + return run_async_celery_task(_run) + + +async def enqueue_telegram_update(account_id: int, raw_update: dict) -> int | None: + session_maker = get_celery_session_maker() + async with session_maker() as session: + parsed = TelegramAdapter("placeholder").parse_inbound(raw_update) + inbox_id = await persist_inbound_event( + session, + account_id=account_id, + platform=ExternalChatPlatform.TELEGRAM, + event_dedupe_key=telegram_event_dedupe_key(raw_update["update_id"]), + external_event_id=str(raw_update["update_id"]), + external_message_id=parsed.external_message_id, + event_kind=parsed.event_kind, + raw_payload=raw_update, + ) + await session.commit() + return inbox_id diff --git a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py index e41251407..e88fb58b9 100644 --- a/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/schedule_checker_task.py @@ -6,7 +6,8 @@ from datetime import UTC, datetime from sqlalchemy.future import select from app.celery_app import celery_app -from app.db import Notification, SearchSourceConnector, SearchSourceConnectorType +from app.db import SearchSourceConnector, SearchSourceConnectorType +from app.notifications.persistence import Notification from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task from app.utils.indexing_locks import is_connector_indexing_locked diff --git a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py index d51c85dee..5bf857d9b 100644 --- a/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/stale_notification_cleanup_task.py @@ -33,7 +33,8 @@ from sqlalchemy.future import select from app.celery_app import celery_app from app.config import config -from app.db import Document, DocumentStatus, Notification +from app.db import Document, DocumentStatus +from app.notifications.persistence import Notification from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py deleted file mode 100644 index e150cf494..000000000 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ /dev/null @@ -1,3064 +0,0 @@ -""" -Streaming task for the new SurfSense deep agent chat. - -This module streams responses from the deep agent using the Vercel AI SDK -Data Stream Protocol (SSE format). - -Supports loading LLM configurations from: -- YAML files (negative IDs for global configs) -- NewLLMConfig database table (positive IDs for user-created configs with prompt settings) -""" - -import asyncio -import contextlib -import gc -import json -import logging -import sys -import time -from collections.abc import AsyncGenerator -from dataclasses import dataclass, field -from functools import partial -from typing import Any, Literal -from uuid import UUID - -import anyio -from langchain_core.messages import HumanMessage -from sqlalchemy.future import select - -from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent -from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent -from app.agents.new_chat.checkpointer import get_checkpointer -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.errors import BusyError -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.llm_config import ( - AgentConfig, - create_chat_litellm_from_agent_config, - create_chat_litellm_from_config, - load_agent_config, - load_global_llm_config_by_id, -) -from app.agents.new_chat.mention_resolver import resolve_mentions, substitute_in_text -from app.agents.new_chat.middleware.busy_mutex import ( - end_turn, - get_cancel_state, - is_cancel_requested, -) -from app.agents.new_chat.middleware.kb_persistence import ( - commit_staged_filesystem_state, -) -from app.db import ( - ChatVisibility, - NewChatMessage, - NewChatThread, - Report, - SearchSourceConnectorType, - async_session_maker, - shielded_async_session, -) -from app.observability import metrics as ot_metrics, otel as ot -from app.prompts import TITLE_GENERATION_PROMPT -from app.services.auto_model_pin_service import ( - mark_runtime_cooldown, - resolve_or_get_pinned_llm_config_id, -) -from app.services.chat_session_state_service import ( - clear_ai_responding, - set_ai_responding, -) -from app.services.connector_service import ConnectorService -from app.services.new_streaming_service import VercelStreamingService -from app.tasks.chat.streaming.graph_stream.event_stream import stream_output -from app.tasks.chat.streaming.helpers.interrupt_inspector import ( - all_interrupt_values, -) -from app.utils.content_utils import bootstrap_history_from_db -from app.utils.perf import get_perf_logger, log_system_snapshot, trim_native_heap -from app.utils.user_message_multimodal import build_human_message_content - -_background_tasks: set[asyncio.Task] = set() -_perf_log = get_perf_logger() -logger = logging.getLogger(__name__) - -TURN_CANCELLING_INITIAL_DELAY_MS = 200 -TURN_CANCELLING_BACKOFF_FACTOR = 2 -TURN_CANCELLING_MAX_DELAY_MS = 1500 - - -def _resume_step_prefix(turn_id: str) -> str: - """Build the per-turn ``step_prefix`` for a resume invocation. - - Each ``_stream_agent_events`` call constructs a fresh - :class:`AgentEventRelayState` with ``thinking_step_counter=0``, so two - consecutive resume turns would otherwise both emit ``thinking-resume-1``, - ``-2`` etc. The frontend rehydrates ``currentThinkingSteps`` from the - immediate prior assistant message at the start of every resume — if the - new stream's IDs collide with the seeded ones, React renders sibling - Timeline rows with the same key. Salting with ``turn_id`` guarantees - disjoint IDs across resumes within one thread. - """ - return f"thinking-resume-{turn_id}" - - -def _compute_turn_cancelling_retry_delay(attempt: int) -> int: - if attempt < 1: - attempt = 1 - delay = TURN_CANCELLING_INITIAL_DELAY_MS * ( - TURN_CANCELLING_BACKOFF_FACTOR ** (attempt - 1) - ) - return min(delay, TURN_CANCELLING_MAX_DELAY_MS) - - -def _extract_chunk_parts(chunk: Any) -> dict[str, Any]: - """Decompose an ``AIMessageChunk`` into typed text/reasoning/tool-call parts. - - Returns a dict with three keys: - - * ``text`` — concatenated string content (empty string if the chunk - contributes none). - * ``reasoning`` — concatenated reasoning content (empty string if the - chunk contributes none). - * ``tool_call_chunks`` — flat list of LangChain ``tool_call_chunk`` - dicts surfaced from either the typed-block list or the - ``tool_call_chunks`` attribute. - - Background - ---------- - ``AIMessageChunk.content`` can be: - - * a ``str`` (most providers), or - * a ``list`` of typed blocks ``{type: 'text' | 'reasoning' | - 'tool_call_chunk' | 'tool_use' | ..., text/content/...}`` for - Anthropic, Bedrock, and several reasoning configurations. - - Reasoning may also live under - ``chunk.additional_kwargs['reasoning_content']`` (some providers - surface it that way instead of as a typed block). Tool-call chunks - may live under ``chunk.tool_call_chunks`` even when ``content`` is a - plain string. - - Earlier versions only handled the ``isinstance(content, str)`` branch - and silently dropped reasoning blocks + tool-call chunks emitted by - LangChain ``AIMessageChunk``s. - """ - out: dict[str, Any] = {"text": "", "reasoning": "", "tool_call_chunks": []} - if chunk is None: - return out - - content = getattr(chunk, "content", None) - if isinstance(content, str): - if content: - out["text"] = content - elif isinstance(content, list): - text_parts: list[str] = [] - reasoning_parts: list[str] = [] - for block in content: - if not isinstance(block, dict): - continue - block_type = block.get("type") - if block_type == "text": - value = block.get("text") or block.get("content") or "" - if isinstance(value, str) and value: - text_parts.append(value) - elif block_type == "reasoning": - value = ( - block.get("reasoning") - or block.get("text") - or block.get("content") - or "" - ) - if isinstance(value, str) and value: - reasoning_parts.append(value) - elif block_type in ("tool_call_chunk", "tool_use"): - out["tool_call_chunks"].append(block) - if text_parts: - out["text"] = "".join(text_parts) - if reasoning_parts: - out["reasoning"] = "".join(reasoning_parts) - - additional = getattr(chunk, "additional_kwargs", None) or {} - if isinstance(additional, dict): - extra_reasoning = additional.get("reasoning_content") - if isinstance(extra_reasoning, str) and extra_reasoning: - existing = out["reasoning"] - out["reasoning"] = ( - (existing + extra_reasoning) if existing else extra_reasoning - ) - - extra_tool_chunks = getattr(chunk, "tool_call_chunks", None) - if isinstance(extra_tool_chunks, list): - for tcc in extra_tool_chunks: - if isinstance(tcc, dict): - out["tool_call_chunks"].append(tcc) - - return out - - -def extract_todos_from_deepagents(command_output) -> dict: - """ - Extract todos from deepagents' TodoListMiddleware Command output. - - deepagents returns a Command object with: - - Command.update['todos'] = [{'content': '...', 'status': '...'}] - - Returns the todos directly (no transformation needed - UI matches deepagents format). - """ - todos_data = [] - if hasattr(command_output, "update"): - # It's a Command object from deepagents - update = command_output.update - todos_data = update.get("todos", []) - elif isinstance(command_output, dict): - # Already a dict - check if it has todos directly or in update - if "todos" in command_output: - todos_data = command_output.get("todos", []) - elif "update" in command_output and isinstance(command_output["update"], dict): - todos_data = command_output["update"].get("todos", []) - - return {"todos": todos_data} - - -@dataclass -class StreamResult: - accumulated_text: str = "" - is_interrupted: bool = False - sandbox_files: list[str] = field(default_factory=list) - request_id: str | None = None - turn_id: str = "" - filesystem_mode: str = "cloud" - client_platform: str = "web" - intent_detected: str = "chat_only" - intent_confidence: float = 0.0 - write_attempted: bool = False - write_succeeded: bool = False - verification_succeeded: bool = False - commit_gate_passed: bool = True - commit_gate_reason: str = "" - # Pre-allocated assistant ``new_chat_messages.id`` for this turn, - # captured by ``persist_assistant_shell`` right after the user row is - # persisted. ``None`` for the legacy / anonymous code paths that don't - # opt in to server-side ``ContentPart[]`` projection. - assistant_message_id: int | None = None - # In-memory mirror of the FE's assistant-ui ``ContentPartsState``, - # populated by the lifecycle methods called from ``_stream_agent_events`` - # at each ``streaming_service.format_*`` yield site. Snapshot in the - # streaming ``finally`` to produce the rich JSONB persisted by - # ``finalize_assistant_turn``. ``repr=False`` keeps the - # log-on-error path (``StreamResult`` is logged in some error - # branches) from dumping a potentially-large parts list. - content_builder: Any | None = field(default=None, repr=False) - - -def _safe_float(value: Any, default: float = 0.0) -> float: - try: - return float(value) - except (TypeError, ValueError): - return default - - -def _tool_output_to_text(tool_output: Any) -> str: - if isinstance(tool_output, dict): - if isinstance(tool_output.get("result"), str): - return tool_output["result"] - if isinstance(tool_output.get("error"), str): - return tool_output["error"] - return json.dumps(tool_output, ensure_ascii=False) - return str(tool_output) - - -def _tool_output_has_error(tool_output: Any) -> bool: - if isinstance(tool_output, dict): - if tool_output.get("error"): - return True - result = tool_output.get("result") - return bool( - isinstance(result, str) and result.strip().lower().startswith("error:") - ) - if isinstance(tool_output, str): - return tool_output.strip().lower().startswith("error:") - return False - - -def _extract_resolved_file_path( - *, tool_name: str, tool_output: Any, tool_input: Any | None = None -) -> str | None: - if isinstance(tool_output, dict): - path_value = tool_output.get("path") - if isinstance(path_value, str) and path_value.strip(): - return path_value.strip() - if tool_name in ("write_file", "edit_file") and isinstance(tool_input, dict): - file_path = tool_input.get("file_path") - if isinstance(file_path, str) and file_path.strip(): - return file_path.strip() - return None - - -def _contract_enforcement_active(result: StreamResult) -> bool: - # Keep policy deterministic with no env-driven progression modes: - # enforce the file-operation contract only in desktop local-folder mode. - return result.filesystem_mode == "desktop_local_folder" - - -def _evaluate_file_contract_outcome(result: StreamResult) -> tuple[bool, str]: - if result.intent_detected != "file_write": - return True, "" - if not result.write_attempted: - return False, "no_write_attempt" - if not result.write_succeeded: - return False, "write_failed" - if not result.verification_succeeded: - return False, "verification_failed" - return True, "" - - -def _log_file_contract(stage: str, result: StreamResult, **extra: Any) -> None: - payload: dict[str, Any] = { - "stage": stage, - "request_id": result.request_id or "unknown", - "turn_id": result.turn_id or "unknown", - "chat_id": result.turn_id.split(":", 1)[0] - if ":" in result.turn_id - else "unknown", - "filesystem_mode": result.filesystem_mode, - "client_platform": result.client_platform, - "intent_detected": result.intent_detected, - "intent_confidence": result.intent_confidence, - "write_attempted": result.write_attempted, - "write_succeeded": result.write_succeeded, - "verification_succeeded": result.verification_succeeded, - "commit_gate_passed": result.commit_gate_passed, - "commit_gate_reason": result.commit_gate_reason or None, - } - payload.update(extra) - _perf_log.info( - "[file_operation_contract] %s", json.dumps(payload, ensure_ascii=False) - ) - - -def _log_chat_stream_error( - *, - flow: Literal["new", "resume", "regenerate"], - error_kind: str, - error_code: str | None, - severity: Literal["info", "warn", "error"], - is_expected: bool, - request_id: str | None, - thread_id: int | None, - search_space_id: int | None, - user_id: str | None, - message: str, - extra: dict[str, Any] | None = None, -) -> None: - payload: dict[str, Any] = { - "event": "chat_stream_error", - "flow": flow, - "error_kind": error_kind, - "error_code": error_code, - "severity": severity, - "is_expected": is_expected, - "request_id": request_id or "unknown", - "thread_id": thread_id, - "search_space_id": search_space_id, - "user_id": user_id, - "message": message, - } - if extra: - payload.update(extra) - - logger = logging.getLogger(__name__) - rendered = json.dumps(payload, ensure_ascii=False) - if severity == "error": - logger.error("[chat_stream_error] %s", rendered) - elif severity == "warn": - logger.warning("[chat_stream_error] %s", rendered) - else: - logger.info("[chat_stream_error] %s", rendered) - - -def _parse_error_payload(message: str) -> dict[str, Any] | None: - candidates = [message] - first_brace_idx = message.find("{") - if first_brace_idx >= 0: - candidates.append(message[first_brace_idx:]) - - for candidate in candidates: - try: - parsed = json.loads(candidate) - if isinstance(parsed, dict): - return parsed - except Exception: - continue - return None - - -def _extract_provider_error_code(parsed: dict[str, Any] | None) -> int | None: - if not isinstance(parsed, dict): - return None - candidates: list[Any] = [parsed.get("code")] - nested = parsed.get("error") - if isinstance(nested, dict): - candidates.append(nested.get("code")) - for value in candidates: - try: - if value is None: - continue - return int(value) - except Exception: - continue - return None - - -def _is_provider_rate_limited(exc: BaseException) -> bool: - """Best-effort detection for provider-side runtime throttling. - - Covers LiteLLM/OpenRouter shapes like: - - class name contains ``RateLimit`` - - nested payload ``{"error": {"code": 429}}`` - - nested payload ``{"error": {"type": "rate_limit_error"}}`` - """ - raw = str(exc) - lowered = raw.lower() - if "ratelimit" in type(exc).__name__.lower(): - return True - parsed = _parse_error_payload(raw) - provider_code = _extract_provider_error_code(parsed) - if provider_code == 429: - return True - - provider_error_type = "" - if parsed: - top_type = parsed.get("type") - if isinstance(top_type, str): - provider_error_type = top_type.lower() - nested = parsed.get("error") - if isinstance(nested, dict): - nested_type = nested.get("type") - if isinstance(nested_type, str): - provider_error_type = nested_type.lower() - if provider_error_type == "rate_limit_error": - return True - - return ( - "rate limited" in lowered - or "rate-limited" in lowered - or "temporarily rate-limited upstream" in lowered - ) - - -async def _build_main_agent_for_thread( - agent_factory: Any, - *, - llm: Any, - search_space_id: int, - db_session: Any, - connector_service: ConnectorService, - checkpointer: Any, - user_id: str | None, - thread_id: int | None, - agent_config: AgentConfig | None, - firecrawl_api_key: str | None, - thread_visibility: ChatVisibility | None, - filesystem_selection: FilesystemSelection | None, - disabled_tools: list[str] | None = None, - mentioned_document_ids: list[int] | None = None, -) -> Any: - """Single (re)build path so the agent factory cannot drift across the - initial build and mid-stream 429 recovery for one ``thread_id``: a - graph swap mid-turn would corrupt checkpointer state.""" - return await agent_factory( - llm=llm, - search_space_id=search_space_id, - db_session=db_session, - connector_service=connector_service, - checkpointer=checkpointer, - user_id=user_id, - thread_id=thread_id, - agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, - thread_visibility=thread_visibility, - filesystem_selection=filesystem_selection, - disabled_tools=disabled_tools, - mentioned_document_ids=mentioned_document_ids, - ) - - -def _classify_stream_exception( - exc: Exception, - *, - flow_label: str, -) -> tuple[ - str, str, Literal["info", "warn", "error"], bool, str, dict[str, Any] | None -]: - raw = str(exc) - if isinstance(exc, BusyError) or "Thread is busy with another request" in raw: - busy_thread_id = str(exc.request_id) if isinstance(exc, BusyError) else None - if busy_thread_id and is_cancel_requested(busy_thread_id): - cancel_state = get_cancel_state(busy_thread_id) - attempt = cancel_state[0] if cancel_state else 1 - retry_after_ms = _compute_turn_cancelling_retry_delay(attempt) - retry_after_at = int(time.time() * 1000) + retry_after_ms - return ( - "thread_busy", - "TURN_CANCELLING", - "info", - True, - "A previous response is still stopping. Please try again in a moment.", - { - "retry_after_ms": retry_after_ms, - "retry_after_at": retry_after_at, - }, - ) - return ( - "thread_busy", - "THREAD_BUSY", - "warn", - True, - "Another response is still finishing for this thread. Please try again in a moment.", - None, - ) - - if _is_provider_rate_limited(exc): - return ( - "rate_limited", - "RATE_LIMITED", - "warn", - True, - "This model is temporarily rate-limited. Please try again in a few seconds or switch models.", - None, - ) - - return ( - "server_error", - "SERVER_ERROR", - "error", - False, - f"Error during {flow_label}: {raw}", - None, - ) - - -def _emit_stream_terminal_error( - *, - streaming_service: VercelStreamingService, - flow: str, - request_id: str | None, - thread_id: int, - search_space_id: int, - user_id: str | None, - message: str, - error_kind: str = "server_error", - error_code: str = "SERVER_ERROR", - severity: Literal["info", "warn", "error"] = "error", - is_expected: bool = False, - extra: dict[str, Any] | None = None, -) -> str: - _log_chat_stream_error( - flow=flow, - error_kind=error_kind, - error_code=error_code, - severity=severity, - is_expected=is_expected, - request_id=request_id, - thread_id=thread_id, - search_space_id=search_space_id, - user_id=user_id, - message=message, - extra=extra, - ) - return streaming_service.format_error(message, error_code=error_code, extra=extra) - - -def _legacy_match_lc_id( - pending_tool_call_chunks: list[dict[str, Any]], - tool_name: str, - run_id: str, - lc_tool_call_id_by_run: dict[str, str], -) -> str | None: - """Best-effort match a buffered ``tool_call_chunk`` to a tool name. - - Pure extract of the in-line match used at ``on_tool_start`` when the - chunk path didn't register an index for this call. Pops the next - id-bearing chunk whose ``name`` - matches ``tool_name`` (or any id-bearing chunk as a fallback) and - returns its id. Mutates ``pending_tool_call_chunks`` and - ``lc_tool_call_id_by_run`` in place. - """ - matched_idx: int | None = None - for idx, tcc in enumerate(pending_tool_call_chunks): - if tcc.get("name") == tool_name and tcc.get("id"): - matched_idx = idx - break - if matched_idx is None: - for idx, tcc in enumerate(pending_tool_call_chunks): - if tcc.get("id"): - matched_idx = idx - break - if matched_idx is None: - return None - matched = pending_tool_call_chunks.pop(matched_idx) - candidate = matched.get("id") - if isinstance(candidate, str) and candidate: - if run_id: - lc_tool_call_id_by_run[run_id] = candidate - return candidate - return None - - -async def _stream_agent_events( - agent: Any, - config: dict[str, Any], - input_data: Any, - streaming_service: VercelStreamingService, - result: StreamResult, - step_prefix: str = "thinking", - initial_step_id: str | None = None, - initial_step_title: str = "", - initial_step_items: list[str] | None = None, - *, - fallback_commit_search_space_id: int | None = None, - fallback_commit_created_by_id: str | None = None, - fallback_commit_filesystem_mode: FilesystemMode = FilesystemMode.CLOUD, - fallback_commit_thread_id: int | None = None, - runtime_context: Any = None, - content_builder: Any | None = None, -) -> AsyncGenerator[str, None]: - """Shared async generator that streams and formats astream_events from the agent. - - Yields SSE-formatted strings. After exhausting, inspect the ``result`` - object for accumulated_text and interrupt state. - - Args: - agent: The compiled LangGraph agent. - config: LangGraph config dict (must include configurable.thread_id). - input_data: The input to pass to agent.astream_events (dict or Command). - streaming_service: VercelStreamingService instance for formatting events. - result: Mutable StreamResult populated with accumulated_text / interrupt info. - step_prefix: Prefix for thinking step IDs (e.g. "thinking" or "thinking-resume"). - initial_step_id: If set, the helper inherits an already-active thinking step. - initial_step_title: Title of the inherited thinking step. - initial_step_items: Items of the inherited thinking step. - content_builder: Optional ``AssistantContentBuilder``. When set, every - ``streaming_service.format_*`` yield site also drives the matching - builder lifecycle method (``on_text_*``, ``on_reasoning_*``, - ``on_tool_*``, ``on_thinking_step``, ``on_step_separator``) so the - in-memory ``ContentPart[]`` projection stays in lockstep with what - the FE renders live. Pure in-memory accumulation — no DB I/O — - consumed by the streaming ``finally`` to produce the rich JSONB - persisted via ``finalize_assistant_turn``. ``None`` (the default) - is used by the anonymous / legacy code paths and is a no-op. - - Yields: - SSE-formatted strings for each event. - """ - async for sse in stream_output( - agent=agent, - config=config, - input_data=input_data, - streaming_service=streaming_service, - result=result, - step_prefix=step_prefix, - initial_step_id=initial_step_id, - initial_step_title=initial_step_title, - initial_step_items=initial_step_items, - content_builder=content_builder, - runtime_context=runtime_context, - ): - yield sse - - accumulated_text = result.accumulated_text - - state = await agent.aget_state(config) - state_values = getattr(state, "values", {}) or {} - - # Safety net: if astream_events was cancelled before - # KnowledgeBasePersistenceMiddleware.aafter_agent ran, any staged work - # (dirty_paths / staged_dirs / pending_moves / pending_deletes / - # pending_dir_deletes) will still be in the checkpointed state. Run - # the SAME shared commit helper here so the turn's writes don't get - # lost on client disconnect, then push the delta back into the graph - # using `as_node=...` so reducers fire as if the after_agent hook - # produced it. - if ( - fallback_commit_filesystem_mode == FilesystemMode.CLOUD - and fallback_commit_search_space_id is not None - and ( - (state_values.get("dirty_paths") or []) - or (state_values.get("staged_dirs") or []) - or (state_values.get("pending_moves") or []) - or (state_values.get("pending_deletes") or []) - or (state_values.get("pending_dir_deletes") or []) - ) - ): - try: - delta = await commit_staged_filesystem_state( - state_values, - search_space_id=fallback_commit_search_space_id, - created_by_id=fallback_commit_created_by_id, - filesystem_mode=fallback_commit_filesystem_mode, - thread_id=fallback_commit_thread_id, - dispatch_events=False, - ) - if delta: - await agent.aupdate_state( - config, - delta, - as_node="KnowledgeBasePersistenceMiddleware.after_agent", - ) - except Exception as exc: - _perf_log.warning("[stream_new_chat] safety-net commit failed: %s", exc) - - contract_state = state_values.get("file_operation_contract") or {} - contract_turn_id = contract_state.get("turn_id") - current_turn_id = config.get("configurable", {}).get("turn_id", "") - intent_value = contract_state.get("intent") - if ( - isinstance(intent_value, str) - and intent_value in ("chat_only", "file_write", "file_read") - and contract_turn_id == current_turn_id - ): - result.intent_detected = intent_value - if ( - isinstance(intent_value, str) - and intent_value - in ( - "chat_only", - "file_write", - "file_read", - ) - and contract_turn_id != current_turn_id - ): - # Ignore stale intent contracts from previous turns/checkpoints. - result.intent_detected = "chat_only" - result.intent_confidence = ( - _safe_float(contract_state.get("confidence"), default=0.0) - if contract_turn_id == current_turn_id - else 0.0 - ) - - if result.intent_detected == "file_write": - result.commit_gate_passed, result.commit_gate_reason = ( - _evaluate_file_contract_outcome(result) - ) - if not result.commit_gate_passed and _contract_enforcement_active(result): - gate_notice = ( - "I could not complete the requested file write because no successful " - "write_file/edit_file operation was confirmed." - ) - gate_text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(gate_text_id) - if content_builder is not None: - content_builder.on_text_start(gate_text_id) - yield streaming_service.format_text_delta(gate_text_id, gate_notice) - if content_builder is not None: - content_builder.on_text_delta(gate_text_id, gate_notice) - yield streaming_service.format_text_end(gate_text_id) - if content_builder is not None: - content_builder.on_text_end(gate_text_id) - yield streaming_service.format_terminal_info(gate_notice, "error") - accumulated_text = gate_notice - else: - result.commit_gate_passed = True - result.commit_gate_reason = "" - - result.accumulated_text = accumulated_text - _log_file_contract("turn_outcome", result) - - pending_values = all_interrupt_values(state) - if pending_values: - result.is_interrupted = True - # One frame per paused subagent so each parallel HITL renders its own - # approval card on the wire. Order matches ``state.interrupts``, which - # the resume slicer in ``checkpointed_subagent_middleware.resume_routing`` - # consumes in the same order — keeping emit and resume in lock-step. - for interrupt_value in pending_values: - yield streaming_service.format_interrupt_request(interrupt_value) - - -async def stream_new_chat( - user_query: str, - search_space_id: int, - chat_id: int, - user_id: str | None = None, - llm_config_id: int = -1, - mentioned_document_ids: list[int] | None = None, - mentioned_folder_ids: list[int] | None = None, - mentioned_connector_ids: list[int] | None = None, - mentioned_connectors: list[dict[str, Any]] | None = None, - mentioned_documents: list[dict[str, Any]] | None = None, - checkpoint_id: str | None = None, - needs_history_bootstrap: bool = False, - thread_visibility: ChatVisibility | None = None, - current_user_display_name: str | None = None, - disabled_tools: list[str] | None = None, - filesystem_selection: FilesystemSelection | None = None, - request_id: str | None = None, - user_image_data_urls: list[str] | None = None, - flow: Literal["new", "regenerate"] = "new", -) -> AsyncGenerator[str, None]: - """ - Stream chat responses from the new SurfSense deep agent. - - This uses the Vercel AI SDK Data Stream Protocol (SSE format) for streaming. - The chat_id is used as LangGraph's thread_id for memory/checkpointing. - - The function creates and manages its own database session to guarantee proper - cleanup even when Starlette's middleware cancels the task on client disconnect. - - Args: - user_query: The user's query - search_space_id: The search space ID - chat_id: The chat ID (used as LangGraph thread_id for memory) - user_id: The current user's UUID string (for memory tools and session state) - llm_config_id: The LLM configuration ID (default: -1 for first global config) - needs_history_bootstrap: If True, load message history from DB (for cloned chats) - mentioned_document_ids: Optional list of document IDs mentioned with @ in the chat - mentioned_folder_ids: Optional list of knowledge-base folder IDs mentioned with @ (cloud mode) - checkpoint_id: Optional checkpoint ID to rewind/fork from (for edit/reload operations) - - Yields: - str: SSE formatted response strings - """ - streaming_service = VercelStreamingService() - stream_result = StreamResult() - _t_total = time.perf_counter() - fs_mode = filesystem_selection.mode.value if filesystem_selection else "cloud" - fs_platform = ( - filesystem_selection.client_platform.value if filesystem_selection else "web" - ) - stream_result.request_id = request_id - stream_result.turn_id = f"{chat_id}:{int(time.time() * 1000)}" - stream_result.filesystem_mode = fs_mode - stream_result.client_platform = fs_platform - chat_agent_mode = "unknown" - chat_outcome = "success" - chat_error_category: str | None = None - chat_span_cm = ot.chat_request_span( - chat_id=chat_id, - search_space_id=search_space_id, - flow=flow, - request_id=request_id, - turn_id=stream_result.turn_id, - filesystem_mode=fs_mode, - client_platform=fs_platform, - agent_mode=chat_agent_mode, - ) - chat_span = chat_span_cm.__enter__() - _log_file_contract("turn_start", stream_result) - _perf_log.info( - "[stream_new_chat] filesystem_mode=%s client_platform=%s", - fs_mode, - fs_platform, - ) - log_system_snapshot("stream_new_chat_START") - - from app.services.token_tracking_service import start_turn - - accumulator = start_turn() - - # Premium credit (USD micro-units) tracking state. Stores the - # amount reserved up front so we can release it on cancellation - # and finalize-debit the actual provider cost reported by LiteLLM. - _premium_reserved_micros = 0 - _premium_request_id: str | None = None - - # ``BusyError`` fires before the lock is acquired; the ``finally`` must - # not release the in-flight caller's lock. - _busy_error_raised = False - - _emit_stream_error = partial( - _emit_stream_terminal_error, - streaming_service=streaming_service, - flow=flow, - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - ) - - session = async_session_maker() - try: - # Mark AI as responding to this user for live collaboration - if user_id: - await set_ai_responding(session, chat_id, UUID(user_id)) - # Load LLM config - supports both YAML (negative IDs) and database (positive IDs) - agent_config: AgentConfig | None = None - requested_llm_config_id = llm_config_id - - async def _load_llm_bundle( - config_id: int, - ) -> tuple[Any, AgentConfig | None, str | None]: - if config_id >= 0: - loaded_agent_config = await load_agent_config( - session=session, - config_id=config_id, - search_space_id=search_space_id, - ) - if not loaded_agent_config: - return ( - None, - None, - f"Failed to load NewLLMConfig with id {config_id}", - ) - return ( - create_chat_litellm_from_agent_config(loaded_agent_config), - loaded_agent_config, - None, - ) - - loaded_llm_config = load_global_llm_config_by_id(config_id) - if not loaded_llm_config: - return None, None, f"Failed to load LLM config with id {config_id}" - return ( - create_chat_litellm_from_config(loaded_llm_config), - AgentConfig.from_yaml_config(loaded_llm_config), - None, - ) - - _t0 = time.perf_counter() - # Image-bearing turns force the Auto-pin resolver to filter the - # candidate pool to vision-capable cfgs (and force-repin a - # text-only existing pin). For explicit selections this flag is - # a no-op — the resolver returns the user's chosen id unchanged. - _requires_image_input = bool(user_image_data_urls) - try: - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=llm_config_id, - requires_image_input=_requires_image_input, - ) - ).resolved_llm_config_id - ot.add_event( - "model.pin.resolved", - { - "pin.requested_id": requested_llm_config_id, - "pin.resolved_id": llm_config_id, - "pin.requires_image_input": _requires_image_input, - }, - ) - except ValueError as pin_error: - # Auto-pin's "no vision-capable cfg" path raises a ValueError - # whose message we map to the friendly image-input SSE error - # so the user sees the same message regardless of whether - # the gate fired in Auto-mode or in the agent_config check - # below. - error_code = ( - "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT" - if _requires_image_input and "vision-capable" in str(pin_error) - else "SERVER_ERROR" - ) - error_kind = ( - "user_error" - if error_code == "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT" - else "server_error" - ) - if error_code == "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT": - ot.add_event( - "quota.denied", - { - "quota.code": error_code, - }, - ) - yield _emit_stream_error( - message=str(pin_error), - error_kind=error_kind, - error_code=error_code, - ) - yield streaming_service.format_done() - return - - llm, agent_config, llm_load_error = await _load_llm_bundle(llm_config_id) - if llm_load_error: - yield _emit_stream_error( - message=llm_load_error, - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - _perf_log.info( - "[stream_new_chat] LLM config loaded in %.3fs (config_id=%s)", - time.perf_counter() - _t0, - llm_config_id, - ) - - # Capability safety net: a turn carrying user-uploaded images - # cannot be routed to a chat config that LiteLLM's authoritative - # model map *explicitly* marks as text-only (``supports_vision`` - # set to False). The check is intentionally narrow — it only - # fires when LiteLLM is *certain* the model can't accept image - # input. Unknown / unmapped / vision-capable models pass - # through. Without this guard a known-text-only model would 404 - # at the provider with ``"No endpoints found that support image - # input"``, surfacing as an opaque ``SERVER_ERROR`` SSE chunk; - # failing here lets us return a friendly message that tells the - # user what to change. - if user_image_data_urls and agent_config is not None: - from app.services.provider_capabilities import ( - is_known_text_only_chat_model, - ) - - agent_litellm_params = agent_config.litellm_params or {} - agent_base_model = ( - agent_litellm_params.get("base_model") - if isinstance(agent_litellm_params, dict) - else None - ) - if is_known_text_only_chat_model( - provider=agent_config.provider, - model_name=agent_config.model_name, - base_model=agent_base_model, - custom_provider=agent_config.custom_provider, - ): - model_label = ( - agent_config.config_name or agent_config.model_name or "model" - ) - ot.add_event( - "quota.denied", - { - "quota.code": "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT", - }, - ) - yield _emit_stream_error( - message=( - f"The selected model ({model_label}) does not support " - "image input. Switch to a vision-capable model " - "(e.g. GPT-4o, Claude, Gemini) or remove the image " - "attachment and try again." - ), - error_kind="user_error", - error_code="MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT", - ) - yield streaming_service.format_done() - return - - # Premium quota reservation for pinned premium model only. - _needs_premium_quota = ( - agent_config is not None and user_id and agent_config.is_premium - ) - if _needs_premium_quota: - import uuid as _uuid - - from app.services.token_quota_service import ( - TokenQuotaService, - estimate_call_reserve_micros, - ) - - _premium_request_id = _uuid.uuid4().hex[:16] - _agent_litellm_params = agent_config.litellm_params or {} - _agent_base_model = ( - _agent_litellm_params.get("base_model") or agent_config.model_name or "" - ) - reserve_amount_micros = estimate_call_reserve_micros( - base_model=_agent_base_model, - quota_reserve_tokens=agent_config.quota_reserve_tokens, - ) - async with shielded_async_session() as quota_session: - quota_result = await TokenQuotaService.premium_reserve( - db_session=quota_session, - user_id=UUID(user_id), - request_id=_premium_request_id, - reserve_micros=reserve_amount_micros, - ) - _premium_reserved_micros = reserve_amount_micros - if not quota_result.allowed: - ot.add_event( - "quota.denied", - { - "quota.code": "PREMIUM_QUOTA_EXHAUSTED", - }, - ) - if requested_llm_config_id == 0: - try: - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=0, - force_repin_free=True, - requires_image_input=_requires_image_input, - ) - ).resolved_llm_config_id - ot.add_event( - "model.repin", - { - "repin.reason": "premium_quota_exhausted", - "repin.to_config_id": llm_config_id, - }, - ) - except ValueError as pin_error: - yield _emit_stream_error( - message=str(pin_error), - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - - llm, agent_config, llm_load_error = await _load_llm_bundle( - llm_config_id - ) - if llm_load_error: - yield _emit_stream_error( - message=llm_load_error, - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - _premium_request_id = None - _premium_reserved_micros = 0 - _log_chat_stream_error( - flow=flow, - error_kind="premium_quota_exhausted", - error_code="PREMIUM_QUOTA_EXHAUSTED", - severity="info", - is_expected=True, - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - message=( - "Premium quota exhausted on pinned model; auto-fallback switched to a free model" - ), - extra={ - "fallback_config_id": llm_config_id, - "auto_fallback": True, - }, - ) - else: - yield _emit_stream_error( - message=( - "Buy more tokens to continue with this model, or switch to a free model" - ), - error_kind="premium_quota_exhausted", - error_code="PREMIUM_QUOTA_EXHAUSTED", - severity="info", - is_expected=True, - extra={ - "resolved_config_id": llm_config_id, - "auto_fallback": False, - }, - ) - yield streaming_service.format_done() - return - - if not llm: - yield _emit_stream_error( - message="Failed to create LLM instance", - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - - # Create connector service - _t0 = time.perf_counter() - connector_service = ConnectorService(session, search_space_id=search_space_id) - - firecrawl_api_key = None - webcrawler_connector = await connector_service.get_connector_by_type( - SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id - ) - if webcrawler_connector and webcrawler_connector.config: - firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") - _perf_log.info( - "[stream_new_chat] Connector service + firecrawl key in %.3fs", - time.perf_counter() - _t0, - ) - - # Get the PostgreSQL checkpointer for persistent conversation memory - _t0 = time.perf_counter() - checkpointer = await get_checkpointer() - _perf_log.info( - "[stream_new_chat] Checkpointer ready in %.3fs", time.perf_counter() - _t0 - ) - - visibility = thread_visibility or ChatVisibility.PRIVATE - from app.config import config as _app_config - - use_multi_agent = bool(_app_config.MULTI_AGENT_CHAT_ENABLED) - chat_agent_mode = "multi" if use_multi_agent else "single" - with contextlib.suppress(Exception): - chat_span.set_attribute("agent.mode", chat_agent_mode) - - _t0 = time.perf_counter() - agent_factory = ( - create_multi_agent_chat_deep_agent - if use_multi_agent - else create_surfsense_deep_agent - ) - # Build the agent inline. Provider 429s surface through the - # in-stream recovery loop below (``_is_provider_rate_limited``), - # which repins the thread to an eligible alternative config and - # rebuilds the agent before the user sees any output. - agent = await _build_main_agent_for_thread( - agent_factory, - llm=llm, - search_space_id=search_space_id, - db_session=session, - connector_service=connector_service, - checkpointer=checkpointer, - user_id=user_id, - thread_id=chat_id, - agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, - thread_visibility=visibility, - filesystem_selection=filesystem_selection, - disabled_tools=disabled_tools, - mentioned_document_ids=mentioned_document_ids, - ) - _perf_log.info( - "[stream_new_chat] Agent created in %.3fs", time.perf_counter() - _t0 - ) - - # Build input with message history - langchain_messages = [] - - _t0 = time.perf_counter() - # Bootstrap history for cloned chats (no LangGraph checkpoint exists yet) - if needs_history_bootstrap: - langchain_messages = await bootstrap_history_from_db( - session, chat_id, thread_visibility=visibility - ) - - thread_result = await session.execute( - select(NewChatThread).filter(NewChatThread.id == chat_id) - ) - thread = thread_result.scalars().first() - if thread: - thread.needs_history_bootstrap = False - await session.commit() - - # Mentioned KB documents are now handled by KnowledgeBaseSearchMiddleware - # which merges them into the scoped filesystem with full document - # structure. Only report context is inlined here. - - # Fetch the most recent report(s) in this thread so the LLM can - # easily find report_id for versioning decisions, instead of - # having to dig through conversation history. - recent_reports_result = await session.execute( - select(Report) - .filter( - Report.thread_id == chat_id, - Report.content.isnot(None), # exclude failed reports - ) - .order_by(Report.id.desc()) - .limit(3) - ) - recent_reports = list(recent_reports_result.scalars().all()) - - # Resolve @-mention chips to canonical virtual paths and rewrite - # the user-typed text so the LLM sees ``\`/documents/...\``` instead - # of bare ``@title``. The substitution lands in ``agent_user_query`` - # ONLY — the original ``user_query`` (with ``@title`` tokens) flows - # untouched into ``persist_user_turn`` below so chip rendering on - # reload still works (``UserTextPart`` → ``parseMentionSegments`` - # matches ``@title``, not ``\`/documents/...\```). It also feeds - # the human-readable surfaces — SSE "Processing X" status, auto - # thread title, memory seed — which all want what the user typed. - # See ``persistence._build_user_content``. - # - # Cloud mode only: local-folder mode keeps the legacy - # ``@title`` text path; mention support there is a follow-up - # task because the path scheme (mount-rooted) and the picker - # UI both need separate work. - agent_user_query = user_query - accepted_folder_ids: list[int] = [] - if fs_mode == FilesystemMode.CLOUD.value and ( - mentioned_document_ids or mentioned_folder_ids or mentioned_documents - ): - from app.schemas.new_chat import ( - MentionedDocumentInfo as _MentionedDocumentInfo, - ) - - chip_objs: list[_MentionedDocumentInfo] | None = None - if mentioned_documents: - chip_objs = [] - for raw in mentioned_documents: - if isinstance(raw, _MentionedDocumentInfo): - chip_objs.append(raw) - continue - try: - chip_objs.append(_MentionedDocumentInfo.model_validate(raw)) - except Exception: - logger.debug( - "stream_new_chat: dropping malformed mention chip %r", - raw, - ) - - resolved = await resolve_mentions( - session, - search_space_id=search_space_id, - mentioned_documents=chip_objs, - mentioned_document_ids=mentioned_document_ids, - mentioned_folder_ids=mentioned_folder_ids, - ) - agent_user_query = substitute_in_text(user_query, resolved.token_to_path) - accepted_folder_ids = resolved.mentioned_folder_ids - - # Format the user query with context (reports only). - # Uses ``agent_user_query`` so the LLM sees backtick-wrapped paths - # instead of bare ``@title`` tokens. - final_query = agent_user_query - context_parts = [] - - if mentioned_connectors: - connector_lines = [] - for connector in mentioned_connectors: - if not isinstance(connector, dict): - continue - connector_id = connector.get("id") - connector_type = connector.get("connector_type") or connector.get( - "document_type" - ) - account_name = connector.get("account_name") or connector.get("title") - if connector_id is None or connector_type is None: - continue - connector_lines.append( - f' - connector_id={connector_id}, connector_type="{connector_type}", ' - f'account_name="{account_name or ""}"' - ) - if connector_lines: - context_parts.append( - "\n" - "The user selected these exact connector accounts with @. " - "These entries are selection metadata, not retrieved connector content. " - "When a connector-backed tool needs an account, use the matching " - "connector_id from this list if the tool supports connector_id:\n" - + "\n".join(connector_lines) - + "\n" - ) - - # Surface report IDs prominently so the LLM doesn't have to - # retrieve them from old tool responses in conversation history. - if recent_reports: - report_lines = [] - for r in recent_reports: - report_lines.append( - f' - report_id={r.id}, title="{r.title}", ' - f'style="{r.report_style or "detailed"}"' - ) - reports_listing = "\n".join(report_lines) - context_parts.append( - "\n" - "Previously generated reports in this conversation:\n" - f"{reports_listing}\n\n" - "If the user wants to MODIFY, REVISE, UPDATE, or ADD to one of " - "these reports, set parent_report_id to the relevant report_id above.\n" - "If the user wants a completely NEW report on a different topic, " - "leave parent_report_id unset.\n" - "" - ) - - if context_parts: - context = "\n\n".join(context_parts) - final_query = f"{context}\n\n{agent_user_query}" - - if visibility == ChatVisibility.SEARCH_SPACE and current_user_display_name: - final_query = f"**[{current_user_display_name}]:** {final_query}" - - # if messages: - # # Convert frontend messages to LangChain format - # for msg in messages: - # if msg.role == "user": - # langchain_messages.append(HumanMessage(content=msg.content)) - # elif msg.role == "assistant": - # langchain_messages.append(AIMessage(content=msg.content)) - # else: - human_content = build_human_message_content( - final_query, list(user_image_data_urls or ()) - ) - langchain_messages.append(HumanMessage(content=human_content)) - - input_state = { - # Lets not pass this message atm because we are using the checkpointer to manage the conversation history - # We will use this to simulate group chat functionality in the future - "messages": langchain_messages, - "search_space_id": search_space_id, - "request_id": request_id or "unknown", - "turn_id": stream_result.turn_id, - } - - _perf_log.info( - "[stream_new_chat] History bootstrap + doc/report queries in %.3fs", - time.perf_counter() - _t0, - ) - - # All pre-streaming DB reads are done. Commit to release the - # transaction and its ACCESS SHARE locks so we don't block DDL - # (e.g. migrations) for the entire duration of LLM streaming. - # Tools that need DB access during streaming will start their own - # short-lived transactions (or use isolated sessions). - await session.commit() - - # Detach heavy ORM objects (documents with chunks, reports, etc.) - # from the session identity map now that we've extracted the data - # we need. This prevents them from accumulating in memory for the - # entire duration of LLM streaming (which can be several minutes). - session.expunge_all() - - _perf_log.info( - "[stream_new_chat] Total pre-stream setup in %.3fs (chat_id=%s)", - time.perf_counter() - _t_total, - chat_id, - ) - - # Configure LangGraph with thread_id for memory - # If checkpoint_id is provided, fork from that checkpoint (for edit/reload) - configurable = {"thread_id": str(chat_id)} - configurable["request_id"] = request_id or "unknown" - configurable["turn_id"] = stream_result.turn_id - if checkpoint_id: - configurable["checkpoint_id"] = checkpoint_id - - config = { - "configurable": configurable, - # Effectively uncapped, matching the agent-level - # ``with_config`` default in ``chat_deepagent.create_agent`` - # and the unbounded ``while(true)`` loop used by OpenCode's - # ``session/processor.ts``. Real circuit-breakers live in - # middleware: ``DoomLoopMiddleware`` (sliding-window tool - # signature check), plus ``enable_tool_call_limit`` / - # ``enable_model_call_limit`` when those flags are set. The - # original LangGraph default of 25 (and our previous 80 - # bump) hit users on legitimate multi-tool plans. - "recursion_limit": 10_000, - } - - # Start the message stream - yield streaming_service.format_message_start() - yield streaming_service.format_start_step() - - # Surface the per-turn correlation id at the very start of the - # stream so the frontend can stamp it onto the in-flight - # assistant message and replay it via ``appendMessage`` - # for durable storage. Tool/action-log events DO carry it later, - # but pure-text turns never produce action-log events; this - # event guarantees the frontend learns the turn id regardless. - yield streaming_service.format_data( - "turn-info", - {"chat_turn_id": stream_result.turn_id}, - ) - yield streaming_service.format_data("turn-status", {"status": "busy"}) - - # Persist the user-side row for this turn before any expensive - # work runs. Closes the "ghost-thread" abuse vector - # (authenticated client hits POST /new_chat then never calls - # /messages — empty new_chat_messages, free LLM completion). - # Idempotent against the unique index in migration 141 so the - # legacy frontend appendMessage call is a no-op on the second - # writer. Hard failure aborts the turn so we never produce a - # title or assistant row that isn't anchored to a persisted - # user message. - from app.tasks.chat.content_builder import AssistantContentBuilder - from app.tasks.chat.persistence import ( - persist_assistant_shell, - persist_user_turn, - ) - - user_message_id = await persist_user_turn( - chat_id=chat_id, - user_id=user_id, - turn_id=stream_result.turn_id, - user_query=user_query, - user_image_data_urls=user_image_data_urls, - mentioned_documents=mentioned_documents, - ) - if user_message_id is None: - yield _emit_stream_error( - message=( - "We couldn't save your message. Please try again in a moment." - ), - error_kind="server_error", - error_code="MESSAGE_PERSIST_FAILED", - ) - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Emit canonical user message id BEFORE any LLM streaming so the - # FE can rename its optimistic ``msg-user-XXX`` placeholder to - # ``msg-{user_message_id}`` and unlock features gated on a real - # DB id (comments, edit-from-this-message). See B4 in - # ``sse-based_message_id_handshake`` plan. - yield streaming_service.format_data( - "user-message-id", - {"message_id": user_message_id, "turn_id": stream_result.turn_id}, - ) - - # Pre-write the assistant row for this turn so we have a stable - # ``message_id`` to anchor mid-stream metadata (token_usage, - # future agent_action_log.message_id correlation) and a - # write-once UPDATE target at finalize time. Idempotent against - # the (thread_id, turn_id, ASSISTANT) partial unique index from - # migration 141 — if the legacy frontend appendMessage races - # this, we recover the existing row's id. - assistant_message_id = await persist_assistant_shell( - chat_id=chat_id, - user_id=user_id, - turn_id=stream_result.turn_id, - ) - if assistant_message_id is None: - # Genuine DB failure — abort the turn rather than stream - # into a void. The user row is already persisted so the - # legacy "ghost-thread" gate isn't reopened. - yield _emit_stream_error( - message=( - "We couldn't initialize the assistant message. Please try again." - ), - error_kind="server_error", - error_code="MESSAGE_PERSIST_FAILED", - ) - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Emit canonical assistant message id BEFORE any LLM streaming - # so the FE can rename its optimistic ``msg-assistant-XXX`` - # placeholder to ``msg-{assistant_message_id}`` and bind - # ``tokenUsageStore`` / ``pendingInterrupt`` to the real id - # immediately. See B4 in ``sse-based_message_id_handshake`` - # plan. - yield streaming_service.format_data( - "assistant-message-id", - {"message_id": assistant_message_id, "turn_id": stream_result.turn_id}, - ) - - stream_result.assistant_message_id = assistant_message_id - stream_result.content_builder = AssistantContentBuilder() - - # Initial thinking step - analyzing the request - initial_title = "Understanding your request" - action_verb = "Processing" - - processing_parts = [] - if user_query.strip(): - query_text = user_query[:80] + ("..." if len(user_query) > 80 else "") - processing_parts.append(query_text) - elif user_image_data_urls: - processing_parts.append(f"[{len(user_image_data_urls)} image(s)]") - else: - processing_parts.append("(message)") - - initial_items = [f"{action_verb}: {' '.join(processing_parts)}"] - initial_step_id = "thinking-1" - - # Drive the builder for this initial thinking step too — the - # ``_emit_thinking_step`` helper lives inside ``_stream_agent_events`` - # so it isn't in scope here, but the FE folds this step into - # the same singleton ``data-thinking-steps`` part as everything - # the agent stream emits later. Mirror that fold server-side. - if stream_result.content_builder is not None: - stream_result.content_builder.on_thinking_step( - initial_step_id, initial_title, "in_progress", initial_items - ) - yield streaming_service.format_thinking_step( - step_id=initial_step_id, - title=initial_title, - status="in_progress", - items=initial_items, - ) - - # These ORM objects can be large. They're only needed to build context - # strings already copied into final_query / langchain_messages — - # release them before streaming. - del recent_reports - del langchain_messages, final_query - - # Check if this is the first assistant response so we can generate - # a title in parallel with the agent stream (better UX than waiting - # until after the full response). - # Use a LIMIT 1 EXISTS-style probe rather than COUNT(*) because - # this is now a hot path executed on every turn, and COUNT scales - # with thread length (server-side persistence can grow rows - # quickly under power users). - # - # IMPORTANT: ``persist_assistant_shell`` above (line ~3112) already - # inserted THIS turn's assistant row. We must therefore exclude - # it from the probe — otherwise the gate fires on every turn - # except the very first, and title generation never runs for new - # threads. Excluding by primary key (``id != assistant_message_id``) - # is bulletproof regardless of ``turn_id`` shape (legacy NULLs, - # resume turns, etc.). - first_assistant_probe = await session.execute( - select(NewChatMessage.id) - .filter( - NewChatMessage.thread_id == chat_id, - NewChatMessage.role == "assistant", - NewChatMessage.id != assistant_message_id, - ) - .limit(1) - ) - is_first_response = first_assistant_probe.scalars().first() is None - - title_task: asyncio.Task[tuple[str | None, dict | None]] | None = None - # Gate title generation on a persisted user message so a stream - # that fails before persistence (we abort above) can never leave - # behind a thread with a generated title and no anchoring rows. - if is_first_response and user_message_id is not None: - - async def _generate_title() -> tuple[str | None, dict | None]: - """Generate a short title via litellm.acompletion. - - Returns (title, usage_dict). Usage is extracted directly from - the response object because litellm fires its async callback - via fire-and-forget ``create_task``, so the - ``TokenTrackingCallback`` would run too late. We also blank - the accumulator in this child-task context so the late callback - doesn't double-count. - """ - try: - from litellm import acompletion - - from app.services.llm_router_service import LLMRouterService - from app.services.provider_api_base import resolve_api_base - from app.services.token_tracking_service import _turn_accumulator - - _turn_accumulator.set(None) - - title_seed = user_query.strip() or ( - f"[{len(user_image_data_urls or [])} image(s)]" - if user_image_data_urls - else "" - ) - prompt = TITLE_GENERATION_PROMPT.replace( - "{user_query}", title_seed[:500] or "(message)" - ) - messages = [{"role": "user", "content": prompt}] - - if getattr(llm, "model", None) == "auto": - router = LLMRouterService.get_router() - response = await router.acompletion( - model="auto", messages=messages - ) - else: - # Apply the same ``api_base`` cascade chat / vision / - # image-gen call sites use so we never inherit - # ``litellm.api_base`` (commonly set by - # ``AZURE_OPENAI_ENDPOINT``) when the chat config - # itself ships an empty ``api_base``. Without this - # the title-gen on an OpenRouter chat config would - # 404 against the inherited Azure endpoint — see - # ``provider_api_base`` docstring for the same - # bug repro on the image-gen / vision paths. - raw_model = getattr(llm, "model", "") or "" - provider_prefix = ( - raw_model.split("/", 1)[0] if "/" in raw_model else None - ) - provider_value = ( - agent_config.provider if agent_config is not None else None - ) - title_api_base = resolve_api_base( - provider=provider_value, - provider_prefix=provider_prefix, - config_api_base=getattr(llm, "api_base", None), - ) - response = await acompletion( - model=raw_model, - messages=messages, - api_key=getattr(llm, "api_key", None), - api_base=title_api_base, - ) - - usage_info = None - usage = getattr(response, "usage", None) - if usage: - raw_model = getattr(llm, "model", "") or "" - model_name = ( - raw_model.split("/", 1)[-1] - if "/" in raw_model - else (raw_model or response.model or "unknown") - ) - usage_info = { - "model": model_name, - "prompt_tokens": getattr(usage, "prompt_tokens", 0) or 0, - "completion_tokens": getattr(usage, "completion_tokens", 0) - or 0, - "total_tokens": getattr(usage, "total_tokens", 0) or 0, - } - - raw_title = response.choices[0].message.content.strip() - if raw_title and len(raw_title) <= 100: - return raw_title.strip("\"'"), usage_info - return None, usage_info - except Exception: - logging.getLogger(__name__).exception( - "[TitleGen] _generate_title failed" - ) - return None, None - - title_task = asyncio.create_task(_generate_title()) - - title_emitted = False - - # Build the per-invocation runtime context (Phase 1.5). - # ``mentioned_document_ids`` is read by ``KnowledgePriorityMiddleware`` - # via ``runtime.context.mentioned_document_ids`` instead of its - # ``__init__`` closure — that way the same compiled-agent instance - # can serve multiple turns with different mention lists. - runtime_context = SurfSenseContextSchema( - search_space_id=search_space_id, - mentioned_document_ids=list(mentioned_document_ids or []), - mentioned_folder_ids=list( - accepted_folder_ids or mentioned_folder_ids or [] - ), - mentioned_connector_ids=list(mentioned_connector_ids or []), - mentioned_connectors=list(mentioned_connectors or []), - request_id=request_id, - turn_id=stream_result.turn_id, - ) - - _t_stream_start = time.perf_counter() - _first_event_logged = False - runtime_rate_limit_recovered = False - while True: - try: - async for sse in _stream_agent_events( - agent=agent, - config=config, - input_data=input_state, - streaming_service=streaming_service, - result=stream_result, - step_prefix="thinking", - initial_step_id=initial_step_id, - initial_step_title=initial_title, - initial_step_items=initial_items, - fallback_commit_search_space_id=search_space_id, - fallback_commit_created_by_id=user_id, - fallback_commit_filesystem_mode=( - filesystem_selection.mode - if filesystem_selection - else FilesystemMode.CLOUD - ), - fallback_commit_thread_id=chat_id, - runtime_context=runtime_context, - content_builder=stream_result.content_builder, - ): - if not _first_event_logged: - _perf_log.info( - "[stream_new_chat] First agent event in %.3fs (time since stream start), " - "%.3fs (total since request start) (chat_id=%s)", - time.perf_counter() - _t_stream_start, - time.perf_counter() - _t_total, - chat_id, - ) - _first_event_logged = True - yield sse - - # Inject title update mid-stream as soon as the background - # task finishes. - if ( - title_task is not None - and title_task.done() - and not title_emitted - ): - generated_title, title_usage = title_task.result() - if title_usage: - accumulator.add(**title_usage) - if generated_title: - async with shielded_async_session() as title_session: - title_thread_result = await title_session.execute( - select(NewChatThread).filter( - NewChatThread.id == chat_id - ) - ) - title_thread = title_thread_result.scalars().first() - if title_thread: - title_thread.title = generated_title - await title_session.commit() - yield streaming_service.format_thread_title_update( - chat_id, generated_title - ) - title_emitted = True - break - except Exception as stream_exc: - can_runtime_recover = ( - not runtime_rate_limit_recovered - and requested_llm_config_id == 0 - and llm_config_id < 0 - and not _first_event_logged - and _is_provider_rate_limited(stream_exc) - ) - if not can_runtime_recover: - raise - - runtime_rate_limit_recovered = True - previous_config_id = llm_config_id - # The failed attempt may still hold the per-thread busy mutex - # (middleware teardown can lag behind raised provider errors). - # Force release before we retry within the same request. - end_turn(str(chat_id)) - mark_runtime_cooldown( - previous_config_id, - reason="provider_rate_limited", - ) - - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=0, - exclude_config_ids={previous_config_id}, - requires_image_input=_requires_image_input, - ) - ).resolved_llm_config_id - - llm, agent_config, llm_load_error = await _load_llm_bundle( - llm_config_id - ) - if llm_load_error: - raise stream_exc - - # Title generation uses the initial llm object. After a runtime - # repin we keep the stream focused on response recovery and skip - # title generation for this turn. - if title_task is not None and not title_task.done(): - title_task.cancel() - title_task = None - - _t0 = time.perf_counter() - agent = await _build_main_agent_for_thread( - agent_factory, - llm=llm, - search_space_id=search_space_id, - db_session=session, - connector_service=connector_service, - checkpointer=checkpointer, - user_id=user_id, - thread_id=chat_id, - agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, - thread_visibility=visibility, - filesystem_selection=filesystem_selection, - disabled_tools=disabled_tools, - mentioned_document_ids=mentioned_document_ids, - ) - _perf_log.info( - "[stream_new_chat] Runtime rate-limit recovery repinned " - "config_id=%s -> %s and rebuilt agent in %.3fs", - previous_config_id, - llm_config_id, - time.perf_counter() - _t0, - ) - ot.add_event( - "chat.rate_limit.recovered", - { - "recovery.reason": "provider_rate_limited", - "recovery.previous_config_id": previous_config_id, - "recovery.fallback_config_id": llm_config_id, - }, - ) - _log_chat_stream_error( - flow=flow, - error_kind="rate_limited", - error_code="RATE_LIMITED", - severity="info", - is_expected=True, - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - message=( - "Auto-pinned model hit runtime rate limit; switched to " - "another eligible model and retried." - ), - extra={ - "auto_runtime_recover": True, - "previous_config_id": previous_config_id, - "fallback_config_id": llm_config_id, - }, - ) - continue - - _perf_log.info( - "[stream_new_chat] Agent stream completed in %.3fs (chat_id=%s)", - time.perf_counter() - _t_stream_start, - chat_id, - ) - log_system_snapshot("stream_new_chat_END") - - if stream_result.is_interrupted: - ot.add_event( - "chat.interrupted", - { - "chat.flow": flow, - }, - ) - if title_task is not None and not title_task.done(): - title_task.cancel() - - usage_summary = accumulator.per_message_summary() - _perf_log.info( - "[token_usage] interrupted new_chat: calls=%d total=%d cost_micros=%d summary=%s", - len(accumulator.calls), - accumulator.grand_total, - accumulator.total_cost_micros, - usage_summary, - ) - if usage_summary: - yield streaming_service.format_data( - "token-usage", - { - "usage": usage_summary, - "prompt_tokens": accumulator.total_prompt_tokens, - "completion_tokens": accumulator.total_completion_tokens, - "total_tokens": accumulator.grand_total, - "cost_micros": accumulator.total_cost_micros, - "call_details": accumulator.serialized_calls(), - }, - ) - - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # If the title task didn't finish during streaming, await it now - if title_task is not None and not title_emitted: - generated_title, title_usage = await title_task - if title_usage: - accumulator.add(**title_usage) - if generated_title: - async with shielded_async_session() as title_session: - title_thread_result = await title_session.execute( - select(NewChatThread).filter(NewChatThread.id == chat_id) - ) - title_thread = title_thread_result.scalars().first() - if title_thread: - title_thread.title = generated_title - await title_session.commit() - yield streaming_service.format_thread_title_update( - chat_id, generated_title - ) - - # Finalize premium credit debit with the actual provider cost - # reported by LiteLLM, summed across every call in the turn. - # Mirrors the pre-cost behaviour of "premium turn → all calls - # count" so free sub-agent calls during a premium turn still - # contribute to the bill (they're $0 in practice anyway). - if _premium_request_id and user_id: - try: - from app.services.token_quota_service import TokenQuotaService - - async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_finalize( - db_session=quota_session, - user_id=UUID(user_id), - request_id=_premium_request_id, - actual_micros=accumulator.total_cost_micros, - reserved_micros=_premium_reserved_micros, - ) - _premium_request_id = None - _premium_reserved_micros = 0 - except Exception: - logging.getLogger(__name__).warning( - "Failed to finalize premium quota for user %s", - user_id, - exc_info=True, - ) - - usage_summary = accumulator.per_message_summary() - _perf_log.info( - "[token_usage] normal new_chat: calls=%d total=%d cost_micros=%d summary=%s", - len(accumulator.calls), - accumulator.grand_total, - accumulator.total_cost_micros, - usage_summary, - ) - if usage_summary: - yield streaming_service.format_data( - "token-usage", - { - "usage": usage_summary, - "prompt_tokens": accumulator.total_prompt_tokens, - "completion_tokens": accumulator.total_completion_tokens, - "total_tokens": accumulator.grand_total, - "cost_micros": accumulator.total_cost_micros, - "call_details": accumulator.serialized_calls(), - }, - ) - - # Finish the step and message - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - # Handle any errors - import traceback - - # ``BusyError`` fires before the agent acquires the lock; the - # cleanup path must skip lock release to avoid freeing the - # in-flight caller's lock. Classification is handled below. - if isinstance(e, BusyError): - _busy_error_raised = True - - ( - error_kind, - error_code, - severity, - is_expected, - user_message, - error_extra, - ) = _classify_stream_exception(e, flow_label="chat") - chat_outcome = error_code or error_kind or "error" - chat_error_category = ot_metrics.categorize_exception(e) - with contextlib.suppress(Exception): - chat_span.set_attribute("chat.outcome", chat_outcome) - chat_span.set_attribute("error.category", chat_error_category) - ot.record_error(chat_span, e) - error_message = f"Error during chat: {e!s}" - print(f"[stream_new_chat] {error_message}") - print(f"[stream_new_chat] Exception type: {type(e).__name__}") - print(f"[stream_new_chat] Traceback:\n{traceback.format_exc()}") - if error_code == "TURN_CANCELLING": - status_payload: dict[str, Any] = {"status": "cancelling"} - if error_extra: - status_payload.update(error_extra) - yield streaming_service.format_data("turn-status", status_payload) - else: - yield streaming_service.format_data("turn-status", {"status": "busy"}) - - yield _emit_stream_error( - message=user_message, - error_kind=error_kind, - error_code=error_code, - severity=severity, - is_expected=is_expected, - extra=error_extra, - ) - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - - finally: - # Shield the ENTIRE async cleanup from anyio cancel-scope - # cancellation. Starlette's BaseHTTPMiddleware uses anyio task - # groups; on client disconnect, it cancels the scope with - # level-triggered cancellation — every unshielded `await` inside - # the cancelled scope raises CancelledError immediately. Without - # this shield the very first `await` (session.rollback) would - # raise CancelledError, `except Exception` wouldn't catch it - # (CancelledError is a BaseException), and the rest of the - # finally block — including session.close() — would never run. - with anyio.CancelScope(shield=True): - # Authoritative fallback cleanup for lock/cancel state. Middleware - # teardown can be skipped on some client-abort paths. - end_turn(str(chat_id)) - - # Release premium reservation if not finalized - if _premium_request_id and _premium_reserved_micros > 0 and user_id: - try: - from app.services.token_quota_service import TokenQuotaService - - async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_release( - db_session=quota_session, - user_id=UUID(user_id), - reserved_micros=_premium_reserved_micros, - ) - _premium_reserved_micros = 0 - except Exception: - logging.getLogger(__name__).warning( - "Failed to release premium quota for user %s", user_id - ) - - try: - await session.rollback() - await clear_ai_responding(session, chat_id) - except Exception: - try: - async with shielded_async_session() as fresh_session: - await clear_ai_responding(fresh_session, chat_id) - except Exception: - logging.getLogger(__name__).warning( - "Failed to clear AI responding state for thread %s", chat_id - ) - - with contextlib.suppress(Exception): - session.expunge_all() - - with contextlib.suppress(Exception): - await session.close() - - # Server-side assistant-message + token_usage finalization. - # Runs after the main session has been closed (uses its own - # shielded session) so we don't fight the same DB connection. - # Idempotent against the legacy frontend appendMessage: - # * the assistant row was already INSERTed by - # ``persist_assistant_shell`` above, so this just UPDATEs - # it with the rich ContentPart[] from the builder. - # * token_usage uses INSERT ... ON CONFLICT DO NOTHING - # against migration 142's partial unique index, so a - # racing append_message recovery branch can never - # double-write. - # ``mark_interrupted`` closes any open text/reasoning blocks - # and flips running tool-calls (no result) to state=aborted - # so the persisted JSONB reflects a coherent end-state even - # on client disconnect. - # Never raises (best-effort, logs only). - if ( - stream_result - and stream_result.turn_id - and stream_result.assistant_message_id - ): - from app.tasks.chat.persistence import finalize_assistant_turn - - builder_stats: dict[str, int] | None = None - if stream_result.content_builder is not None: - stream_result.content_builder.mark_interrupted() - # Snapshot stats BEFORE deepcopy in ``snapshot()`` so - # the perf log records the actual finalised payload - # (post-mark_interrupted), not the live-mutating - # builder state. - builder_stats = stream_result.content_builder.stats() - content_payload = stream_result.content_builder.snapshot() - else: - # Defensive fallback — we always set the builder - # alongside ``assistant_message_id`` above, so this - # branch only fires if a future refactor ever - # decouples them. Persist whatever accumulated - # text we captured so the row at least renders. - content_payload = [ - { - "type": "text", - "text": stream_result.accumulated_text or "", - } - ] - - if builder_stats is not None: - _perf_log.info( - "[stream_new_chat] finalize_payload chat_id=%s " - "message_id=%s parts=%d bytes=%d text=%d " - "reasoning=%d tool_calls=%d " - "tool_calls_completed=%d tool_calls_aborted=%d " - "thinking_step_parts=%d step_separators=%d", - chat_id, - stream_result.assistant_message_id, - builder_stats["parts"], - builder_stats["bytes"], - builder_stats["text"], - builder_stats["reasoning"], - builder_stats["tool_calls"], - builder_stats["tool_calls_completed"], - builder_stats["tool_calls_aborted"], - builder_stats["thinking_step_parts"], - builder_stats["step_separators"], - ) - - await finalize_assistant_turn( - message_id=stream_result.assistant_message_id, - chat_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - turn_id=stream_result.turn_id, - content=content_payload, - accumulator=accumulator, - ) - - # Persist any sandbox-produced files to local storage so they - # remain downloadable after the Daytona sandbox auto-deletes. - if stream_result and stream_result.sandbox_files: - with contextlib.suppress(Exception): - from app.agents.new_chat.sandbox import ( - is_sandbox_enabled, - persist_and_delete_sandbox, - ) - - if is_sandbox_enabled(): - with anyio.CancelScope(shield=True): - await persist_and_delete_sandbox( - chat_id, stream_result.sandbox_files - ) - - # ``aafter_agent`` doesn't fire on ``interrupt()`` or early bailout. - # Skip on ``BusyError`` (caller never acquired the lock). - if not _busy_error_raised: - with contextlib.suppress(Exception): - end_turn(str(chat_id)) - _perf_log.info( - "[stream_new_chat] end_turn cleanup (chat_id=%s)", - chat_id, - ) - - # Break circular refs held by the agent graph, tools, and LLM - # wrappers so the GC can reclaim them in a single pass. - agent = llm = connector_service = None - input_state = stream_result = None - session = None - - collected = gc.collect(0) + gc.collect(1) + gc.collect(2) - if collected: - _perf_log.info( - "[stream_new_chat] gc.collect() reclaimed %d objects (chat_id=%s)", - collected, - chat_id, - ) - trim_native_heap() - log_system_snapshot("stream_new_chat_END") - with contextlib.suppress(Exception): - chat_span.set_attribute("chat.outcome", chat_outcome) - ot_metrics.record_chat_request_duration( - (time.perf_counter() - _t_total) * 1000, - flow=flow, - outcome=chat_outcome, - agent_mode=chat_agent_mode, - ) - ot_metrics.record_chat_request_outcome( - flow=flow, - outcome=chat_outcome, - agent_mode=chat_agent_mode, - error_category=chat_error_category, - ) - chat_span_cm.__exit__(*sys.exc_info()) - - -async def stream_resume_chat( - chat_id: int, - search_space_id: int, - decisions: list[dict], - user_id: str | None = None, - llm_config_id: int = -1, - thread_visibility: ChatVisibility | None = None, - filesystem_selection: FilesystemSelection | None = None, - request_id: str | None = None, - disabled_tools: list[str] | None = None, -) -> AsyncGenerator[str, None]: - streaming_service = VercelStreamingService() - stream_result = StreamResult() - _t_total = time.perf_counter() - fs_mode = filesystem_selection.mode.value if filesystem_selection else "cloud" - fs_platform = ( - filesystem_selection.client_platform.value if filesystem_selection else "web" - ) - stream_result.request_id = request_id - stream_result.turn_id = f"{chat_id}:{int(time.time() * 1000)}" - stream_result.filesystem_mode = fs_mode - stream_result.client_platform = fs_platform - chat_agent_mode = "unknown" - chat_outcome = "success" - chat_error_category: str | None = None - chat_span_cm = ot.chat_request_span( - chat_id=chat_id, - search_space_id=search_space_id, - flow="resume", - request_id=request_id, - turn_id=stream_result.turn_id, - filesystem_mode=fs_mode, - client_platform=fs_platform, - agent_mode=chat_agent_mode, - ) - chat_span = chat_span_cm.__enter__() - _log_file_contract("turn_start", stream_result) - _perf_log.info( - "[stream_resume] filesystem_mode=%s client_platform=%s", - fs_mode, - fs_platform, - ) - from app.services.token_tracking_service import start_turn - - accumulator = start_turn() - - # Skip the finally release on ``BusyError`` (caller never acquired the lock). - _busy_error_raised = False - - _emit_stream_error = partial( - _emit_stream_terminal_error, - streaming_service=streaming_service, - flow="resume", - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - ) - - session = async_session_maker() - try: - if user_id: - await set_ai_responding(session, chat_id, UUID(user_id)) - - agent_config: AgentConfig | None = None - requested_llm_config_id = llm_config_id - - async def _load_llm_bundle( - config_id: int, - ) -> tuple[Any, AgentConfig | None, str | None]: - if config_id >= 0: - loaded_agent_config = await load_agent_config( - session=session, - config_id=config_id, - search_space_id=search_space_id, - ) - if not loaded_agent_config: - return ( - None, - None, - f"Failed to load NewLLMConfig with id {config_id}", - ) - return ( - create_chat_litellm_from_agent_config(loaded_agent_config), - loaded_agent_config, - None, - ) - - loaded_llm_config = load_global_llm_config_by_id(config_id) - if not loaded_llm_config: - return None, None, f"Failed to load LLM config with id {config_id}" - return ( - create_chat_litellm_from_config(loaded_llm_config), - AgentConfig.from_yaml_config(loaded_llm_config), - None, - ) - - _t0 = time.perf_counter() - try: - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=llm_config_id, - ) - ).resolved_llm_config_id - ot.add_event( - "model.pin.resolved", - { - "pin.requested_id": requested_llm_config_id, - "pin.resolved_id": llm_config_id, - "pin.requires_image_input": False, - }, - ) - except ValueError as pin_error: - yield _emit_stream_error( - message=str(pin_error), - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - - llm, agent_config, llm_load_error = await _load_llm_bundle(llm_config_id) - if llm_load_error: - yield _emit_stream_error( - message=llm_load_error, - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - _perf_log.info( - "[stream_resume] LLM config loaded in %.3fs", time.perf_counter() - _t0 - ) - - # Premium credit reservation (same logic as stream_new_chat). - _resume_premium_reserved_micros = 0 - _resume_premium_request_id: str | None = None - _resume_needs_premium = ( - agent_config is not None and user_id and agent_config.is_premium - ) - if _resume_needs_premium: - import uuid as _uuid - - from app.services.token_quota_service import ( - TokenQuotaService, - estimate_call_reserve_micros, - ) - - _resume_premium_request_id = _uuid.uuid4().hex[:16] - _resume_litellm_params = agent_config.litellm_params or {} - _resume_base_model = ( - _resume_litellm_params.get("base_model") - or agent_config.model_name - or "" - ) - reserve_amount_micros = estimate_call_reserve_micros( - base_model=_resume_base_model, - quota_reserve_tokens=agent_config.quota_reserve_tokens, - ) - async with shielded_async_session() as quota_session: - quota_result = await TokenQuotaService.premium_reserve( - db_session=quota_session, - user_id=UUID(user_id), - request_id=_resume_premium_request_id, - reserve_micros=reserve_amount_micros, - ) - _resume_premium_reserved_micros = reserve_amount_micros - if not quota_result.allowed: - ot.add_event( - "quota.denied", - { - "quota.code": "PREMIUM_QUOTA_EXHAUSTED", - }, - ) - if requested_llm_config_id == 0: - try: - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=0, - force_repin_free=True, - ) - ).resolved_llm_config_id - ot.add_event( - "model.repin", - { - "repin.reason": "premium_quota_exhausted", - "repin.to_config_id": llm_config_id, - }, - ) - except ValueError as pin_error: - yield _emit_stream_error( - message=str(pin_error), - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - - llm, agent_config, llm_load_error = await _load_llm_bundle( - llm_config_id - ) - if llm_load_error: - yield _emit_stream_error( - message=llm_load_error, - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - _resume_premium_request_id = None - _resume_premium_reserved_micros = 0 - _log_chat_stream_error( - flow="resume", - error_kind="premium_quota_exhausted", - error_code="PREMIUM_QUOTA_EXHAUSTED", - severity="info", - is_expected=True, - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - message=( - "Premium quota exhausted on pinned model; auto-fallback switched to a free model" - ), - extra={ - "fallback_config_id": llm_config_id, - "auto_fallback": True, - }, - ) - else: - yield _emit_stream_error( - message=( - "Buy more tokens to continue with this model, or switch to a free model" - ), - error_kind="premium_quota_exhausted", - error_code="PREMIUM_QUOTA_EXHAUSTED", - severity="info", - is_expected=True, - extra={ - "resolved_config_id": llm_config_id, - "auto_fallback": False, - }, - ) - yield streaming_service.format_done() - return - - if not llm: - yield _emit_stream_error( - message="Failed to create LLM instance", - error_kind="server_error", - error_code="SERVER_ERROR", - ) - yield streaming_service.format_done() - return - - _t0 = time.perf_counter() - connector_service = ConnectorService(session, search_space_id=search_space_id) - - firecrawl_api_key = None - webcrawler_connector = await connector_service.get_connector_by_type( - SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id - ) - if webcrawler_connector and webcrawler_connector.config: - firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") - _perf_log.info( - "[stream_resume] Connector service + firecrawl key in %.3fs", - time.perf_counter() - _t0, - ) - - _t0 = time.perf_counter() - checkpointer = await get_checkpointer() - _perf_log.info( - "[stream_resume] Checkpointer ready in %.3fs", time.perf_counter() - _t0 - ) - - visibility = thread_visibility or ChatVisibility.PRIVATE - from app.config import config as _app_config - - chat_agent_mode = "multi" if _app_config.MULTI_AGENT_CHAT_ENABLED else "single" - with contextlib.suppress(Exception): - chat_span.set_attribute("agent.mode", chat_agent_mode) - _t0 = time.perf_counter() - agent_factory = ( - create_multi_agent_chat_deep_agent - if _app_config.MULTI_AGENT_CHAT_ENABLED - else create_surfsense_deep_agent - ) - # Build the agent inline. Provider 429s are handled by the - # in-stream recovery loop, which repins to an eligible - # alternative config and rebuilds the agent before the user sees - # any output. - agent = await _build_main_agent_for_thread( - agent_factory, - llm=llm, - search_space_id=search_space_id, - db_session=session, - connector_service=connector_service, - checkpointer=checkpointer, - user_id=user_id, - thread_id=chat_id, - agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, - thread_visibility=visibility, - filesystem_selection=filesystem_selection, - disabled_tools=disabled_tools, - ) - _perf_log.info( - "[stream_resume] Agent created in %.3fs", time.perf_counter() - _t0 - ) - - # Release the transaction before streaming (same rationale as stream_new_chat). - await session.commit() - session.expunge_all() - - _perf_log.info( - "[stream_resume] Total pre-stream setup in %.3fs (chat_id=%s)", - time.perf_counter() - _t_total, - chat_id, - ) - - from langgraph.types import Command - - from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( - build_lg_resume_map, - collect_pending_tool_calls, - slice_decisions_by_tool_call, - ) - - # Each pending interrupt is stamped with its originating ``tool_call_id`` - # (see ``checkpointed_subagent_middleware.propagation``) so we can route - # a flat ``decisions`` list back to the right paused subagent. - parent_state = await agent.aget_state( - {"configurable": {"thread_id": str(chat_id)}} - ) - pending = collect_pending_tool_calls(parent_state) - _perf_log.info( - "[hitl_route] resume_entry chat_id=%s decisions=%d pending_subagents=%d", - chat_id, - len(decisions), - len(pending), - ) - routed_resume_value = slice_decisions_by_tool_call(decisions, pending) - # Langgraph rejects scalar ``Command(resume=...)`` when multiple - # interrupts are pending (parallel HITL); the mapped form works - # for the single-pause case too, so we always use it. - lg_resume_map = build_lg_resume_map(parent_state, routed_resume_value) - - config = { - "configurable": { - "thread_id": str(chat_id), - "request_id": request_id or "unknown", - "turn_id": stream_result.turn_id, - # Per-``tool_call_id`` resume slices read by - # ``SurfSenseCheckpointedSubAgentMiddleware``. Parallel - # siblings each pop their own entry, so they never race. - "surfsense_resume_value": routed_resume_value, - }, - # See ``stream_new_chat`` above for rationale: effectively - # uncapped to mirror the agent default and OpenCode's - # session loop. Doom-loop / call-limit middleware enforce - # the real ceiling. - "recursion_limit": 10_000, - } - - yield streaming_service.format_message_start() - yield streaming_service.format_start_step() - # Same rationale as ``stream_new_chat``: emit the turn id so - # resumed streams can be persisted with their correlation id - # intact. - yield streaming_service.format_data( - "turn-info", - {"chat_turn_id": stream_result.turn_id}, - ) - yield streaming_service.format_data("turn-status", {"status": "busy"}) - - # Pre-write a fresh assistant row for this resume turn. The - # original (interrupted) ``stream_new_chat`` invocation already - # persisted its own assistant row anchored to a different - # ``turn_id``; resume allocates a new ``turn_id`` (above) so we - # need a separate row keyed on the same ``(thread_id, turn_id, - # ASSISTANT)`` invariant. Idempotent against migration 141's - # partial unique index — recovers existing id on retry. - from app.tasks.chat.content_builder import AssistantContentBuilder - from app.tasks.chat.persistence import persist_assistant_shell - - assistant_message_id = await persist_assistant_shell( - chat_id=chat_id, - user_id=user_id, - turn_id=stream_result.turn_id, - ) - if assistant_message_id is None: - yield _emit_stream_error( - message=( - "We couldn't initialize the assistant message. Please try again." - ), - error_kind="server_error", - error_code="MESSAGE_PERSIST_FAILED", - ) - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Emit canonical assistant message id BEFORE any LLM streaming - # so the FE can rename ``pendingInterrupt.assistantMsgId`` to - # ``msg-{assistant_message_id}`` immediately. Resume does NOT - # emit ``data-user-message-id`` because the user row is from - # the original interrupted turn (different ``turn_id``) and is - # never re-persisted here. See B5 in the - # ``sse-based_message_id_handshake`` plan. - yield streaming_service.format_data( - "assistant-message-id", - {"message_id": assistant_message_id, "turn_id": stream_result.turn_id}, - ) - - stream_result.assistant_message_id = assistant_message_id - stream_result.content_builder = AssistantContentBuilder() - - # Resume path doesn't carry new ``mentioned_document_ids`` — - # those are seeded in the original turn. We still pass a - # context so future middleware extensions (Phase 2) can rely on - # ``runtime.context`` always being populated. - runtime_context = SurfSenseContextSchema( - search_space_id=search_space_id, - request_id=request_id, - turn_id=stream_result.turn_id, - ) - - _t_stream_start = time.perf_counter() - _first_event_logged = False - runtime_rate_limit_recovered = False - while True: - try: - async for sse in _stream_agent_events( - agent=agent, - config=config, - input_data=Command(resume=lg_resume_map), - streaming_service=streaming_service, - result=stream_result, - step_prefix=_resume_step_prefix(stream_result.turn_id), - fallback_commit_search_space_id=search_space_id, - fallback_commit_created_by_id=user_id, - fallback_commit_filesystem_mode=( - filesystem_selection.mode - if filesystem_selection - else FilesystemMode.CLOUD - ), - fallback_commit_thread_id=chat_id, - runtime_context=runtime_context, - content_builder=stream_result.content_builder, - ): - if not _first_event_logged: - _perf_log.info( - "[stream_resume] First agent event in %.3fs (stream), %.3fs (total) (chat_id=%s)", - time.perf_counter() - _t_stream_start, - time.perf_counter() - _t_total, - chat_id, - ) - _first_event_logged = True - yield sse - break - except Exception as stream_exc: - can_runtime_recover = ( - not runtime_rate_limit_recovered - and requested_llm_config_id == 0 - and llm_config_id < 0 - and not _first_event_logged - and _is_provider_rate_limited(stream_exc) - ) - if not can_runtime_recover: - raise - - runtime_rate_limit_recovered = True - previous_config_id = llm_config_id - # Ensure the same-request recovery retry does not trip the - # BusyMutex lock retained by the failed attempt. - end_turn(str(chat_id)) - mark_runtime_cooldown( - previous_config_id, - reason="provider_rate_limited", - ) - llm_config_id = ( - await resolve_or_get_pinned_llm_config_id( - session, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - selected_llm_config_id=0, - exclude_config_ids={previous_config_id}, - ) - ).resolved_llm_config_id - - llm, agent_config, llm_load_error = await _load_llm_bundle( - llm_config_id - ) - if llm_load_error: - raise stream_exc - - _t0 = time.perf_counter() - agent = await _build_main_agent_for_thread( - agent_factory, - llm=llm, - search_space_id=search_space_id, - db_session=session, - connector_service=connector_service, - checkpointer=checkpointer, - user_id=user_id, - thread_id=chat_id, - agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, - thread_visibility=visibility, - filesystem_selection=filesystem_selection, - disabled_tools=disabled_tools, - ) - _perf_log.info( - "[stream_resume] Runtime rate-limit recovery repinned " - "config_id=%s -> %s and rebuilt agent in %.3fs", - previous_config_id, - llm_config_id, - time.perf_counter() - _t0, - ) - ot.add_event( - "chat.rate_limit.recovered", - { - "recovery.reason": "provider_rate_limited", - "recovery.previous_config_id": previous_config_id, - "recovery.fallback_config_id": llm_config_id, - }, - ) - _log_chat_stream_error( - flow="resume", - error_kind="rate_limited", - error_code="RATE_LIMITED", - severity="info", - is_expected=True, - request_id=request_id, - thread_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - message=( - "Auto-pinned model hit runtime rate limit; switched to " - "another eligible model and retried." - ), - extra={ - "auto_runtime_recover": True, - "previous_config_id": previous_config_id, - "fallback_config_id": llm_config_id, - }, - ) - continue - _perf_log.info( - "[stream_resume] Agent stream completed in %.3fs (chat_id=%s)", - time.perf_counter() - _t_stream_start, - chat_id, - ) - if stream_result.is_interrupted: - ot.add_event( - "chat.interrupted", - { - "chat.flow": "resume", - }, - ) - usage_summary = accumulator.per_message_summary() - _perf_log.info( - "[token_usage] interrupted resume_chat: calls=%d total=%d cost_micros=%d summary=%s", - len(accumulator.calls), - accumulator.grand_total, - accumulator.total_cost_micros, - usage_summary, - ) - if usage_summary: - yield streaming_service.format_data( - "token-usage", - { - "usage": usage_summary, - "prompt_tokens": accumulator.total_prompt_tokens, - "completion_tokens": accumulator.total_completion_tokens, - "total_tokens": accumulator.grand_total, - "cost_micros": accumulator.total_cost_micros, - "call_details": accumulator.serialized_calls(), - }, - ) - - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Finalize premium credit debit for resume path with the actual - # provider cost reported by LiteLLM (sum of cost across all - # calls in the turn). - if _resume_premium_request_id and user_id: - try: - from app.services.token_quota_service import TokenQuotaService - - async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_finalize( - db_session=quota_session, - user_id=UUID(user_id), - request_id=_resume_premium_request_id, - actual_micros=accumulator.total_cost_micros, - reserved_micros=_resume_premium_reserved_micros, - ) - _resume_premium_request_id = None - _resume_premium_reserved_micros = 0 - except Exception: - logging.getLogger(__name__).warning( - "Failed to finalize premium quota for user %s (resume)", - user_id, - exc_info=True, - ) - - usage_summary = accumulator.per_message_summary() - _perf_log.info( - "[token_usage] normal resume_chat: calls=%d total=%d cost_micros=%d summary=%s", - len(accumulator.calls), - accumulator.grand_total, - accumulator.total_cost_micros, - usage_summary, - ) - if usage_summary: - yield streaming_service.format_data( - "token-usage", - { - "usage": usage_summary, - "prompt_tokens": accumulator.total_prompt_tokens, - "completion_tokens": accumulator.total_completion_tokens, - "total_tokens": accumulator.grand_total, - "cost_micros": accumulator.total_cost_micros, - "call_details": accumulator.serialized_calls(), - }, - ) - - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - import traceback - - # ``BusyError`` fires before the agent acquires the lock; the - # cleanup path must skip lock release to avoid freeing the - # in-flight caller's lock. Classification is handled below. - if isinstance(e, BusyError): - _busy_error_raised = True - - ( - error_kind, - error_code, - severity, - is_expected, - user_message, - error_extra, - ) = _classify_stream_exception(e, flow_label="resume") - chat_outcome = error_code or error_kind or "error" - chat_error_category = ot_metrics.categorize_exception(e) - with contextlib.suppress(Exception): - chat_span.set_attribute("chat.outcome", chat_outcome) - chat_span.set_attribute("error.category", chat_error_category) - ot.record_error(chat_span, e) - error_message = f"Error during resume: {e!s}" - print(f"[stream_resume_chat] {error_message}") - print(f"[stream_resume_chat] Traceback:\n{traceback.format_exc()}") - if error_code == "TURN_CANCELLING": - status_payload: dict[str, Any] = {"status": "cancelling"} - if error_extra: - status_payload.update(error_extra) - yield streaming_service.format_data("turn-status", status_payload) - else: - yield streaming_service.format_data("turn-status", {"status": "busy"}) - yield _emit_stream_error( - message=user_message, - error_kind=error_kind, - error_code=error_code, - severity=severity, - is_expected=is_expected, - extra=error_extra, - ) - yield streaming_service.format_data("turn-status", {"status": "idle"}) - yield streaming_service.format_finish_step() - yield streaming_service.format_finish() - yield streaming_service.format_done() - - finally: - with anyio.CancelScope(shield=True): - # Authoritative fallback cleanup for lock/cancel state. Middleware - # teardown can be skipped on some client-abort paths. - end_turn(str(chat_id)) - - # Release premium reservation if not finalized - if ( - _resume_premium_request_id - and _resume_premium_reserved_micros > 0 - and user_id - ): - try: - from app.services.token_quota_service import TokenQuotaService - - async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_release( - db_session=quota_session, - user_id=UUID(user_id), - reserved_micros=_resume_premium_reserved_micros, - ) - _resume_premium_reserved_micros = 0 - except Exception: - logging.getLogger(__name__).warning( - "Failed to release premium quota for user %s (resume)", user_id - ) - - try: - await session.rollback() - await clear_ai_responding(session, chat_id) - except Exception: - try: - async with shielded_async_session() as fresh_session: - await clear_ai_responding(fresh_session, chat_id) - except Exception: - logging.getLogger(__name__).warning( - "Failed to clear AI responding state for thread %s", chat_id - ) - - with contextlib.suppress(Exception): - session.expunge_all() - - with contextlib.suppress(Exception): - await session.close() - - # Server-side assistant-message + token_usage finalization for - # the resume flow. The original user message was persisted by - # the original (interrupted) ``stream_new_chat`` invocation; - # the resume's own ``persist_assistant_shell`` write lives at - # the new ``turn_id`` above. This finalize updates that row - # with the rich ContentPart[] from the builder and writes - # token_usage idempotently via migration 142's partial - # unique index. Best-effort, never raises. - if ( - stream_result - and stream_result.turn_id - and stream_result.assistant_message_id - ): - from app.tasks.chat.persistence import finalize_assistant_turn - - builder_stats: dict[str, int] | None = None - if stream_result.content_builder is not None: - stream_result.content_builder.mark_interrupted() - builder_stats = stream_result.content_builder.stats() - content_payload = stream_result.content_builder.snapshot() - else: - content_payload = [ - { - "type": "text", - "text": stream_result.accumulated_text or "", - } - ] - - if builder_stats is not None: - _perf_log.info( - "[stream_resume] finalize_payload chat_id=%s " - "message_id=%s parts=%d bytes=%d text=%d " - "reasoning=%d tool_calls=%d " - "tool_calls_completed=%d tool_calls_aborted=%d " - "thinking_step_parts=%d step_separators=%d", - chat_id, - stream_result.assistant_message_id, - builder_stats["parts"], - builder_stats["bytes"], - builder_stats["text"], - builder_stats["reasoning"], - builder_stats["tool_calls"], - builder_stats["tool_calls_completed"], - builder_stats["tool_calls_aborted"], - builder_stats["thinking_step_parts"], - builder_stats["step_separators"], - ) - - await finalize_assistant_turn( - message_id=stream_result.assistant_message_id, - chat_id=chat_id, - search_space_id=search_space_id, - user_id=user_id, - turn_id=stream_result.turn_id, - content=content_payload, - accumulator=accumulator, - ) - - # Release the lock from the original interrupted turn or any - # re-interrupt/bailout. Skip on ``BusyError`` (lock not held here). - if not _busy_error_raised: - with contextlib.suppress(Exception): - end_turn(str(chat_id)) - _perf_log.info( - "[stream_resume] end_turn cleanup (chat_id=%s)", - chat_id, - ) - - agent = llm = connector_service = None - stream_result = None - session = None - - collected = gc.collect(0) + gc.collect(1) + gc.collect(2) - if collected: - _perf_log.info( - "[stream_resume] gc.collect() reclaimed %d objects (chat_id=%s)", - collected, - chat_id, - ) - trim_native_heap() - log_system_snapshot("stream_resume_chat_END") - with contextlib.suppress(Exception): - chat_span.set_attribute("chat.outcome", chat_outcome) - ot_metrics.record_chat_request_duration( - (time.perf_counter() - _t_total) * 1000, - flow="resume", - outcome=chat_outcome, - agent_mode=chat_agent_mode, - ) - ot_metrics.record_chat_request_outcome( - flow="resume", - outcome=chat_outcome, - agent_mode=chat_agent_mode, - error_category=chat_error_category, - ) - chat_span_cm.__exit__(*sys.exc_info()) diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/builder.py b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py index 0db42edbf..dcbd37521 100644 --- a/surfsense_backend/app/tasks/chat/streaming/agent/builder.py +++ b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py @@ -9,8 +9,10 @@ from __future__ import annotations from typing import Any -from app.agents.new_chat.filesystem_selection import FilesystemSelection -from app.agents.new_chat.llm_config import AgentConfig +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemSelection, +) +from app.agents.chat.runtime.llm_config import AgentConfig from app.db import ChatVisibility from app.services.connector_service import ConnectorService diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py b/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py index b77bd3890..d96144bcd 100644 --- a/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py +++ b/surfsense_backend/app/tasks/chat/streaming/agent/event_loop.py @@ -11,10 +11,10 @@ from __future__ import annotations from collections.abc import AsyncGenerator from typing import Any -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.kb_persistence import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import ( commit_staged_filesystem_state, ) +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode from app.services.new_streaming_service import VercelStreamingService from app.tasks.chat.streaming.contract.file_contract import ( contract_enforcement_active, diff --git a/surfsense_backend/app/tasks/chat/streaming/errors/classifier.py b/surfsense_backend/app/tasks/chat/streaming/errors/classifier.py index 3af2b9f9f..6b37df343 100644 --- a/surfsense_backend/app/tasks/chat/streaming/errors/classifier.py +++ b/surfsense_backend/app/tasks/chat/streaming/errors/classifier.py @@ -7,11 +7,11 @@ import logging import time from typing import Any, Literal -from app.agents.new_chat.errors import BusyError -from app.agents.new_chat.middleware.busy_mutex import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import ( get_cancel_state, is_cancel_requested, ) +from app.agents.chat.runtime.errors import BusyError TURN_CANCELLING_INITIAL_DELAY_MS = 200 TURN_CANCELLING_BACKOFF_FACTOR = 2 diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py index af496cee7..dbb8ee2e4 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/auto_pin.py @@ -50,8 +50,14 @@ async def resolve_initial_auto_pin( selected_llm_config_id: int, requires_image_input: bool, requested_llm_config_id: int, + force_repin_free: bool = False, ) -> AutoPinResult: - """Run the resolver and classify any ``ValueError`` for the SSE error path.""" + """Run the resolver and classify any ``ValueError`` for the SSE error path. + + ``force_repin_free`` forces a fresh re-pin to a free-tier config (used on + the premium-quota-exhausted fallback so an out-of-quota user isn't repinned + onto another paid model). + """ try: pinned = await resolve_or_get_pinned_llm_config_id( session, @@ -60,6 +66,7 @@ async def resolve_initial_auto_pin( user_id=user_id, selected_llm_config_id=selected_llm_config_id, requires_image_input=requires_image_input, + force_repin_free=force_repin_free, ) ot.add_event( "model.pin.resolved", diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py index 0c6704bd1..064843aba 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py @@ -9,9 +9,9 @@ Pipeline: can resolve ``report_id`` for versioning without spelunking history. 3. **@-mention resolve** (cloud mode) — substitute ``@title`` tokens in the query with canonical ``\`/documents/...\``` paths the LLM expects. - 4. **Context block render** — XML-wrap recent reports, prepend to the - rewritten query, optionally prefix with display name for SEARCH_SPACE - visibility. + 4. **Context block render** — XML-wrap @-mentioned connectors and recent + reports, prepend to the rewritten query, optionally prefix with display + name for SEARCH_SPACE visibility. 5. **HumanMessage** — multimodal content if images are attached. Returns the assembled ``input_state`` dict plus side-channel data the @@ -28,8 +28,11 @@ from langchain_core.messages import HumanMessage from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.mention_resolver import resolve_mentions, substitute_in_text +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.runtime.mention_resolver import ( + resolve_mentions, + substitute_in_text, +) from app.db import ( ChatVisibility, NewChatThread, @@ -62,6 +65,7 @@ async def build_new_chat_input_state( user_image_data_urls: list[str] | None, mentioned_document_ids: list[int] | None, mentioned_folder_ids: list[int] | None, + mentioned_connectors: list[dict[str, Any]] | None, mentioned_documents: list[dict[str, Any]] | None, needs_history_bootstrap: bool, thread_visibility: ChatVisibility, @@ -110,6 +114,7 @@ async def build_new_chat_input_state( final_query = _render_query_with_context( agent_user_query=agent_user_query, + mentioned_connectors=mentioned_connectors, recent_reports=recent_reports, ) @@ -196,11 +201,19 @@ async def _resolve_mentions_for_query( def _render_query_with_context( *, agent_user_query: str, + mentioned_connectors: list[dict[str, Any]] | None, recent_reports: list[Report], ) -> str: - """Prepend recent-reports XML block to the user query.""" + """Prepend the ```` then ```` blocks. + + Order is load-bearing for legacy parity. + """ context_parts: list[str] = [] + connector_context = _render_mentioned_connectors(mentioned_connectors) + if connector_context: + context_parts.append(connector_context) + if recent_reports: report_lines: list[str] = [] for r in recent_reports: @@ -225,3 +238,40 @@ def _render_query_with_context( return f"{context}\n\n{agent_user_query}" return agent_user_query + + +def _render_mentioned_connectors( + mentioned_connectors: list[dict[str, Any]] | None, +) -> str | None: + """Render selected connector account metadata for connector-backed tools.""" + if not mentioned_connectors: + return None + + connector_lines: list[str] = [] + for connector in mentioned_connectors: + if not isinstance(connector, dict): + continue + connector_id = connector.get("id") + connector_type = connector.get("connector_type") or connector.get( + "document_type" + ) + account_name = connector.get("account_name") or connector.get("title") + if connector_id is None or connector_type is None: + continue + connector_lines.append( + f' - connector_id={connector_id}, connector_type="{connector_type}", ' + f'account_name="{account_name or ""}"' + ) + + if not connector_lines: + return None + + return ( + "\n" + "The user selected these exact connector accounts with @. " + "These entries are selection metadata, not retrieved connector content. " + "When a connector-backed tool needs an account, use the matching " + "connector_id from this list if the tool supports connector_id:\n" + + "\n".join(connector_lines) + + "\n" + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py index 9f4e5d2d8..69b9f4ab8 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/llm_capability.py @@ -15,7 +15,7 @@ tells the user what to change. from __future__ import annotations -from app.agents.new_chat.llm_config import AgentConfig +from app.agents.chat.runtime.llm_config import AgentConfig from app.observability import otel as ot diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py index 1892320d3..e33dca376 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -29,11 +29,12 @@ from typing import Any, Literal import anyio -from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent -from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.middleware.busy_mutex import end_turn -from app.config import config as _app_config +from app.agents.chat.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import end_turn +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) from app.db import ChatVisibility, async_session_maker from app.observability import otel as ot from app.services.new_streaming_service import VercelStreamingService @@ -124,6 +125,8 @@ async def stream_new_chat( llm_config_id: int = -1, mentioned_document_ids: list[int] | None = None, mentioned_folder_ids: list[int] | None = None, + mentioned_connector_ids: list[int] | None = None, + mentioned_connectors: list[dict[str, Any]] | None = None, mentioned_documents: list[dict[str, Any]] | None = None, checkpoint_id: str | None = None, needs_history_bootstrap: bool = False, @@ -272,6 +275,7 @@ async def stream_new_chat( selected_llm_config_id=0, requires_image_input=requires_image_input, requested_llm_config_id=requested_llm_config_id, + force_repin_free=True, ) if pin_fallback.error is not None: message, error_code, error_kind = pin_fallback.error @@ -367,12 +371,6 @@ async def stream_new_chat( mentioned_documents=mentioned_documents, background_tasks=_background_tasks, ) - persist_asst_task = spawn_persist_assistant_shell_task( - chat_id=chat_id, - user_id=user_id, - turn_id=stream_result.turn_id, - background_tasks=_background_tasks, - ) _t0 = time.perf_counter() connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( @@ -390,16 +388,11 @@ async def stream_new_chat( ) visibility = thread_visibility or ChatVisibility.PRIVATE - use_multi_agent = bool(_app_config.MULTI_AGENT_CHAT_ENABLED) - chat_agent_mode = "multi" if use_multi_agent else "single" + chat_agent_mode = "multi" set_agent_mode(chat_span, chat_agent_mode) _t0 = time.perf_counter() - agent_factory = ( - create_multi_agent_chat_deep_agent - if use_multi_agent - else create_surfsense_deep_agent - ) + agent_factory = create_multi_agent_chat_deep_agent # Build the agent inline. Provider 429s surface through the in-stream # recovery loop below, which repins the thread to an eligible # alternative config and rebuilds the agent before the user sees any @@ -435,6 +428,7 @@ async def stream_new_chat( user_image_data_urls=user_image_data_urls, mentioned_document_ids=mentioned_document_ids, mentioned_folder_ids=mentioned_folder_ids, + mentioned_connectors=mentioned_connectors, mentioned_documents=mentioned_documents, needs_history_bootstrap=needs_history_bootstrap, thread_visibility=visibility, @@ -523,6 +517,14 @@ async def stream_new_chat( {"message_id": user_message_id, "turn_id": stream_result.turn_id}, ) + # Spawned only after the user row is confirmed, so a user-persist + # failure can't orphan an assistant shell on the same turn. + persist_asst_task = spawn_persist_assistant_shell_task( + chat_id=chat_id, + user_id=user_id, + turn_id=stream_result.turn_id, + background_tasks=_background_tasks, + ) assistant_message_id = await await_persist_task( persist_asst_task, chat_id=chat_id, @@ -588,6 +590,8 @@ async def stream_new_chat( mentioned_document_ids=mentioned_document_ids, accepted_folder_ids=accepted_folder_ids, mentioned_folder_ids=mentioned_folder_ids, + mentioned_connector_ids=mentioned_connector_ids, + mentioned_connectors=mentioned_connectors, request_id=request_id, turn_id=stream_result.turn_id, ) @@ -825,7 +829,7 @@ async def stream_new_chat( # downloadable after the Daytona sandbox auto-deletes. if stream_result and stream_result.sandbox_files: with contextlib.suppress(Exception): - from app.agents.new_chat.sandbox import ( + from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.sandbox import ( is_sandbox_enabled, persist_and_delete_sandbox, ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py index cf1e8c3fb..195a16b1e 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py @@ -8,7 +8,7 @@ mention lists / request ids / turn ids without rebuilding the graph. from __future__ import annotations -from app.agents.new_chat.context import SurfSenseContextSchema +from app.agents.chat.shared.context import SurfSenseContextSchema def build_new_chat_runtime_context( @@ -17,6 +17,8 @@ def build_new_chat_runtime_context( mentioned_document_ids: list[int] | None, accepted_folder_ids: list[int], mentioned_folder_ids: list[int] | None, + mentioned_connector_ids: list[int] | None, + mentioned_connectors: list[dict[str, object]] | None, request_id: str | None, turn_id: str, ) -> SurfSenseContextSchema: @@ -26,11 +28,16 @@ def build_new_chat_runtime_context( ``mentioned_folder_ids`` from the request: the resolver drops chips that pointed at deleted folders or folders the caller can't see, so middlewares only get authorized ids. + + Connector mentions are set on the schema for legacy parity even though no + middleware reads them yet. """ return SurfSenseContextSchema( search_space_id=search_space_id, mentioned_document_ids=list(mentioned_document_ids or []), mentioned_folder_ids=list(accepted_folder_ids or mentioned_folder_ids or []), + mentioned_connector_ids=list(mentioned_connector_ids or []), + mentioned_connectors=list(mentioned_connectors or []), request_id=request_id, turn_id=turn_id, ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py index 7db45941b..fe3d210bb 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/title_gen.py @@ -30,7 +30,7 @@ from app.prompts import TITLE_GENERATION_PROMPT from app.services.new_streaming_service import VercelStreamingService if TYPE_CHECKING: - from app.agents.new_chat.llm_config import AgentConfig + from app.agents.chat.runtime.llm_config import AgentConfig from app.services.token_tracking_service import TokenAccumulator diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py index e1b95aa63..6d0924850 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py @@ -23,11 +23,12 @@ from uuid import UUID import anyio -from app.agents.multi_agent_chat import create_multi_agent_chat_deep_agent -from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent -from app.agents.new_chat.filesystem_selection import FilesystemMode, FilesystemSelection -from app.agents.new_chat.middleware.busy_mutex import end_turn -from app.config import config as _app_config +from app.agents.chat.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import end_turn +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) from app.db import ChatVisibility, async_session_maker from app.observability import otel as ot from app.services.chat_session_state_service import set_ai_responding @@ -326,16 +327,11 @@ async def stream_resume_chat( ) visibility = thread_visibility or ChatVisibility.PRIVATE - use_multi_agent = bool(_app_config.MULTI_AGENT_CHAT_ENABLED) - chat_agent_mode = "multi" if use_multi_agent else "single" + chat_agent_mode = "multi" set_agent_mode(chat_span, chat_agent_mode) _t0 = time.perf_counter() - agent_factory = ( - create_multi_agent_chat_deep_agent - if use_multi_agent - else create_surfsense_deep_agent - ) + agent_factory = create_multi_agent_chat_deep_agent agent = await build_main_agent_for_thread( agent_factory, llm=llm, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py index 7f4f67aac..d9877c9b0 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/resume_routing.py @@ -41,7 +41,7 @@ async def build_resume_routing( ``surfsense_resume_value`` configurable; parallel siblings each pop their own entry so they never race. """ - from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( + from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py index 59d5d8ca7..54f0dfba0 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/runtime_context.py @@ -7,7 +7,7 @@ can rely on ``runtime.context`` always being populated. from __future__ import annotations -from app.agents.new_chat.context import SurfSenseContextSchema +from app.agents.chat.shared.context import SurfSenseContextSchema def build_resume_chat_runtime_context( diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py index 2f334114c..7e2bc950b 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/llm_bundle.py @@ -14,7 +14,7 @@ from typing import Any from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.llm_config import ( +from app.agents.chat.runtime.llm_config import ( AgentConfig, create_chat_litellm_from_agent_config, create_chat_litellm_from_config, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py index ec92306dd..f717cb325 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py @@ -4,7 +4,7 @@ from __future__ import annotations from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.checkpointer import get_checkpointer +from app.agents.chat.runtime.checkpointer import get_checkpointer from app.db import SearchSourceConnectorType from app.services.connector_service import ConnectorService @@ -33,7 +33,7 @@ async def setup_connector_and_firecrawl( async def get_chat_checkpointer(): """Resolve the PostgreSQL checkpointer for persistent conversation memory. - Thin wrapper around ``app.agents.new_chat.checkpointer.get_checkpointer`` so + Thin wrapper around ``app.agents.chat.runtime.checkpointer.get_checkpointer`` so flow orchestrators can rely on a streaming-local symbol and we have a hook point if the checkpointer source ever needs to vary per flow. """ diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py index cbf44764c..6c08cb29f 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py @@ -19,7 +19,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING from uuid import UUID -from app.agents.new_chat.llm_config import AgentConfig +from app.agents.chat.runtime.llm_config import AgentConfig from app.db import shielded_async_session if TYPE_CHECKING: diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py index 6b3857594..29018fe07 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/rate_limit_recovery.py @@ -17,7 +17,7 @@ from typing import Literal from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.middleware.busy_mutex import end_turn +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import end_turn from app.observability import otel as ot from app.services.auto_model_pin_service import ( mark_runtime_cooldown, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py index 6cf0df855..f455a8ffd 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/stream_loop.py @@ -15,7 +15,7 @@ from __future__ import annotations from collections.abc import AsyncGenerator, Awaitable, Callable from typing import Any -from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode from app.services.new_streaming_service import VercelStreamingService from app.tasks.chat.streaming.agent.event_loop import stream_agent_events from app.tasks.chat.streaming.shared.stream_result import StreamResult diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py index b305dba23..126149cc1 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/terminal_error.py @@ -14,7 +14,7 @@ import traceback from collections.abc import Iterator from typing import Any, Literal -from app.agents.new_chat.errors import BusyError +from app.agents.chat.runtime.errors import BusyError from app.observability import metrics as ot_metrics, otel as ot from app.services.new_streaming_service import VercelStreamingService from app.tasks.chat.streaming.errors.classifier import classify_stream_exception diff --git a/surfsense_backend/app/tasks/chat/streaming/handlers/tool_end.py b/surfsense_backend/app/tasks/chat/streaming/handlers/tool_end.py index 2ff810447..ae04c2823 100644 --- a/surfsense_backend/app/tasks/chat/streaming/handlers/tool_end.py +++ b/surfsense_backend/app/tasks/chat/streaming/handlers/tool_end.py @@ -26,7 +26,7 @@ def _unwrap_command_output(raw_output: Any) -> Any: """Replace a ``Command`` from a tool return with its inner ``ToolMessage``. Tools that participate in receipt-style state writes (see - ``app.agents.shared.receipt_command.with_receipt``) return a + ``app.agents.chat.multi_agent_chat.shared.receipts.command.with_receipt``) return a ``Command(update={"messages": [ToolMessage(...)], "receipts": [...]})``. LangChain's ``on_tool_end`` event surfaces that ``Command`` verbatim as ``data.output``, which the rest of this handler can't introspect: it has diff --git a/surfsense_backend/app/tasks/chat/streaming/handlers/tools/deliverables/generate_video_presentation/emission.py b/surfsense_backend/app/tasks/chat/streaming/handlers/tools/deliverables/generate_video_presentation/emission.py index 51a67f369..34283bcdb 100644 --- a/surfsense_backend/app/tasks/chat/streaming/handlers/tools/deliverables/generate_video_presentation/emission.py +++ b/surfsense_backend/app/tasks/chat/streaming/handlers/tools/deliverables/generate_video_presentation/emission.py @@ -21,7 +21,7 @@ def iter_completion_emission_frames( # ``ready`` is the live success status now that the tool waits for the # Celery worker to reach a terminal state. ``pending`` is retained as a # legacy branch for old saved chats that pre-date the wait-for-terminal - # change (see ``app.agents.shared.deliverable_wait``). + # change (see ``app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.deliverable_wait``). if status == "ready": yield ctx.streaming_service.format_terminal_info( f"Video presentation generated successfully: {out.get('title', 'Presentation')}", diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index f77a0632a..e2a1b109a 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -6,6 +6,7 @@ Implements real-time document status updates using a two-phase approach: - Phase 2: Process each document one by one (pending → processing → ready/failed) """ +import contextlib import time from collections.abc import Awaitable, Callable @@ -14,13 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.airtable_history import AirtableHistoryConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -394,29 +393,10 @@ async def index_airtable_records( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "record_id": item["record_id"], - "created_time": item["record"].get("CREATED_TIME()", ""), - "document_type": "Airtable Record", - "connector_type": "Airtable", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["markdown_content"], - user_llm, - document_metadata_for_summary, - ) - else: - summary_content = f"Airtable Record: {item['record_id']}\n\n{item['markdown_content']}" - summary_embedding = embed_text(summary_content) + summary_content = f"Airtable Record: {item['record_id']}\n\n{item['markdown_content']}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(item["markdown_content"]) @@ -453,10 +433,15 @@ async def index_airtable_records( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index ffc8ab72e..9408874ca 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -2,6 +2,7 @@ Base functionality and shared imports for connector indexers. """ +import contextlib import logging from datetime import UTC, datetime, timedelta @@ -10,6 +11,8 @@ from sqlalchemy.future import select from app.db import ( Document, + DocumentStatus, + DocumentType, SearchSourceConnector, SearchSourceConnectorType, ) @@ -130,6 +133,59 @@ async def check_document_by_unique_identifier( return existing_doc_result.scalars().first() +async def mark_connector_documents_failed( + session: AsyncSession, + *, + document_type: DocumentType, + search_space_id: int, + failures: list[tuple[str, str]], +) -> int: + """Transition placeholder/in-progress documents to ``failed`` by source id. + + Without this, a document whose download/ETL fails stays stuck in + ``pending``/``processing`` forever: undeletable in the UI and never retried. + + ``failures`` is a list of ``(unique_id, reason)``. Best-effort: never raises, + and leaves ``ready`` documents untouched. Returns the number marked failed. + """ + if not failures: + return 0 + + from app.indexing_pipeline.document_hashing import compute_identifier_hash + + marked = 0 + try: + for unique_id, reason in failures: + if not unique_id: + continue + uid_hash = compute_identifier_hash( + document_type.value, unique_id, search_space_id + ) + existing = await check_document_by_unique_identifier(session, uid_hash) + if existing is None: + continue + if DocumentStatus.is_state(existing.status, DocumentStatus.READY): + continue + existing.status = DocumentStatus.failed(reason) + existing.updated_at = datetime.now(UTC) + marked += 1 + + if marked: + await session.commit() + except Exception: + with contextlib.suppress(Exception): + await session.rollback() + logger.warning( + "Failed to mark %d connector document(s) as failed (type=%s)", + len(failures), + getattr(document_type, "value", document_type), + exc_info=True, + ) + return 0 + + return marked + + async def get_connector_by_id( session: AsyncSession, connector_id: int, connector_type: SearchSourceConnectorType ) -> SearchSourceConnector | None: diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index 8e64e56ba..6471ffb00 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -6,6 +6,7 @@ Implements 2-phase document status updates for real-time UI feedback: - Phase 2: Process each page: pending → processing → ready/failed """ +import contextlib import time from collections.abc import Awaitable, Callable from datetime import datetime @@ -15,13 +16,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.bookstack_connector import BookStackConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -384,10 +383,7 @@ async def index_bookstack_pages( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) # Build document metadata doc_metadata = { @@ -403,23 +399,8 @@ async def index_bookstack_pages( "connector_id": connector_id, } - if user_llm and connector.enable_summary: - summary_metadata = { - "page_name": item["page_name"], - "page_id": item["page_id"], - "book_id": item["book_id"], - "document_type": "BookStack Page", - "connector_type": "BookStack", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["full_content"], user_llm, summary_metadata - ) - else: - summary_content = f"BookStack Page: {item['page_name']}\n\nBook ID: {item['book_id']}\n\n{item['full_content']}" - summary_embedding = embed_text(summary_content) + summary_content = f"BookStack Page: {item['page_name']}\n\nBook ID: {item['book_id']}\n\n{item['full_content']}" + summary_embedding = embed_text(summary_content) # Process chunks - using the full page content chunks = await create_document_chunks(item["full_content"]) @@ -452,10 +433,15 @@ async def index_bookstack_pages( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() skipped_pages.append( f"{item.get('page_name', 'Unknown')} (processing error)" ) diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index 5a6cc3485..91763129f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -16,13 +16,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.clickup_history import ClickUpHistoryConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -393,32 +391,10 @@ async def index_clickup_tasks( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "task_id": item["task_id"], - "task_name": item["task_name"], - "task_status": item["task_status"], - "task_priority": item["task_priority"], - "task_list": item["task_list_name"], - "task_space": item["task_space_name"], - "assignees": len(item["task_assignees"]), - "document_type": "ClickUp Task", - "connector_type": "ClickUp", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["task_content"], user_llm, document_metadata_for_summary - ) - else: - summary_content = item["task_content"] - summary_embedding = embed_text(item["task_content"]) + summary_content = item["task_content"] + summary_embedding = embed_text(item["task_content"]) chunks = await create_document_chunks(item["task_content"]) @@ -461,10 +437,15 @@ async def index_clickup_tasks( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index a8c2e3c18..53c438197 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -14,7 +14,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from .base import ( @@ -22,6 +21,7 @@ from .base import ( check_duplicate_document_by_hash, get_connector_by_id, logger, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -36,7 +36,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Confluence page dict to a ConnectorDocument.""" page_id = page.get("id", "") @@ -54,10 +53,6 @@ def _build_connector_doc( "connector_type": "Confluence", } - fallback_summary = ( - f"Confluence Page: {page_title}\n\nSpace ID: {space_id}\n\n{full_content}" - ) - return ConnectorDocument( title=page_title, source_markdown=full_content, @@ -66,8 +61,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -268,7 +261,6 @@ async def index_confluence_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, ) with session.no_autoflush: @@ -297,17 +289,30 @@ async def index_confluence_pages( await pipeline.migrate_legacy_docs(connector_docs) - async def _get_llm(s: AsyncSession): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, ) + # Placeholders for items skipped above (empty/duplicate/unbuildable) would + # otherwise stay stuck in 'pending' and undeletable. Fail them so they're + # recoverable. Leaves already-ready docs untouched. + indexed_ids = {doc.unique_id for doc in connector_docs} + stuck_placeholders = [ + (p.unique_id, "Skipped during sync: no indexable content") + for p in placeholders + if p.unique_id and p.unique_id not in indexed_ids + ] + if stuck_placeholders: + await mark_connector_documents_failed( + session, + document_type=DocumentType.CONFLUENCE_CONNECTOR, + search_space_id=search_space_id, + failures=stuck_placeholders, + ) + await update_connector_last_indexed(session, connector, update_last_indexed) logger.info( diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index 180f21412..8c5bd8f0e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -10,6 +10,7 @@ Uses 2-phase document status updates for real-time UI feedback: """ import asyncio +import contextlib import time from collections.abc import Awaitable, Callable from datetime import UTC, datetime, timedelta @@ -713,10 +714,15 @@ async def index_discord_messages( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py index 9f8c1a33a..7cd3e1613 100644 --- a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py @@ -26,13 +26,14 @@ from app.connectors.dropbox.file_types import should_skip_file as skip_item from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash +from app.indexing_pipeline.exceptions import safe_exception_message from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, get_connector_by_id, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -113,7 +114,12 @@ async def _should_skip_file( logger.info(f"Rename-only update: '{old_name}' -> '{file_name}'") return True, f"File renamed: '{old_name}' -> '{file_name}'" - if not DocumentStatus.is_state(existing.status, DocumentStatus.READY): + state = DocumentStatus.get_state(existing.status) + if state in (DocumentStatus.PENDING, DocumentStatus.PROCESSING): + # Stuck placeholder/in-progress doc (e.g. worker died mid-index): re-index + # instead of skipping, otherwise it never recovers. + return False, None + if state != DocumentStatus.READY: return True, "skipped (previously failed)" return True, "unchanged" @@ -126,7 +132,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: file_id = file.get("id", "") file_name = file.get("name", "Unknown") @@ -138,8 +143,6 @@ def _build_connector_doc( "connector_type": "Dropbox", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -148,8 +151,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -161,19 +162,23 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, -) -> tuple[list[ConnectorDocument], int]: - """Download and ETL files in parallel. Returns (docs, failed_count).""" +) -> tuple[list[ConnectorDocument], list[tuple[str, str]]]: + """Download and ETL files in parallel. + + Returns (docs, failed_files), where failed_files is a list of + (file_id, reason) so callers can mark those placeholders failed. + """ results: list[ConnectorDocument] = [] sem = asyncio.Semaphore(max_concurrency) last_heartbeat = time.time() completed_count = 0 hb_lock = asyncio.Lock() - async def _download_one(file: dict) -> ConnectorDocument | None: + async def _download_one(file: dict) -> ConnectorDocument | str: + # ConnectorDocument on success; failure reason string otherwise. nonlocal last_heartbeat, completed_count async with sem: markdown, db_metadata, error = await download_and_extract_content( @@ -183,7 +188,7 @@ async def _download_files_parallel( file_name = file.get("name", "Unknown") reason = error or "empty content" logger.warning(f"Download/ETL failed for {file_name}: {reason}") - return None + return f"Download/ETL failed: {reason}" doc = _build_connector_doc( file, markdown, @@ -191,7 +196,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -205,14 +209,28 @@ async def _download_files_parallel( tasks = [_download_one(f) for f in files] outcomes = await asyncio.gather(*tasks, return_exceptions=True) - failed = 0 - for outcome in outcomes: - if isinstance(outcome, Exception) or outcome is None: - failed += 1 - else: + failed_files: list[tuple[str, str]] = [] + for file, outcome in zip(files, outcomes, strict=False): + if isinstance(outcome, ConnectorDocument): results.append(outcome) + continue + file_id = file.get("id") + if isinstance(outcome, Exception): + reason = f"Download/ETL error: {safe_exception_message(outcome)}" + logger.warning( + "Download/ETL exception for %s: %s", + file.get("name", "Unknown"), + outcome, + exc_info=outcome, + ) + elif isinstance(outcome, str): + reason = outcome + else: + reason = "Download or extraction failed" + if file_id: + failed_files.append((file_id, reason)) - return results, failed + return results, failed_files async def _download_and_index( @@ -223,38 +241,40 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: """Parallel download then parallel indexing. Returns (batch_indexed, total_failed).""" - connector_docs, download_failed = await _download_files_parallel( + connector_docs, failed_files = await _download_files_parallel( dropbox_client, files, connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) + # Fail rows for files whose download/ETL failed, so they don't stay stuck. + if failed_files: + await mark_connector_documents_failed( + session, + document_type=DocumentType.DROPBOX_FILE, + search_space_id=search_space_id, + failures=failed_files, + ) + batch_indexed = 0 batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) - return batch_indexed, download_failed + batch_failed + return batch_indexed, len(failed_files) + batch_failed async def _remove_document(session: AsyncSession, file_id: str, search_space_id: int): @@ -289,7 +309,6 @@ async def _index_with_delta_sync( log_entry: object, max_files: int, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int, str]: """Delta sync using Dropbox cursor-based change tracking. @@ -361,7 +380,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -388,7 +406,6 @@ async def _index_full_scan( include_subfolders: bool = True, incremental_sync: bool = True, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -473,7 +490,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -502,7 +518,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, incremental_sync: bool = True, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, @@ -563,7 +578,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -629,7 +643,6 @@ async def index_dropbox_files( ) return 0, 0, error_msg, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -664,7 +677,6 @@ async def index_dropbox_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, incremental_sync=incremental_sync, vision_llm=vision_llm, ) @@ -700,7 +712,6 @@ async def index_dropbox_files( task_logger, log_entry, max_files, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) folder_cursors[folder_path] = new_cursor @@ -720,7 +731,6 @@ async def index_dropbox_files( max_files, include_subfolders, incremental_sync=incremental_sync, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_unsupported += unsup diff --git a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py index 3283b41eb..ba0aa3445 100644 --- a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py @@ -6,6 +6,7 @@ Implements 2-phase document status updates for real-time UI feedback: - Phase 2: Process each document: pending → processing → ready/failed """ +import contextlib import json import logging import time @@ -406,10 +407,15 @@ async def index_elasticsearch_documents( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index ae24d750b..ce9b80e5e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -9,6 +9,7 @@ Implements 2-phase document status updates for real-time UI feedback: - Phase 2: Process each document: pending → processing → ready/failed """ +import contextlib import time from collections.abc import Awaitable, Callable from datetime import UTC, datetime @@ -18,13 +19,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.github_connector import GitHubConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -351,42 +350,14 @@ async def index_github_repos( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id + # Heavy processing (embeddings, chunks) + + summary_text = ( + f"# GitHub Repository: {repo_full_name}\n\n" + f"## Summary\n{digest.summary}\n\n" + f"## File Structure\n{digest.tree}" ) - - document_metadata_for_summary = { - "repository": repo_full_name, - "document_type": "GitHub Repository", - "connector_type": "GitHub", - "ingestion_method": "gitingest", - "file_tree": digest.tree[:2000] - if len(digest.tree) > 2000 - else digest.tree, - "estimated_tokens": digest.estimated_tokens, - } - - if user_llm and connector.enable_summary: - # Prepare content for summarization - summary_content = digest.full_digest - if len(summary_content) > MAX_DIGEST_CHARS: - summary_content = ( - f"# Repository: {repo_full_name}\n\n" - f"## File Structure\n\n{digest.tree}\n\n" - f"## File Contents (truncated)\n\n{digest.content[: MAX_DIGEST_CHARS - len(digest.tree) - 200]}..." - ) - - summary_text, summary_embedding = await generate_document_summary( - summary_content, user_llm, document_metadata_for_summary - ) - else: - summary_text = ( - f"# GitHub Repository: {repo_full_name}\n\n" - f"## Summary\n{digest.summary}\n\n" - f"## File Structure\n{digest.tree}" - ) - summary_embedding = embed_text(summary_text) + summary_embedding = embed_text(summary_text) # Chunk the full digest content for granular search try: @@ -443,10 +414,15 @@ async def index_github_repos( try: document.status = DocumentStatus.failed(str(repo_err)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() errors.append(f"Failed processing {repo_full_name}: {repo_err}") documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 3c9f27303..51df39171 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -2,7 +2,7 @@ Google Calendar connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding. +chunking, and embedding. """ from collections.abc import Awaitable, Callable @@ -21,7 +21,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES @@ -29,6 +28,7 @@ from .base import ( check_duplicate_document_by_hash, get_connector_by_id, logger, + mark_connector_documents_failed, parse_date_flexible, update_connector_last_indexed, ) @@ -53,7 +53,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Google Calendar API event dict to a ConnectorDocument.""" event_id = event.get("id", "") @@ -78,8 +77,6 @@ def _build_connector_doc( "connector_type": "Google Calendar", } - fallback_summary = f"Google Calendar Event: {event_summary}\n\n{event_markdown}" - return ConnectorDocument( title=event_summary, source_markdown=event_markdown, @@ -88,8 +85,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -420,7 +415,6 @@ async def index_google_calendar_events( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, ) with session.no_autoflush: @@ -448,18 +442,30 @@ async def index_google_calendar_events( # ── Pipeline: migrate legacy docs + parallel index ───────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, ) + # Placeholders for items skipped above (empty/duplicate/unbuildable) would + # otherwise stay stuck in 'pending' and undeletable. Fail them so they're + # recoverable. Leaves already-ready docs untouched. + indexed_ids = {doc.unique_id for doc in connector_docs} + stuck_placeholders = [ + (p.unique_id, "Skipped during sync: no indexable content") + for p in placeholders + if p.unique_id and p.unique_id not in indexed_ids + ] + if stuck_placeholders: + await mark_connector_documents_failed( + session, + document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR, + search_space_id=search_space_id, + failures=stuck_placeholders, + ) + # ── Finalize ────────────────────────────────────────────────── await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 686f13d9e..b76f84bac 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -35,17 +35,18 @@ from app.connectors.google_drive.file_types import ( from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash +from app.indexing_pipeline.exceptions import safe_exception_message from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, get_connector_by_id, + mark_connector_documents_failed, update_connector_last_indexed, ) from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES @@ -368,7 +369,12 @@ async def _should_skip_file( logger.info(f"Rename-only update: '{old_name}' → '{file_name}'") return True, f"File renamed: '{old_name}' → '{file_name}'" - if not DocumentStatus.is_state(existing.status, DocumentStatus.READY): + state = DocumentStatus.get_state(existing.status) + if state in (DocumentStatus.PENDING, DocumentStatus.PROCESSING): + # Stuck placeholder/in-progress doc (e.g. worker died mid-index): re-index + # instead of skipping, otherwise it never recovers. + return False, None + if state != DocumentStatus.READY: return True, "skipped (previously failed)" return True, "unchanged" @@ -381,7 +387,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Build a ConnectorDocument from Drive file metadata + extracted markdown.""" file_id = file.get("id", "") @@ -394,8 +399,6 @@ def _build_connector_doc( "connector_type": "Google Drive", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -404,8 +407,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -461,14 +462,14 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, -) -> tuple[list[ConnectorDocument], int]: - """Download and ETL files in parallel, returning ConnectorDocuments. +) -> tuple[list[ConnectorDocument], list[tuple[str, str]]]: + """Download and ETL files in parallel. - Returns (connector_docs, download_failed_count). + Returns (connector_docs, failed_files), where failed_files is a list of + (file_id, reason) so callers can mark those placeholders failed. """ results: list[ConnectorDocument] = [] sem = asyncio.Semaphore(max_concurrency) @@ -476,7 +477,8 @@ async def _download_files_parallel( completed_count = 0 hb_lock = asyncio.Lock() - async def _download_one(file: dict) -> ConnectorDocument | None: + async def _download_one(file: dict) -> ConnectorDocument | str: + # ConnectorDocument on success; failure reason string otherwise. nonlocal last_heartbeat, completed_count async with sem: markdown, drive_metadata, error = await download_and_extract_content( @@ -486,7 +488,7 @@ async def _download_files_parallel( file_name = file.get("name", "Unknown") reason = error or "empty content" logger.warning(f"Download/ETL failed for {file_name}: {reason}") - return None + return f"Download/ETL failed: {reason}" doc = _build_connector_doc( file, markdown, @@ -494,7 +496,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -508,14 +509,28 @@ async def _download_files_parallel( tasks = [_download_one(f) for f in files] outcomes = await asyncio.gather(*tasks, return_exceptions=True) - failed = 0 - for outcome in outcomes: - if isinstance(outcome, Exception) or outcome is None: - failed += 1 - else: + failed_files: list[tuple[str, str]] = [] + for file, outcome in zip(files, outcomes, strict=False): + if isinstance(outcome, ConnectorDocument): results.append(outcome) + continue + file_id = file.get("id") + if isinstance(outcome, Exception): + reason = f"Download/ETL error: {safe_exception_message(outcome)}" + logger.warning( + "Download/ETL exception for %s: %s", + file.get("name", "Unknown"), + outcome, + exc_info=outcome, + ) + elif isinstance(outcome, str): + reason = outcome + else: + reason = "Download or extraction failed" + if file_id: + failed_files.append((file_id, reason)) - return results, failed + return results, failed_files async def _process_single_file( @@ -525,7 +540,6 @@ async def _process_single_file( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Download, extract, and index a single Drive file via the pipeline. @@ -551,7 +565,16 @@ async def _process_single_file( drive_client, file, vision_llm=vision_llm ) if error or not markdown: - logger.warning(f"ETL failed for {file_name}: {error}") + reason = error or "empty content" + logger.warning(f"ETL failed for {file_name}: {reason}") + file_id = file.get("id") + if file_id: + await mark_connector_documents_failed( + session, + document_type=DocumentType.GOOGLE_DRIVE_FILE, + search_space_id=search_space_id, + failures=[(file_id, f"Download/ETL failed: {reason}")], + ) return 0, 1, 0 doc = _build_connector_doc( @@ -561,7 +584,6 @@ async def _process_single_file( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) pipeline = IndexingPipelineService(session) @@ -578,10 +600,7 @@ async def _process_single_file( connector_doc = doc_map.get(document.unique_identifier_hash) if not connector_doc: continue - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - await pipeline.index(document, connector_doc, user_llm) + await pipeline.index(document, connector_doc) await page_limit_service.update_page_usage( user_id, estimated_pages, allow_exceed=True @@ -636,7 +655,6 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: @@ -644,33 +662,37 @@ async def _download_and_index( Returns (batch_indexed, total_failed). """ - connector_docs, download_failed = await _download_files_parallel( + connector_docs, failed_files = await _download_files_parallel( drive_client, files, connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) + # Fail the placeholders for files whose download/ETL failed, so they don't + # stay stuck in 'pending'. + if failed_files: + await mark_connector_documents_failed( + session, + document_type=DocumentType.GOOGLE_DRIVE_FILE, + search_space_id=search_space_id, + failures=failed_files, + ) + batch_indexed = 0 batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) - return batch_indexed, download_failed + batch_failed + return batch_indexed, len(failed_files) + batch_failed async def _index_selected_files( @@ -681,7 +703,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int, int, list[str]]: @@ -746,7 +767,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -781,7 +801,6 @@ async def _index_full_scan( max_files: int, include_subfolders: bool = False, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -911,7 +930,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -946,7 +964,6 @@ async def _index_with_delta_sync( max_files: int, include_subfolders: bool = False, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Delta sync using change tracking. @@ -1054,7 +1071,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -1142,7 +1158,6 @@ async def index_google_drive_files( ) return 0, 0, client_error, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1189,7 +1204,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) documents_unsupported += du @@ -1208,7 +1222,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) documents_indexed += ri @@ -1234,7 +1247,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) @@ -1346,7 +1358,6 @@ async def index_google_drive_single_file( ) return 0, client_error - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1370,7 +1381,6 @@ async def index_google_drive_single_file( connector_id, search_space_id, user_id, - connector_enable_summary, vision_llm=vision_llm, ) await session.commit() @@ -1467,7 +1477,6 @@ async def index_google_drive_selected_files( ) return 0, 0, [error_msg] - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1481,7 +1490,6 @@ async def index_google_drive_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 6697c0eb1..25da96b61 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -2,7 +2,7 @@ Google Gmail connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding. +chunking, and embedding. """ from collections.abc import Awaitable, Callable @@ -21,7 +21,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES @@ -30,6 +29,7 @@ from .base import ( check_duplicate_document_by_hash, get_connector_by_id, logger, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -105,7 +105,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Gmail API message dict to a ConnectorDocument.""" message_id = message.get("id", "") @@ -138,12 +137,6 @@ def _build_connector_doc( "connector_type": "Google Gmail", } - fallback_summary = ( - f"Google Gmail Message: {subject}\n\n" - f"From: {sender}\nDate: {date_str}\n\n" - f"{markdown_content}" - ) - return ConnectorDocument( title=subject, source_markdown=markdown_content, @@ -152,8 +145,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -454,7 +445,6 @@ async def index_google_gmail_messages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, ) with session.no_autoflush: @@ -483,18 +473,30 @@ async def index_google_gmail_messages( # ── Pipeline: migrate legacy docs + parallel index ───────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, ) + # Placeholders for items skipped above (empty/duplicate/unbuildable) would + # otherwise stay stuck in 'pending' and undeletable. Fail them so they're + # recoverable. Leaves already-ready docs untouched. + indexed_ids = {doc.unique_id for doc in connector_docs} + stuck_placeholders = [ + (p.unique_id, "Skipped during sync: no indexable content") + for p in placeholders + if p.unique_id and p.unique_id not in indexed_ids + ] + if stuck_placeholders: + await mark_connector_documents_failed( + session, + document_type=DocumentType.GOOGLE_GMAIL_CONNECTOR, + search_space_id=search_space_id, + failures=stuck_placeholders, + ) + # ── Finalize ────────────────────────────────────────────────── await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py deleted file mode 100644 index 4d5cbb9f1..000000000 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ /dev/null @@ -1,364 +0,0 @@ -"""Jira connector indexer using the unified parallel indexing pipeline.""" - -import contextlib -from collections.abc import Awaitable, Callable - -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.connectors.jira_history import JiraHistoryConnector -from app.db import DocumentType, SearchSourceConnectorType -from app.indexing_pipeline.connector_document import ConnectorDocument -from app.indexing_pipeline.document_hashing import compute_content_hash -from app.indexing_pipeline.indexing_pipeline_service import ( - IndexingPipelineService, - PlaceholderInfo, -) -from app.services.llm_service import get_user_long_context_llm -from app.services.task_logging_service import TaskLoggingService - -from .base import ( - calculate_date_range, - check_duplicate_document_by_hash, - get_connector_by_id, - logger, - update_connector_last_indexed, -) - -HeartbeatCallbackType = Callable[[int], Awaitable[None]] -HEARTBEAT_INTERVAL_SECONDS = 30 - - -def _build_connector_doc( - issue: dict, - formatted_issue: dict, - issue_content: str, - *, - connector_id: int, - search_space_id: int, - user_id: str, - enable_summary: bool, -) -> ConnectorDocument: - """Map a raw Jira issue dict to a ConnectorDocument.""" - issue_id = issue.get("key", "") - issue_identifier = issue.get("key", "") - issue_title = issue.get("id", "") - state = formatted_issue.get("status", "Unknown") - priority = formatted_issue.get("priority", "Unknown") - comment_count = len(formatted_issue.get("comments", [])) - - metadata = { - "issue_id": issue_id, - "issue_identifier": issue_identifier, - "issue_title": issue_title, - "state": state, - "priority": priority, - "comment_count": comment_count, - "connector_id": connector_id, - "document_type": "Jira Issue", - "connector_type": "Jira", - } - - fallback_summary = ( - f"Jira Issue {issue_identifier}: {issue_title}\n\n" - f"Status: {state}\n\n{issue_content}" - ) - - return ConnectorDocument( - title=f"{issue_identifier}: {issue_title}", - source_markdown=issue_content, - unique_id=issue_id, - document_type=DocumentType.JIRA_CONNECTOR, - search_space_id=search_space_id, - connector_id=connector_id, - created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, - metadata=metadata, - ) - - -async def index_jira_issues( - session: AsyncSession, - connector_id: int, - search_space_id: int, - user_id: str, - start_date: str | None = None, - end_date: str | None = None, - update_last_indexed: bool = True, - on_heartbeat_callback: HeartbeatCallbackType | None = None, -) -> tuple[int, int, str | None]: - """Index Jira issues and comments.""" - task_logger = TaskLoggingService(session, search_space_id) - log_entry = await task_logger.log_task_start( - task_name="jira_issues_indexing", - source="connector_indexing_task", - message=f"Starting Jira issues indexing for connector {connector_id}", - metadata={ - "connector_id": connector_id, - "user_id": str(user_id), - "start_date": start_date, - "end_date": end_date, - }, - ) - - try: - connector = await get_connector_by_id( - session, connector_id, SearchSourceConnectorType.JIRA_CONNECTOR - ) - - if not connector: - await task_logger.log_task_failure( - log_entry, - f"Connector with ID {connector_id} not found", - "Connector not found", - {"error_type": "ConnectorNotFound"}, - ) - return 0, 0, f"Connector with ID {connector_id} not found" - - await task_logger.log_task_progress( - log_entry, - f"Initializing Jira client for connector {connector_id}", - {"stage": "client_initialization"}, - ) - jira_client = JiraHistoryConnector(session=session, connector_id=connector_id) - - if start_date == "undefined" or start_date == "": - start_date = None - if end_date == "undefined" or end_date == "": - end_date = None - - start_date_str, end_date_str = calculate_date_range( - connector, start_date, end_date, default_days_back=365 - ) - - await task_logger.log_task_progress( - log_entry, - f"Fetching Jira issues from {start_date_str} to {end_date_str}", - { - "stage": "fetching_issues", - "start_date": start_date_str, - "end_date": end_date_str, - }, - ) - - try: - issues, error = await jira_client.get_issues_by_date_range( - start_date=start_date_str, end_date=end_date_str, include_comments=True - ) - - if error: - if "No issues found" in error: - logger.info(f"No Jira issues found: {error}") - if update_last_indexed: - await update_connector_last_indexed( - session, connector, update_last_indexed - ) - await session.commit() - logger.info( - f"Updated last_indexed_at to {connector.last_indexed_at} despite no issues found" - ) - - await task_logger.log_task_success( - log_entry, - f"No Jira issues found in date range {start_date_str} to {end_date_str}", - {"issues_found": 0}, - ) - await jira_client.close() - return 0, 0, None - else: - logger.error(f"Failed to get Jira issues: {error}") - await task_logger.log_task_failure( - log_entry, - f"Failed to get Jira issues: {error}", - "API Error", - {"error_type": "APIError"}, - ) - await jira_client.close() - return 0, 0, f"Failed to get Jira issues: {error}" - - logger.info(f"Retrieved {len(issues)} issues from Jira API") - - except Exception as e: - logger.error(f"Error fetching Jira issues: {e!s}", exc_info=True) - await jira_client.close() - return 0, 0, f"Error fetching Jira issues: {e!s}" - - if not issues: - logger.info("No Jira issues found for the specified date range") - if update_last_indexed: - await update_connector_last_indexed( - session, connector, update_last_indexed - ) - await session.commit() - await jira_client.close() - return 0, 0, None - - # ── Create placeholders for instant UI feedback ─────────────── - pipeline = IndexingPipelineService(session) - placeholders = [ - PlaceholderInfo( - title=f"{issue.get('key', '')}: {issue.get('id', '')}", - document_type=DocumentType.JIRA_CONNECTOR, - unique_id=issue.get("key", ""), - search_space_id=search_space_id, - connector_id=connector_id, - created_by_id=user_id, - metadata={ - "issue_id": issue.get("key", ""), - "connector_id": connector_id, - "connector_type": "Jira", - }, - ) - for issue in issues - if issue.get("key") and issue.get("id") - ] - await pipeline.create_placeholder_documents(placeholders) - - connector_docs: list[ConnectorDocument] = [] - documents_skipped = 0 - duplicate_content_count = 0 - - for issue in issues: - try: - issue_id = issue.get("key") - issue_identifier = issue.get("key", "") - issue_title = issue.get("id", "") - - if not issue_id or not issue_title: - logger.warning( - f"Skipping issue with missing ID or title: {issue_id or 'Unknown'}" - ) - documents_skipped += 1 - continue - - formatted_issue = jira_client.format_issue(issue) - issue_content = jira_client.format_issue_to_markdown(formatted_issue) - - if not issue_content: - logger.warning( - f"Skipping issue with no content: {issue_identifier} - {issue_title}" - ) - documents_skipped += 1 - continue - - doc = _build_connector_doc( - issue, - formatted_issue, - issue_content, - connector_id=connector_id, - search_space_id=search_space_id, - user_id=user_id, - enable_summary=connector.enable_summary, - ) - - with session.no_autoflush: - duplicate_by_content = await check_duplicate_document_by_hash( - session, compute_content_hash(doc) - ) - - if duplicate_by_content: - logger.info( - f"Jira issue {issue_identifier} already indexed by another connector " - f"(existing document ID: {duplicate_by_content.id}, " - f"type: {duplicate_by_content.document_type}). Skipping." - ) - duplicate_content_count += 1 - documents_skipped += 1 - continue - - connector_docs.append(doc) - - except Exception as e: - logger.error( - f"Error building ConnectorDocument for issue {issue_identifier}: {e!s}", - exc_info=True, - ) - documents_skipped += 1 - continue - - await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s: AsyncSession): - return await get_user_long_context_llm(s, user_id, search_space_id) - - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( - connector_docs, - _get_llm, - max_concurrency=3, - on_heartbeat=on_heartbeat_callback, - heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, - ) - - await update_connector_last_indexed(session, connector, update_last_indexed) - - logger.info(f"Final commit: Total {documents_indexed} Jira issues processed") - try: - await session.commit() - logger.info("Successfully committed all JIRA document changes to database") - except Exception as e: - if ( - "duplicate key value violates unique constraint" in str(e).lower() - or "uniqueviolationerror" in str(e).lower() - ): - logger.warning( - f"Duplicate content_hash detected during final commit. " - f"This may occur if the same issue was indexed by multiple connectors. " - f"Rolling back and continuing. Error: {e!s}" - ) - await session.rollback() - # Don't fail the entire task - some documents may have been successfully indexed - else: - raise - - warning_parts = [] - if duplicate_content_count > 0: - warning_parts.append(f"{duplicate_content_count} duplicate") - if documents_failed > 0: - warning_parts.append(f"{documents_failed} failed") - warning_message = ", ".join(warning_parts) if warning_parts else None - - await task_logger.log_task_success( - log_entry, - f"Successfully completed JIRA indexing for connector {connector_id}", - { - "documents_indexed": documents_indexed, - "documents_skipped": documents_skipped, - "documents_failed": documents_failed, - "duplicate_content_count": duplicate_content_count, - }, - ) - logger.info( - f"JIRA indexing completed: {documents_indexed} ready, " - f"{documents_skipped} skipped, {documents_failed} failed " - f"({duplicate_content_count} duplicate content)" - ) - await jira_client.close() - return documents_indexed, documents_skipped, warning_message - - except SQLAlchemyError as db_error: - await session.rollback() - await task_logger.log_task_failure( - log_entry, - f"Database error during JIRA indexing for connector {connector_id}", - str(db_error), - {"error_type": "SQLAlchemyError"}, - ) - logger.error(f"Database error: {db_error!s}", exc_info=True) - if "jira_client" in locals(): - with contextlib.suppress(Exception): - await jira_client.close() - return 0, 0, f"Database error: {db_error!s}" - except Exception as e: - await session.rollback() - await task_logger.log_task_failure( - log_entry, - f"Failed to index JIRA issues for connector {connector_id}", - str(e), - {"error_type": type(e).__name__}, - ) - logger.error(f"Failed to index JIRA issues: {e!s}", exc_info=True) - if "jira_client" in locals(): - with contextlib.suppress(Exception): - await jira_client.close() - return 0, 0, f"Failed to index JIRA issues: {e!s}" diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 8500b700a..2bde77f79 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -2,7 +2,7 @@ Linear connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding with bounded parallel indexing. +chunking, and embedding with bounded parallel indexing. """ from collections.abc import Awaitable, Callable @@ -18,7 +18,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from .base import ( @@ -26,6 +25,7 @@ from .base import ( check_duplicate_document_by_hash, get_connector_by_id, logger, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -41,7 +41,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Linear issue dict to a ConnectorDocument.""" issue_id = issue.get("id", "") @@ -63,11 +62,6 @@ def _build_connector_doc( "connector_type": "Linear", } - fallback_summary = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n" - f"Status: {state}\n\n{issue_content}" - ) - return ConnectorDocument( title=f"{issue_identifier}: {issue_title}", source_markdown=issue_content, @@ -76,8 +70,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -277,7 +269,6 @@ async def index_linear_issues( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, ) with session.no_autoflush: @@ -306,18 +297,30 @@ async def index_linear_issues( # ── Pipeline: migrate legacy docs + parallel index ──────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, ) + # Placeholders for items skipped above (empty/duplicate/unbuildable) would + # otherwise stay stuck in 'pending' and undeletable. Fail them so they're + # recoverable. Leaves already-ready docs untouched. + indexed_ids = {doc.unique_id for doc in connector_docs} + stuck_placeholders = [ + (p.unique_id, "Skipped during sync: no indexable content") + for p in placeholders + if p.unique_id and p.unique_id not in indexed_ids + ] + if stuck_placeholders: + await mark_connector_documents_failed( + session, + document_type=DocumentType.LINEAR_CONNECTOR, + search_space_id=search_space_id, + failures=stuck_placeholders, + ) + # ── Finalize ────────────────────────────────────────────────── await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index 9352b60e0..1cd92dcf8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -33,7 +33,6 @@ from app.db import ( from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitExceededError, PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker @@ -478,7 +477,6 @@ def _build_connector_doc( *, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Build a ConnectorDocument from a local file's extracted content.""" unique_id = f"{folder_name}:{relative_path}" @@ -488,7 +486,6 @@ def _build_connector_doc( "document_type": "Local Folder File", "connector_type": "Local Folder", } - fallback_summary = f"File: {title}\n\n{content[:4000]}" return ConnectorDocument( title=title, @@ -498,8 +495,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=None, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -513,7 +508,6 @@ async def index_local_folder( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, on_heartbeat_callback: HeartbeatCallbackType | None = None, ) -> tuple[int, int, int | None, str | None]: @@ -574,7 +568,6 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_path=target_file_paths[0], - enable_summary=enable_summary, root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, @@ -587,7 +580,6 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_paths=target_file_paths, - enable_summary=enable_summary, root_folder_id=root_folder_id, on_progress_callback=on_heartbeat_callback, ) @@ -774,7 +766,6 @@ async def index_local_folder( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) connector_docs.append(doc) file_meta_map[unique_identifier] = { @@ -845,15 +836,13 @@ async def index_local_folder( doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs} documents = await pipeline.prepare_for_indexing(connector_docs) - llm = await get_user_long_context_llm(session, user_id, search_space_id) - for document in documents: connector_doc = doc_map.get(document.unique_identifier_hash) if connector_doc is None: failed_count += 1 continue - result = await pipeline.index(document, connector_doc, llm) + result = await pipeline.index(document, connector_doc) if DocumentStatus.is_state(result.status, DocumentStatus.READY): indexed_count += 1 @@ -960,7 +949,6 @@ async def _index_batch_files( folder_path: str, folder_name: str, target_file_paths: list[str], - enable_summary: bool, root_folder_id: int | None, on_progress_callback: HeartbeatCallbackType | None = None, ) -> tuple[int, int, str | None]: @@ -995,7 +983,6 @@ async def _index_batch_files( folder_path=folder_path, folder_name=folder_name, target_file_path=file_path, - enable_summary=enable_summary, root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, @@ -1036,7 +1023,6 @@ async def _index_single_file( folder_path: str, folder_name: str, target_file_path: str, - enable_summary: bool, root_folder_id: int | None, task_logger, log_entry, @@ -1125,7 +1111,6 @@ async def _index_single_file( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) if root_folder_id: @@ -1134,7 +1119,6 @@ async def _index_single_file( ) pipeline = IndexingPipelineService(session) - llm = await get_user_long_context_llm(session, user_id, search_space_id) documents = await pipeline.prepare_for_indexing([connector_doc]) if not documents: @@ -1142,7 +1126,7 @@ async def _index_single_file( db_doc = documents[0] - await pipeline.index(db_doc, connector_doc, llm) + await pipeline.index(db_doc, connector_doc) await session.refresh(db_doc) doc_meta = dict(db_doc.document_metadata or {}) @@ -1275,7 +1259,6 @@ async def index_uploaded_files( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], on_heartbeat_callback: HeartbeatCallbackType | None = None, use_vision_llm: bool = False, @@ -1318,7 +1301,6 @@ async def index_uploaded_files( page_limit_service = PageLimitService(session) pipeline = IndexingPipelineService(session) - llm = await get_user_long_context_llm(session, user_id, search_space_id) vision_llm_instance = None if use_vision_llm: @@ -1414,7 +1396,6 @@ async def index_uploaded_files( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) connector_doc.folder_id = await _resolve_folder_for_file( @@ -1432,7 +1413,7 @@ async def index_uploaded_files( db_doc = documents[0] - await pipeline.index(db_doc, connector_doc, llm) + await pipeline.index(db_doc, connector_doc) await session.refresh(db_doc) doc_meta = dict(db_doc.document_metadata or {}) diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 555d60273..eab2c9793 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -7,6 +7,7 @@ Implements 2-phase document status updates for real-time UI feedback: """ import asyncio +import contextlib import time from collections.abc import Awaitable, Callable from datetime import datetime, timedelta @@ -16,13 +17,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.luma_connector import LumaConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -437,38 +436,12 @@ async def index_luma_events( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "event_id": item["event_id"], - "event_name": item["event_name"], - "event_url": item["event_url"], - "start_at": item["start_at"], - "end_at": item["end_at"], - "timezone": item["timezone"], - "location": item["location"] or "No location", - "city": item["city"], - "hosts": item["host_names"], - "document_type": "Luma Event", - "connector_type": "Luma", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["event_markdown"], user_llm, document_metadata_for_summary - ) - else: - summary_content = ( - f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}" - ) - summary_embedding = await asyncio.to_thread( - embed_text, summary_content - ) + summary_content = ( + f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}" + ) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(item["event_markdown"]) @@ -513,10 +486,15 @@ async def index_luma_events( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() skipped_events.append( f"{item.get('event_name', 'Unknown')} (processing error)" ) diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index 77aac795a..9ebafbcdb 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -2,7 +2,7 @@ Notion connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding with bounded parallel indexing. +chunking, and embedding with bounded parallel indexing. """ from collections.abc import Awaitable, Callable @@ -19,7 +19,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.notion_utils import process_blocks @@ -28,6 +27,7 @@ from .base import ( check_duplicate_document_by_hash, get_connector_by_id, logger, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -43,7 +43,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Notion page dict to a ConnectorDocument.""" page_id = page.get("page_id", "") @@ -57,8 +56,6 @@ def _build_connector_doc( "connector_type": "Notion", } - fallback_summary = f"Notion Page: {page_title}\n\n{markdown_content}" - return ConnectorDocument( title=page_title, source_markdown=markdown_content, @@ -67,8 +64,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -314,7 +309,6 @@ async def index_notion_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, ) with session.no_autoflush: @@ -343,18 +337,30 @@ async def index_notion_pages( # ── Pipeline: migrate legacy docs + parallel index ──────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, ) + # Placeholders for items skipped above (empty/duplicate/unbuildable) would + # otherwise stay stuck in 'pending' and undeletable. Fail them so they're + # recoverable. Leaves already-ready docs untouched. + indexed_ids = {doc.unique_id for doc in connector_docs} + stuck_placeholders = [ + (p.unique_id, "Skipped during sync: no indexable content") + for p in placeholders + if p.unique_id and p.unique_id not in indexed_ids + ] + if stuck_placeholders: + await mark_connector_documents_failed( + session, + document_type=DocumentType.NOTION_CONNECTOR, + search_space_id=search_space_id, + failures=stuck_placeholders, + ) + # ── Finalize ────────────────────────────────────────────────── await update_connector_last_indexed(session, connector, update_last_indexed) diff --git a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py index 2def799f3..3fd8a79f2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py @@ -26,13 +26,14 @@ from app.connectors.onedrive.file_types import should_skip_file as skip_item from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash +from app.indexing_pipeline.exceptions import safe_exception_message from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, get_connector_by_id, + mark_connector_documents_failed, update_connector_last_indexed, ) @@ -120,7 +121,12 @@ async def _should_skip_file( logger.info(f"Rename-only update: '{old_name}' -> '{file_name}'") return True, f"File renamed: '{old_name}' -> '{file_name}'" - if not DocumentStatus.is_state(existing.status, DocumentStatus.READY): + state = DocumentStatus.get_state(existing.status) + if state in (DocumentStatus.PENDING, DocumentStatus.PROCESSING): + # Stuck placeholder/in-progress doc (e.g. worker died mid-index): re-index + # instead of skipping, otherwise it never recovers. + return False, None + if state != DocumentStatus.READY: return True, "skipped (previously failed)" return True, "unchanged" @@ -133,7 +139,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: file_id = file.get("id", "") file_name = file.get("name", "Unknown") @@ -145,8 +150,6 @@ def _build_connector_doc( "connector_type": "OneDrive", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -155,8 +158,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -168,19 +169,23 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, -) -> tuple[list[ConnectorDocument], int]: - """Download and ETL files in parallel. Returns (docs, failed_count).""" +) -> tuple[list[ConnectorDocument], list[tuple[str, str]]]: + """Download and ETL files in parallel. + + Returns (docs, failed_files), where failed_files is a list of + (file_id, reason) so callers can mark those placeholders failed. + """ results: list[ConnectorDocument] = [] sem = asyncio.Semaphore(max_concurrency) last_heartbeat = time.time() completed_count = 0 hb_lock = asyncio.Lock() - async def _download_one(file: dict) -> ConnectorDocument | None: + async def _download_one(file: dict) -> ConnectorDocument | str: + # ConnectorDocument on success; failure reason string otherwise. nonlocal last_heartbeat, completed_count async with sem: markdown, od_metadata, error = await download_and_extract_content( @@ -190,7 +195,7 @@ async def _download_files_parallel( file_name = file.get("name", "Unknown") reason = error or "empty content" logger.warning(f"Download/ETL failed for {file_name}: {reason}") - return None + return f"Download/ETL failed: {reason}" doc = _build_connector_doc( file, markdown, @@ -198,7 +203,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -212,14 +216,28 @@ async def _download_files_parallel( tasks = [_download_one(f) for f in files] outcomes = await asyncio.gather(*tasks, return_exceptions=True) - failed = 0 - for outcome in outcomes: - if isinstance(outcome, Exception) or outcome is None: - failed += 1 - else: + failed_files: list[tuple[str, str]] = [] + for file, outcome in zip(files, outcomes, strict=False): + if isinstance(outcome, ConnectorDocument): results.append(outcome) + continue + file_id = file.get("id") + if isinstance(outcome, Exception): + reason = f"Download/ETL error: {safe_exception_message(outcome)}" + logger.warning( + "Download/ETL exception for %s: %s", + file.get("name", "Unknown"), + outcome, + exc_info=outcome, + ) + elif isinstance(outcome, str): + reason = outcome + else: + reason = "Download or extraction failed" + if file_id: + failed_files.append((file_id, reason)) - return results, failed + return results, failed_files async def _download_and_index( @@ -230,38 +248,40 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: """Parallel download then parallel indexing. Returns (batch_indexed, total_failed).""" - connector_docs, download_failed = await _download_files_parallel( + connector_docs, failed_files = await _download_files_parallel( onedrive_client, files, connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) + # Fail rows for files whose download/ETL failed, so they don't stay stuck. + if failed_files: + await mark_connector_documents_failed( + session, + document_type=DocumentType.ONEDRIVE_FILE, + search_space_id=search_space_id, + failures=failed_files, + ) + batch_indexed = 0 batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) - return batch_indexed, download_failed + batch_failed + return batch_indexed, len(failed_files) + batch_failed async def _remove_document(session: AsyncSession, file_id: str, search_space_id: int): @@ -294,7 +314,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int, int, list[str]]: @@ -345,7 +364,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -379,7 +397,6 @@ async def _index_full_scan( max_files: int, include_subfolders: bool = True, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -454,7 +471,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -487,7 +503,6 @@ async def _index_with_delta_sync( log_entry: object, max_files: int, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int, str | None]: """Delta sync using OneDrive change tracking. @@ -579,7 +594,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -651,7 +665,6 @@ async def index_onedrive_files( ) return 0, 0, error_msg, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -681,7 +694,6 @@ async def index_onedrive_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed @@ -711,7 +723,6 @@ async def index_onedrive_files( task_logger, log_entry, max_files, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed @@ -738,7 +749,6 @@ async def index_onedrive_files( log_entry, max_files, include_subfolders, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += ri @@ -758,7 +768,6 @@ async def index_onedrive_files( log_entry, max_files, include_subfolders, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed diff --git a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py index 2c6d0e11e..ac63af38c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py @@ -9,6 +9,7 @@ Uses 2-phase document status updates for real-time UI feedback: - Phase 2: Process each document: pending → processing → ready/failed """ +import contextlib import time from collections.abc import Awaitable, Callable from datetime import datetime @@ -586,10 +587,15 @@ async def index_slack_messages( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( f"Failed to update document status to failed: {status_error}" ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py index 25994895a..e48aedaa5 100644 --- a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py @@ -10,6 +10,7 @@ Uses 2-phase document status updates for real-time UI feedback: """ import asyncio +import contextlib import time from collections.abc import Awaitable, Callable from datetime import UTC, datetime @@ -630,11 +631,16 @@ async def index_teams_messages( try: document.status = DocumentStatus.failed(str(e)) document.updated_at = get_current_timestamp() + # Commit now so the failed status survives a later rollback or + # crash; otherwise the doc stays stuck in pending/processing. + await session.commit() except Exception as status_error: logger.error( "Failed to update document status to failed: %s", str(status_error), ) + with contextlib.suppress(Exception): + await session.rollback() documents_failed += 1 continue diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index ada54e7fc..d81de67c0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -15,13 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.webcrawler_connector import WebCrawlerConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) from app.utils.webcrawler_utils import parse_webcrawler_urls @@ -320,9 +318,7 @@ async def index_crawled_urls( continue # Format content as structured document for summary generation - structured_document = crawler.format_to_structured_document( - crawl_result - ) + crawler.format_to_structured_document(crawl_result) # Generate content hash using a version WITHOUT metadata structured_document_for_hash = crawler.format_to_structured_document( @@ -334,8 +330,8 @@ async def index_crawled_urls( # Extract useful metadata title = metadata.get("title", url) - description = metadata.get("description", "") - language = metadata.get("language", "") + metadata.get("description", "") + metadata.get("language", "") # Update title immediately for better UX document.title = title @@ -372,29 +368,10 @@ async def index_crawled_urls( documents_skipped += 1 continue - # Generate summary with LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Select deterministic document content - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "url": url, - "title": title, - "description": description, - "language": language, - "document_type": "Crawled URL", - "crawler_type": crawler_type, - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - structured_document, user_llm, document_metadata_for_summary - ) - else: - summary_content = f"Crawled URL: {title}\n\nURL: {url}\n\n{content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Crawled URL: {title}\n\nURL: {url}\n\n{content}" + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(content) diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py index d633dd4f6..3b9616cbd 100644 --- a/surfsense_backend/app/tasks/document_processors/_save.py +++ b/surfsense_backend/app/tasks/document_processors/_save.py @@ -1,20 +1,15 @@ -""" -Unified document save/update logic for file processors. -""" +"""Unified document save/update logic for file processors.""" -import asyncio import logging from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) from ._helpers import ( @@ -24,59 +19,6 @@ from ._helpers import ( ) from .base import get_current_timestamp, safe_set_chunks -# --------------------------------------------------------------------------- -# Summary generation -# --------------------------------------------------------------------------- - - -async def _generate_summary( - markdown_content: str, - file_name: str, - etl_service: str, - user_llm, - enable_summary: bool, -) -> tuple[str, list[float]]: - """ - Generate a document summary and embedding. - - Docling uses its own large-document summary strategy; other ETL services - use the standard ``generate_document_summary`` helper. - """ - if not enable_summary: - summary = f"File: {file_name}\n\n{markdown_content[:4000]}" - return summary, await asyncio.to_thread(embed_text, summary) - - if etl_service == "DOCLING": - from app.services.docling_service import create_docling_service - - docling_service = create_docling_service() - summary_text = await docling_service.process_large_document_summary( - content=markdown_content, llm=user_llm, document_title=file_name - ) - - meta = { - "file_name": file_name, - "etl_service": etl_service, - "document_type": "File Document", - } - parts = ["# DOCUMENT METADATA"] - for key, value in meta.items(): - if value: - formatted_key = key.replace("_", " ").title() - parts.append(f"**{formatted_key}:** {value}") - - enhanced = "\n".join(parts) + "\n\n# DOCUMENT SUMMARY\n\n" + summary_text - return enhanced, await asyncio.to_thread(embed_text, enhanced) - - # Standard summary (Unstructured / LlamaCloud / others) - meta = { - "file_name": file_name, - "etl_service": etl_service, - "document_type": "File Document", - } - return await generate_document_summary(markdown_content, user_llm, meta) - - # --------------------------------------------------------------------------- # Unified save function # --------------------------------------------------------------------------- @@ -90,7 +32,6 @@ async def save_file_document( user_id: str, etl_service: str, connector: dict | None = None, - enable_summary: bool = True, ) -> Document | None: """ Process and store a file document with deduplication and migration support. @@ -106,7 +47,6 @@ async def save_file_document( user_id: ID of the user etl_service: Name of the ETL service (UNSTRUCTURED, LLAMACLOUD, DOCLING) connector: Optional connector info for Google Drive files - enable_summary: Whether to generate an AI summary Returns: Document object if successful, None if duplicate detected @@ -133,24 +73,16 @@ async def save_file_document( if should_skip: return doc - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} " - f"in search space {search_space_id}" - ) - - summary_content, summary_embedding = await _generate_summary( - markdown_content, file_name, etl_service, user_llm, enable_summary - ) + document_content = f"File: {file_name}\n\n{markdown_content[:4000]}" + document_embedding = embed_text(document_content) chunks = await create_document_chunks(markdown_content) doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service} if existing_document: existing_document.title = file_name - existing_document.content = summary_content + existing_document.content = document_content existing_document.content_hash = content_hash - existing_document.embedding = summary_embedding + existing_document.embedding = document_embedding existing_document.document_metadata = doc_metadata await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = markdown_content @@ -171,8 +103,8 @@ async def save_file_document( title=file_name, document_type=doc_type, document_metadata=doc_metadata, - content=summary_content, - embedding=summary_embedding, + content=document_content, + embedding=document_embedding, chunks=chunks, content_hash=content_hash, unique_identifier_hash=primary_hash, diff --git a/surfsense_backend/app/tasks/document_processors/circleback_processor.py b/surfsense_backend/app/tasks/document_processors/circleback_processor.py index a6b9568b9..ee36d5bc2 100644 --- a/surfsense_backend/app/tasks/document_processors/circleback_processor.py +++ b/surfsense_backend/app/tasks/document_processors/circleback_processor.py @@ -25,11 +25,10 @@ from app.db import ( SearchSourceConnectorType, SearchSpace, ) -from app.services.llm_service import get_document_summary_llm from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -176,34 +175,8 @@ async def add_circleback_meeting_document( # PHASE 3: Process the document content # ======================================================================= - # Get LLM for generating summary - llm = await get_document_summary_llm(session, search_space_id) - if not llm: - logger.warning( - f"No LLM configured for search space {search_space_id}. Using content as summary." - ) - # Use first 1000 chars as summary if no LLM available - summary_content = ( - markdown_content[:1000] + "..." - if len(markdown_content) > 1000 - else markdown_content - ) - summary_embedding = None - else: - # Generate summary with metadata - summary_metadata = { - "meeting_name": meeting_name, - "meeting_id": meeting_id, - "document_type": "Circleback Meeting", - **{ - k: v - for k, v in metadata.items() - if isinstance(v, str | int | float | bool) - }, - } - summary_content, summary_embedding = await generate_document_summary( - markdown_content, llm, summary_metadata - ) + summary_content = markdown_content + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(markdown_content) @@ -224,8 +197,7 @@ async def add_circleback_meeting_document( document.title = meeting_name document.content = summary_content document.content_hash = content_hash - if summary_embedding is not None: - document.embedding = summary_embedding + document.embedding = summary_embedding document.document_metadata = document_metadata await safe_set_chunks(session, document, chunks) document.source_markdown = markdown_content diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index 7320ec9fa..bdbc985fa 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -9,12 +9,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentType from app.schemas import ExtensionDocumentContent -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -123,26 +122,8 @@ async def add_extension_received_document( f"Content changed for URL {content.metadata.VisitedWebPageURL}. Updating document." ) - # Get user's long context LLM (needed for both create and update) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary with metadata - document_metadata = { - "session_id": content.metadata.BrowsingSessionId, - "url": content.metadata.VisitedWebPageURL, - "title": content.metadata.VisitedWebPageTitle, - "referrer": content.metadata.VisitedWebPageReffererURL, - "timestamp": content.metadata.VisitedWebPageDateWithTimeInISOString, - "duration_ms": content.metadata.VisitedWebPageVisitDurationInMilliseconds, - "document_type": "Browser Extension Capture", - } - summary_content, summary_embedding = await generate_document_summary( - combined_document_string, user_llm, document_metadata - ) + summary_content = combined_document_string + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(content.pageContent) diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 137c27cda..f6929b87c 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -10,13 +10,14 @@ from __future__ import annotations import contextlib import logging import os -from dataclasses import dataclass, field +from dataclasses import dataclass from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Document, Log, Notification -from app.services.notification_service import NotificationService +from app.db import Document, Log +from app.notifications.persistence import Notification +from app.notifications.service import NotificationService from app.services.task_logging_service import TaskLoggingService from ._helpers import update_document_from_connector @@ -48,12 +49,6 @@ class _ProcessingContext: notification: Notification | None = None use_vision_llm: bool = False processing_mode: str = "basic" - enable_summary: bool = field(init=False) - - def __post_init__(self) -> None: - self.enable_summary = ( - self.connector.get("enable_summary", True) if self.connector else True - ) # --------------------------------------------------------------------------- @@ -261,7 +256,6 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None: ctx.user_id, etl_result.etl_service, ctx.connector, - enable_summary=ctx.enable_summary, ) if result: @@ -466,7 +460,6 @@ async def process_file_in_background_with_document( log_entry: Log, connector: dict | None = None, notification: Notification | None = None, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> Document | None: @@ -482,7 +475,6 @@ async def process_file_in_background_with_document( from app.indexing_pipeline.adapters.file_upload_adapter import ( UploadDocumentAdapter, ) - from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import generate_content_hash from .base import check_duplicate_document @@ -522,8 +514,6 @@ async def process_file_in_background_with_document( stage="chunking", ) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - adapter = UploadDocumentAdapter(session) await adapter.index( markdown_content=markdown_content, @@ -531,8 +521,6 @@ async def process_file_in_background_with_document( etl_service=etl_service, search_space_id=search_space_id, user_id=user_id, - llm=user_llm, - should_summarize=should_summarize, ) if billable_pages > 0: diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 0ff340c0e..19a4df87d 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -8,12 +8,11 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, ) from ._helpers import ( @@ -183,21 +182,8 @@ async def add_received_markdown_file_document( return doc # Content changed - continue to update - # Get user's long context LLM (needed for both create and update) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary with metadata - document_metadata = { - "file_name": file_name, - "document_type": "Markdown File Document", - } - summary_content, summary_embedding = await generate_document_summary( - file_in_markdown, user_llm, document_metadata - ) + summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}" + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(file_in_markdown) diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 0ed2e57d2..dde5e4222 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -7,25 +7,25 @@ Implements 2-phase document status updates for real-time UI feedback: """ import logging +import time from urllib.parse import parse_qs, urlparse -import aiohttp from fake_useragent import UserAgent from requests import Session +from scrapling.fetchers import AsyncFetcher from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from youtube_transcript_api import YouTubeTranscriptApi from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) -from app.utils.proxy_config import get_requests_proxies +from app.utils.proxy import get_proxy_url, get_requests_proxies from .base import ( check_document_by_unique_identifier, @@ -219,18 +219,23 @@ async def add_youtube_video_document( } oembed_url = "https://www.youtube.com/oembed" - # Build residential proxy URL (if configured) + # Build residential proxy settings (if configured) residential_proxies = get_requests_proxies() - async with ( - aiohttp.ClientSession() as http_session, - http_session.get( - oembed_url, - params=params, - proxy=residential_proxies["http"] if residential_proxies else None, - ) as response, - ): - video_data = await response.json() + oembed_fetch_start = time.perf_counter() + oembed_page = await AsyncFetcher.get( + oembed_url, + params=params, + proxy=get_proxy_url(), + stealthy_headers=True, + ) + logging.info( + "[youtube][perf] source=oembed video=%s status=%s fetch_ms=%.1f", + video_id, + getattr(oembed_page, "status", None), + (time.perf_counter() - oembed_fetch_start) * 1000, + ) + video_data = oembed_page.json() # Update title immediately for better UX (user sees actual title sooner) document.title = video_data.get("title", f"YouTube Video: {video_id}") @@ -254,6 +259,7 @@ async def add_youtube_video_document( ) try: + transcript_fetch_start = time.perf_counter() ua = UserAgent() http_client = Session() http_client.headers.update({"User-Agent": ua.random}) @@ -266,6 +272,11 @@ async def add_youtube_video_document( transcript_list = ytt_api.list(video_id) transcript = next(iter(transcript_list)) captions = transcript.fetch() + logging.info( + "[youtube][perf] source=transcript video=%s fetch_ms=%.1f", + video_id, + (time.perf_counter() - transcript_fetch_start) * 1000, + ) # Include complete caption information with timestamps transcript_segments = [] @@ -355,40 +366,8 @@ async def add_youtube_video_document( await session.commit() return document - # Get LLM for summary generation - await task_logger.log_task_progress( - log_entry, - f"Preparing for summary generation: {video_data.get('title', 'YouTube Video')}", - {"stage": "llm_setup"}, - ) - - # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary - await task_logger.log_task_progress( - log_entry, - f"Generating summary for video: {video_data.get('title', 'YouTube Video')}", - {"stage": "summary_generation"}, - ) - - # Generate summary with metadata - document_metadata_for_summary = { - "url": url, - "video_id": video_id, - "title": video_data.get("title", "YouTube Video"), - "author": video_data.get("author_name", "Unknown"), - "thumbnail": video_data.get("thumbnail_url", ""), - "document_type": "YouTube Video Document", - "has_transcript": "No captions available" not in transcript_text, - } - summary_content, summary_embedding = await generate_document_summary( - combined_document_string, user_llm, document_metadata_for_summary - ) + summary_content = combined_document_string + summary_embedding = embed_text(summary_content) # Process chunks await task_logger.log_task_progress( diff --git a/surfsense_backend/app/utils/document_converters.py b/surfsense_backend/app/utils/document_converters.py index 059d91806..694ae22ac 100644 --- a/surfsense_backend/app/utils/document_converters.py +++ b/surfsense_backend/app/utils/document_converters.py @@ -9,7 +9,6 @@ from litellm import get_model_info, token_counter from app.config import config from app.db import Chunk, DocumentType -from app.prompts import SUMMARY_PROMPT_TEMPLATE logger = logging.getLogger(__name__) @@ -176,57 +175,6 @@ def optimize_content_for_context_window( return optimized_content -async def generate_document_summary( - content: str, - user_llm, - document_metadata: dict | None = None, -) -> tuple[str, list[float]]: - """ - Generate summary and embedding for document content with metadata. - - Args: - content: Document content - user_llm: User's LLM instance - document_metadata: Optional metadata dictionary to include in summary - - Returns: - Tuple of (enhanced_summary_content, summary_embedding) - """ - # Get model name from user_llm for token counting - model_name = getattr(user_llm, "model", "gpt-3.5-turbo") # Fallback to default - - # Optimize content to fit within context window - optimized_content = optimize_content_for_context_window( - content, document_metadata, model_name - ) - - summary_chain = SUMMARY_PROMPT_TEMPLATE | user_llm - content_with_metadata = f"\n\n{document_metadata}\n\n\n\n\n\n{optimized_content}\n\n" - summary_result = await summary_chain.ainvoke({"document": content_with_metadata}) - summary_content = summary_result.content - - # Combine summary with metadata if provided - if document_metadata: - metadata_parts = [] - metadata_parts.append("# DOCUMENT METADATA") - - for key, value in document_metadata.items(): - if value: # Only include non-empty values - formatted_key = key.replace("_", " ").title() - metadata_parts.append(f"**{formatted_key}:** {value}") - - metadata_section = "\n".join(metadata_parts) - enhanced_summary_content = ( - f"{metadata_section}\n\n# DOCUMENT SUMMARY\n\n{summary_content}" - ) - else: - enhanced_summary_content = summary_content - - summary_embedding = await asyncio.to_thread(embed_text, enhanced_summary_content) - - return enhanced_summary_content, summary_embedding - - async def create_document_chunks(content: str) -> list[Chunk]: """ Create chunks from document content. diff --git a/surfsense_backend/app/utils/proxy/__init__.py b/surfsense_backend/app/utils/proxy/__init__.py new file mode 100644 index 000000000..8ff489a41 --- /dev/null +++ b/surfsense_backend/app/utils/proxy/__init__.py @@ -0,0 +1,40 @@ +"""Modular residential / rotating proxy provider package. + +Selects a provider via the ``PROXY_PROVIDER`` env var (see ``registry.py``) and +exposes proxy settings in the formats different HTTP libraries expect. Add new +vendors by implementing :class:`ProxyProvider` in ``providers/`` and registering +them in ``registry.py``. +""" + +from app.utils.proxy.base import ProxyProvider +from app.utils.proxy.registry import get_active_provider + + +def get_proxy_url() -> str | None: + """Canonical ``http://user:pass@host:port`` URL for Scrapling/curl_cffi.""" + return get_active_provider().get_proxy_url() + + +def get_playwright_proxy() -> dict[str, str] | None: + """Playwright-style proxy dict, or ``None`` when not configured.""" + return get_active_provider().get_playwright_proxy() + + +def get_requests_proxies() -> dict[str, str] | None: + """``{"http": ..., "https": ...}`` dict for requests/aiohttp, or ``None``.""" + return get_active_provider().get_requests_proxies() + + +def get_residential_proxy_url() -> str | None: + """Backward-compatible alias for :func:`get_proxy_url`.""" + return get_proxy_url() + + +__all__ = [ + "ProxyProvider", + "get_active_provider", + "get_playwright_proxy", + "get_proxy_url", + "get_requests_proxies", + "get_residential_proxy_url", +] diff --git a/surfsense_backend/app/utils/proxy/base.py b/surfsense_backend/app/utils/proxy/base.py new file mode 100644 index 000000000..a3e84faf0 --- /dev/null +++ b/surfsense_backend/app/utils/proxy/base.py @@ -0,0 +1,46 @@ +"""Abstract base class for residential / rotating proxy providers. + +Each provider reads its own credentials from the application Config and exposes +proxy settings in the formats the different HTTP stacks expect: + +* ``get_proxy_url`` -> canonical ``http://user:pass@host:port`` string consumed + by Scrapling's fetchers (curl_cffi / patchright / camoufox). +* ``get_requests_proxies`` -> ``{"http": ..., "https": ...}`` dict for + ``requests`` / ``aiohttp``. +* ``get_playwright_proxy`` -> Playwright-style ``{"server", "username", + "password"}`` dict. + +Add a new vendor by subclassing :class:`ProxyProvider` in ``providers/`` and +registering it in ``registry.py``. +""" + +from abc import ABC, abstractmethod + + +class ProxyProvider(ABC): + """Interface every proxy provider must implement.""" + + #: Unique key used to select this provider via the ``PROXY_PROVIDER`` env var. + name: str = "base" + + @abstractmethod + def get_proxy_url(self) -> str | None: + """Return ``http://user:pass@host:port`` (no trailing slash), or ``None``. + + This is the canonical form Scrapling/curl_cffi consume directly. + """ + + @abstractmethod + def get_playwright_proxy(self) -> dict[str, str] | None: + """Return a Playwright proxy dict, or ``None`` when not configured.""" + + def get_requests_proxies(self) -> dict[str, str] | None: + """Return a ``requests``/``aiohttp`` proxies dict, or ``None``. + + Built from :meth:`get_proxy_url` by default; override if a provider needs + different http vs https endpoints. + """ + proxy_url = self.get_proxy_url() + if proxy_url is None: + return None + return {"http": proxy_url, "https": proxy_url} diff --git a/surfsense_backend/app/utils/proxy/providers/__init__.py b/surfsense_backend/app/utils/proxy/providers/__init__.py new file mode 100644 index 000000000..12057c06f --- /dev/null +++ b/surfsense_backend/app/utils/proxy/providers/__init__.py @@ -0,0 +1 @@ +"""Concrete proxy provider implementations.""" diff --git a/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py b/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py new file mode 100644 index 000000000..a005a9e72 --- /dev/null +++ b/surfsense_backend/app/utils/proxy/providers/anonymous_proxies.py @@ -0,0 +1,65 @@ +"""anonymous-proxies.net residential / rotating proxy provider. + +The vendor (``rotating.dnsproxifier.com``) encodes the location and rotation +``type`` options inside a base64-encoded JSON "password". The hostname already +includes the port (e.g. ``rotating.dnsproxifier.com:31230``). +""" + +import base64 +import json +import logging + +from app.config import Config +from app.utils.proxy.base import ProxyProvider + +logger = logging.getLogger(__name__) + + +class AnonymousProxiesProvider(ProxyProvider): + """Provider for anonymous-proxies.net credentials in ``RESIDENTIAL_PROXY_*``.""" + + name = "anonymous_proxies" + + def _password_b64(self) -> str | None: + """Build the base64-encoded password dict required by the vendor. + + Returns ``None`` when the password is not configured. + """ + password = Config.RESIDENTIAL_PROXY_PASSWORD + if not password: + return None + + password_dict = { + "p": password, + "l": Config.RESIDENTIAL_PROXY_LOCATION, + "t": Config.RESIDENTIAL_PROXY_TYPE, + } + return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode( + "utf-8" + ) + + def get_proxy_url(self) -> str | None: + username = Config.RESIDENTIAL_PROXY_USERNAME + hostname = Config.RESIDENTIAL_PROXY_HOSTNAME + password_b64 = self._password_b64() + + if not all([username, hostname, password_b64]): + return None + + # No trailing slash: curl_cffi (Scrapling static fetcher) expects a bare + # ``http://user:pass@host:port`` URL. + return f"http://{username}:{password_b64}@{hostname}" + + def get_playwright_proxy(self) -> dict[str, str] | None: + username = Config.RESIDENTIAL_PROXY_USERNAME + hostname = Config.RESIDENTIAL_PROXY_HOSTNAME + password_b64 = self._password_b64() + + if not all([username, hostname, password_b64]): + return None + + return { + "server": f"http://{hostname}", + "username": username, + "password": password_b64, + } diff --git a/surfsense_backend/app/utils/proxy/registry.py b/surfsense_backend/app/utils/proxy/registry.py new file mode 100644 index 000000000..777dfc049 --- /dev/null +++ b/surfsense_backend/app/utils/proxy/registry.py @@ -0,0 +1,44 @@ +"""Proxy provider registry. + +Maps the ``PROXY_PROVIDER`` config value to a :class:`ProxyProvider` +implementation. To add a new vendor: implement a provider in ``providers/`` and +add a single entry to ``_PROVIDERS`` below - no caller changes required. +""" + +import logging + +from app.config import Config +from app.utils.proxy.base import ProxyProvider +from app.utils.proxy.providers.anonymous_proxies import AnonymousProxiesProvider + +logger = logging.getLogger(__name__) + +# Registered proxy providers, keyed by their ``name``. +_PROVIDERS: dict[str, type[ProxyProvider]] = { + AnonymousProxiesProvider.name: AnonymousProxiesProvider, +} + +_DEFAULT_PROVIDER = AnonymousProxiesProvider.name + +_active_provider: ProxyProvider | None = None + + +def get_active_provider() -> ProxyProvider: + """Return the configured proxy provider instance (cached for the process).""" + global _active_provider + if _active_provider is not None: + return _active_provider + + key = (Config.PROXY_PROVIDER or _DEFAULT_PROVIDER).strip() + provider_cls = _PROVIDERS.get(key) + if provider_cls is None: + logger.warning( + "Unknown PROXY_PROVIDER '%s'; falling back to '%s'. Available: %s", + key, + _DEFAULT_PROVIDER, + ", ".join(sorted(_PROVIDERS)), + ) + provider_cls = _PROVIDERS[_DEFAULT_PROVIDER] + + _active_provider = provider_cls() + return _active_provider diff --git a/surfsense_backend/app/utils/proxy_config.py b/surfsense_backend/app/utils/proxy_config.py index de377e366..7285f3b4e 100644 --- a/surfsense_backend/app/utils/proxy_config.py +++ b/surfsense_backend/app/utils/proxy_config.py @@ -1,86 +1,20 @@ -""" -Residential proxy configuration utility. +"""Backward-compatible shim for the proxy helpers. -Reads proxy credentials from the application Config and provides helper -functions that return proxy configs in the format expected by different -HTTP libraries (requests, httpx, aiohttp, Playwright). +The implementation now lives in the modular :mod:`app.utils.proxy` package. +Existing imports of ``app.utils.proxy_config`` keep working via these re-exports. +Prefer importing from ``app.utils.proxy`` (and ``get_proxy_url``) in new code. """ -import base64 -import json -import logging +from app.utils.proxy import ( + get_playwright_proxy, + get_proxy_url, + get_requests_proxies, + get_residential_proxy_url, +) -from app.config import Config - -logger = logging.getLogger(__name__) - - -def _build_password_b64() -> str | None: - """ - Build the base64-encoded password dict required by anonymous-proxies.net. - - Returns ``None`` when the required config values are not set. - """ - password = Config.RESIDENTIAL_PROXY_PASSWORD - if not password: - return None - - password_dict = { - "p": password, - "l": Config.RESIDENTIAL_PROXY_LOCATION, - "t": Config.RESIDENTIAL_PROXY_TYPE, - } - return base64.b64encode(json.dumps(password_dict).encode("utf-8")).decode("utf-8") - - -def get_residential_proxy_url() -> str | None: - """ - Return the fully-formed residential proxy URL, or ``None`` when not - configured. - - The URL format is:: - - http://:@/ - """ - username = Config.RESIDENTIAL_PROXY_USERNAME - hostname = Config.RESIDENTIAL_PROXY_HOSTNAME - password_b64 = _build_password_b64() - - if not all([username, hostname, password_b64]): - return None - - return f"http://{username}:{password_b64}@{hostname}/" - - -def get_requests_proxies() -> dict[str, str] | None: - """ - Return a ``{"http": …, "https": …}`` dict suitable for - ``requests.Session.proxies`` and ``aiohttp`` ``proxy=`` kwarg, - or ``None`` when not configured. - """ - proxy_url = get_residential_proxy_url() - if proxy_url is None: - return None - return {"http": proxy_url, "https": proxy_url} - - -def get_playwright_proxy() -> dict[str, str] | None: - """ - Return a Playwright-compatible proxy dict:: - - {"server": "http://host:port", "username": "…", "password": "…"} - - or ``None`` when not configured. - """ - username = Config.RESIDENTIAL_PROXY_USERNAME - hostname = Config.RESIDENTIAL_PROXY_HOSTNAME - password_b64 = _build_password_b64() - - if not all([username, hostname, password_b64]): - return None - - return { - "server": f"http://{hostname}", - "username": username, - "password": password_b64, - } +__all__ = [ + "get_playwright_proxy", + "get_proxy_url", + "get_requests_proxies", + "get_residential_proxy_url", +] diff --git a/surfsense_backend/app/zero_publication.py b/surfsense_backend/app/zero_publication.py new file mode 100644 index 000000000..d2755d0a1 --- /dev/null +++ b/surfsense_backend/app/zero_publication.py @@ -0,0 +1,236 @@ +"""Canonical Zero publication definition for SurfSense. + +This module is the single source of truth for ``zero_publication``. Future +publication changes should update ``ZERO_PUBLICATION`` and call +``apply_publication()`` from a migration instead of hand-copying table lists. + +SurfSense runs Zero on Postgres with Zero's event triggers installed, so the +official Zero path is a plain ``ALTER PUBLICATION ... SET TABLE``. If a future +deployment cannot use event triggers, use Zero's documented +``zero_0.update_schemas()`` hook as the fallback instead of COMMENT bookends. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys +from collections.abc import Mapping, Sequence + +from sqlalchemy import text +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import create_async_engine + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + +ZERO_PUBLICATION: Mapping[str, Sequence[str] | None] = { + "notifications": None, + "documents": DOCUMENT_COLS, + "folders": None, + "search_source_connectors": None, + "new_chat_messages": None, + "chat_comments": None, + "chat_session_state": None, + "user": USER_COLS, + "automation_runs": AUTOMATION_RUN_COLS, +} + + +def _quote_identifier(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + +def _column_exists(conn: Connection, table: str, column: str) -> bool: + return ( + conn.execute( + text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_schema = current_schema() " + "AND table_name = :table AND column_name = :column" + ), + {"table": table, "column": column}, + ).fetchone() + is not None + ) + + +def _expected_columns(conn: Connection, table: str) -> list[str] | None: + columns = ZERO_PUBLICATION[table] + if columns is None: + return None + + expected = list(columns) + if table in {"documents", "user"} and _column_exists(conn, table, "_0_version"): + expected.append("_0_version") + return expected + + +def _format_table_entry(conn: Connection, table: str) -> str: + columns = _expected_columns(conn, table) + table_sql = _quote_identifier(table) + if columns is None: + return table_sql + + column_sql = ", ".join(_quote_identifier(column) for column in columns) + return f"{table_sql} ({column_sql})" + + +def build_set_table_sql(conn: Connection) -> str: + """Build the canonical plain SET TABLE statement for Zero's event triggers.""" + + table_list = ", ".join( + _format_table_entry(conn, table) for table in ZERO_PUBLICATION + ) + return f"ALTER PUBLICATION {_quote_identifier(PUBLICATION_NAME)} SET TABLE {table_list}" + + +def apply_publication(conn: Connection) -> None: + """Reconcile ``zero_publication`` to the canonical shape.""" + + exists = conn.execute( + text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + conn.execute(text(build_set_table_sql(conn))) + + +def _actual_publication_shape(conn: Connection) -> dict[str, list[str] | None]: + rows = conn.execute( + text( + "SELECT pt.tablename, pr.prattrs IS NULL AS all_columns, pt.attnames " + "FROM pg_publication_tables pt " + "JOIN pg_publication p ON p.pubname = pt.pubname " + "JOIN pg_class c ON c.relname = pt.tablename " + "JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = pt.schemaname " + "JOIN pg_publication_rel pr ON pr.prpubid = p.oid AND pr.prrelid = c.oid " + "WHERE pt.pubname = :name AND pt.schemaname = current_schema() " + "ORDER BY pt.tablename" + ), + {"name": PUBLICATION_NAME}, + ).mappings() + + return { + str(row["tablename"]): None + if row["all_columns"] + else list(row["attnames"] or []) + for row in rows + } + + +def expected_publication_shape(conn: Connection) -> dict[str, list[str] | None]: + return {table: _expected_columns(conn, table) for table in ZERO_PUBLICATION} + + +def verify_publication(conn: Connection) -> list[str]: + """Return human-readable mismatches between Postgres and the canonical shape.""" + + publication_exists = conn.execute( + text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not publication_exists: + return [f"Publication {PUBLICATION_NAME!r} does not exist"] + + actual = _actual_publication_shape(conn) + expected = expected_publication_shape(conn) + mismatches: list[str] = [] + + for table, expected_columns in expected.items(): + if table not in actual: + mismatches.append(f"{table}: missing from publication") + continue + + actual_columns = actual[table] + actual_key = sorted(actual_columns) if actual_columns is not None else None + expected_key = ( + sorted(expected_columns) if expected_columns is not None else None + ) + if actual_key != expected_key: + mismatches.append( + f"{table}: expected columns {expected_columns or 'ALL'}, " + f"got {actual_columns or 'ALL'}" + ) + + for table in sorted(set(actual) - set(expected)): + mismatches.append(f"{table}: unexpected table in publication") + + return mismatches + + +async def _verify_cli() -> int: + database_url = os.getenv("DATABASE_URL") + if not database_url: + print("DATABASE_URL is required to verify zero_publication.", file=sys.stderr) + return 2 + + engine = create_async_engine(database_url) + async with engine.connect() as async_conn: + + def run_verify(sync_conn: Connection) -> list[str]: + return verify_publication(sync_conn) + + mismatches = await async_conn.run_sync(run_verify) + + await engine.dispose() + + if mismatches: + print("zero_publication shape mismatch:", file=sys.stderr) + for mismatch in mismatches: + print(f" - {mismatch}", file=sys.stderr) + return 1 + + print("zero_publication shape verified.") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Manage SurfSense's Zero publication") + parser.add_argument( + "--verify", action="store_true", help="verify zero_publication shape" + ) + args = parser.parse_args() + + if args.verify: + return asyncio.run(_verify_cli()) + + parser.print_help() + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 51405ec74..16d46445c 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.26" +version = "0.0.27" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ @@ -25,7 +25,6 @@ dependencies = [ "notion-client>=2.3.0", "numpy>=1.24.0", "pgvector>=0.3.6", - "playwright>=1.50.0", "pypdf>=5.1.0", "python-ffmpeg>=2.0.12", "rerankers[flashrank]>=0.7.1", @@ -46,6 +45,7 @@ dependencies = [ "redis>=5.2.1", "firecrawl-py>=4.9.0", "boto3>=1.35.0", + "azure-storage-blob>=12.23.0", "fake-useragent>=2.2.0", "trafilatura>=2.0.0", "fastapi-users[oauth,sqlalchemy]>=15.0.3", @@ -87,9 +87,16 @@ dependencies = [ "opentelemetry-instrumentation-httpx>=0.61b0", "opentelemetry-instrumentation-celery>=0.61b0", "opentelemetry-instrumentation-logging>=0.61b0", + "python-telegram-bot>=22.7", "croniter>=2.0.0", + "scrapling[fetchers]>=0.4.9", ] +[project.optional-dependencies] +cpu = ["torch==2.11.0", "torchvision==0.26.0"] +cu126 = ["torch==2.11.0", "torchvision==0.26.0"] +cu128 = ["torch==2.11.0", "torchvision==0.26.0"] + [dependency-groups] dev = [ "ruff>=0.12.5", @@ -99,6 +106,36 @@ dev = [ "httpx>=0.28.1", ] +[tool.uv] +conflicts = [[{ extra = "cpu" }, { extra = "cu126" }, { extra = "cu128" }]] + +[tool.uv.sources] +torch = [ + { index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" }, + { index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" }, + { index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" }, +] +torchvision = [ + { index = "pytorch-cpu", extra = "cpu", marker = "sys_platform == 'linux'" }, + { index = "pytorch-cu126", extra = "cu126", marker = "sys_platform == 'linux'" }, + { index = "pytorch-cu128", extra = "cu128", marker = "sys_platform == 'linux'" }, +] + +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[[tool.uv.index]] +name = "pytorch-cu126" +url = "https://download.pytorch.org/whl/cu126" +explicit = true + +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true + [tool.ruff] # Exclude a variety of commonly ignored directories. exclude = [ @@ -193,10 +230,11 @@ asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" asyncio_default_test_loop_scope = "session" testpaths = ["tests"] +pythonpath = ["."] python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] -addopts = "-v --tb=short -x --strict-markers -ra --durations=5" +addopts = "-v --tb=short -x --strict-markers -ra --durations=5 --import-mode=importlib" markers = [ "unit: pure logic tests, no DB or external services", "integration: tests that require a real PostgreSQL database" diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index 81db1ae84..2efd78f94 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -49,10 +49,10 @@ trap cleanup SIGTERM SIGINT # ── Database migrations (only for migrate / all) ───────────── # Fail-fast contract: # - alembic upgrade head must succeed within ${MIGRATION_TIMEOUT:-900}s -# - zero_publication must exist in pg_publication afterwards +# - zero_publication must match the canonical app.zero_publication shape # Either failure exits non-zero so the dedicated `migrations` compose # service exits non-zero, halting the rest of the stack instead of -# silently producing a half-built system that crash-loops zero-cache. +# silently producing a drifted Zero publication. run_migrations() { echo "Running database migrations..." for i in {1..30}; do @@ -73,58 +73,13 @@ run_migrations() { fi echo "Migrations completed successfully." - echo "Verifying zero_publication exists in Postgres..." - local pub_oid - pub_oid=$(python <<'PY' 2>/dev/null || true -import asyncio -import sys -from sqlalchemy import text -from app.db import engine - - -async def get_oid(): - async with engine.connect() as conn: - result = await conn.execute( - text("SELECT oid FROM pg_publication WHERE pubname = 'zero_publication'") - ) - row = result.first() - if row is None: - sys.exit(1) - print(int(row[0])) - - -asyncio.run(get_oid()) -PY -) - if [ -z "${pub_oid}" ]; then - echo "ERROR: zero_publication is missing from Postgres after running alembic." >&2 - echo "This usually means migration 116 (or a later publication migration) did not run." >&2 + echo "Verifying zero_publication matches the canonical shape..." + if ! python -m app.zero_publication --verify; then + echo "ERROR: zero_publication does not match the canonical shape." >&2 echo "Inspect alembic state with:" >&2 echo " docker compose exec db psql -U \"\$DB_USER\" -d \"\$DB_NAME\" -c 'SELECT * FROM alembic_version;'" >&2 exit 1 fi - echo "zero_publication verified (oid=${pub_oid})." - - # Stale-replica safety net: if /zero-init is mounted (i.e. we are the - # dedicated `migrations` compose service), drop a marker file when the - # publication oid changed (or on first run) so the wrapped zero-cache - # entrypoint can wipe /data/zero.db before starting. This recovers from - # the case where a previous zero-cache crashed mid-init and left a - # half-built SQLite replica without a `_zero.tableMetadata` table. - if [ -d /zero-init ]; then - local stored_oid="" - [ -f /zero-init/last_pub_oid ] && stored_oid=$(cat /zero-init/last_pub_oid 2>/dev/null || true) - if [ -z "${stored_oid}" ] || [ "${stored_oid}" != "${pub_oid}" ]; then - echo "Publication oid changed (stored=${stored_oid:-}, current=${pub_oid}); writing /zero-init/needs_reset." - : > /zero-init/needs_reset - chmod 666 /zero-init/needs_reset 2>/dev/null || true - fi - echo "${pub_oid}" > /zero-init/last_pub_oid - chmod 666 /zero-init/last_pub_oid 2>/dev/null || true - # World-writable dir so the (possibly non-root) zero-cache container - # can `rm -f /zero-init/needs_reset` after acting on the marker. - chmod 777 /zero-init 2>/dev/null || true - fi } # ── Service starters ───────────────────────────────────────── @@ -140,11 +95,11 @@ start_worker() { if [ -n "${CELERY_QUEUES}" ]; then QUEUE_ARGS="--queues=${CELERY_QUEUES}" else - # When no queues specified, consume from BOTH the default queue and - # the connectors queue. Without --queues, Celery only consumes from - # the default queue, leaving connector indexing tasks stuck. + # When no queues specified, consume from the default, connectors, and + # gateway maintenance queues. Without --queues, Celery only consumes + # from the default queue, leaving connector/gateway maintenance tasks stuck. DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" - QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors" + QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" fi echo "Starting Celery Worker (autoscale=${CELERY_MAX_WORKERS},${CELERY_MIN_WORKERS}, max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD}, queues=${CELERY_QUEUES:-all})..." diff --git a/surfsense_backend/scripts/register_webhook.py b/surfsense_backend/scripts/register_webhook.py new file mode 100644 index 000000000..d1b570943 --- /dev/null +++ b/surfsense_backend/scripts/register_webhook.py @@ -0,0 +1,62 @@ +"""Register the SurfSense Telegram webhook.""" + +from __future__ import annotations + +import asyncio +import os +import re +import sys + +from dotenv import load_dotenv +from telegram import Bot + +from app.db import async_session_maker +from app.gateway.accounts import get_or_create_system_telegram_account + +load_dotenv() + +WEBHOOK_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{1,256}$") + + +async def main() -> int: + token = os.getenv("TELEGRAM_SHARED_BOT_TOKEN") + secret = os.getenv("TELEGRAM_WEBHOOK_SECRET") + base_url = os.getenv("GATEWAY_BASE_URL") or os.getenv("BACKEND_URL") + if not token or not secret or not base_url: + print( + "Missing TELEGRAM_SHARED_BOT_TOKEN, TELEGRAM_WEBHOOK_SECRET, or GATEWAY_BASE_URL/BACKEND_URL", + file=sys.stderr, + ) + return 1 + if not WEBHOOK_SECRET_RE.fullmatch(secret): + print( + "TELEGRAM_WEBHOOK_SECRET must be 1-256 chars and contain only A-Z, a-z, 0-9, '_' or '-'", + file=sys.stderr, + ) + return 1 + + async with async_session_maker() as session: + account = await get_or_create_system_telegram_account(session) + account.webhook_secret = secret + await session.commit() + account_id = int(account.id) + + webhook_url = ( + f"{base_url.rstrip('/')}/api/v1/gateway/webhooks/telegram/{account_id}" + ) + bot = Bot(token=token) + ok = await bot.set_webhook( + url=webhook_url, + secret_token=secret, + allowed_updates=["message", "edited_message"], + drop_pending_updates=True, + ) + if not ok: + print("Telegram rejected webhook registration", file=sys.stderr) + return 1 + print(f"Registered Telegram webhook: {webhook_url}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/surfsense_backend/scripts/whatsapp-bridge/Dockerfile b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile new file mode 100644 index 000000000..42bcd6b21 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/Dockerfile @@ -0,0 +1,15 @@ +FROM node:20-alpine + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --silent + +COPY . . + +ENV WHATSAPP_SESSION_DIR=/data/sessions +EXPOSE 9929 + +HEALTHCHECK --interval=30s --timeout=5s CMD wget -qO- http://127.0.0.1:9929/health || exit 1 + +CMD ["node", "bridge.js"] diff --git a/surfsense_backend/scripts/whatsapp-bridge/bridge.js b/surfsense_backend/scripts/whatsapp-bridge/bridge.js new file mode 100644 index 000000000..017456654 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/bridge.js @@ -0,0 +1,343 @@ +#!/usr/bin/env node + +import { + DisconnectReason, + fetchLatestBaileysVersion, + makeWASocket, + useMultiFileAuthState, +} from "@whiskeysockets/baileys"; +import { Boom } from "@hapi/boom"; +import express from "express"; +import { mkdirSync, readdirSync, rmSync } from "node:fs"; +import path from "node:path"; +import pino from "pino"; +import qrcode from "qrcode-terminal"; + +const PORT = Number(process.env.PORT || "9929"); +const SESSION_DIR = process.env.WHATSAPP_SESSION_DIR || "/data/sessions"; +const SEND_TIMEOUT_MS = Number(process.env.WHATSAPP_SEND_TIMEOUT_MS || "60000"); +const PAIRING_TIMEOUT_MS = Number(process.env.WHATSAPP_PAIRING_TIMEOUT_MS || "30000"); +const MAX_QUEUE_SIZE = Number(process.env.WHATSAPP_MAX_QUEUE_SIZE || "100"); +const WHATSAPP_MODE = process.env.WHATSAPP_MODE || "self-chat"; +const SENT_ECHO_TTL_MS = 60_000; + +mkdirSync(SESSION_DIR, { recursive: true }); + +const app = express(); +app.use(express.json({ limit: "2mb" })); + +const logger = pino({ level: process.env.WHATSAPP_DEBUG ? "debug" : "warn" }); +const messageQueue = []; +const sentKeys = new Map(); +const recentlySentIds = new Set(); + +let sock = null; +let connectionState = "disconnected"; +let latestQr = null; +let starting = null; +let pendingPairing = null; + +function resetSessionState() { + sock = null; + latestQr = null; + sentKeys.clear(); + recentlySentIds.clear(); + mkdirSync(SESSION_DIR, { recursive: true }); + for (const entry of readdirSync(SESSION_DIR)) { + rmSync(path.join(SESSION_DIR, entry), { recursive: true, force: true }); + } +} + +function resolvePendingPairing(payload) { + if (!pendingPairing) return; + clearTimeout(pendingPairing.timer); + pendingPairing.resolve(payload); + pendingPairing = null; +} + +function rejectPendingPairing(error) { + if (!pendingPairing) return; + clearTimeout(pendingPairing.timer); + pendingPairing.reject(error); + pendingPairing = null; +} + +async function maybeRequestPairingCode(update = {}) { + if (!pendingPairing || pendingPairing.inFlight || !sock) return; + + const canRequestPairingCode = + update.connection === "connecting" || + Boolean(update.qr) || + Boolean(latestQr); + + if (!canRequestPairingCode) return; + + pendingPairing.inFlight = true; + connectionState = "pairing"; + try { + const code = await sock.requestPairingCode(pendingPairing.phoneNumber); + resolvePendingPairing({ status: "pairing", pairing_code: code, expires_in: 60 }); + } catch (error) { + rejectPendingPairing(error); + } +} + +function requestPairingCodeWhenReady(phoneNumber) { + if (connectionState === "connected") { + return Promise.resolve({ status: "connected", pairing_code: null, expires_in: 0 }); + } + + if (pendingPairing) { + return Promise.reject(new Error("A WhatsApp pairing request is already in progress")); + } + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pendingPairing = null; + reject(new Error("Timed out waiting for WhatsApp to become ready for pairing")); + }, PAIRING_TIMEOUT_MS); + + pendingPairing = { + phoneNumber, + resolve, + reject, + timer, + inFlight: false, + }; + + void startSocket() + .then(() => maybeRequestPairingCode()) + .catch((error) => rejectPendingPairing(error)); + void maybeRequestPairingCode(); + }); +} + +function normalizeText(message) { + const content = message?.message || {}; + return ( + content.conversation || + content.extendedTextMessage?.text || + content.imageMessage?.caption || + content.videoMessage?.caption || + content.documentMessage?.caption || + "" + ); +} + +function enqueueMessage(message) { + const remoteJid = message?.key?.remoteJid; + const id = message?.key?.id; + if (!remoteJid || !id || !message?.message) return; + if (messageQueue.length >= MAX_QUEUE_SIZE) messageQueue.shift(); + messageQueue.push({ + event: "messages.upsert", + key: message.key, + chatId: remoteJid, + senderId: message.key.participant || remoteJid, + messageId: id, + fromMe: Boolean(message.key.fromMe), + isGroup: remoteJid.endsWith("@g.us"), + body: normalizeText(message), + timestamp: Number(message.messageTimestamp || Date.now() / 1000), + raw: message, + }); +} + +function rememberSentMessage(sent) { + const sentId = sent?.key?.id; + if (!sentId) return; + sentKeys.set(sentId, sent.key); + recentlySentIds.add(sentId); + setTimeout(() => { + recentlySentIds.delete(sentId); + }, SENT_ECHO_TTL_MS).unref?.(); +} + +function withTimeout(promise, timeoutMs) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`sendMessage timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +async function startSocket() { + if (starting) return starting; + starting = (async () => { + connectionState = "connecting"; + const { state, saveCreds } = await useMultiFileAuthState(SESSION_DIR); + const { version } = await fetchLatestBaileysVersion(); + sock = makeWASocket({ + version, + auth: state, + logger, + printQRInTerminal: false, + browser: ["SurfSense", "Chrome", "120.0"], + syncFullHistory: false, + markOnlineOnConnect: false, + getMessage: async () => ({ conversation: "" }), + }); + + sock.ev.on("creds.update", saveCreds); + sock.ev.on("connection.update", (update) => { + const { connection, lastDisconnect, qr } = update; + if (qr) { + latestQr = qr; + connectionState = "qr"; + qrcode.generate(qr, { small: true }); + void maybeRequestPairingCode(update); + } + if (connection === "open") { + latestQr = null; + connectionState = "connected"; + console.log("WhatsApp connected"); + resolvePendingPairing({ status: "connected", pairing_code: null, expires_in: 0 }); + } + if (connection === "close") { + const reason = new Boom(lastDisconnect?.error)?.output?.statusCode; + connectionState = "disconnected"; + if (reason === DisconnectReason.loggedOut) { + console.error("WhatsApp logged out; clearing session and waiting for pairing."); + connectionState = "logged_out"; + resetSessionState(); + setTimeout(() => { + starting = null; + void startSocket(); + }, 1000); + return; + } + setTimeout(() => { + starting = null; + void startSocket(); + }, reason === 515 ? 1000 : 3000); + } + void maybeRequestPairingCode(update); + }); + + sock.ev.on("messages.upsert", ({ messages, type }) => { + if (type !== "notify" && type !== "append") return; + for (const message of messages || []) { + const chatId = message?.key?.remoteJid; + if (!chatId) continue; + if (chatId.endsWith("@g.us") || chatId.includes("status@broadcast")) continue; + + if (message?.key?.fromMe) { + if (WHATSAPP_MODE !== "self-chat") continue; + if (recentlySentIds.has(message.key.id)) continue; + + const myNumber = (sock.user?.id || "").replace(/:.*@/, "@").replace(/@.*/, ""); + const myLid = (sock.user?.lid || "").replace(/:.*@/, "@").replace(/@.*/, ""); + const chatNumber = chatId.replace(/@.*/, ""); + const isSelfChat = + (myNumber && chatNumber === myNumber) || (myLid && chatNumber === myLid); + if (!isSelfChat) continue; + } else if (WHATSAPP_MODE === "self-chat") { + continue; + } + + enqueueMessage(message); + } + }); + })(); + try { + await starting; + } finally { + starting = null; + } +} + +app.get("/health", (_req, res) => { + res.json({ + status: connectionState, + hasQr: Boolean(latestQr), + qr: latestQr, + queueDepth: messageQueue.length, + user: sock?.user || null, + }); +}); + +app.get("/messages", (_req, res) => { + const messages = messageQueue.splice(0, messageQueue.length); + res.json(messages); +}); + +app.post("/send", async (req, res) => { + try { + if (!sock || connectionState !== "connected") { + return res.status(503).json({ error: "WhatsApp is not connected" }); + } + const { chatId, message, replyTo } = req.body || {}; + if (!chatId || !message) { + return res.status(400).json({ error: "chatId and message are required" }); + } + const payload = { text: String(message) }; + if (replyTo) { + payload.contextInfo = { stanzaId: String(replyTo) }; + } + const sent = await withTimeout(sock.sendMessage(chatId, payload), SEND_TIMEOUT_MS); + rememberSentMessage(sent); + res.json({ messageId: sent?.key?.id || null, raw: sent }); + } catch (error) { + res.status(500).json({ error: error?.message || "send failed" }); + } +}); + +app.post("/edit", async (req, res) => { + try { + if (!sock || connectionState !== "connected") { + return res.status(503).json({ error: "WhatsApp is not connected" }); + } + const { chatId, messageId, message } = req.body || {}; + if (!chatId || !messageId || !message) { + return res.status(400).json({ error: "chatId, messageId and message are required" }); + } + const key = sentKeys.get(String(messageId)) || { + remoteJid: chatId, + id: String(messageId), + fromMe: true, + }; + const sent = await withTimeout( + sock.sendMessage(chatId, { text: String(message), edit: key }), + SEND_TIMEOUT_MS, + ); + rememberSentMessage(sent); + res.json({ messageId: sent?.key?.id || messageId, raw: sent }); + } catch (error) { + res.status(500).json({ error: error?.message || "edit failed" }); + } +}); + +app.post("/typing", async (req, res) => { + try { + if (!sock || connectionState !== "connected") return res.status(204).end(); + const { chatId } = req.body || {}; + if (chatId) { + await sock.sendPresenceUpdate("composing", chatId); + } + res.status(204).end(); + } catch { + res.status(204).end(); + } +}); + +app.post("/pair", async (req, res) => { + try { + const phoneNumber = String(req.body?.phoneNumber || req.body?.phone_number || "").replace(/\D/g, ""); + if (!phoneNumber) { + return res.status(400).json({ error: "phoneNumber is required for pairing code" }); + } + res.json(await requestPairingCodeWhenReady(phoneNumber)); + } catch (error) { + res.status(500).json({ error: error?.message || "pairing failed" }); + } +}); + +app.listen(PORT, "0.0.0.0", () => { + console.log( + `SurfSense WhatsApp bridge listening on ${PORT}; session=${path.resolve(SESSION_DIR)}; mode=${WHATSAPP_MODE}`, + ); + void startSocket(); +}); diff --git a/surfsense_backend/scripts/whatsapp-bridge/package-lock.json b/surfsense_backend/scripts/whatsapp-bridge/package-lock.json new file mode 100644 index 000000000..52c6900d7 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/package-lock.json @@ -0,0 +1,2150 @@ +{ + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "dependencies": { + "@hapi/boom": "latest", + "@whiskeysockets/baileys": "latest", + "express": "latest", + "pino": "latest", + "qrcode-terminal": "latest" + } + }, + "node_modules/@borewit/text-codec": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@borewit/text-codec/-/text-codec-0.2.2.tgz", + "integrity": "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.0.9.tgz", + "integrity": "sha512-HdMx6DoGywB30vacDbBsITbIX4pgFqj1zsrV58jZBUw3klzkNoXhj7qOqAgledhxG7YZI5rBSJg7Zp8/VG0DuA==", + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.4.1", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/node-cache": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/@cacheable/node-cache/-/node-cache-1.7.6.tgz", + "integrity": "sha512-6Omk2SgNnjtxB5f/E6bTIWIt5xhdpx39fGNRQgU9lojvRxU68v+qY+SXXLsp3ZGukqoPjsK21wZ6XABFr/Ge3A==", + "license": "MIT", + "dependencies": { + "cacheable": "^2.3.1", + "hookified": "^1.14.0", + "keyv": "^5.5.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.4.1.tgz", + "integrity": "sha512-eiFgzCbIneyMlLOmNG4g9xzF7Hv3Mga4LjxjcSC/ues6VYq2+gUbQI8JqNuw/ZM8tJIeIaBGpswAsqV2V7ApgA==", + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@hapi/boom": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-10.0.1.tgz", + "integrity": "sha512-ERcCZaEjdH3OgSJlyjVk8pHIFeus91CjKP3v+MpgBNp5IvGzP2l/bRiD78nqYcKPaZdbKkK5vDBVPd2ohHBlsA==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^11.0.2" + } + }, + "node_modules/@hapi/hoek": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-11.0.7.tgz", + "integrity": "sha512-HV5undWkKzcB4RZUusqOpcgxOaq6VOAH7zhhIr2g3G8NF/MlFO75SjOr2NfuSx0Mh40+1FqCkagKLJRykUWoFQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "license": "MIT" + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.2.tgz", + "integrity": "sha512-pa0vFRuws4wkvaXKK1uXZMAwAX4/t8ANaJo45iw/oQHNQ9q5xUzwgFmVJGXiga2BeN+zpX7Vf9vmsiIa2J+MUw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@tokenizer/inflate": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.4.1.tgz", + "integrity": "sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "token-types": "^6.1.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "25.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", + "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@whiskeysockets/baileys": { + "version": "7.0.0-rc13", + "resolved": "https://registry.npmjs.org/@whiskeysockets/baileys/-/baileys-7.0.0-rc13.tgz", + "integrity": "sha512-8JPc8gaaCRykkjW2jxLGQ7/RZGrc7awO7WU+QJocf58eSUI9jAdcuYLynzhAbyU4UWvJJsHImZ+5E/JaZj5ypA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@cacheable/node-cache": "^1.4.0", + "@hapi/boom": "^9.1.3", + "async-mutex": "^0.5.0", + "libsignal": "^6.0.0", + "lru-cache": "^11.1.0", + "music-metadata": "^11.12.3", + "p-queue": "^9.0.0", + "pino": "^9.6", + "protobufjs": "^7.5.6", + "whatsapp-rust-bridge": "0.5.4", + "ws": "^8.13.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "audio-decode": "^2.1.3", + "jimp": "^1.6.1", + "link-preview-js": "^3.0.0", + "sharp": "*" + }, + "peerDependenciesMeta": { + "audio-decode": { + "optional": true + }, + "jimp": { + "optional": true + }, + "link-preview-js": { + "optional": true + } + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/@hapi/boom": { + "version": "9.1.4", + "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-9.1.4.tgz", + "integrity": "sha512-Ls1oH8jaN1vNsqcaHVYJrKmgMcKsC1wcp8bujvXrHaAqD2iDYq3HoOwsxwo09Cuda5R5nC0o0IxlrlTuvPuzSw==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "9.x.x" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/@hapi/hoek": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", + "integrity": "sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@whiskeysockets/baileys/node_modules/pino": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", + "integrity": "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/@whiskeysockets/baileys/node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/async-mutex": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.5.0.tgz", + "integrity": "sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacheable": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.3.5.tgz", + "integrity": "sha512-EQfaKe09tl615iNvq/TBRWTFf1AKJNXYQSsMx0Z3EI0nA+pVsVPS8wJhnRlkbdacKPh1d0qVIhwTc2zsQNFEEg==", + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.0.8", + "@cacheable/utils": "^2.4.1", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/curve25519-js": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/curve25519-js/-/curve25519-js-0.0.4.tgz", + "integrity": "sha512-axn2UMEnkhyDUPWOwVKBMVIzSQy2ejH2xRGy1wq81dqRwApXfIzfbE3hIX0ZRFBIihf/KDqK158DLwESu4AK1w==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/file-type": { + "version": "21.3.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-21.3.4.tgz", + "integrity": "sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==", + "license": "MIT", + "dependencies": { + "@tokenizer/inflate": "^0.4.1", + "strtok3": "^10.3.4", + "token-types": "^6.1.1", + "uint8array-extras": "^1.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hasown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/libsignal": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/libsignal/-/libsignal-6.0.0.tgz", + "integrity": "sha512-d/5V3YFtDljbFMufz4ncyUYGYhJl+vzAe+c2EFFBQ6bz1h8Q3IOMEGXYMzlibU60I+e8GagMMpji18iez3P1hA==", + "license": "GPL-3.0", + "dependencies": { + "curve25519-js": "^0.0.4", + "protobufjs": "^7.5.5" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "11.5.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", + "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/music-metadata": { + "version": "11.12.3", + "resolved": "https://registry.npmjs.org/music-metadata/-/music-metadata-11.12.3.tgz", + "integrity": "sha512-n6hSTZkuD59qWgHh6IP5dtDlDZQXoxk/bcA85Jywg8Z1iFrlNgl2+GTFgjZyn52W5UgQpV42V4XqrQZZAMbZTQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + }, + { + "type": "buymeacoffee", + "url": "https://buymeacoffee.com/borewit" + } + ], + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.2.2", + "@tokenizer/token": "^0.3.0", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "file-type": "^21.3.1", + "media-typer": "^1.1.0", + "strtok3": "^10.3.4", + "token-types": "^6.1.2", + "uint8array-extras": "^1.5.0", + "win-guid": "^0.2.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-queue": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.0.tgz", + "integrity": "sha512-7NED7xhQ74Ngp4JP/2e0VZHp7vSWfJfqeiR92jPgxsz6m0Se4P03YoTKa9dDXyZ3r6P616gUXttrB6nnHYKang==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.1.tgz", + "integrity": "sha512-4K0myLaWL5EteuSAro91EGFgcfVgxb64Jx+7oDAY6GOkXD4M69yuSEljNcInGVCA5sOPxmZ/EqDLj2x0Q0+Ygg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "license": "MIT" + }, + "node_modules/qrcode-terminal": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/qrcode-terminal/-/qrcode-terminal-0.12.0.tgz", + "integrity": "sha512-EXtzRZmC+YGmGlDFbXKxQiMZNwCLEO6BANKXG4iCtSIM0yqc/pappSx3RIKr4r0uh5JsBckOXeKrB3Iz7mdQpQ==", + "bin": { + "qrcode-terminal": "bin/qrcode-terminal.js" + } + }, + "node_modules/qs": { + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz", + "integrity": "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==", + "license": "ISC", + "peer": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/strtok3": { + "version": "10.3.5", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-10.3.5.tgz", + "integrity": "sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==", + "license": "MIT", + "dependencies": { + "@tokenizer/token": "^0.3.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/thread-stream/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/token-types": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-6.1.2.tgz", + "integrity": "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==", + "license": "MIT", + "dependencies": { + "@borewit/text-codec": "^0.2.1", + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/uint8array-extras": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/uint8array-extras/-/uint8array-extras-1.5.0.tgz", + "integrity": "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/whatsapp-rust-bridge": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/whatsapp-rust-bridge/-/whatsapp-rust-bridge-0.5.4.tgz", + "integrity": "sha512-yYO1qSs0Fe7tGtnxOFHomocUD6IZtoAgmA4oDFyGIRZ67D3QZk3w7swA6XXFXNQngiyrg2k7tul6IrM3eUFh7A==", + "license": "MIT" + }, + "node_modules/win-guid": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/win-guid/-/win-guid-0.2.1.tgz", + "integrity": "sha512-gEIQU4mkgl2OPeoNrWflcJFJ3Ae2BPd4eCsHHA/XikslkIVms/nHhvnvzIZV7VLmBvtFlDOzLt9rrZT+n6D67A==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/surfsense_backend/scripts/whatsapp-bridge/package.json b/surfsense_backend/scripts/whatsapp-bridge/package.json new file mode 100644 index 000000000..214ebacc6 --- /dev/null +++ b/surfsense_backend/scripts/whatsapp-bridge/package.json @@ -0,0 +1,16 @@ +{ + "name": "surfsense-whatsapp-bridge", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node bridge.js" + }, + "dependencies": { + "@hapi/boom": "latest", + "@whiskeysockets/baileys": "latest", + "express": "latest", + "pino": "latest", + "qrcode-terminal": "latest" + } +} diff --git a/surfsense_backend/tests/README.md b/surfsense_backend/tests/README.md new file mode 100644 index 000000000..5764252a5 --- /dev/null +++ b/surfsense_backend/tests/README.md @@ -0,0 +1,62 @@ +# Tests + +How the backend test suite is organized and the conventions to follow when adding tests. + +## Layout: type-first, module-mirrored + +Tests are split by **type** at the top level, and each type **mirrors the `app/` module tree** inside: + +``` +tests/ +├── conftest.py # global fixtures + DATABASE_URL pinning +├── unit/ # pure logic: no DB, no app, no network +│ └── notifications/ +│ ├── api/test_transform.py +│ └── service/ +│ ├── messages/test_connector_indexing.py +│ └── test_metadata.py +└── integration/ # real PostgreSQL (pgvector) + ├── conftest.py # async engine, transactional db_session, db_user, ... + └── notifications/ + ├── conftest.py # module-scoped fixtures (e.g. transactional client) + └── test_*_handler.py +``` + +To find a feature's tests, look under `tests//`. + +## Unit vs integration + +- `@pytest.mark.unit` — pure, fast, no I/O. Test behavior through a public function's inputs/outputs. +- `@pytest.mark.integration` — requires a real database. Run with `AUTH_TYPE=LOCAL`. + +Maximize logic covered by unit tests; keep integration tests for what genuinely needs the DB (persistence, SQL filters, scoping, HTTP wiring). + +## Principles + +- **Behavior, not implementation.** Assert observable outputs (returned values, persisted rows, HTTP responses), never private helpers. Tests should survive a refactor. +- **Functional core / imperative shell.** Put pure decision logic in a side-effect-free module (e.g. `app/notifications/service/messages/`) so it is unit-testable; keep the persistence shell thin and cover it with a few integration tests. +- **One responsibility per test file**, mirroring the slice it covers. +- **Mock only at system boundaries** (external APIs, brokers), never internal collaborators. Prefer dependency overrides and the transactional `db_session` over mocks. + +## Fixtures + +`conftest.py` is scoped to its directory and below. Keep truly global fixtures in `tests/conftest.py`; put module-specific fixtures in that module's `conftest.py` so a DB fixture never loads for a pure unit test. + +For API integration tests, override `get_async_session` and `current_active_user` to ride the test's transactional `db_session` (see `tests/integration/notifications/conftest.py`): rows seeded in the test and rows read via the endpoint share one transaction that rolls back automatically. + +## Import mode + +The suite uses `--import-mode=importlib` with `pythonpath = ["."]` (see `pyproject.toml`). This lets test files share basenames across modules (e.g. many `test_api.py`) without `__init__.py` boilerplate; new test directories do not need an `__init__.py`. + +## Running + +```bash +# fast unit tests +uv run pytest -m unit + +# integration (needs Postgres + pgvector) +AUTH_TYPE=LOCAL uv run pytest -m integration + +# a single module's tests +uv run pytest tests/unit/notifications +``` diff --git a/surfsense_backend/tests/e2e/auth_mint.py b/surfsense_backend/tests/e2e/auth_mint.py index f489ed274..edbf09f1a 100644 --- a/surfsense_backend/tests/e2e/auth_mint.py +++ b/surfsense_backend/tests/e2e/auth_mint.py @@ -51,7 +51,9 @@ async def mint_test_token( raise HTTPException(status_code=403, detail="invalid e2e mint secret") async with async_session_maker() as session: result = await session.execute(select(User).where(User.email == body.email)) - user = result.scalar_one_or_none() + # ``.unique()`` is required because the User mapper eager-loads a + # collection (oauth_accounts) via joined load. + user = result.unique().scalar_one_or_none() if user is None: raise HTTPException( status_code=404, detail=f"e2e user {body.email!r} not seeded" diff --git a/surfsense_backend/tests/e2e/fakes/chat_llm.py b/surfsense_backend/tests/e2e/fakes/chat_llm.py index fa3a2b158..234a18ec1 100644 --- a/surfsense_backend/tests/e2e/fakes/chat_llm.py +++ b/surfsense_backend/tests/e2e/fakes/chat_llm.py @@ -553,6 +553,49 @@ class FakeChatLLM(BaseChatModel): latest_tool_name = getattr(latest_tool, "name", None) latest_tool_text = _content_to_text(latest_tool.content) if latest_tool else "" + # Marker unique to a connector subagent's prompt: the main agent must + # delegate via ``task``; only the subagent has connector tools registered. + in_connector_subagent = ( + "specialist for the user's connected" in _messages_to_text(messages) + ) + + # Main agent: delegate live-tool connector work to its subagent (which + # then runs the real tools below). Indexed connectors are absent here. + if not in_connector_subagent and latest_tool is None: + connector_delegations = ( + ("gmail", ("gmail", "email", "message", GMAIL_CANARY_SUBJECT)), + ("calendar", ("calendar", "event", "meeting", CALENDAR_CANARY_SUMMARY)), + ( + "jira", + ( + "jira", + "atlassian", + JIRA_CANARY_SUMMARY, + JIRA_CANARY_KEY, + "surfsense-e2e.atlassian.net", + "fake-jira-cloud-001", + ), + ), + ("linear", ("linear", "issue", LINEAR_CANARY_TITLE)), + ("slack", ("slack", SLACK_CANARY_TOKEN)), + ("clickup", ("clickup", CLICKUP_CANARY_TITLE)), + ) + for subagent_type, needles in connector_delegations: + if _contains_any(latest_human, needles): + return AIMessage( + content="", + tool_calls=[ + { + "name": "task", + "args": { + "subagent_type": subagent_type, + "description": latest_human, + }, + "id": f"call_e2e_task_{subagent_type}", + } + ], + ) + if ( latest_tool_name == "search_gmail" and GMAIL_CANARY_MESSAGE_ID in latest_tool_text diff --git a/surfsense_backend/tests/e2e/fakes/llm.py b/surfsense_backend/tests/e2e/fakes/llm.py index 9d2370e2c..8172dd86a 100644 --- a/surfsense_backend/tests/e2e/fakes/llm.py +++ b/surfsense_backend/tests/e2e/fakes/llm.py @@ -7,13 +7,13 @@ The production indexing pipeline summarizes documents with: summary_content = summary_result.content The `llm` parameter is supplied per-document by -`app.services.llm_service.get_user_long_context_llm`. We patch THAT +`app.services.llm_service.get_agent_llm`. We patch THAT function to return a langchain-native FakeListChatModel so the rest of the chain works unchanged. No real LLM provider package is touched. Run-backend / run-celery use unittest.mock.patch.start() to install this at every binding site (the source module + every consumer that -did `from app.services.llm_service import get_user_long_context_llm` +did `from app.services.llm_service import get_agent_llm` at module load time). """ @@ -42,7 +42,7 @@ def _make_fake_llm() -> FakeListChatModel: return fake -async def fake_get_user_long_context_llm(*args: Any, **kwargs: Any) -> Any: - """Drop-in replacement for app.services.llm_service.get_user_long_context_llm.""" +async def fake_get_agent_llm(*args: Any, **kwargs: Any) -> Any: + """Drop-in replacement for app.services.llm_service.get_agent_llm.""" logger.info("[fake-llm] returning FakeListChatModel for E2E indexing") return _make_fake_llm() diff --git a/surfsense_backend/tests/e2e/fakes/mcp_runtime.py b/surfsense_backend/tests/e2e/fakes/mcp_runtime.py index e772bb63a..5cd465af1 100644 --- a/surfsense_backend/tests/e2e/fakes/mcp_runtime.py +++ b/surfsense_backend/tests/e2e/fakes/mcp_runtime.py @@ -137,10 +137,13 @@ def install(active_patches: list[Any]) -> None: """Patch production MCP streamable-HTTP boundaries exactly once.""" targets = [ ( - "app.agents.new_chat.tools.mcp_tool.streamablehttp_client", + "app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.streamablehttp_client", _fake_streamablehttp_client, ), - ("app.agents.new_chat.tools.mcp_tool.ClientSession", _FakeClientSession), + ( + "app.agents.chat.multi_agent_chat.shared.tools.mcp.tool.ClientSession", + _FakeClientSession, + ), ] for target, replacement in targets: p = patch(target, replacement) diff --git a/surfsense_backend/tests/e2e/fakes/native_google.py b/surfsense_backend/tests/e2e/fakes/native_google.py index 73c8cc738..1afcaf9c3 100644 --- a/surfsense_backend/tests/e2e/fakes/native_google.py +++ b/surfsense_backend/tests/e2e/fakes/native_google.py @@ -429,9 +429,18 @@ def install(active_patches: list[Any]) -> None: ("app.connectors.google_drive.client.build", _fake_build), ("app.connectors.google_gmail_connector.build", _fake_build), ("app.connectors.google_calendar_connector.build", _fake_build), - ("app.agents.new_chat.tools.google_calendar.create_event.build", _fake_build), - ("app.agents.new_chat.tools.google_calendar.update_event.build", _fake_build), - ("app.agents.new_chat.tools.google_calendar.delete_event.build", _fake_build), + ( + "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.create_event.build", + _fake_build, + ), + ( + "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.update_event.build", + _fake_build, + ), + ( + "app.agents.chat.multi_agent_chat.subagents.connectors.calendar.tools.delete_event.build", + _fake_build, + ), ("googleapiclient.http.MediaIoBaseDownload", _FakeMediaIoBaseDownload), ( "app.connectors.google_drive.client._build_thread_http", diff --git a/surfsense_backend/tests/e2e/run_backend.py b/surfsense_backend/tests/e2e/run_backend.py index 5a787ac52..87977626f 100644 --- a/surfsense_backend/tests/e2e/run_backend.py +++ b/surfsense_backend/tests/e2e/run_backend.py @@ -72,8 +72,6 @@ def _load_dotenv_and_set_env_defaults() -> None: """ from dotenv import load_dotenv - load_dotenv() - os.environ.setdefault( "DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense", @@ -138,6 +136,11 @@ def _load_dotenv_and_set_env_defaults() -> None: os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id" os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret" + # Load .env last so the E2E defaults above win over a developer's .env + # (e.g. AUTH_TYPE=GOOGLE), while an explicitly exported shell var still + # beats both: setdefault respects it and load_dotenv() never overrides. + load_dotenv() + def _install_synthetic_global_llm_config() -> None: """Materialise a fake ``app/config/global_llm_config.yaml`` for E2E. @@ -206,23 +209,23 @@ def _patch_llm_bindings() -> None: fake_create_chat_litellm_from_agent_config, fake_create_chat_litellm_from_config, ) - from tests.e2e.fakes.llm import fake_get_user_long_context_llm + from tests.e2e.fakes.llm import fake_get_agent_llm targets = [ - "app.services.llm_service.get_user_long_context_llm", - "app.tasks.connector_indexers.confluence_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_drive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_gmail_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.notion_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.onedrive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.dropbox_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.local_folder_indexer.get_user_long_context_llm", - "app.tasks.document_processors._save.get_user_long_context_llm", - "app.tasks.document_processors.markdown_processor.get_user_long_context_llm", + "app.services.llm_service.get_agent_llm", + "app.tasks.connector_indexers.confluence_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_drive_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_gmail_indexer.get_agent_llm", + "app.tasks.connector_indexers.notion_indexer.get_agent_llm", + "app.tasks.connector_indexers.onedrive_indexer.get_agent_llm", + "app.tasks.connector_indexers.dropbox_indexer.get_agent_llm", + "app.tasks.connector_indexers.local_folder_indexer.get_agent_llm", + "app.tasks.document_processors._save.get_agent_llm", + "app.tasks.document_processors.markdown_processor.get_agent_llm", ] for target in targets: try: - p = patch(target, fake_get_user_long_context_llm) + p = patch(target, fake_get_agent_llm) p.start() _active_patches.append(p) logger.info("[fake-llm] patched %s", target) @@ -239,19 +242,19 @@ def _patch_llm_bindings() -> None: chat_targets = [ ( - "app.agents.new_chat.llm_config.create_chat_litellm_from_agent_config", + "app.agents.chat.runtime.llm_config.create_chat_litellm_from_agent_config", fake_create_chat_litellm_from_agent_config, ), ( - "app.agents.new_chat.llm_config.create_chat_litellm_from_config", + "app.agents.chat.runtime.llm_config.create_chat_litellm_from_config", fake_create_chat_litellm_from_config, ), ( - "app.tasks.chat.stream_new_chat.create_chat_litellm_from_agent_config", + "app.tasks.chat.streaming.flows.shared.llm_bundle.create_chat_litellm_from_agent_config", fake_create_chat_litellm_from_agent_config, ), ( - "app.tasks.chat.stream_new_chat.create_chat_litellm_from_config", + "app.tasks.chat.streaming.flows.shared.llm_bundle.create_chat_litellm_from_config", fake_create_chat_litellm_from_config, ), ] diff --git a/surfsense_backend/tests/e2e/run_celery.py b/surfsense_backend/tests/e2e/run_celery.py index e4091d689..bde547083 100644 --- a/surfsense_backend/tests/e2e/run_celery.py +++ b/surfsense_backend/tests/e2e/run_celery.py @@ -57,8 +57,6 @@ def _load_dotenv_and_set_env_defaults() -> None: """ from dotenv import load_dotenv - load_dotenv() - os.environ.setdefault( "DATABASE_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense", @@ -122,6 +120,11 @@ def _load_dotenv_and_set_env_defaults() -> None: os.environ["SLACK_CLIENT_ID"] = "fake-slack-mcp-client-id" os.environ["SLACK_CLIENT_SECRET"] = "fake-slack-mcp-client-secret" + # Load .env last so the E2E defaults above win over a developer's .env + # (e.g. AUTH_TYPE=GOOGLE), while an explicitly exported shell var still + # beats both: setdefault respects it and load_dotenv() never overrides. + load_dotenv() + def _install_synthetic_global_llm_config() -> None: """Materialise a fake ``app/config/global_llm_config.yaml`` for E2E. @@ -183,23 +186,23 @@ def _patch_llm_bindings() -> None: fake_create_chat_litellm_from_agent_config, fake_create_chat_litellm_from_config, ) - from tests.e2e.fakes.llm import fake_get_user_long_context_llm + from tests.e2e.fakes.llm import fake_get_agent_llm targets = [ - "app.services.llm_service.get_user_long_context_llm", - "app.tasks.connector_indexers.confluence_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_drive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_gmail_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.notion_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.onedrive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.dropbox_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.local_folder_indexer.get_user_long_context_llm", - "app.tasks.document_processors._save.get_user_long_context_llm", - "app.tasks.document_processors.markdown_processor.get_user_long_context_llm", + "app.services.llm_service.get_agent_llm", + "app.tasks.connector_indexers.confluence_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_drive_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_gmail_indexer.get_agent_llm", + "app.tasks.connector_indexers.notion_indexer.get_agent_llm", + "app.tasks.connector_indexers.onedrive_indexer.get_agent_llm", + "app.tasks.connector_indexers.dropbox_indexer.get_agent_llm", + "app.tasks.connector_indexers.local_folder_indexer.get_agent_llm", + "app.tasks.document_processors._save.get_agent_llm", + "app.tasks.document_processors.markdown_processor.get_agent_llm", ] for target in targets: try: - p = patch(target, fake_get_user_long_context_llm) + p = patch(target, fake_get_agent_llm) p.start() _active_patches.append(p) logger.info("[fake-llm] patched %s in celery worker", target) @@ -212,19 +215,19 @@ def _patch_llm_bindings() -> None: chat_targets = [ ( - "app.agents.new_chat.llm_config.create_chat_litellm_from_agent_config", + "app.agents.chat.runtime.llm_config.create_chat_litellm_from_agent_config", fake_create_chat_litellm_from_agent_config, ), ( - "app.agents.new_chat.llm_config.create_chat_litellm_from_config", + "app.agents.chat.runtime.llm_config.create_chat_litellm_from_config", fake_create_chat_litellm_from_config, ), ( - "app.tasks.chat.stream_new_chat.create_chat_litellm_from_agent_config", + "app.tasks.chat.streaming.flows.shared.llm_bundle.create_chat_litellm_from_agent_config", fake_create_chat_litellm_from_agent_config, ), ( - "app.tasks.chat.stream_new_chat.create_chat_litellm_from_config", + "app.tasks.chat.streaming.flows.shared.llm_bundle.create_chat_litellm_from_config", fake_create_chat_litellm_from_config, ), ] diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py new file mode 100644 index 000000000..d45570484 --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_agent_turn.py @@ -0,0 +1,140 @@ +"""Guardrail D: the real multi-agent is still assemblable and runnable. + +Builds the production ``create_multi_agent_chat_deep_agent`` factory against a +real (test) DB with a scripted LLM, then drives one turn. This is the only +guard that proves the *assembled* agent — full tool registry, middleware stack, +compiled graph — still executes end to end after files move. A/B/C prove the +parts import, wire, and load; this proves they run together. + +Scripted LLM + faked external tools; everything we own (graph, middleware, +DB-backed connector service) runs for real. +""" + +from __future__ import annotations + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.chat.multi_agent_chat import create_multi_agent_chat_deep_agent +from app.services.connector_service import ConnectorService +from tests.integration.harness import ( + ScriptedTurn, + StubToolSpec, + build_scripted_harness, +) + +pytestmark = pytest.mark.integration + + +def _last_ai_text(messages: list) -> str | None: + for m in reversed(messages): + if isinstance(m, AIMessage): + return m.content if isinstance(m.content, str) else str(m.content) + return None + + +@pytest.mark.asyncio +async def test_agent_runs_a_scripted_text_turn(db_session, db_user, db_search_space): + """A freshly assembled agent streams a scripted final-text turn to completion.""" + harness = build_scripted_harness(turns=[ScriptedTurn(text="done")]) + + agent = await create_multi_agent_chat_deep_agent( + llm=harness.model, + search_space_id=db_search_space.id, + db_session=db_session, + connector_service=ConnectorService(db_session), + checkpointer=InMemorySaver(), + user_id=str(db_user.id), + thread_id=db_search_space.id, + agent_config=None, + ) + + result = await agent.ainvoke( + {"messages": [HumanMessage(content="hello")]}, + config={"configurable": {"thread_id": "guard-d-thread-1"}}, + ) + + assert _last_ai_text(result["messages"]) == "done" + + +@pytest.mark.asyncio +async def test_agent_routes_a_scripted_tool_call(db_session, db_user, db_search_space): + """The compiled graph routes a model tool call to its tool and resumes.""" + harness = build_scripted_harness( + turns=[ + ScriptedTurn( + tool_calls=[{"name": "echo", "args": {"x": 1}, "id": "call_1"}] + ), + ScriptedTurn(text="echoed"), + ], + tools=[ + StubToolSpec( + name="echo", + description="Echo the args back.", + handler=lambda **kwargs: {"echoed": kwargs}, + ), + ], + ) + + agent = await create_multi_agent_chat_deep_agent( + llm=harness.model, + search_space_id=db_search_space.id, + db_session=db_session, + connector_service=ConnectorService(db_session), + checkpointer=InMemorySaver(), + user_id=str(db_user.id), + thread_id=db_search_space.id, + agent_config=None, + additional_tools=harness.tools, + ) + + result = await agent.ainvoke( + {"messages": [HumanMessage(content="echo please")]}, + config={"configurable": {"thread_id": "guard-d-thread-2"}}, + ) + + tool_messages = [m for m in result["messages"] if isinstance(m, ToolMessage)] + assert any("echoed" in str(m.content) for m in tool_messages) + assert _last_ai_text(result["messages"]) == "echoed" + + +@pytest.mark.asyncio +async def test_agent_checkpoint_round_trips_across_turns( + db_session, db_user, db_search_space +): + """Turn 2 sees turn 1's history, proving the checkpoint serializes and reloads. + + Uses InMemorySaver, which serializes via the same ``JsonPlusSerializer`` as + the production Postgres checkpointer — so a state class that became + unserializable after a module move would fail here too. + """ + harness = build_scripted_harness( + turns=[ScriptedTurn(text="ok-one"), ScriptedTurn(text="ok-two")] + ) + checkpointer = InMemorySaver() + config = {"configurable": {"thread_id": "guard-e-thread-1"}} + + async def _build(): + return await create_multi_agent_chat_deep_agent( + llm=harness.model, + search_space_id=db_search_space.id, + db_session=db_session, + connector_service=ConnectorService(db_session), + checkpointer=checkpointer, + user_id=str(db_user.id), + thread_id=db_search_space.id, + agent_config=None, + ) + + agent = await _build() + first = await agent.ainvoke( + {"messages": [HumanMessage(content="remember apple")]}, config + ) + second = await agent.ainvoke( + {"messages": [HumanMessage(content="second turn")]}, config + ) + + texts = [m.content for m in second["messages"] if isinstance(m, HumanMessage)] + assert "remember apple" in texts, "turn 1 history not reloaded from checkpoint" + assert len(second["messages"]) > len(first["messages"]) diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py new file mode 100644 index 000000000..878473f55 --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py @@ -0,0 +1,203 @@ +"""Real-behavior tests for the LIVE knowledge-base filesystem middleware (B) in +cloud mode. + +Cloud mode is the default production filesystem for web chat. Unlike desktop, +cloud writes/edits/moves/deletes are *staged* into LangGraph state during the +turn and committed to Postgres at end-of-turn by the persistence middleware. +These tests drive the production ``build_filesystem_mw`` cloud tools through a +real ``create_agent`` graph and assert the staging contract (namespace policy, +read-from-stage, mkdir staging, duplicate rejection) — all deterministic and +DB-free because cloud ``awrite`` is pure in-state staging. + +The end-of-turn DB commit (``commit_staged_filesystem_state``) is covered +separately; here we lock the per-tool behavior that the reorg could break. +""" + +from __future__ import annotations + +import pytest +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem import ( + build_filesystem_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.resolver import ( + build_backend_resolver, +) +from tests.integration.harness import ScriptedTurn, build_scripted_harness + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio] + +_SEARCH_SPACE_ID = 1 + + +def _build_cloud_fs_mw(): + """Build the production filesystem middleware in cloud mode. + + A non-None ``search_space_id`` makes the resolver hand out a + ``KBPostgresBackend``, exactly as production does. Staging operations never + touch the DB, so a dummy id is sufficient for these tests. + """ + selection = FilesystemSelection(mode=FilesystemMode.CLOUD) + resolver = build_backend_resolver(selection, search_space_id=_SEARCH_SPACE_ID) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=FilesystemMode.CLOUD, + search_space_id=_SEARCH_SPACE_ID, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=_SEARCH_SPACE_ID, + read_only=False, + ) + + +async def _run(turns: list[ScriptedTurn], thread: str): + harness = build_scripted_harness(turns=turns) + agent = create_agent( + harness.model, + tools=[], + middleware=[_build_cloud_fs_mw()], + checkpointer=InMemorySaver(), + ) + return await agent.ainvoke( + {"messages": [HumanMessage(content="do kb work")]}, + config={"configurable": {"thread_id": thread}}, + ) + + +def _tool_text(result, name: str) -> str: + for m in result["messages"]: + if isinstance(m, ToolMessage) and m.name == name: + return str(m.content) + raise AssertionError(f"no ToolMessage from {name!r}") + + +def _write(path: str, content: str, call_id: str) -> ScriptedTurn: + return ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": {"file_path": path, "content": content}, + "id": call_id, + } + ] + ) + + +async def test_cloud_write_then_read_returns_staged_content(): + """A cloud write stages into state and a later read returns that content.""" + result = await _run( + [ + _write("/documents/note.md", "cloud CANARY-CLD-1", "c1"), + ScriptedTurn( + tool_calls=[ + { + "name": "read_file", + "args": {"file_path": "/documents/note.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-write-read", + ) + + assert "Updated file /documents/note.md" in _tool_text(result, "write_file") + assert "CANARY-CLD-1" in _tool_text(result, "read_file") + + +async def test_cloud_write_outside_documents_is_rejected(): + """Cloud namespace policy: writes must target /documents (non-temp paths).""" + result = await _run( + [ + _write("/scratch/note.md", "nope", "c1"), + ScriptedTurn(text="done"), + ], + "fs-cloud-namespace", + ) + + msg = _tool_text(result, "write_file") + assert "must target /documents" in msg + + +async def test_cloud_temp_prefixed_write_is_allowed_anywhere(): + """A ``temp_`` basename escapes the /documents namespace restriction.""" + result = await _run( + [ + _write("/temp_scratch.md", "ephemeral", "c1"), + ScriptedTurn(text="done"), + ], + "fs-cloud-temp", + ) + + msg = _tool_text(result, "write_file") + assert "must target /documents" not in msg + assert "Updated file" in msg + + +async def test_cloud_mkdir_stages_directory(): + """Cloud mkdir stages the directory for end-of-turn creation (no immediate IO).""" + result = await _run( + [ + ScriptedTurn( + tool_calls=[ + { + "name": "mkdir", + "args": {"path": "/documents/projects"}, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-mkdir", + ) + + msg = _tool_text(result, "mkdir") + assert "Staged directory" in msg + assert "/documents/projects" in msg + + +async def test_cloud_mkdir_outside_documents_is_rejected(): + """Cloud mkdir is also restricted to the /documents namespace.""" + result = await _run( + [ + ScriptedTurn( + tool_calls=[ + {"name": "mkdir", "args": {"path": "/elsewhere"}, "id": "c1"} + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-mkdir-bad", + ) + + assert "must target a path under /documents" in _tool_text(result, "mkdir") + + +async def test_cloud_duplicate_write_is_rejected(): + """Writing to a path already staged this turn is rejected (use edit instead).""" + result = await _run( + [ + _write("/documents/dup.md", "first", "c1"), + _write("/documents/dup.md", "second", "c2"), + ScriptedTurn(text="done"), + ], + "fs-cloud-dup", + ) + + # Two write ToolMessages: first succeeds, second is rejected. + write_msgs = [ + str(m.content) + for m in result["messages"] + if isinstance(m, ToolMessage) and m.name == "write_file" + ] + assert len(write_msgs) == 2 + assert "Updated file" in write_msgs[0] + assert "already exists" in write_msgs[1] diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py new file mode 100644 index 000000000..e013ef35b --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py @@ -0,0 +1,349 @@ +"""Real-behavior tests for the LIVE knowledge-base filesystem middleware (B). + +These exercise ``app.agents.chat.multi_agent_chat.shared.middleware.filesystem`` — +the decomposed middleware + tools that production actually mounts on the +knowledge_base subagent (via ``build_filesystem_mw``). The previous +``tests/unit/middleware/test_filesystem_*.py`` suite asserts a *dead twin* +(``app.agents.chat.shared.middleware.filesystem``) that is never instantiated, so the +live tool path had no real coverage. + +Strategy: mount the production ``build_filesystem_mw`` on a minimal +``create_agent`` graph and drive its tools with the scripted harness. Desktop +mode binds a ``MultiRootLocalFolderBackend`` to a real ``tmp_path`` directory, +so every write/edit/move/rm is asserted against the real on-disk filesystem — +no mocks, only the LLM is scripted. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, + LocalFilesystemMount, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem import ( + build_filesystem_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.resolver import ( + build_backend_resolver, +) +from tests.integration.harness import ScriptedTurn, build_scripted_harness + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio] + +_MOUNT_ID = "workspace" + + +def _build_desktop_fs_mw(root: Path): + """Build the production filesystem middleware bound to a real local folder.""" + selection = FilesystemSelection( + mode=FilesystemMode.DESKTOP_LOCAL_FOLDER, + local_mounts=(LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)),), + ) + resolver = build_backend_resolver(selection) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=FilesystemMode.DESKTOP_LOCAL_FOLDER, + search_space_id=1, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=1, + read_only=False, + ) + + +async def _run(root: Path, turns: list[ScriptedTurn], thread: str): + """Assemble a 1-middleware agent and drive the scripted turns to completion.""" + harness = build_scripted_harness(turns=turns) + fs_mw = _build_desktop_fs_mw(root) + agent = create_agent( + harness.model, + tools=[], + middleware=[fs_mw], + checkpointer=InMemorySaver(), + ) + return await agent.ainvoke( + {"messages": [HumanMessage(content="do filesystem work")]}, + config={"configurable": {"thread_id": thread}}, + ) + + +def _tool_messages(result) -> list[ToolMessage]: + return [m for m in result["messages"] if isinstance(m, ToolMessage)] + + +def _tool_text(result, name: str) -> str: + for m in _tool_messages(result): + if m.name == name: + return str(m.content) + raise AssertionError(f"no ToolMessage from {name!r} in {_tool_messages(result)}") + + +async def test_write_then_read_round_trip(tmp_path: Path): + """write_file persists to the real folder and read_file returns the content.""" + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/notes.md", + "content": "hello FS-CANARY-001", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "read_file", + "args": {"file_path": f"/{_MOUNT_ID}/notes.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-write-read", + ) + + # Real on-disk effect, not a mock. + assert (tmp_path / "notes.md").read_text() == "hello FS-CANARY-001" + # The tool actually returned the file content. + assert "FS-CANARY-001" in _tool_text(result, "read_file") + + +async def test_write_then_ls_lists_file(tmp_path: Path): + """ls reflects a freshly written file in the real folder.""" + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/report.md", + "content": "x", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + {"name": "ls", "args": {"path": f"/{_MOUNT_ID}"}, "id": "c2"} + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-ls", + ) + + assert (tmp_path / "report.md").exists() + assert "report.md" in _tool_text(result, "ls") + + +async def test_edit_file_rewrites_on_disk(tmp_path: Path): + """edit_file applies a real string replacement to the on-disk file.""" + await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/doc.md", + "content": "the quick brown fox", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "edit_file", + "args": { + "file_path": f"/{_MOUNT_ID}/doc.md", + "old_string": "brown", + "new_string": "red", + }, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-edit", + ) + + assert (tmp_path / "doc.md").read_text() == "the quick red fox" + + +async def test_write_into_existing_subdir(tmp_path: Path): + """A write into an EXISTING subdirectory lands on disk under that folder.""" + (tmp_path / "sub").mkdir() + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/sub/inner.md", + "content": "nested", + }, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-subdir", + ) + + assert "Error" not in _tool_text(result, "write_file") + assert (tmp_path / "sub" / "inner.md").read_text() == "nested" + + +async def test_write_to_missing_parent_dir_is_rejected(tmp_path: Path): + """Desktop write refuses to create a file under a non-existent directory. + + Real current behavior: the local-folder backend requires the parent to + exist (and ``mkdir`` is a no-op for this backend), so the agent cannot + fabricate new nested folders via ``write_file``. Locking this guards against + a silent behavior change during the agents-module reorg. + """ + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/missing/inner.md", + "content": "nested", + }, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-missing-parent", + ) + + write_msg = _tool_text(result, "write_file") + assert "parent directory" in write_msg.lower() + assert not (tmp_path / "missing").exists() + + +async def test_move_file_relocates_on_disk(tmp_path: Path): + """move_file relocates the real file from source to destination.""" + await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/src.md", + "content": "movable", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "move_file", + "args": { + "source_path": f"/{_MOUNT_ID}/src.md", + "destination_path": f"/{_MOUNT_ID}/dst.md", + }, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-move", + ) + + assert not (tmp_path / "src.md").exists() + assert (tmp_path / "dst.md").read_text() == "movable" + + +async def test_rm_deletes_file_on_disk(tmp_path: Path): + """rm removes the real file (desktop deletes are immediate).""" + await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/trash.md", + "content": "bye", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "rm", + "args": {"path": f"/{_MOUNT_ID}/trash.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-rm", + ) + + assert not (tmp_path / "trash.md").exists() + + +async def test_rmdir_removes_empty_dir_on_disk(tmp_path: Path): + """rmdir removes a real empty directory.""" + (tmp_path / "gone").mkdir() + assert (tmp_path / "gone").is_dir() + + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "rmdir", + "args": {"path": f"/{_MOUNT_ID}/gone"}, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-rmdir", + ) + + assert "Error" not in _tool_text(result, "rmdir") + assert not (tmp_path / "gone").exists() diff --git a/surfsense_backend/tests/integration/chat/test_thread_visibility.py b/surfsense_backend/tests/integration/chat/test_thread_visibility.py new file mode 100644 index 000000000..464d389db --- /dev/null +++ b/surfsense_backend/tests/integration/chat/test_thread_visibility.py @@ -0,0 +1,279 @@ +"""Integration tests for new-chat thread visibility invariants. + +These tests exercise the route handlers directly with real DB-backed +users, memberships, and permissions. The important contract is that a +thread shared with a search space stays shared across normal metadata +updates until the creator explicitly makes it private again. +""" + +from __future__ import annotations + +import uuid + +import pytest +import pytest_asyncio +from fastapi import HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ( + ChatVisibility, + SearchSpace, + SearchSpaceMembership, + SearchSpaceRole, + User, +) +from app.routes import new_chat_routes +from app.schemas.new_chat import ( + NewChatThreadCreate, + NewChatThreadUpdate, + NewChatThreadVisibilityUpdate, +) + +pytestmark = pytest.mark.integration + + +@pytest_asyncio.fixture +async def db_member(db_session: AsyncSession, db_search_space: SearchSpace) -> User: + member = User( + id=uuid.uuid4(), + email="member@surfsense.net", + hashed_password="hashed", + is_active=True, + is_superuser=False, + is_verified=True, + ) + db_session.add(member) + await db_session.flush() + + role = ( + ( + await db_session.execute( + select(SearchSpaceRole).where( + SearchSpaceRole.search_space_id == db_search_space.id, + SearchSpaceRole.name == "Editor", + ) + ) + ) + .scalars() + .one() + ) + db_session.add( + SearchSpaceMembership( + user_id=member.id, + search_space_id=db_search_space.id, + role_id=role.id, + is_owner=False, + ) + ) + await db_session.flush() + return member + + +async def _create_thread( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, + *, + title: str = "Visibility Invariant Chat", +): + return await new_chat_routes.create_thread( + NewChatThreadCreate( + title=title, + archived=False, + search_space_id=db_search_space.id, + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_user, + ) + + +def _active_thread_ids(response) -> set[int]: + return {thread.id for thread in response.threads} + + +def _search_thread_ids(response) -> set[int]: + return {thread.id for thread in response} + + +async def test_private_thread_is_hidden_from_other_search_space_member( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + + assert thread.id not in _active_thread_ids(member_threads) + assert thread.id not in _search_thread_ids(member_search) + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + assert exc_info.value.status_code == 403 + + +async def test_creator_can_share_thread_and_member_can_list_search_read_it( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + + updated = await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + full_thread = await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + + assert updated.visibility == ChatVisibility.SEARCH_SPACE + assert thread.id in _active_thread_ids(member_threads) + assert thread.id in _search_thread_ids(member_search) + assert full_thread["id"] == thread.id + assert full_thread["visibility"] == ChatVisibility.SEARCH_SPACE + + +async def test_rename_and_archive_do_not_reset_shared_visibility( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + renamed = await new_chat_routes.update_thread( + thread_id=thread.id, + thread_update=NewChatThreadUpdate(title="Renamed Shared Chat"), + session=db_session, + user=db_user, + ) + archived = await new_chat_routes.update_thread( + thread_id=thread.id, + thread_update=NewChatThreadUpdate(archived=True), + session=db_session, + user=db_user, + ) + + assert renamed.visibility == ChatVisibility.SEARCH_SPACE + assert archived.visibility == ChatVisibility.SEARCH_SPACE + assert archived.archived is True + + +async def test_non_creator_cannot_change_shared_thread_back_to_private( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_member, + ) + + assert exc_info.value.status_code == 403 + + +async def test_creator_can_make_shared_thread_private_again( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + private_again = await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_user, + ) + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + + assert private_again.visibility == ChatVisibility.PRIVATE + assert thread.id not in _active_thread_ids(member_threads) + assert thread.id not in _search_thread_ids(member_search) + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + assert exc_info.value.status_code == 403 diff --git a/surfsense_backend/tests/integration/conftest.py b/surfsense_backend/tests/integration/conftest.py index e03101e63..19f8e3d0a 100644 --- a/surfsense_backend/tests/integration/conftest.py +++ b/surfsense_backend/tests/integration/conftest.py @@ -1,7 +1,7 @@ import importlib import sys import uuid -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest import pytest_asyncio @@ -123,26 +123,6 @@ async def db_search_space(db_session: AsyncSession, db_user: User) -> SearchSpac return space -@pytest.fixture -def patched_summarize(monkeypatch) -> AsyncMock: - mock = AsyncMock(return_value="Mocked summary.") - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - mock, - ) - return mock - - -@pytest.fixture -def patched_summarize_raises(monkeypatch) -> AsyncMock: - mock = AsyncMock(side_effect=RuntimeError("LLM unavailable")) - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - mock, - ) - return mock - - @pytest.fixture def patched_embed_texts(monkeypatch) -> MagicMock: mock = MagicMock(side_effect=lambda texts: [[0.1] * _EMBEDDING_DIM for _ in texts]) @@ -153,6 +133,16 @@ def patched_embed_texts(monkeypatch) -> MagicMock: return mock +@pytest.fixture +def patched_embed_texts_raises(monkeypatch) -> MagicMock: + mock = MagicMock(side_effect=RuntimeError("Embedding unavailable")) + monkeypatch.setattr( + "app.indexing_pipeline.indexing_pipeline_service.embed_texts", + mock, + ) + return mock + + @pytest.fixture def patched_chunk_text(monkeypatch) -> MagicMock: mock = MagicMock(return_value=["Test chunk content."]) diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index ff44e471a..13e3ab59c 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -68,7 +68,6 @@ class InlineTaskDispatcher: filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: @@ -83,7 +82,6 @@ class InlineTaskDispatcher: filename, search_space_id, user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -266,10 +264,6 @@ async def page_limits(): @pytest.fixture(autouse=True) def _mock_external_apis(monkeypatch): """Mock LLM, embedding, and chunking — these are external API boundaries.""" - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Mocked summary."), - ) monkeypatch.setattr( "app.indexing_pipeline.indexing_pipeline_service.embed_texts", MagicMock(side_effect=lambda texts: [[0.1] * _EMBEDDING_DIM for _ in texts]), diff --git a/surfsense_backend/tests/integration/google_unification/conftest.py b/surfsense_backend/tests/integration/google_unification/conftest.py index de68c7acb..390442fdd 100644 --- a/surfsense_backend/tests/integration/google_unification/conftest.py +++ b/surfsense_backend/tests/integration/google_unification/conftest.py @@ -239,7 +239,7 @@ def patched_shielded_session(async_engine, monkeypatch): yield session monkeypatch.setattr( - "app.agents.new_chat.tools.knowledge_base.shielded_async_session", + "app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.knowledge_base.shielded_async_session", _test_shielded, ) diff --git a/surfsense_backend/tests/integration/google_unification/test_browse_includes_legacy_docs.py b/surfsense_backend/tests/integration/google_unification/test_browse_includes_legacy_docs.py index fc2fec5a8..f0d5c6c6c 100644 --- a/surfsense_backend/tests/integration/google_unification/test_browse_includes_legacy_docs.py +++ b/surfsense_backend/tests/integration/google_unification/test_browse_includes_legacy_docs.py @@ -17,7 +17,9 @@ async def test_browse_recent_documents_with_list_type_returns_both( committed_google_data, patched_shielded_session ): """_browse_recent_documents returns docs of all types when given a list.""" - from app.agents.new_chat.tools.knowledge_base import _browse_recent_documents + from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools.knowledge_base import ( + _browse_recent_documents, + ) space_id = committed_google_data["search_space_id"] diff --git a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py index 6bb1d2094..311716052 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py @@ -7,9 +7,7 @@ from app.indexing_pipeline.adapters.file_upload_adapter import UploadDocumentAda pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful indexing.""" adapter = UploadDocumentAdapter(db_session) @@ -19,7 +17,6 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -30,11 +27,9 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): assert DocumentStatus.is_state(document.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_content_is_summary(db_session, db_search_space, db_user, mocker): - """Document content is set to the LLM-generated summary.""" +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_content_is_source_markdown(db_session, db_search_space, db_user, mocker): + """Document content is set to the extracted source markdown.""" adapter = UploadDocumentAdapter(db_session) await adapter.index( markdown_content="## Hello\n\nSome content.", @@ -42,8 +37,6 @@ async def test_content_is_summary(db_session, db_search_space, db_user, mocker): etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), - should_summarize=True, ) result = await db_session.execute( @@ -51,12 +44,10 @@ async def test_content_is_summary(db_session, db_search_space, db_user, mocker): ) document = result.scalars().first() - assert document.content == "Mocked summary." + assert document.content == "## Hello\n\nSome content." -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker): """Chunks derived from the source markdown are persisted in the DB.""" adapter = UploadDocumentAdapter(db_session) @@ -66,7 +57,6 @@ async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -83,9 +73,7 @@ async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker assert chunks[0].content == "Test chunk content." -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, mocker): """RuntimeError is raised when the indexing step fails so the caller can fire a failure notification.""" adapter = UploadDocumentAdapter(db_session) @@ -96,8 +84,6 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), - should_summarize=True, ) @@ -106,11 +92,9 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, # --------------------------------------------------------------------------- -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_updates_content(db_session, db_search_space, db_user, mocker): - """Document content is updated to the new summary after reindexing.""" + """Document content is updated to the new source markdown after reindexing.""" adapter = UploadDocumentAdapter(db_session) await adapter.index( markdown_content="## Original\n\nOriginal content.", @@ -118,7 +102,6 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -129,15 +112,13 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) - assert document.content == "Mocked summary." + assert document.content == "## Edited\n\nNew content after user edit." -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_updates_content_hash( db_session, db_search_space, db_user, mocker ): @@ -149,7 +130,6 @@ async def test_reindex_updates_content_hash( etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -161,15 +141,13 @@ async def test_reindex_updates_content_hash( document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert document.content_hash != original_hash -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful reindexing.""" adapter = UploadDocumentAdapter(db_session) @@ -179,7 +157,6 @@ async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, m etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -190,13 +167,13 @@ async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, m document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert DocumentStatus.is_state(document.status, DocumentStatus.READY) -@pytest.mark.usefixtures("patched_summarize", "patched_embed_texts") +@pytest.mark.usefixtures("patched_embed_texts") async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, mocker): """Reindexing replaces old chunks with new content rather than appending.""" mocker.patch( @@ -211,7 +188,6 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -223,7 +199,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) chunks_result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -234,9 +210,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc assert chunks[0].content == "Updated chunk." -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_clears_reindexing_flag( db_session, db_search_space, db_user, mocker ): @@ -248,7 +222,6 @@ async def test_reindex_clears_reindexing_flag( etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -260,19 +233,17 @@ async def test_reindex_clears_reindexing_flag( document.content_needs_reindexing = True await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert document.content_needs_reindexing is False @pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") -async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, mocker): +async def test_reindex_raises_on_failure( + db_session, db_search_space, db_user, patched_embed_texts, mocker +): """RuntimeError is raised when reindexing fails so the caller can handle it.""" - mocker.patch( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - return_value="Mocked summary.", - ) adapter = UploadDocumentAdapter(db_session) await adapter.index( @@ -281,7 +252,6 @@ async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, m etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -292,13 +262,10 @@ async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, m document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - mocker.patch( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - side_effect=RuntimeError("LLM unavailable"), - ) + patched_embed_texts.side_effect = RuntimeError("Embedding unavailable") with pytest.raises(RuntimeError, match=r"Embedding failed|Reindexing failed"): - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) async def test_reindex_raises_on_empty_source_markdown( @@ -323,4 +290,4 @@ async def test_reindex_raises_on_empty_source_markdown( adapter = UploadDocumentAdapter(db_session) with pytest.raises(RuntimeError, match="no source_markdown"): - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py index b2dd13e57..8e1ed3752 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py @@ -25,8 +25,6 @@ def _cal_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"Calendar: Event {unique_id}", metadata={ "event_id": unique_id, "start_time": "2025-01-15T10:00:00", @@ -36,9 +34,7 @@ def _cal_doc( ) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_calendar_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -55,7 +51,7 @@ async def test_calendar_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -67,9 +63,7 @@ async def test_calendar_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_calendar_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py index d9900ea87..6e85421ea 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py @@ -25,8 +25,6 @@ def _drive_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.pdf", metadata={ "google_drive_file_id": unique_id, "google_drive_file_name": f"{unique_id}.pdf", @@ -35,9 +33,7 @@ def _drive_doc( ) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_drive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -54,7 +50,7 @@ async def test_drive_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -66,9 +62,7 @@ async def test_drive_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_drive_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker ): @@ -183,3 +177,75 @@ async def test_should_skip_file_skips_failed_document( assert should_skip, "FAILED documents must be skipped during automatic sync" assert "failed" in msg.lower() + + +@pytest.mark.parametrize("stuck_state", ["pending", "processing"]) +async def test_should_skip_file_retries_stuck_document( + db_session, + db_search_space, + db_user, + stuck_state, +): + """A doc stuck in pending/processing (worker died mid-index) must re-index, not skip.""" + import importlib + import sys + import types + + pkg = "app.tasks.connector_indexers" + stub = pkg not in sys.modules + if stub: + mod = types.ModuleType(pkg) + mod.__path__ = ["app/tasks/connector_indexers"] + mod.__package__ = pkg + sys.modules[pkg] = mod + + try: + gdm = importlib.import_module( + "app.tasks.connector_indexers.google_drive_indexer" + ) + _should_skip_file = gdm._should_skip_file + finally: + if stub: + sys.modules.pop(pkg, None) + + space_id = db_search_space.id + file_id = f"file-{stuck_state}-drive" + md5 = "stuck123checksum" + + doc_hash = compute_identifier_hash( + DocumentType.GOOGLE_DRIVE_FILE.value, file_id, space_id + ) + status = ( + DocumentStatus.pending() + if stuck_state == "pending" + else DocumentStatus.processing() + ) + stuck_doc = Document( + title="Stuck File.pdf", + document_type=DocumentType.GOOGLE_DRIVE_FILE, + content="Pending...", + content_hash=f"ch-{doc_hash[:12]}", + unique_identifier_hash=doc_hash, + source_markdown="", + search_space_id=space_id, + created_by_id=str(db_user.id), + status=status, + document_metadata={ + "google_drive_file_id": file_id, + "google_drive_file_name": "Stuck File.pdf", + "md5_checksum": md5, + }, + ) + db_session.add(stuck_doc) + await db_session.flush() + + incoming_file = { + "id": file_id, + "name": "Stuck File.pdf", + "mimeType": "application/pdf", + "md5Checksum": md5, + } + + should_skip, _msg = await _should_skip_file(db_session, incoming_file, space_id) + + assert not should_skip, f"{stuck_state} documents must re-index, not be skipped" diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py index 83e4f7bb4..9faa3db91 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py @@ -24,8 +24,6 @@ def _dropbox_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.docx", metadata={ "dropbox_file_id": unique_id, "dropbox_file_name": f"{unique_id}.docx", @@ -34,9 +32,7 @@ def _dropbox_doc( ) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_dropbox_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -53,7 +49,7 @@ async def test_dropbox_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -65,9 +61,7 @@ async def test_dropbox_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_dropbox_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker ): @@ -86,7 +80,7 @@ async def test_dropbox_duplicate_content_skipped( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py index b74d092c0..2026393c5 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py @@ -28,8 +28,6 @@ def _gmail_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"Gmail: Subject for {unique_id}", metadata={ "message_id": unique_id, "from": "sender@example.com", @@ -38,9 +36,7 @@ def _gmail_doc( ) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_gmail_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -57,7 +53,7 @@ async def test_gmail_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -70,9 +66,7 @@ async def test_gmail_pipeline_creates_ready_document( assert row.source_markdown == doc.source_markdown -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_gmail_legacy_doc_migrated_then_reused( db_session, db_search_space, db_connector, db_user, mocker ): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py index 847f7592c..855676f61 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py @@ -9,9 +9,7 @@ from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineServ pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_index_batch_creates_ready_documents( db_session, db_search_space, make_connector_document, mocker ): @@ -33,7 +31,7 @@ async def test_index_batch_creates_ready_documents( ] service = IndexingPipelineService(session=db_session) - results = await service.index_batch(docs, llm=mocker.Mock()) + results = await service.index_batch(docs) assert len(results) == 2 @@ -49,11 +47,9 @@ async def test_index_batch_creates_ready_documents( assert row.embedding is not None -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_index_batch_empty_returns_empty(db_session, mocker): """index_batch with empty input returns an empty list.""" service = IndexingPipelineService(session=db_session) - results = await service.index_batch([], llm=mocker.Mock()) + results = await service.index_batch([]) assert results == [] diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py index a82148f96..ee895c61b 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py @@ -10,9 +10,7 @@ _EMBEDDING_DIM = app_config.embedding_model_instance.dimension pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_sets_status_ready( db_session, db_search_space, @@ -27,7 +25,7 @@ async def test_sets_status_ready( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -37,16 +35,14 @@ async def test_sets_status_ready( assert DocumentStatus.is_state(reloaded.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_content_is_summary_when_should_summarize_true( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_content_is_source_markdown_by_default( db_session, db_search_space, make_connector_document, mocker, ): - """Document content is set to the LLM-generated summary when should_summarize=True.""" + """Document content is set to source_markdown by default.""" connector_doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) @@ -54,28 +50,25 @@ async def test_content_is_summary_when_should_summarize_true( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) ) reloaded = result.scalars().first() - assert reloaded.content == "Mocked summary." + assert reloaded.content == connector_doc.source_markdown -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_content_is_source_markdown_when_should_summarize_false( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_content_is_source_markdown_when_custom_content( db_session, db_search_space, make_connector_document, ): - """Document content is set to source_markdown verbatim when should_summarize=False.""" + """Document content is set to source_markdown verbatim.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=False, source_markdown="## Raw content", ) service = IndexingPipelineService(session=db_session) @@ -84,7 +77,7 @@ async def test_content_is_source_markdown_when_should_summarize_false( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=None) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -94,9 +87,7 @@ async def test_content_is_source_markdown_when_should_summarize_false( assert reloaded.content == "## Raw content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_chunks_written_to_db( db_session, db_search_space, @@ -111,7 +102,7 @@ async def test_chunks_written_to_db( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -122,9 +113,7 @@ async def test_chunks_written_to_db( assert chunks[0].content == "Test chunk content." -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_embedding_written_to_db( db_session, db_search_space, @@ -139,7 +128,7 @@ async def test_embedding_written_to_db( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -150,9 +139,7 @@ async def test_embedding_written_to_db( assert len(reloaded.embedding) == _EMBEDDING_DIM -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_updated_at_advances_after_indexing( db_session, db_search_space, @@ -172,7 +159,7 @@ async def test_updated_at_advances_after_indexing( ) updated_at_pending = result.scalars().first().updated_at - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -182,18 +169,15 @@ async def test_updated_at_advances_after_indexing( assert updated_at_ready > updated_at_pending -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_no_llm_falls_back_to_source_markdown( db_session, db_search_space, make_connector_document, ): - """When llm=None and no fallback_summary, content falls back to source_markdown.""" + """Content stays deterministic source markdown without an LLM.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=True, source_markdown="## Fallback content", ) service = IndexingPipelineService(session=db_session) @@ -202,7 +186,7 @@ async def test_no_llm_falls_back_to_source_markdown( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=None) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -213,27 +197,23 @@ async def test_no_llm_falls_back_to_source_markdown( assert reloaded.content == "## Fallback content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_fallback_summary_used_when_llm_unavailable( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_source_markdown_used_without_preview( db_session, db_search_space, make_connector_document, ): - """fallback_summary is used as content when llm=None and should_summarize=True.""" + """Source markdown is used without fallback preview fields.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=True, source_markdown="## Full raw content", - fallback_summary="Short pre-built summary.", ) service = IndexingPipelineService(session=db_session) prepared = await service.prepare_for_indexing([connector_doc]) document_id = prepared[0].id - await service.index(prepared[0], connector_doc, llm=None) + await service.index(prepared[0], connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -241,12 +221,10 @@ async def test_fallback_summary_used_when_llm_unavailable( reloaded = result.scalars().first() assert DocumentStatus.is_state(reloaded.status, DocumentStatus.READY) - assert reloaded.content == "Short pre-built summary." + assert reloaded.content == "## Full raw content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_replaces_old_chunks( db_session, db_search_space, @@ -264,14 +242,14 @@ async def test_reindex_replaces_old_chunks( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) updated_doc = make_connector_document( search_space_id=db_search_space.id, source_markdown="## v2", ) re_prepared = await service.prepare_for_indexing([updated_doc]) - await service.index(re_prepared[0], updated_doc, llm=mocker.Mock()) + await service.index(re_prepared[0], updated_doc) result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -281,16 +259,14 @@ async def test_reindex_replaces_old_chunks( assert len(chunks) == 1 -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) -async def test_llm_error_sets_status_failed( +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") +async def test_embedding_error_sets_status_failed( db_session, db_search_space, make_connector_document, mocker, ): - """Document status is FAILED when the LLM raises during indexing.""" + """Document status is FAILED when embedding raises during indexing.""" connector_doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) @@ -298,7 +274,7 @@ async def test_llm_error_sets_status_failed( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -308,10 +284,8 @@ async def test_llm_error_sets_status_failed( assert DocumentStatus.is_state(reloaded.status, DocumentStatus.FAILED) -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) -async def test_llm_error_leaves_no_partial_data( +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") +async def test_embedding_error_leaves_no_partial_data( db_session, db_search_space, make_connector_document, @@ -325,7 +299,7 @@ async def test_llm_error_leaves_no_partial_data( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 4dc5742f7..2cd378343 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -21,7 +21,6 @@ from app.db import ( pytestmark = pytest.mark.integration UNIFIED_FIXTURES = ( - "patched_summarize", "patched_embed_texts", "patched_chunk_text", ) @@ -787,7 +786,7 @@ class TestPipelineIntegration: assert len(prepared) == 1 db_doc = prepared[0] - result = await service.index(db_doc, doc, llm=mocker.Mock()) + result = await service.index(db_doc, doc) assert result is not None docs = ( @@ -1272,7 +1271,7 @@ class TestIndexingProgressFlag: original_index = IndexingPipelineService.index flag_observed = [] - async def patched_index(self_pipe, document, connector_doc, llm): + async def patched_index(self_pipe, document, connector_doc): folder = ( await db_session.execute( select(Folder).where( @@ -1284,7 +1283,7 @@ class TestIndexingProgressFlag: if folder: meta = folder.folder_metadata or {} flag_observed.append(meta.get("indexing_in_progress", False)) - return await original_index(self_pipe, document, connector_doc, llm) + return await original_index(self_pipe, document, connector_doc) IndexingPipelineService.index = patched_index try: diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_mark_connector_documents_failed.py b/surfsense_backend/tests/integration/indexing_pipeline/test_mark_connector_documents_failed.py new file mode 100644 index 000000000..9e3feee1e --- /dev/null +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_mark_connector_documents_failed.py @@ -0,0 +1,110 @@ +"""Integration tests for mark_connector_documents_failed. + +Covers the ETL-failure recovery path: a connector placeholder must move out of +``pending``/``processing`` into ``failed`` so it stays deletable, while a +``ready`` document is never clobbered. +""" + +import hashlib + +import pytest +from sqlalchemy import select + +from app.db import Document, DocumentStatus, DocumentType +from app.indexing_pipeline.document_hashing import compute_identifier_hash +from app.tasks.connector_indexers.base import mark_connector_documents_failed + +pytestmark = pytest.mark.integration + + +async def _make_doc( + db_session, + *, + search_space_id: int, + connector_id: int, + user_id: str, + file_id: str, + status: dict, +) -> Document: + uid_hash = compute_identifier_hash( + DocumentType.GOOGLE_DRIVE_FILE.value, file_id, search_space_id + ) + doc = Document( + title=f"{file_id}.pdf", + document_type=DocumentType.GOOGLE_DRIVE_FILE, + content="Pending...", + content_hash=hashlib.sha256(f"placeholder:{uid_hash}".encode()).hexdigest(), + unique_identifier_hash=uid_hash, + document_metadata={"google_drive_file_id": file_id}, + search_space_id=search_space_id, + connector_id=connector_id, + created_by_id=user_id, + status=status, + ) + db_session.add(doc) + await db_session.flush() + return doc + + +async def test_pending_placeholder_marked_failed( + db_session, db_search_space, db_connector, db_user +): + doc = await _make_doc( + db_session, + search_space_id=db_search_space.id, + connector_id=db_connector.id, + user_id=str(db_user.id), + file_id="file-pending", + status=DocumentStatus.pending(), + ) + + marked = await mark_connector_documents_failed( + db_session, + document_type=DocumentType.GOOGLE_DRIVE_FILE, + search_space_id=db_search_space.id, + failures=[("file-pending", "Download/ETL failed: boom")], + ) + + assert marked == 1 + await db_session.refresh(doc) + assert DocumentStatus.is_state(doc.status, DocumentStatus.FAILED) + assert doc.status.get("reason") == "Download/ETL failed: boom" + + +async def test_ready_document_not_clobbered( + db_session, db_search_space, db_connector, db_user +): + doc = await _make_doc( + db_session, + search_space_id=db_search_space.id, + connector_id=db_connector.id, + user_id=str(db_user.id), + file_id="file-ready", + status=DocumentStatus.ready(), + ) + + marked = await mark_connector_documents_failed( + db_session, + document_type=DocumentType.GOOGLE_DRIVE_FILE, + search_space_id=db_search_space.id, + failures=[("file-ready", "should be ignored")], + ) + + assert marked == 0 + await db_session.refresh(doc) + assert DocumentStatus.is_state(doc.status, DocumentStatus.READY) + + +async def test_missing_document_is_noop(db_session, db_search_space): + marked = await mark_connector_documents_failed( + db_session, + document_type=DocumentType.GOOGLE_DRIVE_FILE, + search_space_id=db_search_space.id, + failures=[("does-not-exist", "reason")], + ) + + assert marked == 0 + result = await db_session.execute( + select(Document).filter(Document.search_space_id == db_search_space.id) + ) + assert result.scalars().first() is None diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py index 541e3a38e..e368ec256 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py @@ -24,8 +24,6 @@ def _onedrive_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.docx", metadata={ "onedrive_file_id": unique_id, "onedrive_file_name": f"{unique_id}.docx", @@ -34,9 +32,7 @@ def _onedrive_doc( ) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_onedrive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker ): @@ -53,7 +49,7 @@ async def test_onedrive_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -65,9 +61,7 @@ async def test_onedrive_pipeline_creates_ready_document( assert DocumentStatus.is_state(row.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_onedrive_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker ): @@ -86,7 +80,7 @@ async def test_onedrive_duplicate_content_skipped( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py index 776180b9a..4b6662fc8 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py @@ -32,9 +32,7 @@ async def test_new_document_is_persisted_with_pending_status( assert reloaded.source_markdown == doc.source_markdown -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_unchanged_ready_document_is_skipped( db_session, db_search_space, @@ -47,7 +45,7 @@ async def test_unchanged_ready_document_is_skipped( # Index fully so the document reaches ready state prepared = await service.prepare_for_indexing([doc]) - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) # Same content on the next run — a ready document must be skipped results = await service.prepare_for_indexing([doc]) @@ -55,9 +53,7 @@ async def test_unchanged_ready_document_is_skipped( assert results == [] -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_title_only_change_updates_title_in_db( db_session, db_search_space, @@ -72,7 +68,7 @@ async def test_title_only_change_updates_title_in_db( prepared = await service.prepare_for_indexing([original]) document_id = prepared[0].id - await service.index(prepared[0], original, llm=mocker.Mock()) + await service.index(prepared[0], original) renamed = make_connector_document( search_space_id=db_search_space.id, title="Updated Title" @@ -338,9 +334,7 @@ async def test_same_content_from_different_source_is_skipped( assert len(result.scalars().all()) == 1 -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") async def test_failed_document_with_unchanged_content_is_requeued( db_session, db_search_space, @@ -351,10 +345,10 @@ async def test_failed_document_with_unchanged_content_is_requeued( doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) - # First run: document is created and indexing crashes → status = failed + # First run: document is created and indexing crashes, so status becomes failed. prepared = await service.prepare_for_indexing([doc]) document_id = prepared[0].id - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) diff --git a/surfsense_backend/tests/integration/notifications/conftest.py b/surfsense_backend/tests/integration/notifications/conftest.py new file mode 100644 index 000000000..17a44a51d --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/conftest.py @@ -0,0 +1,53 @@ +"""Notifications integration fixtures. + +The app's DB session and current-user dependencies are overridden to ride the +test's transactional `db_session`, so API calls and seeded rows share one +transaction that rolls back per test. Overriding `current_active_user` also +bypasses real JWT auth, so these tests don't depend on AUTH_TYPE. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator + +import httpx +import pytest +import pytest_asyncio +from httpx import ASGITransport +from sqlalchemy.ext.asyncio import AsyncSession + +from app.app import app, limiter +from app.db import User, get_async_session +from app.users import current_active_user + +pytestmark = pytest.mark.integration + +limiter.enabled = False + + +@pytest_asyncio.fixture +async def client( + db_session: AsyncSession, + db_user: User, +) -> AsyncGenerator[httpx.AsyncClient, None]: + async def override_session() -> AsyncGenerator[AsyncSession, None]: + yield db_session + + async def override_user() -> User: + return db_user + + previous_overrides = app.dependency_overrides.copy() + app.dependency_overrides[get_async_session] = override_session + app.dependency_overrides[current_active_user] = override_user + + try: + async with httpx.AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + timeout=30.0, + follow_redirects=False, + ) as test_client: + yield test_client + finally: + app.dependency_overrides.clear() + app.dependency_overrides.update(previous_overrides) diff --git a/surfsense_backend/tests/integration/notifications/test_base_handler.py b/surfsense_backend/tests/integration/notifications/test_base_handler.py new file mode 100644 index 000000000..ef7d9ee6c --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_base_handler.py @@ -0,0 +1,164 @@ +"""Behavior guard for the shared find/upsert/update logic (BaseNotificationHandler). + +Uses the connector-indexing handler instance to drive the base methods against +real Postgres, pinning upsert dedup, search-space scoping, and status stamping. +""" + +from __future__ import annotations + +import pytest +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.persistence import Notification +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + +handler = NotificationService.connector_indexing + + +async def test_find_or_create_creates_with_progress_metadata( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Creating a notification seeds operation id, in-progress status, and start time.""" + notification = await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-create", + title="Title", + message="Message", + search_space_id=db_search_space.id, + ) + + assert notification.notification_metadata["operation_id"] == "op-create" + assert notification.notification_metadata["status"] == "in_progress" + assert "started_at" in notification.notification_metadata + + +async def test_find_or_create_upserts_same_operation( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Reusing an operation id updates the same row instead of creating a duplicate.""" + first = await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-upsert", + title="First", + message="First message", + search_space_id=db_search_space.id, + ) + + second = await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-upsert", + title="Second", + message="Second message", + search_space_id=db_search_space.id, + ) + + assert second.id == first.id + assert second.title == "Second" + assert second.message == "Second message" + + count = await db_session.scalar( + select(func.count(Notification.id)).where( + Notification.user_id == db_user.id, + Notification.notification_metadata["operation_id"].astext == "op-upsert", + ) + ) + assert count == 1 + + +async def test_find_by_operation_is_scoped_to_search_space( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Operation-id lookup is scoped per search space, so other spaces don't match.""" + await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-scoped", + title="Title", + message="Message", + search_space_id=db_search_space.id, + ) + + other_space = SearchSpace(name="Other Space", user_id=db_user.id) + db_session.add(other_space) + await db_session.flush() + + found_other = await handler.find_notification_by_operation( + session=db_session, + user_id=db_user.id, + operation_id="op-scoped", + search_space_id=other_space.id, + ) + assert found_other is None + + found_same = await handler.find_notification_by_operation( + session=db_session, + user_id=db_user.id, + operation_id="op-scoped", + search_space_id=db_search_space.id, + ) + assert found_same is not None + + +async def test_update_notification_completed_stamps_completed_at( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Completing a notification stamps completed_at and merges metadata updates.""" + notification = await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-complete", + title="Title", + message="Message", + search_space_id=db_search_space.id, + ) + + updated = await handler.update_notification( + session=db_session, + notification=notification, + status="completed", + metadata_updates={"indexed_count": 7}, + ) + + assert updated.notification_metadata["status"] == "completed" + assert "completed_at" in updated.notification_metadata + assert updated.notification_metadata["indexed_count"] == 7 + + +async def test_update_notification_failed_stamps_completed_at( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Failing a notification also stamps completed_at for the terminal state.""" + notification = await handler.find_or_create_notification( + session=db_session, + user_id=db_user.id, + operation_id="op-fail", + title="Title", + message="Message", + search_space_id=db_search_space.id, + ) + + updated = await handler.update_notification( + session=db_session, + notification=notification, + status="failed", + ) + + assert updated.notification_metadata["status"] == "failed" + assert "completed_at" in updated.notification_metadata diff --git a/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py new file mode 100644 index 000000000..894f036f0 --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_comment_reply_handler.py @@ -0,0 +1,64 @@ +"""Behavior guard for the comment-reply notification handler.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + +handler = NotificationService.comment_reply + + +async def _notify(db_session, db_user, db_search_space, *, reply_id=1, preview="hi"): + """Raise a comment-reply notification for the assertions in the tests below.""" + return await handler.notify_comment_reply( + session=db_session, + user_id=db_user.id, + reply_id=reply_id, + parent_comment_id=10, + message_id=20, + thread_id=30, + thread_title="Thread", + author_id="author-1", + author_name="Bob", + author_avatar_url=None, + author_email="bob@surfsense.net", + content_preview=preview, + search_space_id=db_search_space.id, + ) + + +async def test_comment_reply_title_and_message( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A reply notification names the author and carries the comment preview.""" + notification = await _notify(db_session, db_user, db_search_space, preview="thanks") + + assert notification.type == "comment_reply" + assert notification.title == "Bob replied in a thread" + assert notification.message == "thanks" + + +async def test_comment_reply_truncates_long_preview( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A long comment preview is truncated in the reply message.""" + notification = await _notify( + db_session, db_user, db_search_space, preview="y" * 150 + ) + + assert notification.message == "y" * 100 + "..." + + +async def test_comment_reply_is_idempotent( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """Re-notifying the same reply id reuses the existing notification row.""" + first = await _notify(db_session, db_user, db_search_space, reply_id=5) + second = await _notify(db_session, db_user, db_search_space, reply_id=5) + + assert second.id == first.id diff --git a/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py b/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py new file mode 100644 index 000000000..a882716b9 --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_connector_indexing_handler.py @@ -0,0 +1,235 @@ +"""Behavior guard for the connector-indexing notification handler. + +Exercises the real handler against Postgres via the transactional db_session, +pinning the title/message/status/metadata it produces so the upcoming +functional-core extraction cannot drift. +""" + +from __future__ import annotations + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + + +async def test_indexing_started_opens_notification( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Starting indexing opens an unread notification with connecting-stage metadata.""" + notification = await NotificationService.connector_indexing.notify_indexing_started( + session=db_session, + user_id=db_user.id, + connector_id=42, + connector_name="Notion - My Workspace", + connector_type="NOTION_CONNECTOR", + search_space_id=db_search_space.id, + ) + + assert notification.id is not None + assert notification.type == "connector_indexing" + assert notification.title == "Syncing: Notion - My Workspace" + assert notification.message == "Connecting to your account" + assert notification.read is False + + metadata = notification.notification_metadata + assert metadata["connector_id"] == 42 + assert metadata["connector_type"] == "NOTION_CONNECTOR" + assert metadata["indexed_count"] == 0 + assert metadata["sync_stage"] == "connecting" + assert metadata["status"] == "in_progress" + assert "operation_id" in metadata + assert "started_at" in metadata + + +async def _started( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, + *, + connector_name: str = "Notion - My Workspace", +): + """Open a connector-indexing notification to update in the tests below.""" + return await NotificationService.connector_indexing.notify_indexing_started( + session=db_session, + user_id=db_user.id, + connector_id=42, + connector_name=connector_name, + connector_type="NOTION_CONNECTOR", + search_space_id=db_search_space.id, + ) + + +async def test_indexing_progress_reports_stage_and_percent( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Progress updates surface the stage message and compute a percent complete.""" + notification = await _started(db_session, db_user, db_search_space) + + updated = await NotificationService.connector_indexing.notify_indexing_progress( + session=db_session, + notification=notification, + indexed_count=5, + total_count=10, + stage="fetching", + ) + + assert updated.message == "Fetching your content" + metadata = updated.notification_metadata + assert metadata["indexed_count"] == 5 + assert metadata["total_count"] == 10 + assert metadata["progress_percent"] == 50 + assert metadata["sync_stage"] == "fetching" + assert metadata["status"] == "in_progress" + + +async def test_indexing_completed_clean_success( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """A clean multi-file sync reports ready/completed with plural wording.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await NotificationService.connector_indexing.notify_indexing_completed( + session=db_session, + notification=notification, + indexed_count=3, + ) + + assert done.title == "Ready: Notion - My Workspace" + assert done.message == "Now searchable! 3 files synced." + assert done.notification_metadata["status"] == "completed" + assert done.notification_metadata["sync_stage"] == "completed" + + +async def test_indexing_completed_singular_file( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """A single synced file uses singular 'file' wording.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await NotificationService.connector_indexing.notify_indexing_completed( + session=db_session, + notification=notification, + indexed_count=1, + ) + + assert done.message == "Now searchable! 1 file synced." + + +async def test_indexing_completed_nothing_to_sync( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """Completing with nothing new reports 'Already up to date!'.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await NotificationService.connector_indexing.notify_indexing_completed( + session=db_session, + notification=notification, + indexed_count=0, + ) + + assert done.title == "Ready: Notion - My Workspace" + assert done.message == "Already up to date!" + assert done.notification_metadata["status"] == "completed" + + +async def test_indexing_completed_hard_failure( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """An error with nothing synced reports a hard failure.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await NotificationService.connector_indexing.notify_indexing_completed( + session=db_session, + notification=notification, + indexed_count=0, + error_message="boom", + ) + + assert done.title == "Failed: Notion - My Workspace" + assert done.message == "Sync failed: boom" + assert done.notification_metadata["status"] == "failed" + assert done.notification_metadata["sync_stage"] == "failed" + + +async def test_indexing_completed_partial_with_error_note( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """An error after partial progress still completes, with an appended note.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await NotificationService.connector_indexing.notify_indexing_completed( + session=db_session, + notification=notification, + indexed_count=2, + error_message="partial outage", + ) + + assert done.title == "Ready: Notion - My Workspace" + assert done.message == "Now searchable! 2 files synced. Note: partial outage" + assert done.notification_metadata["status"] == "completed" + + +async def test_retry_progress_frames_delay_as_providers( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """A retry message frames the delay as the provider's, using its short name.""" + notification = await _started(db_session, db_user, db_search_space) + + retry = await NotificationService.connector_indexing.notify_retry_progress( + session=db_session, + notification=notification, + indexed_count=0, + retry_reason="rate_limit", + attempt=1, + max_attempts=3, + ) + + # service_name is derived from the connector name, stripping the workspace suffix. + assert retry.message == "Notion rate limit reached. Retrying..." + assert retry.notification_metadata["sync_stage"] == "waiting_retry" + assert retry.notification_metadata["retry_attempt"] == 1 + assert retry.notification_metadata["retry_reason"] == "rate_limit" + + +async def test_retry_progress_shows_wait_and_synced_count( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + """A retry surfaces the wait time and how many items synced so far.""" + notification = await _started(db_session, db_user, db_search_space) + + retry = await NotificationService.connector_indexing.notify_retry_progress( + session=db_session, + notification=notification, + indexed_count=2, + retry_reason="rate_limit", + attempt=2, + max_attempts=3, + wait_seconds=10, + ) + + assert ( + retry.message + == "Notion rate limit reached. Retrying in 10s... (2 items synced so far)" + ) diff --git a/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py b/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py new file mode 100644 index 000000000..f602f2e66 --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_document_processing_handler.py @@ -0,0 +1,80 @@ +"""Behavior guard for the document-processing notification handler.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + +handler = NotificationService.document_processing + + +async def _started(db_session, db_user, db_search_space, *, name="report.pdf"): + """Open a document-processing notification to update in the tests below.""" + return await handler.notify_processing_started( + session=db_session, + user_id=db_user.id, + document_type="FILE", + document_name=name, + search_space_id=db_search_space.id, + ) + + +async def test_processing_started_queues( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """Starting processing queues a notification in the 'queued' stage.""" + notification = await _started(db_session, db_user, db_search_space) + + assert notification.type == "document_processing" + assert notification.title == "Processing: report.pdf" + assert notification.message == "Waiting in queue" + assert notification.notification_metadata["processing_stage"] == "queued" + + +async def test_processing_progress_maps_stage( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A progress update maps the stage to its user-facing message.""" + notification = await _started(db_session, db_user, db_search_space) + + updated = await handler.notify_processing_progress( + session=db_session, notification=notification, stage="parsing" + ) + + assert updated.message == "Reading your file" + assert updated.notification_metadata["processing_stage"] == "parsing" + + +async def test_processing_completed_success( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """Successful processing reports ready/searchable and a completed status.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await handler.notify_processing_completed( + session=db_session, notification=notification, document_id=99 + ) + + assert done.title == "Ready: report.pdf" + assert done.message == "Now searchable!" + assert done.notification_metadata["status"] == "completed" + + +async def test_processing_completed_failure( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """Failed processing reports a failed status with the error in the message.""" + notification = await _started(db_session, db_user, db_search_space) + + done = await handler.notify_processing_completed( + session=db_session, notification=notification, error_message="bad file" + ) + + assert done.title == "Failed: report.pdf" + assert done.message == "Processing failed: bad file" + assert done.notification_metadata["status"] == "failed" diff --git a/surfsense_backend/tests/integration/notifications/test_inbox_api.py b/surfsense_backend/tests/integration/notifications/test_inbox_api.py new file mode 100644 index 000000000..524a0ba60 --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_inbox_api.py @@ -0,0 +1,221 @@ +"""Behavior guard for the notifications inbox HTTP API. + +Rows are seeded through the transactional db_session and read back through the +real endpoints (auth + DB bound to the same transaction), pinning list filters, +counts, mark-read semantics, and response mapping. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import User +from app.notifications.persistence import Notification + +pytestmark = pytest.mark.integration + +BASE = "/api/v1/notifications" + + +async def _seed( + db_session: AsyncSession, + user: User, + *, + type: str = "document_processing", + title: str = "Title", + message: str = "Message", + read: bool = False, + search_space_id: int | None = None, + metadata: dict | None = None, + created_at: datetime | None = None, +) -> Notification: + """Insert a notification row directly for the API tests to read back.""" + notification = Notification( + user_id=user.id, + search_space_id=search_space_id, + type=type, + title=title, + message=message, + read=read, + notification_metadata=metadata or {}, + ) + if created_at is not None: + notification.created_at = created_at + db_session.add(notification) + await db_session.flush() + return notification + + +async def test_list_returns_user_notifications_mapped(client, db_session, db_user): + """GET / returns the caller's notifications mapped to the response shape.""" + seeded = await _seed( + db_session, db_user, type="document_processing", title="Doc done" + ) + + resp = await client.get(BASE) + + assert resp.status_code == 200 + body = resp.json() + assert body["total"] == 1 + item = body["items"][0] + assert item["id"] == seeded.id + assert item["user_id"] == str(db_user.id) + assert item["type"] == "document_processing" + assert item["title"] == "Doc done" + assert item["read"] is False + assert item["created_at"] # ISO string present + + +async def test_list_orders_newest_first(client, db_session, db_user): + """The list is ordered by creation time, newest first.""" + now = datetime.now(UTC) + await _seed(db_session, db_user, title="older", created_at=now - timedelta(hours=2)) + await _seed(db_session, db_user, title="newer", created_at=now) + + resp = await client.get(BASE) + + titles = [item["title"] for item in resp.json()["items"]] + assert titles == ["newer", "older"] + + +async def test_list_filters_by_category(client, db_session, db_user): + """The category filter narrows results to that category's notification types.""" + await _seed(db_session, db_user, type="connector_indexing", title="status item") + await _seed(db_session, db_user, type="comment_reply", title="comment item") + + resp = await client.get(BASE, params={"category": "comments"}) + + titles = [item["title"] for item in resp.json()["items"]] + assert titles == ["comment item"] + + +async def test_list_filters_unread_only(client, db_session, db_user): + """The unread filter returns only notifications that haven't been read.""" + await _seed(db_session, db_user, title="unread one", read=False) + await _seed(db_session, db_user, title="read one", read=True) + + resp = await client.get(BASE, params={"filter": "unread"}) + + titles = [item["title"] for item in resp.json()["items"]] + assert titles == ["unread one"] + + +async def test_list_filters_by_connector_source_type(client, db_session, db_user): + """A 'connector:' source filter selects only that connector's notifications.""" + await _seed( + db_session, + db_user, + type="connector_indexing", + title="github", + metadata={"connector_type": "GITHUB_CONNECTOR"}, + ) + await _seed( + db_session, + db_user, + type="connector_indexing", + title="notion", + metadata={"connector_type": "NOTION_CONNECTOR"}, + ) + + resp = await client.get(BASE, params={"source_type": "connector:GITHUB_CONNECTOR"}) + + titles = [item["title"] for item in resp.json()["items"]] + assert titles == ["github"] + + +async def test_list_rejects_invalid_before_date(client, db_session, db_user): + """A malformed before_date is rejected with a 400.""" + await _seed(db_session, db_user) + + resp = await client.get(BASE, params={"before_date": "not-a-date"}) + + assert resp.status_code == 400 + + +async def test_list_paginates_with_has_more(client, db_session, db_user): + """Pagination caps the page and reports has_more plus the next offset.""" + now = datetime.now(UTC) + for i in range(3): + await _seed( + db_session, db_user, title=f"n{i}", created_at=now - timedelta(minutes=i) + ) + + resp = await client.get(BASE, params={"limit": 2, "offset": 0}) + + body = resp.json() + assert len(body["items"]) == 2 + assert body["has_more"] is True + assert body["next_offset"] == 2 + + +async def test_unread_count_splits_total_and_recent(client, db_session, db_user): + """The unread count reports total unread and a recent-window subset.""" + now = datetime.now(UTC) + await _seed(db_session, db_user, read=False, created_at=now) + await _seed(db_session, db_user, read=False, created_at=now - timedelta(days=30)) + await _seed(db_session, db_user, read=True, created_at=now) + + resp = await client.get(f"{BASE}/unread-count") + + body = resp.json() + assert body["total_unread"] == 2 + assert body["recent_unread"] == 1 + + +async def test_unread_counts_batch_by_category(client, db_session, db_user): + """The batch endpoint breaks unread counts down per category.""" + await _seed(db_session, db_user, type="comment_reply", read=False) + await _seed(db_session, db_user, type="connector_indexing", read=False) + + resp = await client.get(f"{BASE}/unread-counts-batch") + + body = resp.json() + assert body["comments"]["total_unread"] == 1 + assert body["status"]["total_unread"] == 1 + + +async def test_mark_read_then_idempotent(client, db_session, db_user): + """Marking read succeeds, and a repeat call is a no-op reporting already-read.""" + notification = await _seed(db_session, db_user, read=False) + + first = await client.patch(f"{BASE}/{notification.id}/read") + assert first.status_code == 200 + assert first.json()["success"] is True + + second = await client.patch(f"{BASE}/{notification.id}/read") + assert second.status_code == 200 + assert second.json()["message"] == "Notification already marked as read" + + +async def test_mark_read_foreign_notification_404(client, db_session, db_user): + """Marking another user's notification read returns 404, not a cross-user write.""" + other = User( + email="other@surfsense.net", + hashed_password="hashed", + is_active=True, + is_superuser=False, + is_verified=True, + ) + db_session.add(other) + await db_session.flush() + foreign = await _seed(db_session, other, read=False) + + resp = await client.patch(f"{BASE}/{foreign.id}/read") + + assert resp.status_code == 404 + + +async def test_mark_all_read_returns_count(client, db_session, db_user): + """Mark-all-read flips only the unread rows and returns how many changed.""" + await _seed(db_session, db_user, read=False) + await _seed(db_session, db_user, read=False) + await _seed(db_session, db_user, read=True) + + resp = await client.patch(f"{BASE}/read-all") + + body = resp.json() + assert body["success"] is True + assert body["updated_count"] == 2 diff --git a/surfsense_backend/tests/integration/notifications/test_mention_handler.py b/surfsense_backend/tests/integration/notifications/test_mention_handler.py new file mode 100644 index 000000000..3254d737c --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_mention_handler.py @@ -0,0 +1,64 @@ +"""Behavior guard for the @mention notification handler.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + +handler = NotificationService.mention + + +async def _notify(db_session, db_user, db_search_space, *, mention_id=1, preview="hi"): + """Raise an @mention notification for the assertions in the tests below.""" + return await handler.notify_new_mention( + session=db_session, + mentioned_user_id=db_user.id, + mention_id=mention_id, + comment_id=10, + message_id=20, + thread_id=30, + thread_title="Thread", + author_id="author-1", + author_name="Alice", + author_avatar_url=None, + author_email="alice@surfsense.net", + content_preview=preview, + search_space_id=db_search_space.id, + ) + + +async def test_new_mention_title_and_message( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A mention notification names the author and carries the comment preview.""" + notification = await _notify(db_session, db_user, db_search_space, preview="hello") + + assert notification.type == "new_mention" + assert notification.title == "Alice mentioned you" + assert notification.message == "hello" + + +async def test_new_mention_truncates_long_preview( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A long comment preview is truncated in the mention message.""" + notification = await _notify( + db_session, db_user, db_search_space, preview="x" * 150 + ) + + assert notification.message == "x" * 100 + "..." + + +async def test_new_mention_is_idempotent( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """Re-notifying the same mention id reuses the existing notification row.""" + first = await _notify(db_session, db_user, db_search_space, mention_id=7) + second = await _notify(db_session, db_user, db_search_space, mention_id=7) + + assert second.id == first.id diff --git a/surfsense_backend/tests/integration/notifications/test_page_limit_handler.py b/surfsense_backend/tests/integration/notifications/test_page_limit_handler.py new file mode 100644 index 000000000..ab89d63c9 --- /dev/null +++ b/surfsense_backend/tests/integration/notifications/test_page_limit_handler.py @@ -0,0 +1,61 @@ +"""Behavior guard for the page-limit notification handler.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import SearchSpace, User +from app.notifications.service import NotificationService + +pytestmark = pytest.mark.integration + +handler = NotificationService.page_limit + + +async def test_page_limit_message_and_action( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A page-limit notification states usage and carries an upgrade action link.""" + notification = await handler.notify_page_limit_exceeded( + session=db_session, + user_id=db_user.id, + document_name="short.pdf", + document_type="FILE", + search_space_id=db_search_space.id, + pages_used=95, + pages_limit=100, + pages_to_add=10, + ) + + assert notification.type == "page_limit_exceeded" + assert notification.title == "Page limit exceeded: short.pdf" + assert notification.message == ( + "This document has ~10 page(s) but you've used 95/100 pages. " + "Upgrade to process more documents." + ) + assert notification.notification_metadata["status"] == "failed" + assert notification.notification_metadata["action_label"] == "Upgrade Plan" + assert notification.notification_metadata["action_url"] == ( + f"/dashboard/{db_search_space.id}/more-pages" + ) + + +async def test_page_limit_truncates_long_name( + db_session: AsyncSession, db_user: User, db_search_space: SearchSpace +): + """A long document name is truncated in the notification title.""" + long_name = "a" * 50 + + notification = await handler.notify_page_limit_exceeded( + session=db_session, + user_id=db_user.id, + document_name=long_name, + document_type="FILE", + search_space_id=db_search_space.id, + pages_used=95, + pages_limit=100, + pages_to_add=10, + ) + + assert notification.title == f"Page limit exceeded: {'a' * 40}..." diff --git a/surfsense_backend/tests/integration/retriever/test_knowledge_search_date_filters.py b/surfsense_backend/tests/integration/retriever/test_knowledge_search_date_filters.py index 910d882a7..ce076b147 100644 --- a/surfsense_backend/tests/integration/retriever/test_knowledge_search_date_filters.py +++ b/surfsense_backend/tests/integration/retriever/test_knowledge_search_date_filters.py @@ -8,7 +8,10 @@ from datetime import UTC, datetime, timedelta import numpy as np import pytest -from app.agents.new_chat.middleware.knowledge_search import search_knowledge_base +from app.agents.chat.multi_agent_chat.shared.middleware import knowledge_search as ks +from app.agents.chat.multi_agent_chat.shared.middleware.knowledge_search import ( + search_knowledge_base, +) from .conftest import DUMMY_EMBEDDING @@ -26,13 +29,9 @@ async def test_search_knowledge_base_applies_date_filters( async def fake_shielded_async_session(): yield db_session + monkeypatch.setattr(ks, "shielded_async_session", fake_shielded_async_session) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.shielded_async_session", - fake_shielded_async_session, - ) - monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.embed_texts", - lambda texts: [np.array(DUMMY_EMBEDDING) for _ in texts], + ks, "embed_texts", lambda texts: [np.array(DUMMY_EMBEDDING) for _ in texts] ) space_id = seed_date_filtered_docs["search_space"].id diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_hitl_bridge.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_hitl_bridge.py index 72408a5d9..45db9c901 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_hitl_bridge.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_hitl_bridge.py @@ -14,16 +14,16 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Command, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.config import ( - subagent_invoke_config, -) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) +from app.agents.chat.multi_agent_chat.subagents.shared.invocation import ( + subagent_invoke_config, +) class _SubagentState(TypedDict, total=False): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_heterogeneous_decisions.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_heterogeneous_decisions.py index d4a68939e..dd895c54e 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_heterogeneous_decisions.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_heterogeneous_decisions.py @@ -40,12 +40,12 @@ from langgraph.graph.message import add_messages from langgraph.types import Command, Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_partial_pause_routing.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_partial_pause_routing.py index 1aba0c480..7ac7686e9 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_partial_pause_routing.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_partial_pause_routing.py @@ -47,12 +47,12 @@ from langgraph.graph.message import add_messages from langgraph.types import Command, Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_reject_only_routing.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_reject_only_routing.py index 5810d5394..a1bbb9e7a 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_reject_only_routing.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_reject_only_routing.py @@ -37,12 +37,12 @@ from langgraph.graph.message import add_messages from langgraph.types import Command, Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_resume_command_keying.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_resume_command_keying.py index 839cb7564..b082119e3 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_resume_command_keying.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_resume_command_keying.py @@ -37,12 +37,12 @@ from langgraph.graph.message import add_messages from langgraph.types import Command, Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_self_and_middleware_gated.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_self_and_middleware_gated.py index 921c4a9eb..2c098ef8a 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_self_and_middleware_gated.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_self_and_middleware_gated.py @@ -35,21 +35,21 @@ from langgraph.graph.message import add_messages from langgraph.types import Command, Send from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( build_lg_resume_map, collect_pending_tool_calls, slice_decisions_by_tool_call, ) -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) -from app.agents.multi_agent_chat.middleware.shared.permissions.ask.request import ( +from app.agents.chat.multi_agent_chat.shared.permissions import Rule +from app.agents.chat.multi_agent_chat.shared.permissions.ask.request import ( request_permission_decision, ) -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) -from app.agents.new_chat.permissions import Rule class _SubState(TypedDict, total=False): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_tasks.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_tasks.py index 81be7d1ac..836822d34 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_tasks.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_parallel_tasks.py @@ -18,7 +18,7 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Command from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_pending_interrupt.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_pending_interrupt.py index 75242689d..ec757bcf0 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_pending_interrupt.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_pending_interrupt.py @@ -9,7 +9,7 @@ from __future__ import annotations from types import SimpleNamespace -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume import ( get_first_pending_subagent_interrupt, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_decision_routing.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_decision_routing.py index ceb0df830..62f33addc 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_decision_routing.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_decision_routing.py @@ -17,7 +17,7 @@ from types import SimpleNamespace import pytest -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.resume_routing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.resume_routing import ( collect_pending_tool_calls, slice_decisions_by_tool_call, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_helpers.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_helpers.py index e8aacfc5d..ba9d163a4 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_helpers.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_resume_helpers.py @@ -4,7 +4,7 @@ from __future__ import annotations from langchain.tools import ToolRuntime -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.config import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.config import ( consume_surfsense_resume, has_surfsense_resume, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_interrupt_stamping.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_interrupt_stamping.py index 7df9dedc6..4bc0ecace 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_interrupt_stamping.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_interrupt_stamping.py @@ -30,7 +30,7 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_invoke_config.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_invoke_config.py index 3465dd1d8..5044d8fbe 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_invoke_config.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/checkpointed_subagent_middleware/test_subagent_invoke_config.py @@ -16,7 +16,7 @@ from __future__ import annotations from langchain.tools import ToolRuntime -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.config import ( +from app.agents.chat.multi_agent_chat.subagents.shared.invocation import ( subagent_invoke_config, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_lc_hitl_wire.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_lc_hitl_wire.py index a331190b2..3f89a9707 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_lc_hitl_wire.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_lc_hitl_wire.py @@ -16,10 +16,10 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Command from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.shared.permissions.ask.request import ( +from app.agents.chat.multi_agent_chat.shared.permissions import Rule +from app.agents.chat.multi_agent_chat.shared.permissions.ask.request import ( request_permission_decision, ) -from app.agents.new_chat.permissions import Rule class _State(TypedDict, total=False): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_permission_ask_mcp_context.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_permission_ask_mcp_context.py index c9bd4e142..33256c2ff 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_permission_ask_mcp_context.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_permission_ask_mcp_context.py @@ -13,14 +13,15 @@ from langgraph.graph.message import add_messages from pydantic import BaseModel from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.shared.permissions import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.permissions import ( + Rule, + Ruleset, build_permission_mw, ) -from app.agents.multi_agent_chat.middleware.shared.permissions.ask.payload import ( +from app.agents.chat.multi_agent_chat.shared.permissions.ask.payload import ( build_permission_ask_payload, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.permissions import Rule, Ruleset class _NoArgs(BaseModel): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_subagent_owned_ruleset.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_subagent_owned_ruleset.py index b9ac6cd15..66dec22b0 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_subagent_owned_ruleset.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_subagent_owned_ruleset.py @@ -23,11 +23,12 @@ from langgraph.graph.message import add_messages from langgraph.types import Command from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.shared.permissions import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.permissions import ( + Rule, + Ruleset, build_permission_mw, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.permissions import Rule, Ruleset def _kb_style_ruleset() -> Ruleset: diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_trusted_tool_save_on_always.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_trusted_tool_save_on_always.py index 47d3704ac..479d607f7 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_trusted_tool_save_on_always.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/middleware/shared/permissions/test_trusted_tool_save_on_always.py @@ -14,11 +14,12 @@ from langgraph.types import Command from pydantic import BaseModel from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.shared.permissions import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.permissions import ( + Rule, + Ruleset, build_permission_mw, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.permissions import Rule, Ruleset class _NoArgs(BaseModel): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/test_lc_hitl_wire.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/test_lc_hitl_wire.py index 195b1bc01..a33d11358 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/test_lc_hitl_wire.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/approvals/self_gated/test_lc_hitl_wire.py @@ -22,7 +22,7 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Command from typing_extensions import TypedDict -from app.agents.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.approvals.self_gated import ( request_approval, ) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/wire/test_hitl_wire.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/wire/test_hitl_wire.py index c06f9a627..cdaa4d71d 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/wire/test_hitl_wire.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/hitl/wire/test_hitl_wire.py @@ -18,7 +18,7 @@ These tests pin the shape: from __future__ import annotations -from app.agents.multi_agent_chat.subagents.shared.hitl.wire import ( +from app.agents.chat.multi_agent_chat.subagents.shared.hitl.wire import ( LC_DECISION_APPROVE, LC_DECISION_EDIT, LC_DECISION_REJECT, diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py index 062ea92ec..2f3553a27 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py @@ -19,14 +19,14 @@ from langchain_core.language_models.fake_chat_models import ( from langchain_core.messages import AIMessage, BaseMessage, HumanMessage from langchain_core.outputs import ChatGeneration, ChatResult -from app.agents.multi_agent_chat.middleware.shared.permissions.middleware.core import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset, evaluate +from app.agents.chat.multi_agent_chat.shared.permissions.middleware.core import ( PermissionMiddleware, ) -from app.agents.multi_agent_chat.subagents.shared.subagent_builder import ( +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( pack_subagent, ) -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.permissions import Rule, Ruleset, evaluate class RateLimitError(Exception): diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py new file mode 100644 index 000000000..ccdfc0b98 --- /dev/null +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_prompt_resources.py @@ -0,0 +1,59 @@ +"""Guardrail C: package-relative prompt/snippet resources must resolve. + +Prompt fragments are loaded by *package name* via ``importlib.resources`` — not +by import, so the import-all smoke test (guardrail A) cannot see them, and not +by mocked unit tests. A move that relocates a package without its ``.md`` files, +or that leaves a hardcoded package string stale, returns an empty string and +silently degrades the system prompt. These tests assert the resources still +resolve to non-empty content. + +(Builtin skill resources are covered separately by ``test_skills_backends.py``.) +""" + +from __future__ import annotations + +import pytest + +from app.agents.chat.multi_agent_chat.main_agent.system_prompt.builder.load_md import ( + read_prompt_md, +) +from app.agents.chat.multi_agent_chat.subagents.registry import ( + SUBAGENT_BUILDERS_BY_NAME, + _route_resource_package, +) +from app.agents.chat.multi_agent_chat.subagents.shared.md_file_reader import ( + read_md_file, + read_shared_snippet, +) + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize("name", sorted(SUBAGENT_BUILDERS_BY_NAME)) +def test_every_subagent_has_description_md(name: str): + """Each specialist ships a non-empty ``description.md`` next to its agent.""" + package = _route_resource_package(SUBAGENT_BUILDERS_BY_NAME[name]) + assert read_md_file(package, "description").strip(), ( + f"{name}: description.md missing/empty at package {package}" + ) + + +# Real fragments under the hardcoded main-agent prompts package, including a +# nested path — guards both the package string and nested resource resolution. +@pytest.mark.parametrize( + "filename", + [ + "core_behavior.md", + "routing.md", + "tools/web_search/description.md", + ], +) +def test_main_agent_prompt_fragments_resolve(filename: str): + """Main-agent prompt fragments resolve to non-empty content.""" + assert read_prompt_md(filename).strip(), f"prompt fragment {filename} is empty" + + +@pytest.mark.parametrize("snippet", ["output_contract_base", "verifiable_handle"]) +def test_shared_snippets_resolve(snippet: str): + """Shared subagent snippets resolve from the snippets package.""" + assert read_shared_snippet(snippet).strip(), f"snippet {snippet} is empty" diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py new file mode 100644 index 000000000..157f1703b --- /dev/null +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_subagent_composition.py @@ -0,0 +1,72 @@ +"""Guardrail B: the subagent registry composition must stay intact. + +A structural move can silently drop, rename, or mis-wire a subagent builder +(e.g. a forgotten import line). The compiled agent would then quietly lose a +specialist with no ImportError. This test pins the exact registry contents and +their cross-references so any such drift fails loudly. +""" + +from __future__ import annotations + +import pytest + +from app.agents.chat.multi_agent_chat.constants import ( + SUBAGENT_TO_REQUIRED_CONNECTOR_MAP, +) +from app.agents.chat.multi_agent_chat.subagents.registry import ( + SUBAGENT_BUILDERS_BY_NAME, +) + +pytestmark = pytest.mark.unit + +# The full specialist roster the main agent composes from: 4 builtins + 15 +# connector routes. Adding/removing a specialist is a deliberate product change +# and must be reflected here. +_EXPECTED_SUBAGENTS = frozenset( + { + "airtable", + "calendar", + "clickup", + "confluence", + "deliverables", + "discord", + "dropbox", + "gmail", + "google_drive", + "jira", + "knowledge_base", + "linear", + "luma", + "memory", + "notion", + "onedrive", + "research", + "slack", + "teams", + } +) + +# Specialists that are always available regardless of connected sources, so they +# carry no required-connector entry. +_CONNECTORLESS = frozenset({"memory", "research"}) + + +def test_registry_contains_exactly_expected_subagents(): + """No specialist is silently added, dropped, or renamed by a move.""" + assert set(SUBAGENT_BUILDERS_BY_NAME) == _EXPECTED_SUBAGENTS + + +def test_every_builder_is_callable_route_agent(): + """Each registry value is a callable defined in its route's ``agent`` module.""" + for name, builder in SUBAGENT_BUILDERS_BY_NAME.items(): + assert callable(builder), f"{name} builder is not callable" + assert builder.__module__.endswith(".agent"), ( + f"{name} builder lives in {builder.__module__}, expected a *.agent module" + ) + + +def test_required_connector_map_covers_connector_subagents(): + """The connector-gating map stays in lockstep with the registry.""" + assert set(SUBAGENT_TO_REQUIRED_CONNECTOR_MAP) == ( + _EXPECTED_SUBAGENTS - _CONNECTORLESS + ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/middleware/test_scoped_model_fallback.py b/surfsense_backend/tests/unit/agents/new_chat/middleware/test_scoped_model_fallback.py index 80b9862e7..361a23f41 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/middleware/test_scoped_model_fallback.py +++ b/surfsense_backend/tests/unit/agents/new_chat/middleware/test_scoped_model_fallback.py @@ -87,7 +87,7 @@ class RateLimitError(Exception): def _build_agent(primary: BaseChatModel, fallback: BaseChatModel): from langchain.agents import create_agent - from app.agents.new_chat.middleware.scoped_model_fallback import ( + from app.agents.chat.multi_agent_chat.shared.middleware.resilience.scoped_model_fallback import ( ScopedModelFallbackMiddleware, ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py index 36fe04aa2..4f0369e12 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py +++ b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py @@ -6,12 +6,12 @@ from datetime import UTC, datetime import pytest -from app.agents.new_chat.prompts.composer import ( +from app.db import ChatVisibility +from app.prompts.system_prompt_composer.composer import ( ALL_TOOL_NAMES_ORDERED, compose_system_prompt, detect_provider_variant, ) -from app.db import ChatVisibility pytestmark = pytest.mark.unit @@ -64,7 +64,7 @@ class TestProviderVariantDetection: ``gpt-5`` reasoning regex first. Codex is the more specialised prompt and mirrors OpenCode's dispatch order. """ - from app.agents.new_chat.prompts.composer import detect_provider_variant + from app.prompts.system_prompt_composer.composer import detect_provider_variant assert detect_provider_variant("openai:gpt-5-codex") == "openai_codex" assert detect_provider_variant("openai:gpt-5") == "openai_reasoning" diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py b/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py index 8ef1430a9..e476538bd 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py @@ -10,9 +10,11 @@ import pytest from langchain_core.messages import ToolMessage from langchain_core.tools import tool -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.middleware.action_log import ActionLogMiddleware -from app.agents.new_chat.tools.registry import ToolDefinition +from app.agents.chat.multi_agent_chat.main_agent.middleware.action_log.middleware import ( + ActionLogMiddleware, + ToolDefinition, +) +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags @dataclass @@ -58,7 +60,7 @@ def _disabled_flags() -> AgentFeatureFlags: def patch_get_flags(): def _patch(flags: AgentFeatureFlags): return patch( - "app.agents.new_chat.middleware.action_log.get_flags", + "app.agents.chat.multi_agent_chat.main_agent.middleware.action_log.middleware.get_flags", return_value=flags, ) @@ -360,7 +362,7 @@ class TestActionLogDispatch: patch_get_flags(_enabled_flags()), patch("app.db.shielded_async_session", side_effect=lambda: factory()), patch( - "app.agents.new_chat.middleware.action_log.adispatch_custom_event", + "app.agents.chat.multi_agent_chat.main_agent.middleware.action_log.middleware.adispatch_custom_event", dispatch_mock, ), ): @@ -395,7 +397,7 @@ class TestActionLogDispatch: patch_get_flags(_enabled_flags()), patch("app.db.shielded_async_session", side_effect=_exploding_session), patch( - "app.agents.new_chat.middleware.action_log.adispatch_custom_event", + "app.agents.chat.multi_agent_chat.main_agent.middleware.action_log.middleware.adispatch_custom_event", dispatch_mock, ), ): diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_agent_cache.py b/surfsense_backend/tests/unit/agents/new_chat/test_agent_cache.py index 9b3de2db7..ecc5a1a83 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_agent_cache.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_agent_cache.py @@ -16,7 +16,7 @@ from dataclasses import dataclass import pytest -from app.agents.new_chat.agent_cache import ( +from app.agents.chat.multi_agent_chat.main_agent.runtime.agent_cache_store import ( flags_signature, reload_for_tests, stable_hash, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_busy_mutex.py b/surfsense_backend/tests/unit/agents/new_chat/test_busy_mutex.py index f0161f605..5a39c6e66 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_busy_mutex.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_busy_mutex.py @@ -4,8 +4,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.errors import BusyError -from app.agents.new_chat.middleware.busy_mutex import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.busy_mutex import ( BusyMutexMiddleware, end_turn, get_cancel_event, @@ -14,6 +13,7 @@ from app.agents.new_chat.middleware.busy_mutex import ( request_cancel, reset_cancel, ) +from app.agents.chat.runtime.errors import BusyError pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py b/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py index c6d4cc452..2ac462959 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py @@ -10,7 +10,7 @@ from langchain_core.messages import ( ToolMessage, ) -from app.agents.new_chat.middleware.compaction import ( +from app.agents.chat.shared.middleware.compaction import ( PROTECTED_SYSTEM_PREFIXES, _is_protected_system_message, _sanitize_message_content, @@ -72,7 +72,7 @@ class TestPartitionMessages: # SurfSenseCompactionMiddleware without a real model, but the # override path needs ``_lc_helper`` to delegate to. We mock # that with a simple slicing partitioner equivalent to the real one. - from app.agents.new_chat.middleware.compaction import ( + from app.agents.chat.shared.middleware.compaction import ( SurfSenseCompactionMiddleware, ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py b/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py index ba2246413..9632fd14d 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py @@ -7,7 +7,7 @@ from typing import Any import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from app.agents.new_chat.middleware.context_editing import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.context_editing.middleware import ( SpillToBackendEdit, _build_spill_placeholder, ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py b/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py index 61d9b499f..61a04c1c1 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py @@ -6,7 +6,7 @@ import pytest from langchain_core.messages import AIMessage from langchain_core.tools import StructuredTool -from app.agents.new_chat.middleware.dedup_tool_calls import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.dedup_hitl import ( DedupHITLToolCallsMiddleware, ) @@ -91,10 +91,9 @@ def test_no_agent_tools_means_no_dedup() -> None: """After the cleanup tier removed the legacy ``_NATIVE_HITL_TOOL_DEDUP_KEYS`` map, dedup is purely declarative — no resolvers means no dedup runs. - Coverage for the previously hardcoded native HITL tools now lives on - each :class:`ToolDefinition.dedup_key` in - :mod:`app.agents.new_chat.tools.registry`, which is wired through to - ``tool.metadata`` by :func:`build_tools`. + Dedup is purely declarative: tools opt in by carrying a ``dedup_key`` + (callable) or ``hitl_dedup_key`` (arg name) in their ``metadata``. With no + agent tools, there are no resolvers and dedup is a no-op. """ mw = DedupHITLToolCallsMiddleware(agent_tools=None) state = { @@ -109,27 +108,6 @@ def test_no_agent_tools_means_no_dedup() -> None: assert out is None -def test_registry_propagates_dedup_key_to_tool_metadata() -> None: - """Smoke-check the wiring path that replaced the legacy native map. - - ``ToolDefinition.dedup_key`` set in the registry must be copied onto - the constructed tool's ``metadata`` so :class:`DedupHITLToolCallsMiddleware` - can pick it up at agent build time. - """ - from app.agents.new_chat.tools.registry import ( - BUILTIN_TOOLS, - wrap_dedup_key_by_arg_name, - ) - - notion_tool_defs = [t for t in BUILTIN_TOOLS if t.name == "create_notion_page"] - assert notion_tool_defs, "registry should still expose create_notion_page" - tool_def = notion_tool_defs[0] - assert tool_def.dedup_key is not None - # Same wrapping helper used in the registry — sanity check identity - sample = wrap_dedup_key_by_arg_name("title")({"title": "Plan"}) - assert sample == "plan" - - def test_full_args_dedup_keeps_distinct_calls_sharing_a_field() -> None: """Regression: MCP tools (e.g. ``createJiraIssue``) used to dedup on the schema's first required field, which is often the workspace / @@ -137,7 +115,9 @@ def test_full_args_dedup_keeps_distinct_calls_sharing_a_field() -> None: With :func:`dedup_key_full_args` only fully identical arg dicts dedup. """ - from app.agents.new_chat.middleware.dedup_tool_calls import dedup_key_full_args + from app.agents.chat.multi_agent_chat.shared.middleware.dedup_tool_calls import ( + dedup_key_full_args, + ) tool = _make_tool("createJiraIssue", dedup_key=dedup_key_full_args) mw = DedupHITLToolCallsMiddleware(agent_tools=[tool]) @@ -179,7 +159,9 @@ def test_full_args_dedup_keeps_distinct_calls_sharing_a_field() -> None: def test_full_args_dedup_drops_only_exact_duplicates() -> None: - from app.agents.new_chat.middleware.dedup_tool_calls import dedup_key_full_args + from app.agents.chat.multi_agent_chat.shared.middleware.dedup_tool_calls import ( + dedup_key_full_args, + ) tool = _make_tool("createJiraIssue", dedup_key=dedup_key_full_args) mw = DedupHITLToolCallsMiddleware(agent_tools=[tool]) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py index 2f222e148..b6341bfec 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py @@ -17,7 +17,7 @@ caused two production-painful behaviors: read-only tool calls, raising ``RejectedError("ls")``. * Mutating connector tools got *double* prompted — once via the middleware ``ask`` and again via the per-tool ``interrupt()`` in - ``app.agents.new_chat.tools.hitl``. + ``app.agents.chat.multi_agent_chat.shared.tools.hitl``. These tests pin the layering so a refactor that drops the default ruleset fails loud. @@ -27,7 +27,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions import ( Rule, Ruleset, aggregate_action, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_desktop_safety_rules.py b/surfsense_backend/tests/unit/agents/new_chat/test_desktop_safety_rules.py index 653175eab..62712e797 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_desktop_safety_rules.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_desktop_safety_rules.py @@ -10,8 +10,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.middleware.permission import PermissionMiddleware -from app.agents.new_chat.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions import ( Rule, Ruleset, aggregate_action, @@ -87,36 +86,3 @@ class TestDesktopSafetyOverridesAllowDefault: # Correct order: defaults < desktop_safety -> ask wins. action = _action_for("rm", SURFSENSE_DEFAULTS, DESKTOP_SAFETY_RULESET) assert action == "ask" - - -class TestPermissionMiddlewareIntegration: - def test_middleware_raises_interrupt_for_rm_in_desktop_mode(self) -> None: - from langchain_core.messages import AIMessage - - from app.agents.new_chat.errors import RejectedError - - mw = PermissionMiddleware(rulesets=[SURFSENSE_DEFAULTS, DESKTOP_SAFETY_RULESET]) - # Stub the interrupt to a "reject" decision so we can assert the - # ask path was taken without spinning up the LangGraph runtime. - mw._raise_interrupt = lambda **kw: {"decision_type": "reject"} # type: ignore[assignment] - - state = { - "messages": [ - AIMessage( - content="", - tool_calls=[ - { - "name": "rm", - "args": {"path": "/Users/me/Documents/important.docx"}, - "id": "tc-rm", - } - ], - ) - ] - } - - class _FakeRuntime: - config: dict = {"configurable": {"thread_id": "test"}} - - with pytest.raises(RejectedError): - mw.after_model(state, _FakeRuntime()) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py b/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py index 802129bf6..47e962242 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py @@ -5,7 +5,10 @@ from __future__ import annotations import pytest from langchain_core.messages import AIMessage -from app.agents.new_chat.middleware.doom_loop import DoomLoopMiddleware, _signature +from app.agents.chat.multi_agent_chat.main_agent.middleware.doom_loop.middleware import ( + DoomLoopMiddleware, + _signature, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_feature_flags.py b/surfsense_backend/tests/unit/agents/new_chat/test_feature_flags.py index 099aea882..e715a80c6 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_feature_flags.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_feature_flags.py @@ -4,7 +4,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.feature_flags import ( +from app.agents.chat.multi_agent_chat.shared.feature_flags import ( AgentFeatureFlags, reload_for_tests, ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_flatten_system.py b/surfsense_backend/tests/unit/agents/new_chat/test_flatten_system.py deleted file mode 100644 index 6c323d920..000000000 --- a/surfsense_backend/tests/unit/agents/new_chat/test_flatten_system.py +++ /dev/null @@ -1,344 +0,0 @@ -"""Tests for ``FlattenSystemMessageMiddleware``. - -The middleware exists to defend against Anthropic's "Found 5 cache_control -blocks" 400 when our deepagent middleware stack stacks 5+ text blocks on -the system message and the OpenRouter→Anthropic adapter redistributes -``cache_control`` across all of them. The flattening collapses every -all-text system content list to a single string before the LLM call. -""" - -from __future__ import annotations - -from typing import Any -from unittest.mock import MagicMock - -import pytest -from langchain_core.messages import HumanMessage, SystemMessage - -from app.agents.new_chat.middleware.flatten_system import ( - FlattenSystemMessageMiddleware, - _flatten_text_blocks, - _flattened_request, -) - -pytestmark = pytest.mark.unit - - -# --------------------------------------------------------------------------- -# _flatten_text_blocks — pure helper, the heart of the middleware. -# --------------------------------------------------------------------------- - - -class TestFlattenTextBlocks: - def test_joins_text_blocks_with_double_newline(self) -> None: - blocks = [ - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - ] - assert ( - _flatten_text_blocks(blocks) - == "\n\n\n\n" - ) - - def test_handles_single_text_block(self) -> None: - blocks = [{"type": "text", "text": "only one"}] - assert _flatten_text_blocks(blocks) == "only one" - - def test_handles_empty_list(self) -> None: - assert _flatten_text_blocks([]) == "" - - def test_passes_through_bare_string_blocks(self) -> None: - # LangChain content can mix bare strings and dict blocks. - blocks = ["raw string", {"type": "text", "text": "dict block"}] - assert _flatten_text_blocks(blocks) == "raw string\n\ndict block" - - def test_returns_none_for_image_block(self) -> None: - # System messages with images are rare — but we never want to - # silently lose the image payload by joining as text. - blocks = [ - {"type": "text", "text": "look at this"}, - {"type": "image_url", "image_url": {"url": "data:image/png..."}}, - ] - assert _flatten_text_blocks(blocks) is None - - def test_returns_none_for_non_dict_non_str_block(self) -> None: - blocks = [{"type": "text", "text": "hi"}, 42] # type: ignore[list-item] - assert _flatten_text_blocks(blocks) is None - - def test_returns_none_when_text_field_missing(self) -> None: - blocks = [{"type": "text"}] # no ``text`` key - assert _flatten_text_blocks(blocks) is None - - def test_returns_none_when_text_is_not_string(self) -> None: - blocks = [{"type": "text", "text": ["nested", "list"]}] - assert _flatten_text_blocks(blocks) is None - - def test_drops_cache_control_from_inner_blocks(self) -> None: - # The whole point: existing cache_control on inner blocks is - # discarded so LiteLLM's ``cache_control_injection_points`` can - # re-attach exactly one breakpoint after flattening. - blocks = [ - {"type": "text", "text": "first"}, - { - "type": "text", - "text": "second", - "cache_control": {"type": "ephemeral"}, - }, - ] - flattened = _flatten_text_blocks(blocks) - assert flattened == "first\n\nsecond" - assert "cache_control" not in flattened # type: ignore[operator] - - -# --------------------------------------------------------------------------- -# _flattened_request — decides when to override and when to no-op. -# --------------------------------------------------------------------------- - - -def _make_request(system_message: SystemMessage | None) -> Any: - """Build a minimal ModelRequest stub. We only need .system_message - and .override(system_message=...) — the middleware never touches - other fields. - """ - request = MagicMock() - request.system_message = system_message - - def override(**kwargs: Any) -> Any: - new_request = MagicMock() - new_request.system_message = kwargs.get( - "system_message", request.system_message - ) - new_request.messages = kwargs.get("messages", getattr(request, "messages", [])) - new_request.tools = kwargs.get("tools", getattr(request, "tools", [])) - return new_request - - request.override = override - return request - - -class TestFlattenedRequest: - def test_collapses_multi_block_system_to_string(self) -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - ] - ) - request = _make_request(sys) - flattened = _flattened_request(request) - - assert flattened is not None - assert isinstance(flattened.system_message, SystemMessage) - assert flattened.system_message.content == ( - "\n\n\n\n\n\n\n\n" - ) - - def test_no_op_for_string_content(self) -> None: - sys = SystemMessage(content="already a string") - request = _make_request(sys) - assert _flattened_request(request) is None - - def test_no_op_for_single_block_list(self) -> None: - # One block already produces one breakpoint — no need to flatten. - sys = SystemMessage(content=[{"type": "text", "text": "single"}]) - request = _make_request(sys) - assert _flattened_request(request) is None - - def test_no_op_when_system_message_missing(self) -> None: - request = _make_request(None) - assert _flattened_request(request) is None - - def test_no_op_when_list_contains_non_text_block(self) -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": "look"}, - {"type": "image_url", "image_url": {"url": "data:..."}}, - ] - ) - request = _make_request(sys) - assert _flattened_request(request) is None - - def test_preserves_additional_kwargs_and_metadata(self) -> None: - # Defensive: nothing in the current chain sets these on a system - # message, but losing them silently when something does in the - # future would be a regression. ``name`` in particular is the only - # ``additional_kwargs`` field that ChatLiteLLM's - # ``_convert_message_to_dict`` propagates onto the wire. - sys = SystemMessage( - content=[ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ], - additional_kwargs={"name": "surfsense_system", "x": 1}, - response_metadata={"tokens": 42}, - ) - sys.id = "sys-msg-1" - request = _make_request(sys) - - flattened = _flattened_request(request) - assert flattened is not None - assert flattened.system_message.content == "a\n\nb" - assert flattened.system_message.additional_kwargs == { - "name": "surfsense_system", - "x": 1, - } - assert flattened.system_message.response_metadata == {"tokens": 42} - assert flattened.system_message.id == "sys-msg-1" - - def test_idempotent_when_run_twice(self) -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ] - ) - request = _make_request(sys) - first = _flattened_request(request) - assert first is not None - - # Second pass on the already-flattened request should be a no-op. - # We re-wrap in a request stub since the helper inspects - # ``request.system_message.content``. - second_request = _make_request(first.system_message) - assert _flattened_request(second_request) is None - - -# --------------------------------------------------------------------------- -# Middleware integration — verify the handler sees a flattened request. -# --------------------------------------------------------------------------- - - -class TestMiddlewareWrap: - @pytest.mark.asyncio - async def test_async_passes_flattened_request_to_handler(self) -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": "alpha"}, - {"type": "text", "text": "beta"}, - ] - ) - request = _make_request(sys) - captured: dict[str, Any] = {} - - async def handler(req: Any) -> str: - captured["request"] = req - return "ok" - - mw = FlattenSystemMessageMiddleware() - result = await mw.awrap_model_call(request, handler) - - assert result == "ok" - assert isinstance(captured["request"].system_message, SystemMessage) - assert captured["request"].system_message.content == "alpha\n\nbeta" - - @pytest.mark.asyncio - async def test_async_passes_through_when_already_string(self) -> None: - sys = SystemMessage(content="just a string") - request = _make_request(sys) - captured: dict[str, Any] = {} - - async def handler(req: Any) -> str: - captured["request"] = req - return "ok" - - mw = FlattenSystemMessageMiddleware() - await mw.awrap_model_call(request, handler) - - # Same request object: no override happened. - assert captured["request"] is request - - def test_sync_passes_flattened_request_to_handler(self) -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": "alpha"}, - {"type": "text", "text": "beta"}, - ] - ) - request = _make_request(sys) - captured: dict[str, Any] = {} - - def handler(req: Any) -> str: - captured["request"] = req - return "ok" - - mw = FlattenSystemMessageMiddleware() - result = mw.wrap_model_call(request, handler) - - assert result == "ok" - assert captured["request"].system_message.content == "alpha\n\nbeta" - - def test_sync_passes_through_when_no_system_message(self) -> None: - request = _make_request(None) - captured: dict[str, Any] = {} - - def handler(req: Any) -> str: - captured["request"] = req - return "ok" - - mw = FlattenSystemMessageMiddleware() - mw.wrap_model_call(request, handler) - assert captured["request"] is request - - -# --------------------------------------------------------------------------- -# Regression guard — pin the worst-case shape that triggered the -# "Found 5" 400 in production. Confirms we collapse 5 blocks to 1 so the -# downstream cache_control_injection_points can only place 1 breakpoint -# on the system message regardless of provider redistribution quirks. -# --------------------------------------------------------------------------- - - -def test_regression_five_block_system_collapses_to_one_block() -> None: - sys = SystemMessage( - content=[ - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - {"type": "text", "text": ""}, - ] - ) - request = _make_request(sys) - flattened = _flattened_request(request) - - assert flattened is not None - assert isinstance(flattened.system_message.content, str) - # The exact join doesn't matter for the cache_control accounting — - # only that there is exactly ONE content block when LiteLLM's - # AnthropicCacheControlHook later targets ``role: system``. - assert " None: - # Sanity: the middleware MUST NOT touch user messages — only the - # system message. Multi-block user content is the path that carries - # image attachments and would lose its image_url block on - # accidental flatten. - sys = SystemMessage( - content=[ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ] - ) - user = HumanMessage( - content=[ - {"type": "text", "text": "look at this"}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}, - ] - ) - request = _make_request(sys) - request.messages = [user] - - flattened = _flattened_request(request) - assert flattened is not None - # System flattened to string … - assert isinstance(flattened.system_message.content, str) - # … user message is untouched (the helper does not even look at it). - assert flattened.messages == [user] - assert isinstance(user.content, list) - assert len(user.content) == 2 diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_hitl_auto_approve.py b/surfsense_backend/tests/unit/agents/new_chat/test_hitl_auto_approve.py index d0ea73376..9c19cbd6b 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_hitl_auto_approve.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_hitl_auto_approve.py @@ -10,7 +10,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.tools.hitl import ( +from app.agents.chat.multi_agent_chat.shared.tools.hitl import ( DEFAULT_AUTO_APPROVED_TOOLS, HITLResult, request_approval, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_mention_resolver.py b/surfsense_backend/tests/unit/agents/new_chat/test_mention_resolver.py index 1f8d35841..4130c9d4e 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_mention_resolver.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_mention_resolver.py @@ -15,14 +15,17 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from app.agents.new_chat import mention_resolver -from app.agents.new_chat.mention_resolver import ( +from app.agents.chat.runtime import mention_resolver +from app.agents.chat.runtime.mention_resolver import ( ResolvedMention, ResolvedMentionSet, resolve_mentions, substitute_in_text, ) -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT, PathIndex +from app.agents.chat.runtime.path_resolver import ( + DOCUMENTS_ROOT, + PathIndex, +) from app.schemas.new_chat import MentionedDocumentInfo pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py b/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py index 346271f4b..42df4eecf 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py @@ -5,7 +5,7 @@ from __future__ import annotations import pytest from langchain_core.messages import AIMessage, HumanMessage -from app.agents.new_chat.middleware.noop_injection import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.noop_injection.middleware import ( NOOP_TOOL_NAME, NoopInjectionMiddleware, _last_ai_has_tool_calls, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py index dc59c6dac..e2978d277 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock import pytest from langchain_core.messages import AIMessage, ToolMessage -from app.agents.new_chat.middleware.otel_span import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.otel_span.middleware import ( OtelSpanMiddleware, _annotate_model_response, _annotate_tool_result, @@ -206,13 +206,13 @@ class TestMiddlewareIntegration: duration_calls: list[dict[str, Any]] = [] token_calls: list[dict[str, Any]] = [] monkeypatch.setattr( - "app.agents.new_chat.middleware.otel_span.ot_metrics.record_model_call_duration", + "app.agents.chat.multi_agent_chat.main_agent.middleware.otel_span.middleware.ot_metrics.record_model_call_duration", lambda duration_ms, **attrs: duration_calls.append( {"duration_ms": duration_ms, **attrs} ), ) monkeypatch.setattr( - "app.agents.new_chat.middleware.otel_span.ot_metrics.record_model_token_usage", + "app.agents.chat.multi_agent_chat.main_agent.middleware.otel_span.middleware.ot_metrics.record_model_token_usage", lambda **attrs: token_calls.append(attrs), ) @@ -257,11 +257,11 @@ class TestMiddlewareIntegration: errors: list[str] = [] monkeypatch.setattr( - "app.agents.new_chat.middleware.otel_span.ot_metrics.record_tool_call_error", + "app.agents.chat.multi_agent_chat.main_agent.middleware.otel_span.middleware.ot_metrics.record_tool_call_error", lambda *, tool_name: errors.append(tool_name), ) monkeypatch.setattr( - "app.agents.new_chat.middleware.otel_span.ot_metrics.record_tool_call_duration", + "app.agents.chat.multi_agent_chat.main_agent.middleware.otel_span.middleware.ot_metrics.record_tool_call_duration", lambda *args, **kwargs: None, ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py b/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py index ac6f61767..2617bff8e 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_path_resolver.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from app.agents.new_chat.path_resolver import ( +from app.agents.chat.runtime.path_resolver import ( DOCUMENTS_ROOT, PathIndex, doc_to_virtual_path, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py b/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py deleted file mode 100644 index 68db11ba6..000000000 --- a/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Tests for PermissionMiddleware end-to-end behavior.""" - -from __future__ import annotations - -import pytest -from langchain_core.messages import AIMessage, ToolMessage - -from app.agents.new_chat.errors import CorrectedError, RejectedError -from app.agents.new_chat.middleware.permission import ( - PermissionMiddleware, - _normalize_permission_decision, -) -from app.agents.new_chat.permissions import Rule, Ruleset - -pytestmark = pytest.mark.unit - - -class _FakeRuntime: - config: dict = {"configurable": {"thread_id": "test"}} - - -def _msg(*tool_calls: dict) -> AIMessage: - return AIMessage(content="", tool_calls=list(tool_calls)) - - -class TestAllow: - def test_passthrough_when_allow(self) -> None: - rs = Ruleset(rules=[Rule("send_email", "*", "allow")]) - mw = PermissionMiddleware(rulesets=[rs]) - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - out = mw.after_model(state, _FakeRuntime()) - assert out is None # no change - - -class TestDeny: - def test_replaces_with_deny_tool_message(self) -> None: - rs = Ruleset(rules=[Rule("send_email", "*", "deny")]) - mw = PermissionMiddleware(rulesets=[rs]) - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - out = mw.after_model(state, _FakeRuntime()) - assert out is not None - msgs = out["messages"] - # Find the deny ToolMessage - deny_msgs = [m for m in msgs if isinstance(m, ToolMessage)] - assert len(deny_msgs) == 1 - assert deny_msgs[0].status == "error" - assert "permission_denied" in str(deny_msgs[0].additional_kwargs) - # AIMessage's tool_calls should now be empty (denied call removed) - ai_msg = next(m for m in msgs if isinstance(m, AIMessage)) - assert ai_msg.tool_calls == [] - - def test_mixed_allow_deny(self) -> None: - rs = Ruleset( - rules=[ - Rule("send_email", "*", "deny"), - Rule("read", "*", "allow"), - ] - ) - mw = PermissionMiddleware(rulesets=[rs]) - state = { - "messages": [ - _msg( - {"name": "send_email", "args": {}, "id": "1"}, - {"name": "read", "args": {}, "id": "2"}, - ) - ] - } - out = mw.after_model(state, _FakeRuntime()) - assert out is not None - ai_msg = next(m for m in out["messages"] if isinstance(m, AIMessage)) - assert len(ai_msg.tool_calls) == 1 - assert ai_msg.tool_calls[0]["name"] == "read" - - -class TestAsk: - def test_reject_without_feedback_raises(self) -> None: - # Default: nothing matches -> ask - rs = Ruleset(rules=[]) - mw = PermissionMiddleware(rulesets=[rs]) - - # Bypass real interrupt — patch the helper - mw._raise_interrupt = lambda **kw: {"decision_type": "reject"} # type: ignore[assignment] - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - with pytest.raises(RejectedError): - mw.after_model(state, _FakeRuntime()) - - def test_reject_with_feedback_raises_corrected(self) -> None: - rs = Ruleset(rules=[]) - mw = PermissionMiddleware(rulesets=[rs]) - mw._raise_interrupt = lambda **kw: { # type: ignore[assignment] - "decision_type": "reject", - "feedback": "use a different subject line", - } - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - with pytest.raises(CorrectedError) as excinfo: - mw.after_model(state, _FakeRuntime()) - assert excinfo.value.feedback == "use a different subject line" - - def test_once_proceeds_without_persisting(self) -> None: - mw = PermissionMiddleware(rulesets=[]) - mw._raise_interrupt = lambda **kw: {"decision_type": "once"} # type: ignore[assignment] - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - out = mw.after_model(state, _FakeRuntime()) - # No state change because all calls kept - assert out is None - # No new rule persisted - assert mw._runtime_ruleset.rules == [] - - def test_approve_always_persists_runtime_rule(self) -> None: - mw = PermissionMiddleware(rulesets=[]) - mw._raise_interrupt = lambda **kw: {"decision_type": "approve_always"} # type: ignore[assignment] - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - out = mw.after_model(state, _FakeRuntime()) - assert out is None # call kept - # Runtime ruleset got the always-allow rule - new_rules = [r for r in mw._runtime_ruleset.rules if r.action == "allow"] - assert any(r.permission == "send_email" for r in new_rules) - - -class TestNormalizeDecision: - """Resume shapes ``_normalize_permission_decision`` must accept.""" - - def test_legacy_decision_type_dict_passes_through(self) -> None: - decision = {"decision_type": "once"} - assert _normalize_permission_decision(decision) == {"decision_type": "once"} - - def test_legacy_decision_type_with_feedback_passes_through(self) -> None: - decision = {"decision_type": "reject", "feedback": "no thanks"} - assert _normalize_permission_decision(decision) == decision - - def test_plain_string_wrapped(self) -> None: - assert _normalize_permission_decision("once") == {"decision_type": "once"} - assert _normalize_permission_decision("reject") == {"decision_type": "reject"} - - def test_lc_envelope_approve_maps_to_once(self) -> None: - decision = {"decisions": [{"type": "approve"}]} - assert _normalize_permission_decision(decision) == {"decision_type": "once"} - - def test_lc_envelope_reject_maps_to_reject(self) -> None: - decision = {"decisions": [{"type": "reject"}]} - assert _normalize_permission_decision(decision) == {"decision_type": "reject"} - - def test_lc_envelope_reject_with_message_carries_feedback(self) -> None: - decision = {"decisions": [{"type": "reject", "message": "wrong recipient"}]} - out = _normalize_permission_decision(decision) - assert out == {"decision_type": "reject", "feedback": "wrong recipient"} - - def test_lc_envelope_reject_with_feedback_field(self) -> None: - decision = { - "decisions": [{"type": "reject", "feedback": "tighten the subject"}] - } - out = _normalize_permission_decision(decision) - assert out == {"decision_type": "reject", "feedback": "tighten the subject"} - - def test_lc_envelope_edit_maps_to_once(self) -> None: - # Pins the contract: edited args are NOT merged by permission. - decision = { - "decisions": [ - { - "type": "edit", - "edited_action": { - "name": "send_email", - "args": {"subject": "edited"}, - }, - } - ] - } - assert _normalize_permission_decision(decision) == {"decision_type": "once"} - - def test_lc_single_decision_without_envelope(self) -> None: - assert _normalize_permission_decision({"type": "approve"}) == { - "decision_type": "once" - } - - def test_unknown_type_falls_back_to_reject(self) -> None: - decision = {"decisions": [{"type": "totally_unknown"}]} - assert _normalize_permission_decision(decision) == {"decision_type": "reject"} - - def test_missing_type_falls_back_to_reject(self) -> None: - assert _normalize_permission_decision({"decisions": [{}]}) == { - "decision_type": "reject" - } - - def test_non_dict_non_string_falls_back_to_reject(self) -> None: - assert _normalize_permission_decision(None) == {"decision_type": "reject"} - assert _normalize_permission_decision(42) == {"decision_type": "reject"} - - def test_empty_decisions_list_falls_back_to_reject(self) -> None: - # Fail-closed on a malformed reply rather than treat it as approve. - assert _normalize_permission_decision({"decisions": []}) == { - "decision_type": "reject" - } - - -class TestResumeShapesEndToEnd: - """LangChain HITL envelope reaches ``_process`` correctly via ``_raise_interrupt``.""" - - def test_lc_approve_envelope_keeps_call(self) -> None: - mw = PermissionMiddleware(rulesets=[]) - mw._raise_interrupt = lambda **kw: { # type: ignore[assignment] - "decisions": [{"type": "approve"}] - } - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - original = mw._raise_interrupt - mw._raise_interrupt = lambda **kw: _normalize_permission_decision( # type: ignore[assignment] - original(**kw) - ) - out = mw.after_model(state, _FakeRuntime()) - assert out is None - - def test_lc_reject_envelope_raises(self) -> None: - mw = PermissionMiddleware(rulesets=[]) - original = lambda **kw: {"decisions": [{"type": "reject"}]} # noqa: E731 - mw._raise_interrupt = lambda **kw: _normalize_permission_decision( # type: ignore[assignment] - original(**kw) - ) - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - with pytest.raises(RejectedError): - mw.after_model(state, _FakeRuntime()) - - def test_lc_reject_with_message_raises_corrected(self) -> None: - mw = PermissionMiddleware(rulesets=[]) - original = lambda **kw: { # noqa: E731 - "decisions": [{"type": "reject", "message": "wrong recipient"}] - } - mw._raise_interrupt = lambda **kw: _normalize_permission_decision( # type: ignore[assignment] - original(**kw) - ) - state = {"messages": [_msg({"name": "send_email", "args": {}, "id": "1"})]} - with pytest.raises(CorrectedError) as excinfo: - mw.after_model(state, _FakeRuntime()) - assert excinfo.value.feedback == "wrong recipient" - - def test_lc_edit_envelope_keeps_call_with_original_args(self) -> None: - # Pins the "edit -> once, args unchanged" contract. - mw = PermissionMiddleware(rulesets=[]) - original = lambda **kw: { # noqa: E731 - "decisions": [ - { - "type": "edit", - "edited_action": { - "name": "send_email", - "args": {"to": "edited@example.com"}, - }, - } - ] - } - mw._raise_interrupt = lambda **kw: _normalize_permission_decision( # type: ignore[assignment] - original(**kw) - ) - state = { - "messages": [ - _msg( - { - "name": "send_email", - "args": {"to": "original@example.com"}, - "id": "1", - } - ) - ] - } - out = mw.after_model(state, _FakeRuntime()) - assert out is None diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py b/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py index 8ec16617a..e680a955b 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py @@ -4,7 +4,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.permissions import ( +from app.agents.chat.multi_agent_chat.shared.permissions import ( Rule, Ruleset, aggregate_action, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py index 5dbf765a7..9b3931549 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py @@ -6,13 +6,13 @@ from unittest.mock import MagicMock, patch from langchain.agents.middleware import AgentMiddleware -from app.agents.new_chat.plugin_loader import ( +from app.agents.chat.multi_agent_chat.main_agent.plugins.loader import ( PLUGIN_ENTRY_POINT_GROUP, PluginContext, load_allowed_plugin_names_from_env, load_plugin_middlewares, ) -from app.agents.new_chat.plugins.year_substituter import ( +from app.agents.chat.multi_agent_chat.main_agent.plugins.year_substituter import ( _YearSubstituterMiddleware, make_middleware as year_substituter_factory, ) @@ -66,7 +66,7 @@ class TestPluginLoaderBasics: ep = _FakeEntryPoint("dangerous_plugin", factory) with patch( - "app.agents.new_chat.plugin_loader.entry_points", + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=[ep], ): result = load_plugin_middlewares( @@ -78,7 +78,7 @@ class TestPluginLoaderBasics: def test_loads_allowlisted_plugin(self) -> None: ep = _FakeEntryPoint("year_substituter", year_substituter_factory) with patch( - "app.agents.new_chat.plugin_loader.entry_points", + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=[ep], ): result = load_plugin_middlewares( @@ -95,7 +95,7 @@ class TestPluginLoaderIsolation: ep = _FakeEntryPoint("buggy", crashing_factory) with patch( - "app.agents.new_chat.plugin_loader.entry_points", + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=[ep], ): result = load_plugin_middlewares(_ctx(), allowed_plugin_names={"buggy"}) @@ -107,7 +107,7 @@ class TestPluginLoaderIsolation: ep = _FakeEntryPoint("liar", bad_factory) with patch( - "app.agents.new_chat.plugin_loader.entry_points", + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=[ep], ): result = load_plugin_middlewares(_ctx(), allowed_plugin_names={"liar"}) @@ -121,7 +121,7 @@ class TestPluginLoaderIsolation: raise ImportError("cannot import") with patch( - "app.agents.new_chat.plugin_loader.entry_points", + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", return_value=[_BrokenEP()], ): result = load_plugin_middlewares(_ctx(), allowed_plugin_names={"broken"}) @@ -137,7 +137,10 @@ class TestPluginLoaderIsolation: _FakeEntryPoint("crashing", crashing_factory), _FakeEntryPoint("ok", year_substituter_factory), ] - with patch("app.agents.new_chat.plugin_loader.entry_points", return_value=eps): + with patch( + "app.agents.chat.multi_agent_chat.main_agent.plugins.loader.entry_points", + return_value=eps, + ): result = load_plugin_middlewares( _ctx(), allowed_plugin_names={"crashing", "ok"} ) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_prompt_caching.py b/surfsense_backend/tests/unit/agents/new_chat/test_prompt_caching.py index c3de15c58..6fbe39349 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_prompt_caching.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_prompt_caching.py @@ -1,5 +1,5 @@ r"""Tests for ``apply_litellm_prompt_caching`` in -:mod:`app.agents.new_chat.prompt_caching`. +:mod:`app.agents.chat.runtime.prompt_caching`. The helper replaces the legacy ``AnthropicPromptCachingMiddleware`` (which never activated for our LiteLLM stack) with LiteLLM-native multi-provider @@ -34,8 +34,10 @@ from typing import Any import pytest -from app.agents.new_chat.llm_config import AgentConfig -from app.agents.new_chat.prompt_caching import apply_litellm_prompt_caching +from app.agents.chat.runtime.llm_config import AgentConfig +from app.agents.chat.runtime.prompt_caching import ( + apply_litellm_prompt_caching, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_resolve_prompt_model_name.py b/surfsense_backend/tests/unit/agents/new_chat/test_resolve_prompt_model_name.py deleted file mode 100644 index ffe3dbaa4..000000000 --- a/surfsense_backend/tests/unit/agents/new_chat/test_resolve_prompt_model_name.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Tests for ``_resolve_prompt_model_name`` in :mod:`app.agents.new_chat.chat_deepagent`. - -The helper picks the model id fed to ``detect_provider_variant`` so the -right ```` block lands in the system prompt. The tests -below pin its preference order: - -1. ``agent_config.litellm_params["base_model"]`` (Azure-correct). -2. ``agent_config.model_name``. -3. ``getattr(llm, "model", None)``. - -Without (1) an Azure deployment named e.g. ``"prod-chat-001"`` would -silently miss every provider regex. -""" - -from __future__ import annotations - -import pytest - -from app.agents.new_chat.chat_deepagent import _resolve_prompt_model_name -from app.agents.new_chat.llm_config import AgentConfig - -pytestmark = pytest.mark.unit - - -def _make_cfg(**overrides) -> AgentConfig: - """Build an ``AgentConfig`` with sensible defaults for the helper test.""" - defaults = { - "provider": "OPENAI", - "model_name": "x", - "api_key": "k", - } - return AgentConfig(**{**defaults, **overrides}) - - -class _FakeLLM: - """Stand-in for a ``ChatLiteLLM`` / ``ChatLiteLLMRouter`` instance. - - The resolver only reads the ``.model`` attribute via ``getattr``, - matching the established idiom in ``knowledge_search.py`` / - ``stream_new_chat.py`` / ``document_summarizer.py``. - """ - - def __init__(self, model: str | None) -> None: - self.model = model - - -def test_prefers_litellm_params_base_model_over_deployment_name() -> None: - """Azure deployment slug must NOT shadow the underlying model family. - - This is the failure mode the helper exists to prevent: a deployment - named ``"azure/prod-chat-001"`` would not match any provider regex - on its own, but the family ``"gpt-4o"`` lives in - ``litellm_params["base_model"]`` and routes to ``openai_classic``. - """ - cfg = _make_cfg( - model_name="azure/prod-chat-001", - litellm_params={"base_model": "gpt-4o"}, - ) - assert _resolve_prompt_model_name(cfg, _FakeLLM("azure/prod-chat-001")) == "gpt-4o" - - -def test_falls_back_to_model_name_when_litellm_params_is_none() -> None: - cfg = _make_cfg( - model_name="anthropic/claude-3-5-sonnet", - litellm_params=None, - ) - got = _resolve_prompt_model_name(cfg, _FakeLLM("anthropic/claude-3-5-sonnet")) - assert got == "anthropic/claude-3-5-sonnet" - - -def test_handles_litellm_params_without_base_model_key() -> None: - cfg = _make_cfg( - model_name="openai/gpt-4o", - litellm_params={"temperature": 0.5}, - ) - assert _resolve_prompt_model_name(cfg, _FakeLLM("openai/gpt-4o")) == "openai/gpt-4o" - - -def test_ignores_blank_base_model() -> None: - """Whitespace-only ``base_model`` must not shadow ``model_name``.""" - cfg = _make_cfg( - model_name="openai/gpt-4o", - litellm_params={"base_model": " "}, - ) - assert _resolve_prompt_model_name(cfg, _FakeLLM("openai/gpt-4o")) == "openai/gpt-4o" - - -def test_ignores_non_string_base_model() -> None: - """Defensive: a non-string ``base_model`` should not crash the resolver.""" - cfg = _make_cfg( - model_name="openai/gpt-4o", - litellm_params={"base_model": 42}, - ) - assert _resolve_prompt_model_name(cfg, _FakeLLM("openai/gpt-4o")) == "openai/gpt-4o" - - -def test_falls_back_to_llm_model_when_no_agent_config() -> None: - """No ``agent_config`` -> use ``llm.model`` directly. Defensive path - for direct callers; production callers always supply a config.""" - assert ( - _resolve_prompt_model_name(None, _FakeLLM("openai/gpt-4o-mini")) - == "openai/gpt-4o-mini" - ) - - -def test_returns_none_when_nothing_available() -> None: - """``compose_system_prompt`` treats ``None`` as the ``"default"`` - variant and emits no provider block.""" - assert _resolve_prompt_model_name(None, _FakeLLM(None)) is None - - -def test_auto_mode_resolves_to_auto_string() -> None: - """Auto mode -> ``"auto"``. ``detect_provider_variant("auto")`` - returns ``"default"``, which is correct: the child model isn't - known until the LiteLLM Router dispatches.""" - cfg = AgentConfig.from_auto_mode() - assert _resolve_prompt_model_name(cfg, _FakeLLM("auto")) == "auto" diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py b/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py index d23fd693b..b70718ff9 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py @@ -4,7 +4,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.middleware.retry_after import ( +from app.agents.chat.shared.middleware.retry_after import ( RetryAfterMiddleware, _extract_retry_after_seconds, _is_non_retryable, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_skills_backends.py b/surfsense_backend/tests/unit/agents/new_chat/test_skills_backends.py index eb9cf396c..1c497d99b 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_skills_backends.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_skills_backends.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from app.agents.new_chat.middleware.skills_backends import ( +from app.agents.chat.multi_agent_chat.main_agent.skills.backends import ( SKILLS_BUILTIN_PREFIX, SKILLS_SPACE_PREFIX, BuiltinSkillsBackend, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py b/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py deleted file mode 100644 index 3c7fe5336..000000000 --- a/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Tests for the specialized subagents (explore / report_writer / connector_negotiator).""" - -from __future__ import annotations - -from langchain_core.tools import tool - -from app.agents.new_chat.middleware.permission import PermissionMiddleware -from app.agents.new_chat.subagents import ( - build_connector_negotiator_subagent, - build_explore_subagent, - build_report_writer_subagent, - build_specialized_subagents, -) -from app.agents.new_chat.subagents.config import ( - EXPLORE_READ_TOOLS, - REPORT_WRITER_TOOLS, - WRITE_TOOL_DENY_PATTERNS, -) - -# --------------------------------------------------------------------------- -# Fake tools used to verify filtering & permission behavior -# --------------------------------------------------------------------------- - - -@tool -def web_search(query: str) -> str: - """Search the public web.""" - return "" - - -@tool -def scrape_webpage(url: str) -> str: - """Scrape a single webpage.""" - return "" - - -@tool -def read_file(path: str) -> str: - """Read a file.""" - return "" - - -@tool -def ls_tree(path: str) -> str: - """List a tree.""" - return "" - - -@tool -def grep(pattern: str) -> str: - """Grep.""" - return "" - - -@tool -def update_memory(content: str) -> str: - """Update the user's memory.""" - return "" - - -@tool -def edit_file(path: str, old: str, new: str) -> str: - """Edit a file.""" - return "" - - -@tool -def linear_create_issue(title: str) -> str: - """Create a Linear issue.""" - return "" - - -@tool -def slack_send_message(channel: str, text: str) -> str: - """Send a Slack message.""" - return "" - - -@tool -def get_connected_accounts() -> str: - """List connected accounts.""" - return "" - - -@tool -def generate_report(topic: str) -> str: - """Generate a report artifact.""" - return "" - - -ALL_TOOLS = [ - web_search, - scrape_webpage, - read_file, - ls_tree, - grep, - update_memory, - edit_file, - linear_create_issue, - slack_send_message, - get_connected_accounts, - generate_report, -] - - -class TestExploreSubagent: - def test_only_read_tools_are_exposed(self) -> None: - spec = build_explore_subagent(tools=ALL_TOOLS) - names = {t.name for t in spec["tools"]} # type: ignore[index] - assert names == EXPLORE_READ_TOOLS & {t.name for t in ALL_TOOLS} - assert "update_memory" not in names - assert "linear_create_issue" not in names - assert "edit_file" not in names - - def test_includes_permission_middleware_with_deny_rules(self) -> None: - spec = build_explore_subagent(tools=ALL_TOOLS) - permission_mws = [ - m - for m in spec["middleware"] - if isinstance(m, PermissionMiddleware) # type: ignore[index] - ] - assert len(permission_mws) == 1 - ruleset = permission_mws[0]._static_rulesets[0] - assert ruleset.origin == "subagent_explore" - deny_patterns = {r.permission for r in ruleset.rules if r.action == "deny"} - assert "update_memory" in deny_patterns - assert "edit_file" in deny_patterns - assert "*create*" in deny_patterns - assert "*send*" in deny_patterns - - def test_skills_inherits_default_sources(self) -> None: - spec = build_explore_subagent(tools=ALL_TOOLS) - assert spec["skills"] == ["/skills/builtin/", "/skills/space/"] # type: ignore[index] - - def test_name_and_description_match_contract(self) -> None: - spec = build_explore_subagent(tools=ALL_TOOLS) - assert spec["name"] == "explore" - assert "read-only" in spec["description"].lower() - - def test_includes_dedup_and_patch_middleware(self) -> None: - from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware - - from app.agents.new_chat.middleware import DedupHITLToolCallsMiddleware - - spec = build_explore_subagent(tools=ALL_TOOLS) - types = {type(m) for m in spec["middleware"]} # type: ignore[index] - assert PatchToolCallsMiddleware in types - assert DedupHITLToolCallsMiddleware in types - - -class TestReportWriterSubagent: - def test_exposes_only_report_writing_tools(self) -> None: - spec = build_report_writer_subagent(tools=ALL_TOOLS) - names = {t.name for t in spec["tools"]} # type: ignore[index] - assert names == REPORT_WRITER_TOOLS & {t.name for t in ALL_TOOLS} - assert "generate_report" in names - assert "read_file" in names - - def test_deny_rules_block_writes_but_allow_generate_report(self) -> None: - spec = build_report_writer_subagent(tools=ALL_TOOLS) - permission_mws = [ - m - for m in spec["middleware"] - if isinstance(m, PermissionMiddleware) # type: ignore[index] - ] - ruleset = permission_mws[0]._static_rulesets[0] - deny_patterns = {r.permission for r in ruleset.rules if r.action == "deny"} - assert "update_memory" in deny_patterns - # generate_report MUST not be denied — it's the whole point of the subagent. - assert "generate_report" not in deny_patterns - # No deny pattern should match `generate_report` either. - assert all( - not _wildcard_matches(pattern, "generate_report") - for pattern in deny_patterns - ) - - -class TestConnectorNegotiatorSubagent: - def test_inherits_all_parent_tools(self) -> None: - spec = build_connector_negotiator_subagent(tools=ALL_TOOLS) - names = {t.name for t in spec["tools"]} # type: ignore[index] - # Every parent tool is inherited; the deny ruleset enforces behavior - # at execution time instead of trimming the tool list. - assert names == {t.name for t in ALL_TOOLS} - - def test_get_connected_accounts_is_present(self) -> None: - spec = build_connector_negotiator_subagent(tools=ALL_TOOLS) - names = {t.name for t in spec["tools"]} # type: ignore[index] - assert "get_connected_accounts" in names - - def test_deny_ruleset_blocks_mutating_connector_tools(self) -> None: - spec = build_connector_negotiator_subagent(tools=ALL_TOOLS) - permission_mws = [ - m - for m in spec["middleware"] - if isinstance(m, PermissionMiddleware) # type: ignore[index] - ] - ruleset = permission_mws[0]._static_rulesets[0] - deny_patterns = {r.permission for r in ruleset.rules if r.action == "deny"} - # `linear_create_issue` matches the `*_create` deny pattern. - assert any(_wildcard_matches(p, "linear_create_issue") for p in deny_patterns) - assert any(_wildcard_matches(p, "slack_send_message") for p in deny_patterns) - - -class TestBuildSpecializedSubagents: - def test_returns_five_specs(self) -> None: - specs = build_specialized_subagents(tools=ALL_TOOLS) - names = [s["name"] for s in specs] # type: ignore[index] - assert names == [ - "explore", - "report_writer", - "linear_specialist", - "slack_specialist", - "connector_negotiator", - ] - - def test_all_specs_have_unique_names(self) -> None: - specs = build_specialized_subagents(tools=ALL_TOOLS) - names = [s["name"] for s in specs] # type: ignore[index] - assert len(set(names)) == len(names) - - def test_extra_middleware_is_prepended_to_each_spec(self) -> None: - """Sentinel middleware passed via ``extra_middleware`` must appear - in each subagent's ``middleware`` list, before the local rules. - - This guards against the regression where specialized subagents - promised filesystem tools (``read_file``, ``ls``, ``grep``) in - their system prompts but had no filesystem middleware mounted. - """ - - class _Sentinel: - pass - - sentinel = _Sentinel() - specs = build_specialized_subagents( - tools=ALL_TOOLS, extra_middleware=[sentinel] - ) - for spec in specs: - mws = spec["middleware"] # type: ignore[index] - assert sentinel in mws - # The sentinel must appear *before* the permission middleware - # (subagent-local rules), preserving the documented composition - # order: extra → custom → patch → dedup. - sentinel_idx = mws.index(sentinel) - perm_idx = next( - (i for i, m in enumerate(mws) if isinstance(m, PermissionMiddleware)), - None, - ) - assert perm_idx is not None - assert sentinel_idx < perm_idx - - -class TestFilterToolsWarningSuppression: - """Names provided by middleware (read_file, ls, grep, …) must not - trigger the spurious "missing" warning in :func:`_filter_tools`.""" - - def test_middleware_provided_names_are_silent(self, caplog) -> None: - import logging - - from app.agents.new_chat.subagents.config import _filter_tools - - with caplog.at_level( - logging.INFO, logger="app.agents.new_chat.subagents.config" - ): - # Allowed set asks for two registry tools (one present, one - # not) plus a bunch of middleware-provided names. - _filter_tools( - [web_search], - allowed_names={ - "web_search", - "scrape_webpage", # legitimately missing → should warn - "read_file", # mw-provided → suppressed - "ls", - "grep", - "glob", - "write_todos", - }, - ) - - warnings = [r.message for r in caplog.records if r.levelno >= logging.INFO] - # Exactly one warning, and it should mention scrape_webpage but not - # any middleware-provided name. Inspect the rendered "missing" - # list (between the brackets) so we don't false-match substrings - # like ``ls`` inside ``available``. - assert len(warnings) == 1, warnings - msg = warnings[0] - assert "scrape_webpage" in msg - bracket_section = msg.split("missing: ", 1)[1] - for noisy in ("read_file", "ls", "grep", "glob", "write_todos"): - assert f"'{noisy}'" not in bracket_section, msg - - -class TestDenyPatternsCoverage: - def test_deny_patterns_cover_canonical_write_tools(self) -> None: - canonical_writes = [ - "update_memory", - "edit_file", - "write_file", - "move_file", - "mkdir", - "linear_create_issue", - "linear_update_issue", - "linear_delete_issue", - "slack_send_message", - "create_index", - "update_account", - "delete_record", - "send_email", - ] - for tool_name in canonical_writes: - assert any( - _wildcard_matches(pattern, tool_name) - for pattern in WRITE_TOOL_DENY_PATTERNS - ), f"no deny pattern matches {tool_name!r}" - - def test_deny_patterns_do_not_match_safe_read_tools(self) -> None: - canonical_reads = [ - "read_file", - "ls_tree", - "grep", - "web_search", - "scrape_webpage", - "get_connected_accounts", - "generate_report", - ] - for tool_name in canonical_reads: - assert not any( - _wildcard_matches(pattern, tool_name) - for pattern in WRITE_TOOL_DENY_PATTERNS - ), f"deny pattern incorrectly matches read tool {tool_name!r}" - - -def _wildcard_matches(pattern: str, value: str) -> bool: - """Helper using the same matcher the rule evaluator does.""" - from app.agents.new_chat.permissions import wildcard_match - - return wildcard_match(value, pattern) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_state_reducers.py b/surfsense_backend/tests/unit/agents/new_chat/test_state_reducers.py index 185753990..637a10704 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_state_reducers.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_state_reducers.py @@ -4,7 +4,7 @@ from __future__ import annotations import pytest -from app.agents.new_chat.state_reducers import ( +from app.agents.chat.multi_agent_chat.shared.state.reducers import ( _CLEAR, _add_unique_reducer, _dict_merge_with_tombstones_reducer, diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py b/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py index e02a04774..1e11e39ce 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py @@ -5,10 +5,12 @@ from __future__ import annotations import pytest from langchain_core.messages import AIMessage -from app.agents.new_chat.middleware.tool_call_repair import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.tool_call_repair.middleware import ( ToolCallNameRepairMiddleware, ) -from app.agents.new_chat.tools.invalid_tool import INVALID_TOOL_NAME +from app.agents.chat.multi_agent_chat.main_agent.tools.invalid_tool import ( + INVALID_TOOL_NAME, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/agents/new_chat/tools/test_mcp_tools_cache.py b/surfsense_backend/tests/unit/agents/new_chat/tools/test_mcp_tools_cache.py index bae97ba9f..7d9d35b55 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/tools/test_mcp_tools_cache.py +++ b/surfsense_backend/tests/unit/agents/new_chat/tools/test_mcp_tools_cache.py @@ -7,7 +7,7 @@ from types import SimpleNamespace import pytest -from app.agents.new_chat.tools.mcp_tools_cache import ( +from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( CachedMCPToolDef, CachedMCPTools, read_cached_tools, diff --git a/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py b/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py index 4f93ad732..61fa87b76 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py +++ b/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py @@ -1,17 +1,58 @@ -"""Unit tests for resume page-limit helpers and enforcement flow.""" +"""Unit tests for resume page-limit helpers and enforcement flow. + +Targets the live deliverables resume tool. The tool returns a +``Command`` (payload JSON-encoded in ``update["messages"][0].content`` +plus a receipt), so flow tests invoke it via a ToolCall dict and unwrap +the payload. +""" import io +import json from types import SimpleNamespace from unittest.mock import AsyncMock import pypdf import pytest +from langchain.tools import ToolRuntime -from app.agents.new_chat.tools import resume as resume_tool +from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools import ( + resume as resume_tool, +) pytestmark = pytest.mark.unit +@pytest.fixture(autouse=True) +def _silence_progress_events(monkeypatch): + """The live tool emits ``dispatch_custom_event`` progress updates that + require a langgraph run context; neutralize them for direct unit calls.""" + monkeypatch.setattr(resume_tool, "dispatch_custom_event", lambda *a, **k: None) + + +def _runtime(tool_call_id: str = "call-1") -> ToolRuntime: + """Minimal ToolRuntime; the resume tool only reads ``tool_call_id``.""" + return ToolRuntime( + state={}, + context=None, + config={}, + stream_writer=None, + tool_call_id=tool_call_id, + store=None, + ) + + +async def _invoke(tool, args: dict) -> dict: + """Drive a Command-returning tool and return its decoded payload. + + These tools take an injected ``ToolRuntime`` and return a + ``Command``; invoke the raw coroutine with a hand-built runtime + (the repo's pattern for unit-testing such tools) and decode the + ToolMessage payload. + """ + command = await tool.coroutine(runtime=_runtime(), **args) + return json.loads(command.update["messages"][0].content) + + class _FakeReport: _next_id = 1000 @@ -101,14 +142,14 @@ async def test_generate_resume_defaults_to_one_page_target(monkeypatch) -> None: llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=_llm_invoke)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: 1) tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1) - result = await tool.ainvoke({"user_info": "Jane Doe experience"}) + result = await _invoke(tool, {"user_info": "Jane Doe experience"}) assert result["status"] == "ready" assert prompts @@ -130,7 +171,7 @@ async def test_generate_resume_compresses_when_over_limit(monkeypatch) -> None: llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -138,7 +179,7 @@ async def test_generate_resume_compresses_when_over_limit(monkeypatch) -> None: monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts)) tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1) - result = await tool.ainvoke({"user_info": "Jane Doe experience", "max_pages": 1}) + result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1}) assert result["status"] == "ready" assert write_session.added, "Expected successful report write" @@ -165,7 +206,7 @@ async def test_generate_resume_returns_ready_when_target_not_met(monkeypatch) -> llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -173,7 +214,7 @@ async def test_generate_resume_returns_ready_when_target_not_met(monkeypatch) -> monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts)) tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1) - result = await tool.ainvoke({"user_info": "Jane Doe experience", "max_pages": 1}) + result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1}) assert result["status"] == "ready" assert "could not fit the target" in (result["message"] or "").lower() @@ -198,7 +239,7 @@ async def test_generate_resume_fails_when_hard_limit_exceeded(monkeypatch) -> No llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -206,7 +247,7 @@ async def test_generate_resume_fails_when_hard_limit_exceeded(monkeypatch) -> No monkeypatch.setattr(resume_tool, "_count_pdf_pages", lambda _pdf: next(page_counts)) tool = resume_tool.create_generate_resume_tool(search_space_id=1, thread_id=1) - result = await tool.ainvoke({"user_info": "Jane Doe experience", "max_pages": 1}) + result = await _invoke(tool, {"user_info": "Jane Doe experience", "max_pages": 1}) assert result["status"] == "failed" assert "hard page limit" in (result["error"] or "").lower() diff --git a/surfsense_backend/tests/unit/agents/test_import_all.py b/surfsense_backend/tests/unit/agents/test_import_all.py new file mode 100644 index 000000000..b45bf3359 --- /dev/null +++ b/surfsense_backend/tests/unit/agents/test_import_all.py @@ -0,0 +1,53 @@ +"""Guardrail A: every agent module (and its prod entrypoints) must import. + +Static reachability analysis and mocked unit tests cannot catch a module that +fails to import after files move or imports are rewritten. This smoke test +imports every submodule under ``app.agents`` plus the production entrypoints +that consume agents, turning a move-time ``ImportError`` into a fast, local CI +signal instead of a runtime failure in prod. +""" + +from __future__ import annotations + +import importlib +import pkgutil + +import pytest + +import app.agents as agents_pkg + +pytestmark = pytest.mark.unit + +# Prod consumers of app.agents that live OUTSIDE the agents tree; a broken +# importer here would not be caught by walking app.agents alone. +_PROD_ENTRYPOINTS = [ + "app.tasks.chat.streaming.flows.new_chat.orchestrator", + "app.tasks.chat.streaming.agent.builder", + "app.gateway.agent_invoke", + "app.routes.new_chat_routes", +] + + +def _iter_agent_modules() -> list[str]: + names: list[str] = [] + + def _record(name: str) -> None: + names.append(name) + + for info in pkgutil.walk_packages( + agents_pkg.__path__, prefix=agents_pkg.__name__ + ".", onerror=_record + ): + names.append(info.name) + return sorted(set(names)) + + +@pytest.mark.parametrize("module_name", _iter_agent_modules()) +def test_agent_module_imports(module_name: str) -> None: + """Importing the module must not raise (no broken or missed imports).""" + importlib.import_module(module_name) + + +@pytest.mark.parametrize("module_name", _PROD_ENTRYPOINTS) +def test_prod_entrypoint_imports(module_name: str) -> None: + """The production code paths that build/invoke agents must import.""" + importlib.import_module(module_name) diff --git a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py index ac20b2608..79da12933 100644 --- a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py +++ b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py @@ -14,8 +14,8 @@ from typing import Any import pytest -import app.automations.actions.agent_task.dependencies as deps_mod -from app.automations.actions.agent_task.dependencies import ( +import app.automations.actions.builtin.agent_task.dependencies as deps_mod +from app.automations.actions.builtin.agent_task.dependencies import ( DependencyError, build_dependencies, ) diff --git a/surfsense_backend/tests/unit/automations/services/test_model_policy.py b/surfsense_backend/tests/unit/automations/services/test_model_policy.py index 2a471b4e9..8e0806151 100644 --- a/surfsense_backend/tests/unit/automations/services/test_model_policy.py +++ b/surfsense_backend/tests/unit/automations/services/test_model_policy.py @@ -44,7 +44,7 @@ def patched_globals(monkeypatch: pytest.MonkeyPatch): -2: {"id": -2, "billing_tier": "free"}, } monkeypatch.setattr( - "app.agents.new_chat.llm_config.load_global_llm_config_by_id", + "app.agents.chat.runtime.llm_config.load_global_llm_config_by_id", lambda cid: llm_configs.get(cid), ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py index a8cf05269..ff85096d4 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py @@ -71,7 +71,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "Engineering Handbook" @@ -81,7 +80,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["page_id"] == "abc-123" assert doc.metadata["page_title"] == "Engineering Handbook" assert doc.metadata["space_id"] == "ENG" @@ -89,21 +87,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Confluence Page" assert doc.metadata["connector_type"] == "Confluence" - assert doc.fallback_summary is not None - assert "Engineering Handbook" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - doc = _build_connector_doc( - _make_page(), - _to_markdown(_make_page()), - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -111,10 +94,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py index 9ba87207a..b87d1be42 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py @@ -71,11 +71,10 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 0 + assert failed == [] assert docs[0].title == "test.txt" assert docs[0].unique_id == "f1" assert docs[0].document_type == DocumentType.DROPBOX_FILE @@ -97,11 +96,10 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert {d.unique_id for d in docs} == {"f0", "f1", "f2"} @@ -125,11 +123,10 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 - assert failed == 1 + assert len(failed) == 1 assert {d.unique_id for d in docs} == {"f0", "f2"} @@ -152,11 +149,10 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 1 + assert len(failed) == 1 # Slice 5: Semaphore bound @@ -191,12 +187,11 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) assert len(docs) == 6 - assert failed == 0 + assert failed == [] assert peak <= 2, f"Peak concurrency was {peak}, expected <= 2" @@ -231,12 +226,11 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert len(heartbeat_calls) >= 1, "Heartbeat should have fired at least once" @@ -324,7 +318,6 @@ async def _run_full_scan(mocks, monkeypatch, page_files, *, max_files=500): mocks["task_logger"], mocks["log_entry"], max_files, - enable_summary=True, ) @@ -434,7 +427,6 @@ async def _run_selected(mocks, file_tuples): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -569,7 +561,6 @@ async def test_delta_sync_deletions_call_remove_document(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["id:del1", "id:del2"] @@ -608,7 +599,6 @@ async def test_delta_sync_upserts_filtered_and_downloaded(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert indexed == 2 @@ -670,7 +660,6 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["id:del1", "id:del2"] @@ -704,7 +693,6 @@ async def test_delta_sync_returns_new_cursor(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert cursor == "brand-new-cursor-xyz" @@ -725,7 +713,7 @@ def orchestrator_mocks(monkeypatch): mock_connector = MagicMock() mock_connector.config = {"_token_encrypted": False} mock_connector.last_indexed_at = None - mock_connector.enable_summary = True + mock_connector.enable_vision_llm = True monkeypatch.setattr( _mod, diff --git a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py index 7e968514c..9a13e4525 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py @@ -66,11 +66,10 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 0 + assert failed == [] assert docs[0].title == "test.txt" assert docs[0].unique_id == "f1" @@ -91,11 +90,10 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert {d.unique_id for d in docs} == {"f0", "f1", "f2"} @@ -119,11 +117,10 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 - assert failed == 1 + assert len(failed) == 1 assert {d.unique_id for d in docs} == {"f0", "f2"} @@ -146,11 +143,10 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 1 + assert len(failed) == 1 async def test_concurrency_bounded_by_semaphore( @@ -186,12 +182,11 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) assert len(docs) == 6 - assert failed == 0 + assert failed == [] assert peak <= 2, f"Peak concurrency was {peak}, expected <= 2" @@ -226,12 +221,11 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert len(heartbeat_calls) >= 1, "Heartbeat should have fired at least once" @@ -287,7 +281,7 @@ def full_scan_mocks(mock_drive_client, monkeypatch): monkeypatch.setattr(_mod, "_should_skip_file", _fake_skip) - download_mock = AsyncMock(return_value=([], 0)) + download_mock = AsyncMock(return_value=([], [])) monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) batch_mock = AsyncMock(return_value=([], 0, 0)) @@ -300,12 +294,6 @@ def full_scan_mocks(mock_drive_client, monkeypatch): MagicMock(return_value=pipeline_mock), ) - monkeypatch.setattr( - _mod, - "get_user_long_context_llm", - AsyncMock(return_value=MagicMock()), - ) - return { "drive_client": mock_drive_client, "session": mock_session, @@ -333,7 +321,6 @@ async def _run_full_scan(mocks, *, max_files=500, include_subfolders=False): mocks["log_entry"], max_files, include_subfolders=include_subfolders, - enable_summary=True, ) @@ -363,7 +350,7 @@ async def test_full_scan_three_phase_counts(full_scan_mocks, monkeypatch): ) mock_docs = [MagicMock(), MagicMock()] - full_scan_mocks["download_mock"].return_value = (mock_docs, 0) + full_scan_mocks["download_mock"].return_value = (mock_docs, []) full_scan_mocks["batch_mock"].return_value = ([], 2, 0) indexed, skipped, _unsupported = await _run_full_scan(full_scan_mocks) @@ -389,7 +376,7 @@ async def test_full_scan_respects_max_files(full_scan_mocks, monkeypatch): AsyncMock(return_value=(page_files, None, None)), ) - full_scan_mocks["download_mock"].return_value = ([], 0) + full_scan_mocks["download_mock"].return_value = ([], []) full_scan_mocks["batch_mock"].return_value = ([], 0, 0) await _run_full_scan(full_scan_mocks, max_files=3) @@ -413,7 +400,7 @@ async def test_full_scan_uses_max_concurrency_3_for_indexing( ) mock_docs = [MagicMock()] - full_scan_mocks["download_mock"].return_value = (mock_docs, 0) + full_scan_mocks["download_mock"].return_value = (mock_docs, []) full_scan_mocks["batch_mock"].return_value = ([], 1, 0) await _run_full_scan(full_scan_mocks) @@ -475,7 +462,7 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): ) mock_docs = [MagicMock(), MagicMock()] - download_mock = AsyncMock(return_value=(mock_docs, 0)) + download_mock = AsyncMock(return_value=(mock_docs, [])) monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) batch_mock = AsyncMock(return_value=([], 2, 0)) @@ -487,12 +474,6 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): "IndexingPipelineService", MagicMock(return_value=pipeline_mock), ) - monkeypatch.setattr( - _mod, - "get_user_long_context_llm", - AsyncMock(return_value=MagicMock()), - ) - mock_session, _ = _make_page_limit_session() mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -509,7 +490,6 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["del1", "del2", "trash1"] @@ -577,7 +557,6 @@ async def _run_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_jira_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_jira_parallel.py deleted file mode 100644 index 494991074..000000000 --- a/surfsense_backend/tests/unit/connector_indexers/test_jira_parallel.py +++ /dev/null @@ -1,387 +0,0 @@ -"""Tests for Jira indexer migrated to the unified parallel pipeline.""" - -from unittest.mock import AsyncMock, MagicMock - -import pytest - -import app.tasks.connector_indexers.jira_indexer as _mod -from app.db import DocumentType -from app.tasks.connector_indexers.jira_indexer import ( - _build_connector_doc, - index_jira_issues, -) - -pytestmark = pytest.mark.unit - -_USER_ID = "00000000-0000-0000-0000-000000000001" -_CONNECTOR_ID = 42 -_SEARCH_SPACE_ID = 1 - - -def _make_issue( - issue_key: str = "ENG-1", - issue_id: str = "10001", - title: str = "Fix login", -): - return {"key": issue_key, "id": issue_id, "title": title} - - -def _make_formatted_issue( - issue_key: str = "ENG-1", - issue_id: str = "10001", - title: str = "Fix login", - status: str = "In Progress", - priority: str = "High", - comments=None, -): - return { - "key": issue_key, - "id": issue_id, - "title": title, - "status": status, - "priority": priority, - "comments": comments or [], - } - - -# --------------------------------------------------------------------------- -# Slice 1: _build_connector_doc tracer bullet -# --------------------------------------------------------------------------- - - -async def test_build_connector_doc_produces_correct_fields(): - issue = _make_issue(issue_key="ENG-42", issue_id="4242", title="Fix auth bug") - formatted = _make_formatted_issue( - issue_key="ENG-42", - issue_id="4242", - title="Fix auth bug", - status="Done", - priority="Urgent", - comments=[{"id": "c1"}], - ) - markdown = "# ENG-42: Fix auth bug\n\nBody" - - doc = _build_connector_doc( - issue, - formatted, - markdown, - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=True, - ) - - assert doc.title == "ENG-42: 4242" - assert doc.unique_id == "ENG-42" - assert doc.document_type == DocumentType.JIRA_CONNECTOR - assert doc.source_markdown == markdown - assert doc.search_space_id == _SEARCH_SPACE_ID - assert doc.connector_id == _CONNECTOR_ID - assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True - assert doc.metadata["issue_id"] == "ENG-42" - assert doc.metadata["issue_identifier"] == "ENG-42" - assert doc.metadata["issue_title"] == "4242" - assert doc.metadata["state"] == "Done" - assert doc.metadata["priority"] == "Urgent" - assert doc.metadata["comment_count"] == 1 - assert doc.metadata["connector_id"] == _CONNECTOR_ID - assert doc.metadata["document_type"] == "Jira Issue" - assert doc.metadata["connector_type"] == "Jira" - assert doc.fallback_summary is not None - assert "ENG-42" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - doc = _build_connector_doc( - _make_issue(), - _make_formatted_issue(), - "# content", - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - assert doc.should_summarize is False - - -# --------------------------------------------------------------------------- -# Shared fixtures for Slices 2-7 -# --------------------------------------------------------------------------- - - -def _mock_connector(enable_summary: bool = True): - c = MagicMock() - c.config = {"access_token": "tok"} - c.enable_summary = enable_summary - c.last_indexed_at = None - return c - - -def _mock_jira_client(issues=None, error=None): - client = MagicMock() - client.get_issues_by_date_range = AsyncMock( - return_value=(issues if issues is not None else [], error), - ) - client.format_issue = MagicMock( - side_effect=lambda i: _make_formatted_issue( - issue_key=i.get("key", ""), - issue_id=i.get("id", ""), - title=i.get("title", ""), - ) - ) - client.format_issue_to_markdown = MagicMock( - side_effect=lambda fi: f"# {fi.get('key', '')}: {fi.get('id', '')}\n\nContent" - ) - client.close = AsyncMock() - return client - - -@pytest.fixture -def jira_mocks(monkeypatch): - mock_session = AsyncMock() - mock_session.no_autoflush = MagicMock() - - mock_connector = _mock_connector() - monkeypatch.setattr( - _mod, - "get_connector_by_id", - AsyncMock(return_value=mock_connector), - ) - - jira_client = _mock_jira_client(issues=[_make_issue()]) - monkeypatch.setattr( - _mod, - "JiraHistoryConnector", - MagicMock(return_value=jira_client), - ) - - monkeypatch.setattr( - _mod, - "check_duplicate_document_by_hash", - AsyncMock(return_value=None), - ) - monkeypatch.setattr( - _mod, - "update_connector_last_indexed", - AsyncMock(), - ) - monkeypatch.setattr( - _mod, - "calculate_date_range", - MagicMock(return_value=("2025-01-01", "2025-12-31")), - ) - - mock_task_logger = MagicMock() - mock_task_logger.log_task_start = AsyncMock(return_value=MagicMock()) - mock_task_logger.log_task_progress = AsyncMock() - mock_task_logger.log_task_success = AsyncMock() - mock_task_logger.log_task_failure = AsyncMock() - monkeypatch.setattr( - _mod, - "TaskLoggingService", - MagicMock(return_value=mock_task_logger), - ) - - batch_mock = AsyncMock(return_value=([], 1, 0)) - pipeline_mock = MagicMock() - pipeline_mock.index_batch_parallel = batch_mock - pipeline_mock.migrate_legacy_docs = AsyncMock() - pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) - monkeypatch.setattr( - _mod, - "IndexingPipelineService", - MagicMock(return_value=pipeline_mock), - ) - - return { - "session": mock_session, - "connector": mock_connector, - "jira_client": jira_client, - "task_logger": mock_task_logger, - "pipeline_mock": pipeline_mock, - "batch_mock": batch_mock, - } - - -async def _run_index(mocks, **overrides): - return await index_jira_issues( - session=mocks["session"], - connector_id=overrides.get("connector_id", _CONNECTOR_ID), - search_space_id=overrides.get("search_space_id", _SEARCH_SPACE_ID), - user_id=overrides.get("user_id", _USER_ID), - start_date=overrides.get("start_date", "2025-01-01"), - end_date=overrides.get("end_date", "2025-12-31"), - update_last_indexed=overrides.get("update_last_indexed", True), - on_heartbeat_callback=overrides.get("on_heartbeat_callback"), - ) - - -# --------------------------------------------------------------------------- -# Slice 2: Full pipeline wiring -# --------------------------------------------------------------------------- - - -async def test_one_issue_calls_pipeline_and_returns_indexed_count(jira_mocks): - indexed, skipped, warning = await _run_index(jira_mocks) - assert indexed == 1 - assert skipped == 0 - assert warning is None - - jira_mocks["batch_mock"].assert_called_once() - connector_docs = jira_mocks["batch_mock"].call_args[0][0] - assert len(connector_docs) == 1 - assert connector_docs[0].document_type == DocumentType.JIRA_CONNECTOR - - -async def test_pipeline_called_with_max_concurrency_3(jira_mocks): - await _run_index(jira_mocks) - call_kwargs = jira_mocks["batch_mock"].call_args[1] - assert call_kwargs.get("max_concurrency") == 3 - - -async def test_migrate_legacy_docs_called_before_indexing(jira_mocks): - await _run_index(jira_mocks) - jira_mocks["pipeline_mock"].migrate_legacy_docs.assert_called_once() - - -# --------------------------------------------------------------------------- -# Slice 3: Issue skipping (missing key/title/content) -# --------------------------------------------------------------------------- - - -async def test_issues_with_missing_key_are_skipped(jira_mocks): - issues = [ - _make_issue(issue_key="ENG-1", issue_id="10001"), - {"key": "", "id": "10002", "title": "No key"}, - ] - jira_mocks["jira_client"].get_issues_by_date_range.return_value = (issues, None) - - _, skipped, _ = await _run_index(jira_mocks) - connector_docs = jira_mocks["batch_mock"].call_args[0][0] - assert len(connector_docs) == 1 - assert skipped == 1 - - -async def test_issues_with_missing_title_are_skipped(jira_mocks): - issues = [ - _make_issue(issue_key="ENG-1", issue_id="10001"), - {"key": "ENG-2", "id": "", "title": "Missing id used as title"}, - ] - jira_mocks["jira_client"].get_issues_by_date_range.return_value = (issues, None) - - _, skipped, _ = await _run_index(jira_mocks) - connector_docs = jira_mocks["batch_mock"].call_args[0][0] - assert len(connector_docs) == 1 - assert skipped == 1 - - -async def test_issues_with_no_content_are_skipped(jira_mocks): - issues = [ - _make_issue(issue_key="ENG-1", issue_id="10001"), - _make_issue(issue_key="ENG-2", issue_id="10002"), - ] - jira_mocks["jira_client"].get_issues_by_date_range.return_value = (issues, None) - - jira_mocks["jira_client"].format_issue_to_markdown.side_effect = [ - "# ENG-1: 10001\n\nContent", - "", - ] - _, skipped, _ = await _run_index(jira_mocks) - connector_docs = jira_mocks["batch_mock"].call_args[0][0] - assert len(connector_docs) == 1 - assert skipped == 1 - - -# --------------------------------------------------------------------------- -# Slice 4: Duplicate content skipping -# --------------------------------------------------------------------------- - - -async def test_duplicate_content_issues_are_skipped(jira_mocks, monkeypatch): - issues = [ - _make_issue(issue_key="ENG-1", issue_id="10001"), - _make_issue(issue_key="ENG-2", issue_id="10002"), - ] - jira_mocks["jira_client"].get_issues_by_date_range.return_value = (issues, None) - - call_count = 0 - - async def _check_dup(session, content_hash): - nonlocal call_count - call_count += 1 - if call_count == 2: - dup = MagicMock() - dup.id = 99 - dup.document_type = "OTHER" - return dup - return None - - monkeypatch.setattr(_mod, "check_duplicate_document_by_hash", _check_dup) - - _, skipped, _ = await _run_index(jira_mocks) - connector_docs = jira_mocks["batch_mock"].call_args[0][0] - assert len(connector_docs) == 1 - assert skipped == 1 - - -# --------------------------------------------------------------------------- -# Slice 5: Heartbeat callback forwarding -# --------------------------------------------------------------------------- - - -async def test_heartbeat_callback_forwarded_to_pipeline(jira_mocks): - heartbeat_cb = AsyncMock() - await _run_index(jira_mocks, on_heartbeat_callback=heartbeat_cb) - call_kwargs = jira_mocks["batch_mock"].call_args[1] - assert call_kwargs.get("on_heartbeat") is heartbeat_cb - - -# --------------------------------------------------------------------------- -# Slice 6: Empty issues and no-data success tuple -# --------------------------------------------------------------------------- - - -async def test_empty_issues_returns_zero_tuple(jira_mocks): - jira_mocks["jira_client"].get_issues_by_date_range.return_value = ([], None) - indexed, skipped, warning = await _run_index(jira_mocks) - assert indexed == 0 - assert skipped == 0 - assert warning is None - jira_mocks["batch_mock"].assert_not_called() - - -async def test_no_issues_error_message_returns_success_tuple(jira_mocks): - jira_mocks["jira_client"].get_issues_by_date_range.return_value = ( - [], - "No issues found in date range", - ) - indexed, skipped, warning = await _run_index(jira_mocks) - assert indexed == 0 - assert skipped == 0 - assert warning is None - - -async def test_api_error_still_returns_3_tuple(jira_mocks): - jira_mocks["jira_client"].get_issues_by_date_range.return_value = ( - [], - "API exploded", - ) - result = await _run_index(jira_mocks) - assert len(result) == 3 - assert result[0] == 0 - assert result[1] == 0 - assert "Failed to get Jira issues" in result[2] - - -# --------------------------------------------------------------------------- -# Slice 7: Failed docs warning -# --------------------------------------------------------------------------- - - -async def test_failed_docs_warning_in_result(jira_mocks): - jira_mocks["batch_mock"].return_value = ([], 0, 2) - _, _, warning = await _run_index(jira_mocks) - assert warning is not None - assert "2 failed" in warning diff --git a/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py index ef17aae06..f057a6352 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py @@ -70,7 +70,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "ENG-42: Fix login bug" @@ -80,7 +79,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["issue_id"] == "abc-123" assert doc.metadata["issue_identifier"] == "ENG-42" assert doc.metadata["issue_title"] == "Fix login bug" @@ -90,24 +88,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Linear Issue" assert doc.metadata["connector_type"] == "Linear" - assert doc.fallback_summary is not None - assert "ENG-42" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - """When enable_summary is False, should_summarize is False.""" - doc = _build_connector_doc( - _make_issue(), - _make_formatted_issue(), - "# content", - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -115,10 +95,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py index 651524015..e40f739d8 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py @@ -41,7 +41,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "My Notion Page" @@ -51,29 +50,11 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["page_title"] == "My Notion Page" assert doc.metadata["page_id"] == "abc-123" assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Notion Page" assert doc.metadata["connector_type"] == "Notion" - assert doc.fallback_summary is not None - assert "My Notion Page" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - """When enable_summary is False, should_summarize is False.""" - doc = _build_connector_doc( - _make_page(), - "# content", - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -81,10 +62,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py index 396d79e73..01e81da17 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py @@ -65,11 +65,10 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 0 + assert failed == [] assert docs[0].title == "test.txt" assert docs[0].unique_id == "f1" assert docs[0].document_type == DocumentType.ONEDRIVE_FILE @@ -91,11 +90,10 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert {d.unique_id for d in docs} == {"f0", "f1", "f2"} @@ -119,11 +117,10 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 - assert failed == 1 + assert len(failed) == 1 assert {d.unique_id for d in docs} == {"f0", "f2"} @@ -146,11 +143,10 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 - assert failed == 1 + assert len(failed) == 1 # Slice 5: Semaphore bound @@ -185,12 +181,11 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) assert len(docs) == 6 - assert failed == 0 + assert failed == [] assert peak <= 2, f"Peak concurrency was {peak}, expected <= 2" @@ -225,10 +220,9 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) assert len(docs) == 3 - assert failed == 0 + assert failed == [] assert len(heartbeat_calls) >= 1, "Heartbeat should have fired at least once" diff --git a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py index 573ee43d8..66722ffd7 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py @@ -180,7 +180,6 @@ async def _run_gdrive_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -326,7 +325,7 @@ def gdrive_full_scan_mocks(monkeypatch): _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) ) - download_mock = AsyncMock(return_value=([], 0)) + download_mock = AsyncMock(return_value=([], [])) monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) batch_mock = AsyncMock(return_value=([], 0, 0)) @@ -336,10 +335,6 @@ def gdrive_full_scan_mocks(monkeypatch): monkeypatch.setattr( _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) ) - monkeypatch.setattr( - _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) - ) - return { "mod": _mod, "session": session, @@ -366,7 +361,6 @@ async def _run_gdrive_full_scan(mocks, max_files=500): MagicMock(), max_files, include_subfolders=False, - enable_summary=True, ) @@ -383,7 +377,7 @@ async def test_gdrive_full_scan_skips_over_quota(gdrive_full_scan_mocks, monkeyp "get_files_in_folder", AsyncMock(return_value=(page_files, None, None)), ) - m["download_mock"].return_value = ([], 0) + m["download_mock"].return_value = ([], []) m["batch_mock"].return_value = ([], 2, 0) _indexed, skipped, _unsup = await _run_gdrive_full_scan(m) @@ -409,7 +403,7 @@ async def test_gdrive_full_scan_deducts_after_indexing( AsyncMock(return_value=(page_files, None, None)), ) mock_docs = [MagicMock() for _ in range(3)] - m["download_mock"].return_value = (mock_docs, 0) + m["download_mock"].return_value = (mock_docs, []) m["batch_mock"].return_value = ([], 3, 0) await _run_gdrive_full_scan(m) @@ -444,7 +438,7 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) ) - download_mock = AsyncMock(return_value=([], 0)) + download_mock = AsyncMock(return_value=([], [])) monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) batch_mock = AsyncMock(return_value=([], 2, 0)) @@ -454,10 +448,6 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): monkeypatch.setattr( _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) ) - monkeypatch.setattr( - _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) - ) - mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -473,7 +463,6 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) call_files = download_mock.call_args[0][1] @@ -539,7 +528,6 @@ async def _run_onedrive_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -641,7 +629,6 @@ async def _run_dropbox_selected(mocks, file_paths): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) diff --git a/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py new file mode 100644 index 000000000..de4386abb --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_byo_long_poll_lifespan.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import asyncio + +import pytest +import pytest_asyncio + +from app.gateway import byo_long_poll, runner + + +class ScalarResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + return self + + def __iter__(self): + return iter(self._rows) + + +class SessionContext: + def __init__(self, session): + self.session = session + + async def __aenter__(self): + return self.session + + async def __aexit__(self, exc_type, exc, tb): + return False + + +@pytest_asyncio.fixture(autouse=True) +async def cleanup_supervisors(): + yield + await byo_long_poll.stop_byo_long_poll_supervisors() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_noops_when_mode_is_webhook(monkeypatch): + monkeypatch.setattr(byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook") + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_noops_when_no_byo_accounts(mocker, monkeypatch): + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult([]) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_start_byo_long_poll_spawns_one_supervisor_per_account( + mocker, monkeypatch +): + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) + accounts = [mocker.Mock(id=1), mocker.Mock(id=2)] + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult(accounts) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + monkeypatch.setattr( + byo_long_poll, "account_token", lambda account: f"token-{account.id}" + ) + + async def forever(_account_id: int, _token: str) -> None: + await asyncio.Event().wait() + + monkeypatch.setattr(byo_long_poll, "_byo_account_supervisor", forever) + + await byo_long_poll.start_byo_long_poll_supervisors() + + assert len(byo_long_poll._tasks) == 2 + + +@pytest.mark.asyncio +async def test_supervisor_retries_after_run_returns(mocker, monkeypatch): + byo_long_poll._shutdown_event = asyncio.Event() + run = mocker.AsyncMock(side_effect=[None, None]) + monkeypatch.setattr(byo_long_poll, "_run_telegram_account", run) + sleep_count = 0 + + async def fake_sleep(_seconds: float) -> None: + nonlocal sleep_count + sleep_count += 1 + if sleep_count >= 2: + assert byo_long_poll._shutdown_event is not None + byo_long_poll._shutdown_event.set() + + monkeypatch.setattr(byo_long_poll, "_sleep_or_shutdown", fake_sleep) + + await byo_long_poll._byo_account_supervisor(7, "token") + + assert run.await_count == 2 + + +@pytest.mark.asyncio +async def test_shutdown_cancels_running_supervisors(mocker, monkeypatch): + monkeypatch.setattr( + byo_long_poll.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "longpoll" + ) + session = mocker.AsyncMock() + session.execute.return_value = ScalarResult([mocker.Mock(id=1)]) + monkeypatch.setattr( + byo_long_poll, + "async_session_maker", + lambda: SessionContext(session), + ) + monkeypatch.setattr(byo_long_poll, "account_token", lambda _account: "token") + + async def forever(_account_id: int, _token: str) -> None: + await asyncio.Event().wait() + + monkeypatch.setattr(byo_long_poll, "_byo_account_supervisor", forever) + + await byo_long_poll.start_byo_long_poll_supervisors() + await byo_long_poll.stop_byo_long_poll_supervisors() + + assert byo_long_poll._tasks == set() + + +@pytest.mark.asyncio +async def test_run_telegram_account_persists_for_fastapi_inbox_worker( + mocker, monkeypatch +): + class ConnectionContext: + async def __aenter__(self): + conn = mocker.AsyncMock() + conn.scalar.return_value = True + return conn + + async def __aexit__(self, exc_type, exc, tb): + return False + + class EngineStub: + def connect(self): + return ConnectionContext() + + class AdapterStub: + def __init__(self, _token: str) -> None: + pass + + async def fetch_updates(self, *, offset: int | None): + yield {"update_id": 11, "message": {"message_id": 5}} + + def parse_inbound(self, update): + return mocker.Mock(external_message_id="5", event_kind="message") + + first_session = mocker.AsyncMock() + first_session.get.return_value = mocker.Mock(cursor_state={}) + second_session = mocker.AsyncMock() + contexts = iter([SessionContext(first_session), SessionContext(second_session)]) + monkeypatch.setattr(runner, "engine", EngineStub()) + monkeypatch.setattr(runner, "async_session_maker", lambda: next(contexts)) + monkeypatch.setattr(runner, "TelegramAdapter", AdapterStub) + persist = mocker.AsyncMock(return_value=42) + monkeypatch.setattr(runner, "persist_inbound_event", persist) + + await runner._run_telegram_account(123, "token") + + second_session.commit.assert_awaited_once() + persist.assert_awaited_once() + assert persist.await_args.kwargs["request_id"].startswith("gateway_") diff --git a/surfsense_backend/tests/unit/gateway/test_discord_adapter.py b/surfsense_backend/tests/unit/gateway/test_discord_adapter.py new file mode 100644 index 000000000..c6790f20b --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_discord_adapter.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import pytest + +from app.gateway.base.adapter import PlatformSendResult +from app.gateway.discord.adapter import DiscordAdapter + + +def _discord_payload(content: str = "<@999> summarize this channel"): + return { + "type": "message", + "bot_user_id": "999", + "event": { + "type": "message", + "id": "111", + "guild_id": "222", + "guild_name": "SurfSense Guild", + "channel_id": "333", + "channel_name": "general", + "content": content, + "author": {"id": "444", "username": "anish", "bot": False}, + "mentions": [{"id": "999", "username": "SurfSense"}], + }, + } + + +def test_discord_adapter_parses_mention_and_strips_bot_mention(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound(_discord_payload()) + + assert parsed.platform == "discord" + assert parsed.text == "summarize this channel" + assert parsed.external_peer_id == "discord_thread:222:333:111" + assert parsed.metadata["discord_user_peer_id"] == "discord_user:222:444" + assert parsed.metadata["discord_thread_peer_id"] == "discord_thread:222:333:111" + assert parsed.metadata["mentions_bot"] is True + + +def test_discord_adapter_strips_nickname_mention(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound(_discord_payload("<@!999> continue")) + + assert parsed.text == "continue" + + +def test_discord_adapter_uses_message_reference_as_thread_key(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + payload = _discord_payload("<@999> continue") + payload["event"]["id"] = "112" + payload["event"]["message_reference"] = { + "message_id": "111", + "channel_id": "333", + "guild_id": "222", + } + + parsed = adapter.parse_inbound(payload) + + assert parsed.external_peer_id == "discord_thread:222:333:111" + assert parsed.metadata["message_id"] == "112" + assert parsed.metadata["thread_key"] == "111" + + +def test_discord_adapter_returns_missing_peer_for_incomplete_payload(): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + + parsed = adapter.parse_inbound({"event": {"id": "111"}}) + + assert parsed.external_peer_id is None + assert parsed.external_peer_kind == "unknown" + + +@pytest.mark.asyncio +async def test_discord_adapter_sends_message(mocker): + adapter = DiscordAdapter("discord-token", bot_user_id="999") + adapter.client.send_message = mocker.AsyncMock( + return_value=PlatformSendResult(external_message_id="555") + ) + + result = await adapter.send_message( + external_peer_id="333", + text="hello", + reply_to_message_id="111", + ) + + assert result.external_message_id == "555" + adapter.client.send_message.assert_awaited_once_with( + channel_id="333", + content="hello", + reply_to_message_id="111", + ) diff --git a/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py new file mode 100644 index 000000000..8be8942cc --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_enqueue_received_sweep.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from app.tasks.celery_tasks import gateway_tasks + + +def test_enqueue_received_sweep_is_noop_guard(mocker): + apply_async = mocker.Mock() + mocker.patch.object( + gateway_tasks.process_inbound_event_task, "apply_async", apply_async + ) + info = mocker.patch.object(gateway_tasks.logger, "info") + + replayed = gateway_tasks.enqueue_received_sweep_task.run() + + apply_async.assert_not_called() + assert replayed == 0 + info.assert_called_once() diff --git a/surfsense_backend/tests/unit/gateway/test_formatting.py b/surfsense_backend/tests/unit/gateway/test_formatting.py new file mode 100644 index 000000000..4d842e169 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_formatting.py @@ -0,0 +1,17 @@ +from app.gateway.telegram.formatting import chunk_message, escape_markdown_v2 + + +def test_escape_markdown_v2_reserved_chars(): + text = r"_*[]()~`>#+-=|{}.!" + + assert escape_markdown_v2(text) == r"\_\*\[\]\(\)\~\`\>\#\+\-\=\|\{\}\.\!" + + +def test_chunk_message_preserves_content_and_limits_size(): + text = "First paragraph.\n\n" + ("x" * 5000) + + chunks = chunk_message(text, max_units=4096) + + assert "".join(chunks) == text + assert len(chunks) > 1 + assert all(len(chunk.encode("utf-16-le")) // 2 <= 4096 for chunk in chunks) diff --git a/surfsense_backend/tests/unit/gateway/test_hitl_filter.py b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py new file mode 100644 index 000000000..04766b986 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_hitl_filter.py @@ -0,0 +1,14 @@ +from app.gateway.hitl_filter import filter_hitl_tools + + +class Tool: + def __init__(self, name: str) -> None: + self.name = name + + +def test_filter_hitl_tools_removes_known_approval_tools(): + tools = [Tool("delete_document"), Tool("search"), "send_email", "summarize"] + + filtered = filter_hitl_tools(tools) + + assert [getattr(tool, "name", tool) for tool in filtered] == ["search", "summarize"] diff --git a/surfsense_backend/tests/unit/gateway/test_inbox_worker.py b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py new file mode 100644 index 000000000..1e5b2a184 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_inbox_worker.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from app.gateway import inbox_worker + + +@pytest.mark.asyncio +async def test_inbox_worker_claims_and_processes_in_fastapi_process( + mocker, monkeypatch +): + claim = mocker.AsyncMock(return_value=7) + process = mocker.AsyncMock(side_effect=asyncio.CancelledError) + monkeypatch.setattr(inbox_worker, "claim_next_inbound_event", claim) + monkeypatch.setattr(inbox_worker, "process_inbound_event", process) + + with pytest.raises(asyncio.CancelledError): + await inbox_worker._process_inbox_forever() + + claim.assert_awaited_once() + process.assert_awaited_once_with(7) + + +@pytest.mark.asyncio +async def test_start_stop_gateway_inbox_worker(mocker, monkeypatch): + started = asyncio.Event() + stopped = asyncio.Event() + + async def run_forever(): + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + monkeypatch.setattr(inbox_worker, "_process_inbox_forever", run_forever) + inbox_worker._task = None + + await inbox_worker.start_gateway_inbox_worker() + await asyncio.wait_for(started.wait(), timeout=1) + await inbox_worker.stop_gateway_inbox_worker() + + assert stopped.is_set() + assert inbox_worker._task is None diff --git a/surfsense_backend/tests/unit/gateway/test_pairing.py b/surfsense_backend/tests/unit/gateway/test_pairing.py new file mode 100644 index 000000000..9f90fa259 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_pairing.py @@ -0,0 +1,40 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from app.db import ExternalChatBindingState +from app.gateway.pairing import generate_pairing_code, redeem_pairing_code + + +def test_generate_pairing_code_is_short_display_token(): + code = generate_pairing_code() + + assert len(code) >= 8 + assert "\n" not in code + + +@pytest.mark.asyncio +async def test_redeem_pairing_code_binds_pending_row(mocker): + binding = mocker.Mock() + binding.state = ExternalChatBindingState.PENDING + binding.pairing_code_expires_at = datetime.now(UTC) + timedelta(minutes=1) + scalars = mocker.Mock() + scalars.first.return_value = binding + result = mocker.Mock() + result.scalars.return_value = scalars + session = mocker.AsyncMock() + session.execute.return_value = result + + redeemed = await redeem_pairing_code( + session, + code="abc", + external_peer_id="telegram:123", + external_peer_kind="direct", + external_display_name="Anish", + external_username="anish", + ) + + assert redeemed is binding + assert binding.state == ExternalChatBindingState.BOUND + assert binding.external_peer_id == "telegram:123" + assert binding.pairing_code is None diff --git a/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py new file mode 100644 index 000000000..a929ff12a --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_process_inbound_event_task.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from app.tasks.celery_tasks import gateway_tasks + + +def test_process_inbound_event_task_is_noop_guard(mocker): + warning = mocker.patch.object(gateway_tasks.logger, "warning") + + assert gateway_tasks.process_inbound_event_task.run(123) is None + + warning.assert_called_once() + assert ( + "FastAPI owns external chat agent turn processing" in warning.call_args.args[0] + ) diff --git a/surfsense_backend/tests/unit/gateway/test_slack_adapter.py b/surfsense_backend/tests/unit/gateway/test_slack_adapter.py new file mode 100644 index 000000000..8742a6bf4 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_slack_adapter.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from app.gateway.slack.adapter import SlackAdapter + + +def test_slack_adapter_parses_app_mention_and_strips_bot_mention(): + adapter = SlackAdapter("xoxb-test", bot_user_id="U_BOT") + + parsed = adapter.parse_inbound( + { + "team_id": "T123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> summarize this thread", + "ts": "1717000000.000100", + }, + } + ) + + assert parsed.platform == "slack" + assert parsed.text == "summarize this thread" + assert parsed.external_peer_id == "slack_thread:T123:C123:1717000000.000100" + assert parsed.metadata["slack_user_peer_id"] == "slack_user:T123:U123" + assert parsed.metadata["thread_ts"] == "1717000000.000100" + + +def test_slack_adapter_uses_existing_thread_ts(): + adapter = SlackAdapter("xoxb-test", bot_user_id="U_BOT") + + parsed = adapter.parse_inbound( + { + "team_id": "T123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> continue", + "ts": "1717000001.000200", + "thread_ts": "1717000000.000100", + }, + } + ) + + assert parsed.external_peer_id == "slack_thread:T123:C123:1717000000.000100" + assert parsed.metadata["message_ts"] == "1717000001.000200" diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py new file mode 100644 index 000000000..aa8bd3a89 --- /dev/null +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import hashlib +import hmac +import inspect +import json +import time +from types import SimpleNamespace + +import pytest + +from app.db import ExternalChatAccount, ExternalChatAccountMode, ExternalChatPlatform +from app.routes import gateway_webhook_routes as routes + + +@pytest.fixture(autouse=True) +def _enable_gateways(monkeypatch): + """Turn on the Telegram/Slack/Discord gateway flags the routes gate on. + + The routes early-return when their integration is unconfigured, so without + this the handlers never reach the logic these tests assert on. + """ + monkeypatch.setattr(routes.config, "GATEWAY_TELEGRAM_INTAKE_MODE", "webhook") + monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_TOKEN", "telegram-token") + monkeypatch.setattr(routes.config, "TELEGRAM_SHARED_BOT_USERNAME", "surf_bot") + monkeypatch.setattr( + routes.config, "TELEGRAM_WEBHOOK_SECRET", "telegram-webhook-secret" + ) + + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_ENABLED", True) + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_ID", "slack-client") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_SECRET", "slack-secret") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + + monkeypatch.setattr(routes.config, "GATEWAY_DISCORD_ENABLED", True) + monkeypatch.setattr(routes.config, "DISCORD_CLIENT_ID", "discord-client") + monkeypatch.setattr(routes.config, "DISCORD_CLIENT_SECRET", "discord-secret") + monkeypatch.setattr(routes.config, "DISCORD_BOT_TOKEN", "discord-bot-token") + + +class RequestStub: + def __init__( + self, payload=None, *, headers=None, json_exc: Exception | None = None + ): + self.headers = headers or {} + self._payload = payload + self._json_exc = json_exc + + async def json(self): + if self._json_exc is not None: + raise self._json_exc + return self._payload + + async def body(self): + return json.dumps(self._payload).encode() + + +def _account(secret: str = "secret") -> ExternalChatAccount: + return ExternalChatAccount( + id=123, + platform=ExternalChatPlatform.TELEGRAM, + webhook_secret=secret, + bot_username="surf_bot", + ) + + +def _slack_account() -> ExternalChatAccount: + return ExternalChatAccount( + id=456, + platform=ExternalChatPlatform.SLACK, + mode=ExternalChatAccountMode.CLOUD_SHARED, + is_system_account=True, + cursor_state={"team_id": "T123", "bot_user_id": "U_BOT"}, + ) + + +def _signed_slack_request( + payload: dict, *, secret: str = "signing-secret" +) -> RequestStub: + body = json.dumps(payload).encode() + timestamp = str(int(time.time())) + digest = hmac.new( + secret.encode(), + b"v0:" + timestamp.encode() + b":" + body, + hashlib.sha256, + ).hexdigest() + + class SlackRequestStub(RequestStub): + async def body(self): + return body + + return SlackRequestStub( + payload, + headers={ + "X-Slack-Request-Timestamp": timestamp, + "X-Slack-Signature": f"v0={digest}", + }, + ) + + +def _enable_slack_gateway(monkeypatch): + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_ENABLED", True) + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_ID", "client-id") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_SECRET", "client-secret") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + + +async def _call_webhook(*, request: RequestStub, account_id: int, session): + return await routes.telegram_webhook( + request=request, + account_id=account_id, + session=session, + ) + + +@pytest.mark.asyncio +async def test_telegram_webhook_returns_200_on_null_update_id(mocker): + session = mocker.AsyncMock() + session.get.return_value = _account() + request = RequestStub( + {"message": {"message_id": 7}}, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + session.commit.assert_not_called() + + +@pytest.mark.asyncio +async def test_telegram_webhook_returns_200_on_bad_json(mocker, monkeypatch): + parse_metric = mocker.Mock() + monkeypatch.setattr(routes, "record_gateway_webhook_parse_error", parse_metric) + request = RequestStub(json_exc=ValueError("bad json")) + + response = await _call_webhook( + request=request, + account_id=123, + session=mocker.AsyncMock(), + ) + + assert response.status_code == 200 + parse_metric.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_resolve_webhook_account_rejects_missing_or_wrong_header(mocker): + session = mocker.AsyncMock() + session.get.return_value = _account() + + with pytest.raises(routes.HTTPException) as missing: + await routes._resolve_webhook_account( + session, + account_id=123, + header_secret=None, + ) + assert missing.value.status_code == 403 + + with pytest.raises(routes.HTTPException) as wrong: + await routes._resolve_webhook_account( + session, + account_id=123, + header_secret="wrong", + ) + assert wrong.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_telegram_webhook_persists_for_fastapi_inbox_worker(mocker, monkeypatch): + session = mocker.AsyncMock() + session.get.return_value = _account() + persist = mocker.AsyncMock(return_value=99) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + + request = RequestStub( + { + "update_id": 10, + "message": {"message_id": 7, "chat": {"id": 1}, "from": {"id": 2}}, + }, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + persist.assert_awaited_once() + session.commit.assert_awaited_once() + assert persist.await_args.kwargs["request_id"].startswith("gateway_") + + +@pytest.mark.asyncio +async def test_telegram_webhook_commits_dedup_without_enqueue(mocker, monkeypatch): + session = mocker.AsyncMock() + session.get.return_value = _account() + monkeypatch.setattr( + routes, "persist_inbound_event", mocker.AsyncMock(return_value=None) + ) + + request = RequestStub( + {"update_id": 10, "message": {"message_id": 7}}, + headers={"X-Telegram-Bot-Api-Secret-Token": "secret"}, + ) + + response = await _call_webhook( + request=request, + account_id=123, + session=session, + ) + + assert response.status_code == 200 + session.commit.assert_awaited_once() + + +def test_telegram_webhook_does_not_use_slowapi_limiter(): + route_source = inspect.getsource(routes) + + assert "@limiter.limit" not in route_source + + +def test_verify_slack_signature_accepts_valid_signature(): + payload = b'{"type":"event_callback"}' + timestamp = str(int(time.time())) + digest = hmac.new( + b"secret", + b"v0:" + timestamp.encode() + b":" + payload, + hashlib.sha256, + ).hexdigest() + + assert routes.verify_slack_signature( + signing_secret="secret", + timestamp=timestamp, + signature=f"v0={digest}", + body=payload, + ) + + +@pytest.mark.asyncio +async def test_slack_webhook_url_verification(monkeypatch, mocker): + _enable_slack_gateway(monkeypatch) + request = _signed_slack_request({"type": "url_verification", "challenge": "abc123"}) + + response = await routes.slack_webhook(request=request, session=mocker.AsyncMock()) + + assert response.status_code == 200 + assert json.loads(response.body)["challenge"] == "abc123" + + +@pytest.mark.asyncio +async def test_slack_webhook_persists_event(monkeypatch, mocker): + _enable_slack_gateway(monkeypatch) + session = mocker.AsyncMock() + monkeypatch.setattr( + routes, + "get_slack_account_by_team", + mocker.AsyncMock(return_value=_slack_account()), + ) + persist = mocker.AsyncMock(return_value=100) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + payload = { + "type": "event_callback", + "team_id": "T123", + "event_id": "Ev123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U123", + "text": "<@U_BOT> hello", + "ts": "1717000000.000100", + }, + } + request = _signed_slack_request(payload) + + response = await routes.slack_webhook(request=request, session=session) + + assert response.status_code == 200 + persist.assert_awaited_once() + assert persist.await_args.kwargs["event_dedupe_key"] == "slack_event:Ev123" + assert persist.await_args.kwargs["platform"] == ExternalChatPlatform.SLACK + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): + _enable_slack_gateway(monkeypatch) + session = mocker.AsyncMock() + monkeypatch.setattr( + routes, + "get_slack_account_by_team", + mocker.AsyncMock(return_value=_slack_account()), + ) + persist = mocker.AsyncMock(return_value=100) + monkeypatch.setattr(routes, "persist_inbound_event", persist) + request = _signed_slack_request( + { + "type": "event_callback", + "team_id": "T123", + "event_id": "Ev123", + "event": { + "type": "app_mention", + "channel": "C123", + "user": "U_BOT", + "text": "loop", + "ts": "1717000000.000100", + }, + } + ) + + response = await routes.slack_webhook(request=request, session=session) + + assert response.status_code == 200 + persist.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_discord_gateway_install_returns_oauth_url(monkeypatch, mocker): + monkeypatch.setattr(routes.config, "DISCORD_CLIENT_ID", "discord-client") + monkeypatch.setattr( + routes.config, + "GATEWAY_DISCORD_REDIRECT_URI", + "http://localhost:8000/api/v1/gateway/discord/callback", + ) + monkeypatch.setattr(routes.config, "SECRET_KEY", "test-secret") + monkeypatch.setattr(routes, "check_search_space_access", mocker.AsyncMock()) + + response = await routes.install_discord_gateway( + search_space_id=123, + user=SimpleNamespace(id="00000000-0000-0000-0000-000000000001"), + session=mocker.AsyncMock(), + ) + + assert response["auth_url"].startswith("https://discord.com/api/oauth2/authorize?") + assert "client_id=discord-client" in response["auth_url"] + assert "gateway%2Fdiscord%2Fcallback" in response["auth_url"] + assert "scope=identify+guilds+bot" in response["auth_url"] + + +def test_discord_gateway_callback_does_not_create_search_source_connector(): + callback_source = inspect.getsource(routes.discord_gateway_callback) + + assert "SearchSourceConnector" not in callback_source diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py index 2136f2152..f85c632ef 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py @@ -18,7 +18,6 @@ def test_valid_document_created_with_required_fields(): connector_id=42, created_by_id="00000000-0000-0000-0000-000000000001", ) - assert doc.should_summarize is True assert doc.should_use_code_chunker is False assert doc.metadata == {} assert doc.connector_id == 42 diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py b/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py deleted file mode 100644 index eee32357f..000000000 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py +++ /dev/null @@ -1,41 +0,0 @@ -from unittest.mock import MagicMock - -import pytest - -from app.indexing_pipeline.document_summarizer import summarize_document - -pytestmark = pytest.mark.unit - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_without_metadata_returns_raw_summary(): - """Summarizer returns the LLM output directly when no metadata is provided.""" - result = await summarize_document("# Content", llm=MagicMock(model="gpt-4")) - - assert result == "The summary." - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_with_metadata_includes_metadata_values_in_output(): - """Non-empty metadata values are prepended to the summary output.""" - result = await summarize_document( - "# Content", - llm=MagicMock(model="gpt-4"), - metadata={"author": "Alice", "source": "Notion"}, - ) - - assert "Alice" in result - assert "Notion" in result - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_with_metadata_omits_empty_fields_from_output(): - """Empty metadata fields are omitted from the summary output.""" - result = await summarize_document( - "# Content", - llm=MagicMock(model="gpt-4"), - metadata={"author": "Alice", "description": ""}, - ) - - assert "Alice" in result - assert "description" not in result.lower() diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py index dd9940503..963ac6792 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py @@ -37,12 +37,10 @@ async def test_calls_prepare_then_index_per_document(pipeline, make_connector_do orm2 = MagicMock(spec=Document) orm2.unique_identifier_hash = compute_unique_identifier_hash(doc2) - mock_llm = MagicMock() - pipeline.prepare_for_indexing = AsyncMock(return_value=[orm1, orm2]) - pipeline.index = AsyncMock(side_effect=lambda doc, cdoc, llm: doc) + pipeline.index = AsyncMock(side_effect=lambda doc, cdoc: doc) - results = await pipeline.index_batch([doc1, doc2], mock_llm) + results = await pipeline.index_batch([doc1, doc2]) pipeline.prepare_for_indexing.assert_awaited_once_with([doc1, doc2]) assert pipeline.index.await_count == 2 @@ -53,7 +51,7 @@ async def test_empty_input_returns_empty(pipeline): """Empty connector_docs list returns empty result.""" pipeline.prepare_for_indexing = AsyncMock(return_value=[]) - results = await pipeline.index_batch([], MagicMock()) + results = await pipeline.index_batch([]) assert results == [] @@ -74,7 +72,7 @@ async def test_skips_document_without_matching_connector_doc( pipeline.prepare_for_indexing = AsyncMock(return_value=[orphan_orm]) pipeline.index = AsyncMock() - results = await pipeline.index_batch([doc1], MagicMock()) + results = await pipeline.index_batch([doc1]) pipeline.index.assert_not_awaited() assert results == [] diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py index 07e388836..3a1b77d90 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py @@ -51,11 +51,6 @@ async def test_index_calls_embed_and_chunk_via_to_thread( return await original_to_thread(func, *args, **kwargs) monkeypatch.setattr(asyncio, "to_thread", tracking_to_thread) - - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Summary."), - ) mock_chunk_hybrid = MagicMock(return_value=["chunk1"]) mock_chunk_hybrid.__name__ = "chunk_text_hybrid" monkeypatch.setattr( @@ -85,7 +80,7 @@ async def test_index_calls_embed_and_chunk_via_to_thread( document.id = 1 document.status = DocumentStatus.pending() - await pipeline.index(document, connector_doc, llm=MagicMock()) + await pipeline.index(document, connector_doc) # Either chunker entry point satisfies the "chunking runs off the event # loop" contract this test guards. Routing between the two is verified @@ -104,10 +99,6 @@ async def test_non_code_documents_use_hybrid_chunker( mid-row. Only documents flagged with ``should_use_code_chunker=True`` should take the ``chunk_text`` path. """ - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Summary."), - ) mock_chunk_hybrid = MagicMock(return_value=["chunk1"]) mock_chunk_hybrid.__name__ = "chunk_text_hybrid" monkeypatch.setattr( @@ -139,7 +130,7 @@ async def test_non_code_documents_use_hybrid_chunker( document.id = 1 document.status = DocumentStatus.pending() - await pipeline.index(document, connector_doc, llm=MagicMock()) + await pipeline.index(document, connector_doc) mock_chunk_hybrid.assert_called_once() mock_chunk_code.assert_not_called() @@ -192,19 +183,14 @@ async def test_batch_parallel_indexes_all_documents( index_calls = [] - async def fake_index(self, document, connector_doc, llm): + async def fake_index(self, document, connector_doc): index_calls.append(document.id) document.status = DocumentStatus.ready() return document monkeypatch.setattr(IndexingPipelineService, "index", fake_index) - async def mock_get_llm(session): - return MagicMock() - - _, indexed, failed = await pipeline.index_batch_parallel( - docs, mock_get_llm, max_concurrency=2 - ) + _, indexed, failed = await pipeline.index_batch_parallel(docs, max_concurrency=2) assert indexed == 3 assert failed == 0 @@ -233,20 +219,15 @@ async def test_batch_parallel_one_failure_does_not_affect_others( _mock_session_factory(orm_by_id), ) - async def failing_index(self, document, connector_doc, llm): + async def failing_index(self, document, connector_doc): if document.id == 2: - raise RuntimeError("LLM exploded") + raise RuntimeError("Indexing exploded") document.status = DocumentStatus.ready() return document monkeypatch.setattr(IndexingPipelineService, "index", failing_index) - async def mock_get_llm(session): - return MagicMock() - - _, indexed, failed = await pipeline.index_batch_parallel( - docs, mock_get_llm, max_concurrency=4 - ) + _, indexed, failed = await pipeline.index_batch_parallel(docs, max_concurrency=4) assert indexed == 2 assert failed == 1 diff --git a/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py new file mode 100644 index 000000000..1e648a9c9 --- /dev/null +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py @@ -0,0 +1,293 @@ +"""Path/cwd/namespace + multi-root mount-normalization tests for LIVE filesystem. + +Ported from the dead-twin suites: +* ``tests/unit/middleware/test_filesystem_middleware.py`` (cwd defaults, + relative resolution, cloud write-namespace policy) +* ``tests/unit/middleware/test_filesystem_verification.py`` (desktop + multi-root mount-prefix normalization) + +Both exercised ``app.agents.chat.shared.middleware.filesystem`` (dead). This drives +the production free functions in +``app.agents.chat.multi_agent_chat.shared.middleware.filesystem.middleware`` instead. +The functions only touch ``mw._filesystem_mode`` and ``mw._get_backend`` so we +pass a lightweight fake ``mw`` rather than constructing the full middleware. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.multi_root_local_folder import ( + MultiRootLocalFolderBackend, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.middleware.mode import ( + default_cwd, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.middleware.namespace_policy import ( + check_cloud_write_namespace, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.middleware.path_resolution import ( + current_cwd, + get_contract_suggested_path, + normalize_local_mount_path, + resolve_relative, +) + +pytestmark = pytest.mark.unit + + +def _mw(mode: FilesystemMode = FilesystemMode.CLOUD, backend=None): + return SimpleNamespace(_filesystem_mode=mode, _get_backend=lambda _rt: backend) + + +def _runtime(state: dict | None = None) -> SimpleNamespace: + return SimpleNamespace(state=state or {}) + + +# --------------------------------------------------------------------------- +# cwd defaults +# --------------------------------------------------------------------------- + + +class TestCwdDefaults: + def test_default_cwd_in_cloud_is_documents_root(self): + assert default_cwd(FilesystemMode.CLOUD) == "/documents" + + def test_default_cwd_in_desktop_is_root(self): + assert default_cwd(FilesystemMode.DESKTOP_LOCAL_FOLDER) == "/" + + def test_current_cwd_uses_state_when_set(self): + assert ( + current_cwd(_mw(), _runtime({"cwd": "/documents/notes"})) + == "/documents/notes" + ) + + def test_current_cwd_falls_back_to_default(self): + assert current_cwd(_mw(), _runtime({})) == "/documents" + + def test_current_cwd_ignores_invalid(self): + assert current_cwd(_mw(), _runtime({"cwd": "not-absolute"})) == "/documents" + + +# --------------------------------------------------------------------------- +# relative resolution +# --------------------------------------------------------------------------- + + +class TestRelativePathResolution: + def test_relative_path_resolves_against_cwd(self): + assert ( + resolve_relative( + _mw(), "notes.md", _runtime({"cwd": "/documents/projects"}) + ) + == "/documents/projects/notes.md" + ) + + def test_relative_path_with_dotdot(self): + assert ( + resolve_relative(_mw(), "../c.md", _runtime({"cwd": "/documents/a/b"})) + == "/documents/a/c.md" + ) + + def test_absolute_path_is_kept(self): + assert ( + resolve_relative(_mw(), "/other/x.md", _runtime({"cwd": "/documents"})) + == "/other/x.md" + ) + + def test_empty_path_returns_cwd(self): + assert ( + resolve_relative(_mw(), "", _runtime({"cwd": "/documents/projects"})) + == "/documents/projects" + ) + + +# --------------------------------------------------------------------------- +# contract suggested-path fallback +# --------------------------------------------------------------------------- + + +class TestContractSuggestedPath: + def test_falls_back_to_documents_notes_md_in_cloud(self): + suggested = get_contract_suggested_path( + _mw(FilesystemMode.CLOUD), + _runtime({"file_operation_contract": {}}), + ) + assert suggested == "/documents/notes.md" + + def test_falls_back_to_root_notes_md_in_desktop(self): + suggested = get_contract_suggested_path( + _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER), + _runtime({"file_operation_contract": {}}), + ) + assert suggested == "/notes.md" + + +# --------------------------------------------------------------------------- +# cloud write-namespace policy +# --------------------------------------------------------------------------- + + +class TestCloudWriteNamespacePolicy: + def test_documents_path_allowed(self): + assert ( + check_cloud_write_namespace(_mw(), "/documents/foo.md", _runtime()) is None + ) + + def test_documents_root_allowed(self): + assert check_cloud_write_namespace(_mw(), "/documents", _runtime()) is None + + def test_temp_basename_anywhere_allowed(self): + assert ( + check_cloud_write_namespace(_mw(), "/temp_scratch.md", _runtime()) is None + ) + assert check_cloud_write_namespace(_mw(), "/foo/temp_x.md", _runtime()) is None + assert ( + check_cloud_write_namespace(_mw(), "/documents/temp_x.md", _runtime()) + is None + ) + + def test_other_paths_rejected(self): + err = check_cloud_write_namespace(_mw(), "/foo/bar.md", _runtime()) + assert err is not None + assert "must target /documents" in err + + def test_anon_doc_path_is_read_only(self): + runtime = _runtime( + { + "kb_anon_doc": { + "path": "/documents/uploaded.xml", + "title": "uploaded", + "content": "", + "chunks": [], + } + } + ) + err = check_cloud_write_namespace(_mw(), "/documents/uploaded.xml", runtime) + assert err is not None + assert "read-only" in err + + def test_desktop_mode_skips_namespace_policy(self): + assert ( + check_cloud_write_namespace( + _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER), "/random/path.md", _runtime() + ) + is None + ) + + +# --------------------------------------------------------------------------- +# desktop multi-root mount normalization +# --------------------------------------------------------------------------- + + +def _desktop_mw(backend) -> SimpleNamespace: + return _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER, backend) + + +class TestNormalizeLocalMountPath: + def test_prefixes_default_mount(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/random-note.md" + + def test_keeps_explicit_mount(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/pc_backups/notes/random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_windows_backslashes(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\notes\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_normalizes_mixed_separators(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\\notes//nested\\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/nested/random-note.md" + + def test_keeps_explicit_mount_with_backslashes(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\pc_backups\notes\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_prefixes_posix_absolute_path(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/var/log/app.log", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/var/log/app.log" + + def test_prefers_unique_existing_parent_mount(self, tmp_path: Path): + root_a = tmp_path / "RootA" + root_b = tmp_path / "RootB" + (root_a / "other").mkdir(parents=True) + (root_b / "nested" / "deep").mkdir(parents=True) + backend = MultiRootLocalFolderBackend( + (("root_a", str(root_a)), ("root_b", str(root_b))) + ) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/nested/deep/new-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/root_b/nested/deep/new-note.md" + + def test_uses_suggested_mount_when_ambiguous(self, tmp_path: Path): + root_a = tmp_path / "RootA" + root_b = tmp_path / "RootB" + root_a.mkdir(parents=True) + root_b.mkdir(parents=True) + backend = MultiRootLocalFolderBackend( + (("root_a", str(root_a)), ("root_b", str(root_b))) + ) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/brand-new-note.md", + _runtime( + { + "file_operation_contract": { + "suggested_path": "/root_b/notes/context.md" + } + } + ), + ) + assert resolved == "/root_b/brand-new-note.md" diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py similarity index 61% rename from surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py rename to surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py index 7cabb6524..898ec3765 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py @@ -1,15 +1,14 @@ -"""Cloud-mode behavior tests for the new ``rm`` and ``rmdir`` filesystem tools. +"""Cloud-mode ``rm``/``rmdir`` staging tests for the LIVE filesystem middleware. -The tools build ``Command(update=...)`` payloads that the persistence -middleware applies at end of turn. These tests stub out the backend and -runtime to assert the staging payload shape: - -* ``rm`` queues into ``pending_deletes`` and tombstones state files. -* ``rm`` rejects directories, ``/documents``, root, and the anonymous doc. -* ``rmdir`` queues into ``pending_dir_deletes`` and rejects non-empty dirs. -* ``rmdir`` un-stages a same-turn ``mkdir`` rather than queuing a delete. -* ``rmdir`` refuses to drop the cwd or any of its ancestors. -* ``KBPostgresBackend`` view-helpers honor staged deletes. +Ported from the former ``tests/unit/agents/new_chat/test_rm_rmdir_cloud.py``, +which exercised the *dead twin* ``app.agents.chat.shared.middleware.filesystem``. +This drives the production decomposed tools +(``app.agents.chat.multi_agent_chat.shared.middleware.filesystem``) instead: it +builds the real middleware via ``build_filesystem_mw``, pulls the real ``rm`` / +``rmdir`` tools off it, and invokes their coroutines with a stubbed +``KBPostgresBackend`` + runtime so we can assert the end-of-turn staging +payloads (``pending_deletes`` / ``pending_dir_deletes``) and the destructive-op +guard rails (root, /documents, anon doc, non-empty, cwd/ancestor, file vs dir). """ from __future__ import annotations @@ -20,18 +19,38 @@ from unittest.mock import AsyncMock import pytest -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware -from app.agents.new_chat.middleware.kb_postgres_backend import KBPostgresBackend +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem import ( + build_filesystem_mw, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.kb_postgres import ( + KBPostgresBackend, +) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.resolver import ( + build_backend_resolver, +) +from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR pytestmark = pytest.mark.unit def _make_middleware(mode: FilesystemMode = FilesystemMode.CLOUD): - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = mode - middleware._custom_tool_descriptions = {} - return middleware + selection = FilesystemSelection(mode=mode) + resolver = build_backend_resolver(selection, search_space_id=1) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=mode, + search_space_id=1, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=1, + ) + + +def _tool(mw, name: str): + return next(t for t in mw.tools if t.name == name) def _runtime(state: dict[str, Any] | None = None, *, tool_call_id: str = "tc-abc"): @@ -41,13 +60,12 @@ def _runtime(state: dict[str, Any] | None = None, *, tool_call_id: str = "tc-abc class _KBBackendStub(KBPostgresBackend): - """Construct-able subclass of :class:`KBPostgresBackend` for tests. + """Construct-able ``KBPostgresBackend`` subclass for tests. - We bypass the real ``__init__`` (which expects a runtime + DB session) - and inject just the methods the rm/rmdir tools touch. The class - inheritance keeps ``isinstance(backend, KBPostgresBackend)`` checks - inside the tools happy, which is what gates them from the desktop - code path. + Bypasses the real ``__init__`` (which expects a runtime + DB session) and + injects only the async methods the rm/rmdir tools touch. The class + inheritance keeps the ``isinstance(backend, KBPostgresBackend)`` checks in + the tools on the cloud path. """ def __init__(self, *, children=None, file_data=None) -> None: @@ -61,9 +79,8 @@ def _make_backend_stub(*, children=None, file_data=None) -> KBPostgresBackend: return _KBBackendStub(children=children, file_data=file_data) -def _bind_backend(middleware, backend): - """Inject a backend resolver onto the middleware test instance.""" - middleware._get_backend = lambda runtime: backend +def _bind_backend(mw, backend): + mw._get_backend = lambda runtime: backend return backend @@ -86,8 +103,7 @@ class TestRmStaging: tool_call_id="tc-1", ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents/notes.md", runtime=runtime) assert hasattr(result, "update"), f"expected Command, got {result!r}" update = result.update @@ -100,31 +116,22 @@ class TestRmStaging: @pytest.mark.asyncio async def test_rejects_documents_root(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/documents", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents", runtime=_runtime()) assert isinstance(result, str) assert "refusing to rm" in result @pytest.mark.asyncio async def test_rejects_root(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/", runtime=runtime) + result = await _tool(m, "rm").coroutine("/", runtime=_runtime()) assert isinstance(result, str) assert "refusing to rm" in result @pytest.mark.asyncio async def test_rejects_directory_via_staged_dirs(self): m = _make_middleware() - runtime = _runtime( - { - "staged_dirs": ["/documents/team-x"], - } - ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/team-x", runtime=runtime) + runtime = _runtime({"staged_dirs": ["/documents/team-x"]}) + result = await _tool(m, "rm").coroutine("/documents/team-x", runtime=runtime) assert isinstance(result, str) assert "directory" in result.lower() assert "rmdir" in result @@ -138,9 +145,7 @@ class TestRmStaging: children=[{"path": "/documents/foo/x.md", "is_dir": False}] ), ) - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/foo", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents/foo", runtime=_runtime()) assert isinstance(result, str) assert "directory" in result.lower() @@ -157,8 +162,9 @@ class TestRmStaging: } } ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/uploaded.xml", runtime=runtime) + result = await _tool(m, "rm").coroutine( + "/documents/uploaded.xml", runtime=runtime + ) assert isinstance(result, str) assert "read-only" in result @@ -173,12 +179,9 @@ class TestRmStaging: "dirty_paths": ["/documents/notes.md"], } ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) - update = result.update - # First element is _CLEAR sentinel; the rest must NOT contain the - # rm'd path. - dirty = update.get("dirty_paths") or [] + result = await _tool(m, "rm").coroutine("/documents/notes.md", runtime=runtime) + dirty = result.update.get("dirty_paths") or [] + # First element is the _CLEAR sentinel; the rm'd path must not survive. assert "/documents/notes.md" not in dirty[1:] @@ -192,30 +195,19 @@ class TestRmdirStaging: async def test_stages_dir_delete_when_empty_and_db_backed(self): m = _make_middleware() backend = _bind_backend(m, _make_backend_stub(children=[])) - # Override _load_file_data to return None (folder, not a file) and - # parent listing to claim the folder exists. backend._load_file_data = AsyncMock(return_value=None) backend.als_info = AsyncMock( side_effect=[ [], # children of /documents/proj - [ - {"path": "/documents/proj", "is_dir": True}, - ], # parent listing + [{"path": "/documents/proj", "is_dir": True}], # parent listing ] ) - runtime = _runtime( - { - "cwd": "/documents", - }, - tool_call_id="tc-rd", - ) + runtime = _runtime({"cwd": "/documents"}, tool_call_id="tc-rd") - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert hasattr(result, "update") - update = result.update - assert update["pending_dir_deletes"] == [ + assert result.update["pending_dir_deletes"] == [ {"path": "/documents/proj", "tool_call_id": "tc-rd"} ] @@ -228,9 +220,9 @@ class TestRmdirStaging: children=[{"path": "/documents/proj/x.md", "is_dir": False}] ), ) - runtime = _runtime() - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/proj", runtime=_runtime() + ) assert isinstance(result, str) assert "not empty" in result @@ -239,30 +231,25 @@ class TestRmdirStaging: m = _make_middleware() _bind_backend(m, _make_backend_stub(children=[])) runtime = _runtime( - { - "cwd": "/documents", - "staged_dirs": ["/documents/scratch"], - }, + {"cwd": "/documents", "staged_dirs": ["/documents/scratch"]}, tool_call_id="tc-rd", ) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/scratch", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/scratch", runtime=runtime + ) assert hasattr(result, "update") update = result.update assert "pending_dir_deletes" not in update - # _CLEAR sentinel + remaining items (in this case, none). staged_after = update["staged_dirs"] - assert staged_after[0] == "\x00__SURFSENSE_FILESYSTEM_CLEAR__\x00" + assert staged_after[0] == _CLEAR assert "/documents/scratch" not in staged_after[1:] @pytest.mark.asyncio - async def test_rejects_root(self): + async def test_rejects_root_and_documents(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rmdir_tool() for victim in ("/", "/documents"): - result = await tool.coroutine(victim, runtime=runtime) + result = await _tool(m, "rmdir").coroutine(victim, runtime=_runtime()) assert isinstance(result, str) assert "refusing to rmdir" in result @@ -270,8 +257,7 @@ class TestRmdirStaging: async def test_rejects_cwd(self): m = _make_middleware() runtime = _runtime({"cwd": "/documents/proj"}) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert isinstance(result, str) assert "cwd" in result.lower() @@ -279,8 +265,7 @@ class TestRmdirStaging: async def test_rejects_ancestor_of_cwd(self): m = _make_middleware() runtime = _runtime({"cwd": "/documents/proj/sub"}) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert isinstance(result, str) assert "cwd" in result.lower() @@ -288,34 +273,31 @@ class TestRmdirStaging: async def test_rejects_files(self): m = _make_middleware() _bind_backend(m, _make_backend_stub(children=[], file_data={"content": ["x"]})) - runtime = _runtime() - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/notes.md", runtime=_runtime() + ) assert isinstance(result, str) assert "is a file" in result # --------------------------------------------------------------------------- -# KBPostgresBackend view filter +# KBPostgresBackend staged-delete view filter (already the live backend) # --------------------------------------------------------------------------- class TestKBPostgresBackendDeleteFilter: - """als_info / glob / grep should suppress paths queued for delete.""" + """``als_info`` / glob / grep must suppress paths queued for delete.""" def _make_backend(self, state: dict[str, Any]) -> KBPostgresBackend: runtime = SimpleNamespace(state=state) - backend = KBPostgresBackend(search_space_id=1, runtime=runtime) - return backend + return KBPostgresBackend(search_space_id=1, runtime=runtime) def test_pending_filesystem_view_returns_deleted_paths(self): backend = self._make_backend( { - "pending_deletes": [ - {"path": "/documents/x.md", "tool_call_id": "t1"}, - ], + "pending_deletes": [{"path": "/documents/x.md", "tool_call_id": "t1"}], "pending_dir_deletes": [ - {"path": "/documents/d1", "tool_call_id": "t2"}, + {"path": "/documents/d1", "tool_call_id": "t2"} ], } ) diff --git a/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py new file mode 100644 index 000000000..b68dc5b4b --- /dev/null +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py @@ -0,0 +1,54 @@ +"""Mode-specific system-prompt assembly tests for the LIVE filesystem middleware. + +Ported from ``TestModeSpecificPrompts`` in the former +``tests/unit/middleware/test_filesystem_middleware.py`` (which exercised the +dead twin ``app.agents.chat.shared.middleware.filesystem._build_filesystem_system_prompt``). + +These drive the production ``build_system_prompt`` so the prompt the model +actually receives stays mode-scoped: cloud rules don't leak into desktop +sessions and vice-versa, and the sandbox section appears only when available. + +The per-tool *description* assertions from the old suite are intentionally NOT +ported: they assert exact prompt copy (tightly coupled to the old wording) and +guard prompt token hygiene rather than the code-movement refactor this suite +protects. +""" + +from __future__ import annotations + +import pytest + +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.system_prompt import ( + build_system_prompt, +) + +pytestmark = pytest.mark.unit + + +class TestModeSpecificPrompts: + def test_cloud_prompt_omits_desktop_section(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=False) + assert "Local Folder Mode" not in prompt + assert "mount-prefixed" not in prompt + assert "Persistence Rules" in prompt + assert "/documents" in prompt + assert "temp_" in prompt + + def test_desktop_prompt_omits_cloud_persistence_rules(self): + prompt = build_system_prompt( + FilesystemMode.DESKTOP_LOCAL_FOLDER, sandbox_available=False + ) + assert "Persistence Rules" not in prompt + assert "Workspace Tree" not in prompt + assert "Local Folder Mode" in prompt + assert "mount-prefixed" in prompt + + def test_sandbox_addendum_appended_when_available(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=True) + assert "execute_code" in prompt + assert "Code Execution" in prompt + + def test_sandbox_addendum_absent_when_unavailable(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=False) + assert "execute_code" not in prompt diff --git a/surfsense_backend/tests/unit/middleware/test_dedup_hitl_tool_calls.py b/surfsense_backend/tests/unit/middleware/test_dedup_hitl_tool_calls.py index 467ba6d5f..91b6bcf3c 100644 --- a/surfsense_backend/tests/unit/middleware/test_dedup_hitl_tool_calls.py +++ b/surfsense_backend/tests/unit/middleware/test_dedup_hitl_tool_calls.py @@ -2,8 +2,10 @@ import pytest from langchain_core.messages import AIMessage from langchain_core.tools import StructuredTool -from app.agents.new_chat.middleware.dedup_tool_calls import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.dedup_hitl import ( DedupHITLToolCallsMiddleware, +) +from app.agents.chat.multi_agent_chat.shared.middleware.dedup_tool_calls import ( wrap_dedup_key_by_arg_name, ) diff --git a/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py b/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py deleted file mode 100644 index 7fd3fe4a7..000000000 --- a/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py +++ /dev/null @@ -1,214 +0,0 @@ -import pytest -from langchain_core.messages import AIMessage, HumanMessage - -from app.agents.new_chat.middleware.file_intent import ( - FileIntentMiddleware, - FileOperationIntent, - _fallback_path, -) - -pytestmark = pytest.mark.unit - - -class _FakeLLM: - def __init__(self, response_text: str): - self._response_text = response_text - - async def ainvoke(self, *_args, **_kwargs): - return AIMessage(content=self._response_text) - - -@pytest.mark.asyncio -async def test_file_write_intent_injects_contract_message(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.93,"suggested_filename":"ideas.md"}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="Create another random note for me")], - "turn_id": "123:456", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/ideas.md" - assert contract["turn_id"] == "123:456" - assert any( - "file_operation_contract" in str(msg.content) - for msg in result["messages"] - if hasattr(msg, "content") - ) - - -@pytest.mark.asyncio -async def test_non_write_intent_does_not_inject_contract_message(): - llm = _FakeLLM('{"intent":"file_read","confidence":0.88,"suggested_filename":null}') - middleware = FileIntentMiddleware(llm=llm) - original_messages = [HumanMessage(content="Read /notes.md")] - state = {"messages": original_messages, "turn_id": "abc:def"} - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - assert ( - result["file_operation_contract"]["intent"] - == FileOperationIntent.FILE_READ.value - ) - assert "messages" not in result - - -@pytest.mark.asyncio -async def test_file_write_null_filename_uses_semantic_default_path(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.74,"suggested_filename":null}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create a random markdown file")], - "turn_id": "turn:1", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/notes.md" - - -@pytest.mark.asyncio -async def test_file_write_null_filename_defaults_to_markdown_path(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.71,"suggested_filename":null}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create a sample json config file")], - "turn_id": "turn:2", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/notes.md" - - -@pytest.mark.asyncio -async def test_file_write_txt_suggestion_is_normalized_to_markdown(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.82,"suggested_filename":"random.txt"}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create a random file")], - "turn_id": "turn:3", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/random.md" - - -@pytest.mark.asyncio -async def test_file_write_with_suggested_directory_preserves_folder(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.86,"suggested_filename":"random.md","suggested_directory":"pc backups","suggested_path":null}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create a random file in pc backups folder")], - "turn_id": "turn:4", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/pc_backups/random.md" - - -@pytest.mark.asyncio -async def test_file_write_with_suggested_path_takes_precedence(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.9,"suggested_filename":"ignored.md","suggested_directory":"docs","suggested_path":"/reports/q2/summary.md"}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create report")], - "turn_id": "turn:5", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/reports/q2/summary.md" - - -@pytest.mark.asyncio -async def test_file_write_infers_directory_from_user_text_when_missing(): - llm = _FakeLLM( - '{"intent":"file_write","confidence":0.83,"suggested_filename":"random.md","suggested_directory":null,"suggested_path":null}' - ) - middleware = FileIntentMiddleware(llm=llm) - state = { - "messages": [HumanMessage(content="create a random file in pc backups folder")], - "turn_id": "turn:6", - } - - result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] - - assert result is not None - contract = result["file_operation_contract"] - assert contract["intent"] == FileOperationIntent.FILE_WRITE.value - assert contract["suggested_path"] == "/pc_backups/random.md" - - -def test_fallback_path_normalizes_windows_slashes() -> None: - resolved = _fallback_path( - suggested_filename="summary.md", - suggested_path=r"\reports\q2\summary.md", - user_text="create report", - ) - - assert resolved == "/reports/q2/summary.md" - - -def test_fallback_path_normalizes_windows_drive_path() -> None: - resolved = _fallback_path( - suggested_filename=None, - suggested_path=r"C:\Users\anish\notes\todo.md", - user_text="create note", - ) - - assert resolved == "/C/Users/anish/notes/todo.md" - - -def test_fallback_path_normalizes_mixed_separators_and_duplicate_slashes() -> None: - resolved = _fallback_path( - suggested_filename="summary.md", - suggested_path=r"\\reports\\q2//summary.md", - user_text="create report", - ) - - assert resolved == "/reports/q2/summary.md" - - -def test_fallback_path_keeps_posix_style_absolute_path_for_linux_and_macos() -> None: - resolved = _fallback_path( - suggested_filename=None, - suggested_path="/var/log/surfsense/notes.md", - user_text="create note", - ) - - assert resolved == "/var/log/surfsense/notes.md" diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_backends.py b/surfsense_backend/tests/unit/middleware/test_filesystem_backends.py index c71b5efde..dafda17d2 100644 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_backends.py +++ b/surfsense_backend/tests/unit/middleware/test_filesystem_backends.py @@ -2,16 +2,18 @@ from pathlib import Path import pytest -from app.agents.new_chat.filesystem_backends import build_backend_resolver -from app.agents.new_chat.filesystem_selection import ( +from app.agents.chat.multi_agent_chat.shared.filesystem_selection import ( ClientPlatform, FilesystemMode, FilesystemSelection, LocalFilesystemMount, ) -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.multi_root_local_folder import ( MultiRootLocalFolderBackend, ) +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.resolver import ( + build_backend_resolver, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py b/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py deleted file mode 100644 index 70430f4ca..000000000 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Unit tests for the SurfSense filesystem middleware new behaviors. - -Covers: -* cloud cwd defaults to ``/documents`` and relative paths resolve under it -* cloud writes outside ``/documents/`` are rejected unless basename starts - with ``temp_`` -* cloud writes/edits to the anonymous document are rejected (read-only) -* helper methods on the middleware (``_resolve_relative``, - ``_check_cloud_write_namespace``, ``_default_cwd``) - -These tests use ``__new__`` to bypass the heavy ``__init__`` and exercise -the helper methods directly so the test surface stays narrow and fast. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -import pytest - -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.filesystem import ( - SurfSenseFilesystemMiddleware, - _build_filesystem_system_prompt, - _build_tool_descriptions, -) - -pytestmark = pytest.mark.unit - - -def _make_middleware(mode: FilesystemMode = FilesystemMode.CLOUD): - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = mode - return middleware - - -def _runtime(state: dict | None = None) -> SimpleNamespace: - return SimpleNamespace(state=state or {}) - - -class TestCloudCwdDefaults: - def test_default_cwd_in_cloud_is_documents_root(self): - m = _make_middleware() - assert m._default_cwd() == "/documents" - - def test_default_cwd_in_desktop_is_root(self): - m = _make_middleware(FilesystemMode.DESKTOP_LOCAL_FOLDER) - assert m._default_cwd() == "/" - - def test_current_cwd_uses_state_when_set(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/notes"}) - assert m._current_cwd(runtime) == "/documents/notes" - - def test_current_cwd_falls_back_to_default(self): - m = _make_middleware() - runtime = _runtime({}) - assert m._current_cwd(runtime) == "/documents" - - def test_current_cwd_ignores_invalid(self): - m = _make_middleware() - runtime = _runtime({"cwd": "not-absolute"}) - assert m._current_cwd(runtime) == "/documents" - - -class TestRelativePathResolution: - def test_relative_path_resolves_against_cwd(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/projects"}) - assert ( - m._resolve_relative("notes.md", runtime) == "/documents/projects/notes.md" - ) - - def test_relative_path_with_dotdot(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/a/b"}) - assert m._resolve_relative("../c.md", runtime) == "/documents/a/c.md" - - def test_absolute_path_is_kept(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents"}) - assert m._resolve_relative("/other/x.md", runtime) == "/other/x.md" - - def test_empty_path_returns_cwd(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/projects"}) - assert m._resolve_relative("", runtime) == "/documents/projects" - - -class TestCloudWriteNamespacePolicy: - def test_documents_path_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/documents/foo.md", runtime) is None - - def test_documents_root_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/documents", runtime) is None - - def test_temp_basename_anywhere_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/temp_scratch.md", runtime) is None - assert m._check_cloud_write_namespace("/foo/temp_x.md", runtime) is None - assert m._check_cloud_write_namespace("/documents/temp_x.md", runtime) is None - - def test_other_paths_rejected(self): - m = _make_middleware() - runtime = _runtime() - err = m._check_cloud_write_namespace("/foo/bar.md", runtime) - assert err is not None - assert "must target /documents" in err - - def test_anon_doc_path_is_read_only(self): - m = _make_middleware() - runtime = _runtime( - { - "kb_anon_doc": { - "path": "/documents/uploaded.xml", - "title": "uploaded", - "content": "", - "chunks": [], - } - } - ) - err = m._check_cloud_write_namespace("/documents/uploaded.xml", runtime) - assert err is not None - assert "read-only" in err - - def test_desktop_mode_skips_namespace_policy(self): - m = _make_middleware(FilesystemMode.DESKTOP_LOCAL_FOLDER) - runtime = _runtime() - assert m._check_cloud_write_namespace("/random/path.md", runtime) is None - - -class TestModeSpecificPrompts: - """The prompt and tool descriptions must only describe the active mode. - - Cross-mode noise wastes tokens and confuses the model with rules it - cannot use this session. - """ - - def test_cloud_prompt_omits_desktop_section(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=False - ) - assert "Local Folder Mode" not in prompt - assert "mount-prefixed" not in prompt - assert "Persistence Rules" in prompt - assert "/documents" in prompt - assert "temp_" in prompt - - def test_desktop_prompt_omits_cloud_persistence_rules(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.DESKTOP_LOCAL_FOLDER, sandbox_available=False - ) - assert "Persistence Rules" not in prompt - assert "Workspace Tree" not in prompt - assert "" not in prompt - assert "Local Folder Mode" in prompt - assert "mount-prefixed" in prompt - - def test_cloud_tool_descs_omit_desktop_phrases(self): - descs = _build_tool_descriptions(FilesystemMode.CLOUD) - for name in ( - "write_file", - "edit_file", - "move_file", - "mkdir", - "rm", - "rmdir", - "list_tree", - "grep", - ): - text = descs[name] - assert "Desktop" not in text, f"{name} leaks desktop hints" - assert "Cloud mode:" not in text, f"{name} qualifies a cloud-only desc" - - def test_desktop_tool_descs_omit_cloud_phrases(self): - descs = _build_tool_descriptions(FilesystemMode.DESKTOP_LOCAL_FOLDER) - for name in ( - "write_file", - "edit_file", - "move_file", - "mkdir", - "rm", - "rmdir", - "list_tree", - "grep", - ): - text = descs[name] - assert "Cloud" not in text, f"{name} leaks cloud hints" - assert "/documents/" not in text, f"{name} mentions cloud namespace" - assert "temp_" not in text, f"{name} mentions cloud temp_ semantics" - - def test_cloud_descs_include_rm_and_rmdir(self): - descs = _build_tool_descriptions(FilesystemMode.CLOUD) - assert "rm" in descs and "rmdir" in descs - assert "Deletes a single file" in descs["rm"] - assert "Deletes an empty directory" in descs["rmdir"] - assert "rmdir" in descs["rmdir"] and "POSIX" in descs["rmdir"] - - def test_desktop_descs_warn_about_irreversibility(self): - descs = _build_tool_descriptions(FilesystemMode.DESKTOP_LOCAL_FOLDER) - assert "NOT reversible" in descs["rm"] - assert "NOT reversible" in descs["rmdir"] - - def test_sandbox_addendum_appended_when_available(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=True - ) - assert "execute_code" in prompt - assert "Code Execution" in prompt - - def test_sandbox_addendum_absent_when_unavailable(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=False - ) - assert "execute_code" not in prompt diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py b/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py deleted file mode 100644 index 81cf590d3..000000000 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py +++ /dev/null @@ -1,173 +0,0 @@ -from pathlib import Path - -import pytest - -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( - MultiRootLocalFolderBackend, -) - -pytestmark = pytest.mark.unit - - -class _RuntimeNoSuggestedPath: - state = {"file_operation_contract": {}} - - -class _RuntimeWithSuggestedPath: - def __init__(self, suggested_path: str) -> None: - self.state = {"file_operation_contract": {"suggested_path": suggested_path}} - - -def test_contract_suggested_path_falls_back_to_documents_notes_md() -> None: - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = FilesystemMode.CLOUD - suggested = middleware._get_contract_suggested_path(_RuntimeNoSuggestedPath()) # type: ignore[arg-type] - # Cloud default cwd is /documents so the fallback lands in the KB. - assert suggested == "/documents/notes.md" - - -def test_contract_suggested_path_falls_back_to_root_notes_md_in_desktop() -> None: - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = FilesystemMode.DESKTOP_LOCAL_FOLDER - suggested = middleware._get_contract_suggested_path(_RuntimeNoSuggestedPath()) # type: ignore[arg-type] - assert suggested == "/notes.md" - - -def test_normalize_local_mount_path_prefixes_default_mount(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path("/random-note.md", runtime) # type: ignore[arg-type] - - assert resolved == "/pc_backups/random-note.md" - - -def test_normalize_local_mount_path_keeps_explicit_mount(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/pc_backups/notes/random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_windows_backslashes(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\notes\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_normalizes_mixed_separators(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\\notes//nested\\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/nested/random-note.md" - - -def test_normalize_local_mount_path_keeps_explicit_mount_with_backslashes( - tmp_path: Path, -) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\pc_backups\notes\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_prefixes_posix_absolute_path_for_linux_and_macos( - tmp_path: Path, -) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path("/var/log/app.log", runtime) # type: ignore[arg-type] - - assert resolved == "/pc_backups/var/log/app.log" - - -def test_normalize_local_mount_path_prefers_unique_existing_parent_mount( - tmp_path: Path, -) -> None: - root_a = tmp_path / "RootA" - root_b = tmp_path / "RootB" - (root_a / "other").mkdir(parents=True) - (root_b / "nested" / "deep").mkdir(parents=True) - backend = MultiRootLocalFolderBackend( - (("root_a", str(root_a)), ("root_b", str(root_b))) - ) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/nested/deep/new-note.md", - runtime, - ) - - assert resolved == "/root_b/nested/deep/new-note.md" - - -def test_normalize_local_mount_path_uses_suggested_mount_when_ambiguous( - tmp_path: Path, -) -> None: - root_a = tmp_path / "RootA" - root_b = tmp_path / "RootB" - root_a.mkdir(parents=True) - root_b.mkdir(parents=True) - backend = MultiRootLocalFolderBackend( - (("root_a", str(root_a)), ("root_b", str(root_b))) - ) - runtime = _RuntimeWithSuggestedPath("/root_b/notes/context.md") - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/brand-new-note.md", - runtime, - ) - - assert resolved == "/root_b/brand-new-note.md" diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py index ef95434bf..e78db1e76 100644 --- a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py @@ -15,7 +15,9 @@ from unittest.mock import AsyncMock import numpy as np import pytest -from app.agents.new_chat.middleware import kb_persistence +from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import ( + middleware as kb_persistence, +) from app.db import Document diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py index feca23d27..023213aaa 100644 --- a/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_revisions.py @@ -21,7 +21,9 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from app.agents.new_chat.middleware import kb_persistence +from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence import ( + middleware as kb_persistence, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/middleware/test_knowledge_search.py b/surfsense_backend/tests/unit/middleware/test_knowledge_search.py index 3529a946b..027738fba 100644 --- a/surfsense_backend/tests/unit/middleware/test_knowledge_search.py +++ b/surfsense_backend/tests/unit/middleware/test_knowledge_search.py @@ -5,10 +5,13 @@ import json import pytest from langchain_core.messages import AIMessage, HumanMessage -from app.agents.new_chat.document_xml import build_document_xml as _build_document_xml -from app.agents.new_chat.middleware.knowledge_search import ( +from app.agents.chat.multi_agent_chat.shared.middleware import knowledge_search as ks +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.document_xml import ( + build_document_xml as _build_document_xml, +) +from app.agents.chat.multi_agent_chat.shared.middleware.knowledge_search import ( KBSearchPlan, - KnowledgeBaseSearchMiddleware, + KnowledgePriorityMiddleware, _normalize_optional_date_range, _parse_kb_search_plan_response, _render_recent_conversation, @@ -201,7 +204,7 @@ class FakeBudgetLLM: return sum(len(msg.get("content", "")) for msg in messages) -class TestKnowledgeBaseSearchMiddlewarePlanner: +class TestKnowledgePriorityMiddlewarePlanner: @pytest.fixture(autouse=True) def _disable_planner_runnable(self, monkeypatch): # ``FakeLLM`` is a duck-typed mock; ``create_agent`` (used when the @@ -258,7 +261,8 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -271,7 +275,7 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: } ) ) - middleware = KnowledgeBaseSearchMiddleware(llm=llm, search_space_id=37) + middleware = KnowledgePriorityMiddleware(llm=llm, search_space_id=37) result = await middleware.abefore_agent( { @@ -301,11 +305,12 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) - middleware = KnowledgeBaseSearchMiddleware( + middleware = KnowledgePriorityMiddleware( llm=FakeLLM("not json"), search_space_id=37, ) @@ -330,11 +335,12 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) - middleware = KnowledgeBaseSearchMiddleware( + middleware = KnowledgePriorityMiddleware( llm=FakeLLM( json.dumps( { @@ -375,11 +381,13 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.browse_recent_documents", + ks, + "browse_recent_documents", fake_browse_recent_documents, ) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -393,7 +401,7 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: } ) ) - middleware = KnowledgeBaseSearchMiddleware(llm=llm, search_space_id=42) + middleware = KnowledgePriorityMiddleware(llm=llm, search_space_id=42) result = await middleware.abefore_agent( {"messages": [HumanMessage(content="what's my latest file?")]}, @@ -422,11 +430,13 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.browse_recent_documents", + ks, + "browse_recent_documents", fake_browse_recent_documents, ) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) @@ -440,7 +450,7 @@ class TestKnowledgeBaseSearchMiddlewarePlanner: } ) ) - middleware = KnowledgeBaseSearchMiddleware(llm=llm, search_space_id=42) + middleware = KnowledgePriorityMiddleware(llm=llm, search_space_id=42) await middleware.abefore_agent( {"messages": [HumanMessage(content="find the quarterly revenue report")]}, @@ -549,15 +559,17 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) - middleware = KnowledgeBaseSearchMiddleware( + middleware = KnowledgePriorityMiddleware( llm=self._planner_llm(), search_space_id=42, mentioned_document_ids=[1, 2, 3], @@ -597,17 +609,19 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) # Simulate a cached middleware instance whose closure was seeded # by a previous turn's cache-miss build (mentions=[1,2,3]). - middleware = KnowledgeBaseSearchMiddleware( + middleware = KnowledgePriorityMiddleware( llm=self._planner_llm(), search_space_id=42, mentioned_document_ids=[1, 2, 3], @@ -642,15 +656,17 @@ class TestKnowledgePriorityMentionDrain: return [] monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.fetch_mentioned_documents", + ks, + "fetch_mentioned_documents", fake_fetch_mentioned_documents, ) monkeypatch.setattr( - "app.agents.new_chat.middleware.knowledge_search.search_knowledge_base", + ks, + "search_knowledge_base", fake_search_knowledge_base, ) - middleware = KnowledgeBaseSearchMiddleware( + middleware = KnowledgePriorityMiddleware( llm=self._planner_llm(), search_space_id=42, mentioned_document_ids=[7, 8], diff --git a/surfsense_backend/tests/unit/middleware/test_knowledge_tree.py b/surfsense_backend/tests/unit/middleware/test_knowledge_tree.py index caaec3114..c14eca080 100644 --- a/surfsense_backend/tests/unit/middleware/test_knowledge_tree.py +++ b/surfsense_backend/tests/unit/middleware/test_knowledge_tree.py @@ -9,8 +9,10 @@ contract cannot silently regress. from __future__ import annotations -from app.agents.new_chat.middleware.knowledge_tree import KnowledgeTreeMiddleware -from app.agents.new_chat.path_resolver import DOCUMENTS_ROOT +from app.agents.chat.multi_agent_chat.main_agent.middleware.knowledge_tree.middleware import ( + KnowledgeTreeMiddleware, +) +from app.agents.chat.runtime.path_resolver import DOCUMENTS_ROOT def _compute(folder_paths: list[str], doc_paths: list[str]) -> set[str]: @@ -86,7 +88,7 @@ class TestFormatTreeRendering: folder_paths: list[str], doc_specs: list[dict], ) -> str: - from app.agents.new_chat.path_resolver import PathIndex + from app.agents.chat.runtime.path_resolver import PathIndex index = PathIndex( folder_paths={i + 1: p for i, p in enumerate(folder_paths)}, diff --git a/surfsense_backend/tests/unit/middleware/test_local_folder_backend.py b/surfsense_backend/tests/unit/middleware/test_local_folder_backend.py index 6e81ecf8e..aaa3b47fb 100644 --- a/surfsense_backend/tests/unit/middleware/test_local_folder_backend.py +++ b/surfsense_backend/tests/unit/middleware/test_local_folder_backend.py @@ -2,7 +2,9 @@ from pathlib import Path import pytest -from app.agents.new_chat.middleware.local_folder_backend import LocalFolderBackend +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.local_folder import ( + LocalFolderBackend, +) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/middleware/test_multi_root_local_folder_backend.py b/surfsense_backend/tests/unit/middleware/test_multi_root_local_folder_backend.py index 43a671178..b2d545f27 100644 --- a/surfsense_backend/tests/unit/middleware/test_multi_root_local_folder_backend.py +++ b/surfsense_backend/tests/unit/middleware/test_multi_root_local_folder_backend.py @@ -2,7 +2,7 @@ from pathlib import Path import pytest -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( +from app.agents.chat.multi_agent_chat.shared.middleware.filesystem.backends.multi_root_local_folder import ( MultiRootLocalFolderBackend, ) diff --git a/surfsense_backend/tests/unit/notifications/api/test_transform.py b/surfsense_backend/tests/unit/notifications/api/test_transform.py new file mode 100644 index 000000000..96624fe61 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/api/test_transform.py @@ -0,0 +1,94 @@ +"""Unit tests for pure notifications API request/response helpers.""" + +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +import pytest + +from app.notifications.api.transform import ( + parse_before_date, + parse_source_type, + to_response, +) +from app.notifications.persistence import Notification + +pytestmark = pytest.mark.unit + + +class TestParseSourceType: + def test_connector_prefix(self): + """A 'connector:' filter selects the connector types and JSONB facet.""" + parsed = parse_source_type("connector:GITHUB_CONNECTOR") + assert parsed.types == ("connector_indexing", "connector_deletion") + assert parsed.metadata_key == "connector_type" + assert parsed.value == "GITHUB_CONNECTOR" + + def test_doctype_prefix(self): + """A 'doctype:' filter selects the document type and JSONB facet.""" + parsed = parse_source_type("doctype:FILE") + assert parsed.types == ("document_processing",) + assert parsed.metadata_key == "document_type" + assert parsed.value == "FILE" + + def test_unknown_prefix_returns_none(self): + """An unrecognized prefix yields no filter.""" + assert parse_source_type("mystery:thing") is None + + +class TestParseBeforeDate: + def test_parses_iso_with_zulu(self): + """An ISO date with a 'Z' suffix parses to a UTC datetime.""" + parsed = parse_before_date("2024-01-15T00:00:00Z") + assert parsed == datetime(2024, 1, 15, tzinfo=UTC) + + def test_invalid_raises_value_error(self): + """A malformed date raises ValueError for the endpoint to turn into a 400.""" + with pytest.raises(ValueError): + parse_before_date("not-a-date") + + +def _notification(**overrides) -> Notification: + defaults = { + "id": 1, + "user_id": uuid.uuid4(), + "search_space_id": 3, + "type": "document_processing", + "title": "Title", + "message": "Message", + "read": False, + "notification_metadata": {"k": "v"}, + "created_at": datetime(2024, 1, 1, tzinfo=UTC), + "updated_at": datetime(2024, 1, 2, tzinfo=UTC), + } + defaults.update(overrides) + return Notification(**defaults) + + +class TestToResponse: + def test_maps_core_fields(self): + """A persisted notification maps its core fields onto the response shape.""" + notification = _notification() + response = to_response(notification) + assert response.id == 1 + assert response.user_id == str(notification.user_id) + assert response.type == "document_processing" + assert response.metadata == {"k": "v"} + assert response.created_at == "2024-01-01T00:00:00+00:00" + assert response.updated_at == "2024-01-02T00:00:00+00:00" + + def test_missing_updated_at_maps_to_none(self): + """A missing updated_at is represented as None in the response.""" + response = to_response(_notification(updated_at=None)) + assert response.updated_at is None + + def test_missing_created_at_maps_to_empty_string(self): + """A missing created_at is represented as an empty string in the response.""" + response = to_response(_notification(created_at=None)) + assert response.created_at == "" + + def test_null_metadata_maps_to_empty_dict(self): + """Null metadata is normalized to an empty dict in the response.""" + response = to_response(_notification(notification_metadata=None)) + assert response.metadata == {} diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_connector_indexing.py b/surfsense_backend/tests/unit/notifications/service/messages/test_connector_indexing.py new file mode 100644 index 000000000..391ce4466 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_connector_indexing.py @@ -0,0 +1,176 @@ +"""Unit tests for connector-indexing presentation logic.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.messages import connector_indexing as msg + +pytestmark = pytest.mark.unit + + +class TestOperationId: + def test_encodes_connector_id(self): + """The operation id embeds the connector id.""" + assert msg.operation_id(42).startswith("connector_42_") + + def test_appends_date_range_when_given(self): + """A start/end date range is appended to the operation id.""" + op = msg.operation_id(42, start_date="2024-01-01", end_date="2024-02-01") + assert op.endswith("_2024-01-01_2024-02-01") + + def test_uses_none_placeholder_for_open_ended_range(self): + """A missing range bound is encoded as the 'none' placeholder.""" + assert msg.operation_id(42, start_date="2024-01-01").endswith( + "_2024-01-01_none" + ) + + def test_google_drive_encodes_counts(self): + """The Drive operation id embeds connector id plus folder/file counts.""" + op = msg.google_drive_operation_id(7, folder_count=2, file_count=5) + assert op.startswith("drive_7_") + assert op.endswith("_2f_5files") + + +class TestProgress: + def test_known_stage_maps_to_message(self): + """A known stage maps to its user-facing message and is recorded.""" + message, meta = msg.progress(3, stage="fetching") + assert message == "Fetching your content" + assert meta["indexed_count"] == 3 + assert meta["sync_stage"] == "fetching" + + def test_unknown_stage_falls_back_to_processing(self): + """An unrecognized stage falls back to a generic 'Processing' message.""" + message, _ = msg.progress(1, stage="weird") + assert message == "Processing" + + def test_stage_message_overrides_mapping(self): + """An explicit stage message overrides the stage-to-message mapping.""" + message, _ = msg.progress(1, stage="fetching", stage_message="Custom") + assert message == "Custom" + + def test_no_stage_uses_legacy_default(self): + """With neither stage nor message, the legacy default message is used.""" + message, meta = msg.progress(1) + assert message == "Fetching your content" + assert "sync_stage" not in meta + + def test_total_count_yields_percent(self): + """Supplying a total count produces a progress percentage.""" + _, meta = msg.progress(5, total_count=10) + assert meta["total_count"] == 10 + assert meta["progress_percent"] == 50 + + +class TestRetry: + def test_strips_workspace_suffix_from_connector_name(self): + """The provider name is derived by stripping the workspace suffix.""" + message, _ = msg.retry("Notion - My Workspace", 0, "rate_limit", 1, 3) + assert message == "Notion rate limit reached. Retrying..." + + def test_explicit_service_name_wins(self): + """An explicit service name overrides the connector-derived name.""" + message, _ = msg.retry( + "Notion - WS", 0, "rate_limit", 1, 3, service_name="Slack" + ) + assert message.startswith("Slack rate limit reached") + + @pytest.mark.parametrize( + ("reason", "expected"), + [ + ("rate_limit", "Notion rate limit reached"), + ("server_error", "Notion is slow to respond"), + ("timeout", "Notion took too long"), + ("temporary_error", "Notion temporarily unavailable"), + ("something_else", "Waiting for Notion"), + ], + ) + def test_reason_wording(self, reason, expected): + """Each retry reason maps to its wording; unknown reasons get a fallback.""" + message, _ = msg.retry("Notion", 0, reason, 1, 3) + assert message.startswith(expected) + + def test_long_wait_shows_seconds(self): + """A wait longer than the threshold surfaces the retry delay in seconds.""" + message, _ = msg.retry("Notion", 0, "rate_limit", 1, 3, wait_seconds=10) + assert "Retrying in 10s..." in message + + def test_short_wait_is_hidden(self): + """A short wait is not worth showing, so no seconds are surfaced.""" + message, _ = msg.retry("Notion", 0, "rate_limit", 1, 3, wait_seconds=3) + assert message.endswith("Retrying...") + + def test_synced_count_suffix_singular_and_plural(self): + """Already-synced items are appended with correct singular/plural wording.""" + one, _ = msg.retry("Notion", 1, "rate_limit", 1, 3) + many, _ = msg.retry("Notion", 2, "rate_limit", 1, 3) + assert one.endswith("(1 item synced so far)") + assert many.endswith("(2 items synced so far)") + + def test_metadata_records_retry_state(self): + """Retry metadata captures the attempt, reason, and wait state.""" + _, meta = msg.retry("Notion", 0, "rate_limit", 2, 5, wait_seconds=8) + assert meta["sync_stage"] == "waiting_retry" + assert meta["retry_attempt"] == 2 + assert meta["retry_max_attempts"] == 5 + assert meta["retry_reason"] == "rate_limit" + assert meta["retry_wait_seconds"] == 8 + + +class TestCompletion: + def test_clean_success_plural(self): + """A clean multi-file sync reports ready/completed with plural wording.""" + title, message, status, meta = msg.completion("GitHub", 3) + assert title == "Ready: GitHub" + assert message == "Now searchable! 3 files synced." + assert status == "completed" + assert meta["sync_stage"] == "completed" + + def test_clean_success_singular(self): + """A single synced file uses singular 'file' wording.""" + _, message, _, _ = msg.completion("GitHub", 1) + assert message == "Now searchable! 1 file synced." + + def test_nothing_to_sync(self): + """Zero new items with no error reports 'Already up to date!'.""" + _, message, status, _ = msg.completion("GitHub", 0) + assert message == "Already up to date!" + assert status == "completed" + + def test_hard_failure(self): + """An error with nothing synced reports a hard failure.""" + title, message, status, meta = msg.completion("GitHub", 0, error_message="boom") + assert title == "Failed: GitHub" + assert message == "Sync failed: boom" + assert status == "failed" + assert meta["sync_stage"] == "failed" + + def test_partial_success_with_error_note(self): + """An error after partial progress still completes, with an appended note.""" + title, message, status, _ = msg.completion("GitHub", 2, error_message="flaky") + assert title == "Ready: GitHub" + assert message == "Now searchable! 2 files synced. Note: flaky" + assert status == "completed" + + def test_warning_is_treated_as_complete(self): + """A warning-level error completes the run rather than failing it.""" + title, message, status, _ = msg.completion( + "GitHub", 0, error_message="partial", is_warning=True + ) + assert title == "Ready: GitHub" + assert message == "Sync complete. partial" + assert status == "completed" + + def test_unsupported_files_note_singular_and_plural(self): + """Unsupported-file counts are described with correct singular/plural wording.""" + _, one, _, _ = msg.completion("GitHub", 2, unsupported_count=1) + _, many, _, _ = msg.completion("GitHub", 2, unsupported_count=3) + assert "1 file was not supported." in one + assert "3 files were not supported." in many + + def test_zero_indexed_with_unsupported_reports_complete(self): + """Nothing synced but some unsupported files still reports completion.""" + _, message, status, _ = msg.completion("GitHub", 0, unsupported_count=2) + assert message == "Sync complete. 2 files were not supported." + assert status == "completed" diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_document_processing.py b/surfsense_backend/tests/unit/notifications/service/messages/test_document_processing.py new file mode 100644 index 000000000..2f0a6a9d3 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_document_processing.py @@ -0,0 +1,63 @@ +"""Unit tests for document-processing presentation logic.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.messages import document_processing as msg + +pytestmark = pytest.mark.unit + + +def test_operation_id_encodes_type_and_space(): + """The operation id embeds the document type and search space id.""" + op = msg.operation_id("FILE", "report.pdf", 9) + assert op.startswith("doc_FILE_9_") + + +@pytest.mark.parametrize( + ("stage", "expected"), + [ + ("parsing", "Reading your file"), + ("chunking", "Preparing for search"), + ("embedding", "Preparing for search"), + ("storing", "Finalizing"), + ("unknown", "Processing"), + ], +) +def test_progress_stage_messages(stage, expected): + """Each processing stage maps to its message; unknown stages get a fallback.""" + message, meta = msg.progress(stage) + assert message == expected + assert meta["processing_stage"] == stage + + +def test_progress_records_chunks_count(): + """A provided chunk count is stored in metadata for debugging.""" + _, meta = msg.progress("chunking", chunks_count=12) + assert meta["chunks_count"] == 12 + + +def test_progress_message_override(): + """An explicit stage message overrides the stage mapping.""" + message, _ = msg.progress("parsing", stage_message="Scanning") + assert message == "Scanning" + + +def test_completion_success(): + """A successful run reports ready/completed and records the document id.""" + title, message, status, meta = msg.completion("report.pdf", document_id=5) + assert title == "Ready: report.pdf" + assert message == "Now searchable!" + assert status == "completed" + assert meta["document_id"] == 5 + assert meta["processing_stage"] == "completed" + + +def test_completion_failure(): + """An error reports failed status with the error surfaced in the message.""" + title, message, status, meta = msg.completion("report.pdf", error_message="bad") + assert title == "Failed: report.pdf" + assert message == "Processing failed: bad" + assert status == "failed" + assert meta["processing_stage"] == "failed" diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py b/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py new file mode 100644 index 000000000..606e985f2 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py @@ -0,0 +1,32 @@ +"""Unit tests for page-limit presentation logic.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.messages import page_limit as msg + +pytestmark = pytest.mark.unit + + +def test_operation_id_encodes_search_space(): + """The operation id embeds the search space id.""" + assert msg.operation_id("doc.pdf", 9).startswith("page_limit_9_") + + +def test_summary_title_and_message(): + """The summary states the document and the used/limit page counts.""" + title, message = msg.summary( + "short.pdf", pages_used=95, pages_limit=100, pages_to_add=10 + ) + assert title == "Page limit exceeded: short.pdf" + assert message == ( + "This document has ~10 page(s) but you've used 95/100 pages. " + "Upgrade to process more documents." + ) + + +def test_summary_truncates_long_name(): + """A long document name is truncated in the title.""" + title, _ = msg.summary("a" * 50, pages_used=1, pages_limit=2, pages_to_add=1) + assert title == f"Page limit exceeded: {'a' * 40}..." diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_text.py b/surfsense_backend/tests/unit/notifications/service/messages/test_text.py new file mode 100644 index 000000000..bf3611607 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_text.py @@ -0,0 +1,24 @@ +"""Unit tests for shared notification text helpers.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.messages.text import truncate + +pytestmark = pytest.mark.unit + + +def test_truncate_leaves_short_text_unchanged(): + """Text under the limit is returned verbatim, with no ellipsis.""" + assert truncate("hello", 100) == "hello" + + +def test_truncate_keeps_text_at_exact_limit(): + """Text exactly at the limit is not truncated.""" + assert truncate("a" * 40, 40) == "a" * 40 + + +def test_truncate_appends_ellipsis_when_over_limit(): + """Text past the limit is cut to the limit and gains an ellipsis.""" + assert truncate("a" * 41, 40) == "a" * 40 + "..." diff --git a/surfsense_backend/tests/unit/notifications/service/test_metadata.py b/surfsense_backend/tests/unit/notifications/service/test_metadata.py new file mode 100644 index 000000000..56f1dc583 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/test_metadata.py @@ -0,0 +1,63 @@ +"""Unit tests for pure notification metadata transitions.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.metadata import apply_update, start_metadata + +pytestmark = pytest.mark.unit + + +class TestStartMetadata: + def test_seeds_operation_and_progress_fields(self): + """A new notification is seeded with operation id, in-progress status, and start time.""" + meta = start_metadata("op-1") + assert meta["operation_id"] == "op-1" + assert meta["status"] == "in_progress" + assert "started_at" in meta + + def test_preserves_initial_fields(self): + """Caller-provided initial metadata is carried through.""" + meta = start_metadata("op-1", {"connector_id": 7}) + assert meta["connector_id"] == 7 + + def test_does_not_mutate_caller_dict(self): + """Seeding returns a new dict without mutating the caller's input.""" + initial = {"connector_id": 7} + start_metadata("op-1", initial) + assert initial == {"connector_id": 7} + + +class TestApplyUpdate: + def test_completed_stamps_completed_at(self): + """A completed status records a completion timestamp.""" + meta = apply_update({"status": "in_progress"}, status="completed") + assert meta["status"] == "completed" + assert "completed_at" in meta + + def test_failed_stamps_completed_at(self): + """A failed status also records a completion timestamp.""" + meta = apply_update({}, status="failed") + assert "completed_at" in meta + + def test_in_progress_does_not_stamp_completed_at(self): + """A non-terminal status leaves the completion timestamp unset.""" + meta = apply_update({}, status="in_progress") + assert "completed_at" not in meta + + def test_merges_metadata_updates(self): + """Metadata updates are merged into the existing metadata.""" + meta = apply_update({"a": 1}, metadata_updates={"b": 2}) + assert meta == {"a": 1, "b": 2} + + def test_updates_override_existing_keys(self): + """Updates take precedence over existing keys on conflict.""" + meta = apply_update({"a": 1}, metadata_updates={"a": 9}) + assert meta["a"] == 9 + + def test_does_not_mutate_caller_dict(self): + """Applying updates returns a new dict without mutating the caller's input.""" + current = {"a": 1} + apply_update(current, status="completed", metadata_updates={"b": 2}) + assert current == {"a": 1} diff --git a/surfsense_backend/tests/unit/routes/test_revert_turn_route.py b/surfsense_backend/tests/unit/routes/test_revert_turn_route.py index 1e1cbffb3..35d409a40 100644 --- a/surfsense_backend/tests/unit/routes/test_revert_turn_route.py +++ b/surfsense_backend/tests/unit/routes/test_revert_turn_route.py @@ -18,7 +18,7 @@ from unittest.mock import AsyncMock, patch import pytest -from app.agents.new_chat.feature_flags import AgentFeatureFlags +from app.agents.chat.multi_agent_chat.shared.feature_flags import AgentFeatureFlags from app.routes import agent_revert_route from app.services.revert_service import RevertOutcome diff --git a/surfsense_backend/tests/unit/services/test_image_gen_api_base_defense.py b/surfsense_backend/tests/unit/services/test_image_gen_api_base_defense.py index 9d5fdb190..571e7d15b 100644 --- a/surfsense_backend/tests/unit/services/test_image_gen_api_base_defense.py +++ b/surfsense_backend/tests/unit/services/test_image_gen_api_base_defense.py @@ -20,6 +20,7 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest +from langchain.tools import ToolRuntime pytestmark = pytest.mark.unit @@ -90,7 +91,9 @@ async def test_global_openrouter_image_gen_sets_api_base_when_config_empty(): async def test_generate_image_tool_global_sets_api_base_when_config_empty(): """Same defense at the agent tool entry point — both surfaces share the same OpenRouter config payloads.""" - from app.agents.new_chat.tools import generate_image as gi_module + from app.agents.chat.multi_agent_chat.subagents.builtins.deliverables.tools import ( + generate_image as gi_module, + ) cfg = { "id": -20_001, @@ -150,7 +153,19 @@ async def test_generate_image_tool_global_sets_api_base_when_config_empty(): tool = gi_module.create_generate_image_tool( search_space_id=1, db_session=MagicMock() ) - await tool.ainvoke({"prompt": "a cat", "n": 1}) + # The live tool takes an injected ToolRuntime and returns a Command; + # drive the raw coroutine with a minimal runtime (the tool only reads + # ``tool_call_id``). We assert on what was forwarded to litellm, not + # on the return value. + runtime = ToolRuntime( + state={}, + context=None, + config={}, + stream_writer=None, + tool_call_id="call-1", + store=None, + ) + await tool.coroutine(prompt="a cat", n=1, runtime=runtime) assert captured.get("api_base") == "https://openrouter.ai/api/v1" assert captured["model"] == "openrouter/openai/gpt-image-1" diff --git a/surfsense_backend/tests/unit/services/test_supports_image_input.py b/surfsense_backend/tests/unit/services/test_supports_image_input.py index 71fdee1c7..fabb3587a 100644 --- a/surfsense_backend/tests/unit/services/test_supports_image_input.py +++ b/surfsense_backend/tests/unit/services/test_supports_image_input.py @@ -227,7 +227,7 @@ global_llm_configs: def test_agent_config_from_yaml_explicit_overrides_resolver(): - from app.agents.new_chat.llm_config import AgentConfig + from app.agents.chat.runtime.llm_config import AgentConfig cfg_text_only = AgentConfig.from_yaml_config( { @@ -256,7 +256,7 @@ def test_agent_config_from_yaml_explicit_overrides_resolver(): def test_agent_config_from_yaml_unannotated_uses_resolver(): """Without an explicit YAML key, AgentConfig defers to the catalog resolver — for ``gpt-4o`` LiteLLM's map says supports_vision=True.""" - from app.agents.new_chat.llm_config import AgentConfig + from app.agents.chat.runtime.llm_config import AgentConfig cfg = AgentConfig.from_yaml_config( { @@ -275,7 +275,7 @@ def test_agent_config_auto_mode_supports_image_input(): so users can keep their selection on Auto with a vision-capable deployment somewhere in the pool. The router's own `allowed_fails` handles non-vision deployments via fallback.""" - from app.agents.new_chat.llm_config import AgentConfig + from app.agents.chat.runtime.llm_config import AgentConfig auto = AgentConfig.from_auto_mode() assert auto.supports_image_input is True diff --git a/surfsense_backend/tests/unit/services/test_vision_llm_api_base_defense.py b/surfsense_backend/tests/unit/services/test_vision_llm_api_base_defense.py index b8ba9d80c..5e3aa6eda 100644 --- a/surfsense_backend/tests/unit/services/test_vision_llm_api_base_defense.py +++ b/surfsense_backend/tests/unit/services/test_vision_llm_api_base_defense.py @@ -61,7 +61,7 @@ async def test_get_vision_llm_global_openrouter_sets_api_base(): return_value=cfg, ), patch( - "app.agents.new_chat.llm_config.SanitizedChatLiteLLM", + "app.agents.chat.runtime.llm_config.SanitizedChatLiteLLM", new=FakeSanitized, ), ): diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_interrupt_inspector_all.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_interrupt_inspector_all.py index 15ab89b73..4457f4768 100644 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_interrupt_inspector_all.py +++ b/surfsense_backend/tests/unit/tasks/chat/streaming/test_interrupt_inspector_all.py @@ -18,7 +18,7 @@ from langgraph.graph import END, START, StateGraph from langgraph.types import Send, interrupt from typing_extensions import TypedDict -from app.agents.multi_agent_chat.middleware.main_agent.checkpointed_subagent_middleware.task_tool import ( +from app.agents.chat.multi_agent_chat.main_agent.middleware.checkpointed_subagent_middleware.task_tool import ( build_task_tool_with_parent_config, ) from app.tasks.chat.streaming.helpers.interrupt_inspector import ( diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py deleted file mode 100644 index e014bb911..000000000 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py +++ /dev/null @@ -1,557 +0,0 @@ -"""Parity gate for the parallel refactor of ``stream_new_chat.py``. - -The new tree under ``app.tasks.chat.streaming.flows`` is built side-by-side with -the legacy monolithic ``app.tasks.chat.stream_new_chat`` so we can cut over -atomically. This file pins externally-observable behaviour at module -boundaries so a divergence between the two trees fails loudly *before* the -cutover. - -What we verify: - - 1. **Signature parity** — ``stream_new_chat`` / ``stream_resume_chat`` from - the new tree have the same call signature as the originals. - 2. **Helper extraction parity** — the SRP modules in ``flows/`` produce the - same outputs as the inline code in the legacy file for representative - inputs (initial thinking step, image-capability gate, runtime context, - SSE frame sequences, token-usage frame shape, persistence guards). - 3. **Wrapper delegation** — wrappers like ``load_llm_bundle`` / - ``can_recover_provider_rate_limit`` exist and are addressable. - -Delete this file along with ``stream_new_chat.py`` once the cutover is done -(see the parent refactor plan). -""" - -from __future__ import annotations - -import asyncio -import inspect -from typing import Any -from unittest.mock import AsyncMock, patch - -import pytest - -from app.agents.new_chat.context import SurfSenseContextSchema -from app.services.new_streaming_service import VercelStreamingService -from app.tasks.chat.stream_new_chat import ( - stream_new_chat as old_stream_new_chat, - stream_resume_chat as old_stream_resume_chat, -) -from app.tasks.chat.streaming.flows import ( - stream_new_chat as new_stream_new_chat, - stream_resume_chat as new_stream_resume_chat, -) -from app.tasks.chat.streaming.flows.new_chat.initial_thinking_step import ( - build_initial_thinking_step, -) -from app.tasks.chat.streaming.flows.new_chat.llm_capability import ( - check_image_input_capability, -) -from app.tasks.chat.streaming.flows.new_chat.persistence_spawn import ( - await_persist_task, - spawn_persist_assistant_shell_task, - spawn_persist_user_task, - spawn_set_ai_responding_bg, -) -from app.tasks.chat.streaming.flows.new_chat.runtime_context import ( - build_new_chat_runtime_context, -) -from app.tasks.chat.streaming.flows.resume_chat.runtime_context import ( - build_resume_chat_runtime_context, -) -from app.tasks.chat.streaming.flows.shared.finalize_emit import iter_token_usage_frame -from app.tasks.chat.streaming.flows.shared.first_frames import ( - iter_final_frames, - iter_initial_frames, -) -from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle -from app.tasks.chat.streaming.flows.shared.premium_quota import ( - PremiumReservation, - needs_premium_quota, -) -from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( - can_recover_provider_rate_limit, -) - -pytestmark = pytest.mark.unit - - -# --------------------------------------------------------------------- signature - - -def _normalize_annotation(ann: Any) -> str: - """Compare-friendly form for an annotation. - - The legacy ``stream_new_chat.py`` does NOT use ``from __future__ import - annotations``, so its annotations are evaluated at import time and come - back as type objects / typing generics. The new tree DOES use it, so its - annotations are PEP-563 strings. - - Both reprs describe the same types — strip the module prefixes / typing - namespace + the ```` wrapper so we compare the canonical - declared form. - """ - if ann is inspect.Signature.empty: - return "" - raw = ann if isinstance(ann, str) else repr(ann) - cleaned = ( - raw.replace("typing.", "") - .replace("collections.abc.", "") - .replace("app.db.", "") - .replace("app.agents.new_chat.filesystem_selection.", "") - .replace("app.agents.new_chat.context.", "") - ) - # Unwrap ```` → ``int`` (legacy-side type objects). - if cleaned.startswith(""): - cleaned = cleaned[len("")] - return cleaned - - -def _normalize_sig(sig: inspect.Signature) -> list[tuple[str, Any, str]]: - return [ - (p.name, p.default, _normalize_annotation(p.annotation)) - for p in sig.parameters.values() - ] - - -def test_stream_new_chat_signature_matches_legacy() -> None: - old = inspect.signature(old_stream_new_chat) - new = inspect.signature(new_stream_new_chat) - assert _normalize_sig(new) == _normalize_sig(old) - assert _normalize_annotation(new.return_annotation) == _normalize_annotation( - old.return_annotation - ) - - -def test_stream_resume_chat_signature_matches_legacy() -> None: - old = inspect.signature(old_stream_resume_chat) - new = inspect.signature(new_stream_resume_chat) - assert _normalize_sig(new) == _normalize_sig(old) - assert _normalize_annotation(new.return_annotation) == _normalize_annotation( - old.return_annotation - ) - - -def test_orchestrators_are_async_generator_functions() -> None: - assert inspect.isasyncgenfunction(new_stream_new_chat) - assert inspect.isasyncgenfunction(new_stream_resume_chat) - - -# ------------------------------------------------------------ initial thinking - - -@pytest.mark.parametrize( - "user_query, image_urls, expected_title, expected_action", - [ - ("hello world", None, "Understanding your request", "Processing"), - ( - "", - ["data:image/png;base64,AAA"], - "Understanding your request", - "Processing", - ), - ("", None, "Understanding your request", "Processing"), - ], -) -def test_initial_thinking_step_branches( - user_query: str, - image_urls: list[str] | None, - expected_title: str, - expected_action: str, -) -> None: - step = build_initial_thinking_step( - user_query=user_query, - user_image_data_urls=image_urls, - ) - assert step.step_id == "thinking-1" - assert step.title == expected_title - assert len(step.items) == 1 - assert step.items[0].startswith(f"{expected_action}: ") - - -def test_initial_thinking_step_truncates_long_query() -> None: - long_query = "x" * 200 - step = build_initial_thinking_step( - user_query=long_query, - user_image_data_urls=None, - ) - # 80-char truncation + ellipsis, sandwiched after "Processing: ". - assert "..." in step.items[0] - item = step.items[0] - payload = item[len("Processing: ") :] - assert payload.startswith("x" * 80) and payload.endswith("...") - - -# ------------------------------------------------------------ capability gate - - -def test_image_capability_passes_without_images() -> None: - assert ( - check_image_input_capability(user_image_data_urls=None, agent_config=None) - is None - ) - - -def test_image_capability_passes_when_capability_unknown() -> None: - """Unknown / unmapped models are not blocked — only models LiteLLM has - *explicitly* marked text-only trip the gate.""" - - class _AgentConfig: - provider = "openrouter" - model_name = "unknown-mystery-model" - custom_provider = None - config_name = "Unknown" - litellm_params: dict[str, Any] = {} - - with patch( - "app.services.provider_capabilities.is_known_text_only_chat_model", - return_value=False, - ): - assert ( - check_image_input_capability( - user_image_data_urls=["data:image/png;base64,AAA"], - agent_config=_AgentConfig(), # type: ignore[arg-type] - ) - is None - ) - - -def test_image_capability_blocks_known_text_only_models() -> None: - class _AgentConfig: - provider = "openai" - model_name = "gpt-3.5-turbo" - custom_provider = None - config_name = "GPT-3.5" - litellm_params: dict[str, Any] = {"base_model": "gpt-3.5-turbo"} - - with patch( - "app.services.provider_capabilities.is_known_text_only_chat_model", - return_value=True, - ): - result = check_image_input_capability( - user_image_data_urls=["data:image/png;base64,AAA"], - agent_config=_AgentConfig(), # type: ignore[arg-type] - ) - assert result is not None - message, error_code = result - assert error_code == "MODEL_DOES_NOT_SUPPORT_IMAGE_INPUT" - assert "GPT-3.5" in message - - -# ---------------------------------------------------------------- runtime ctx - - -def test_new_chat_runtime_context_prefers_accepted_folder_ids() -> None: - ctx = build_new_chat_runtime_context( - search_space_id=7, - mentioned_document_ids=[1, 2], - accepted_folder_ids=[10], - mentioned_folder_ids=[20, 30], - request_id="req", - turn_id="t1", - ) - assert isinstance(ctx, SurfSenseContextSchema) - assert ctx.search_space_id == 7 - assert list(ctx.mentioned_document_ids) == [1, 2] - assert list(ctx.mentioned_folder_ids) == [10] - assert ctx.request_id == "req" - assert ctx.turn_id == "t1" - - -def test_new_chat_runtime_context_falls_back_to_mentioned_folder_ids() -> None: - ctx = build_new_chat_runtime_context( - search_space_id=7, - mentioned_document_ids=None, - accepted_folder_ids=[], - mentioned_folder_ids=[20, 30], - request_id=None, - turn_id="t2", - ) - assert list(ctx.mentioned_folder_ids) == [20, 30] - - -def test_resume_chat_runtime_context_empty_mention_lists() -> None: - ctx = build_resume_chat_runtime_context( - search_space_id=42, request_id="req-r", turn_id="t-r" - ) - assert ctx.search_space_id == 42 - assert ctx.request_id == "req-r" - assert ctx.turn_id == "t-r" - - -# ---------------------------------------------------------------- SSE frames - - -def test_iter_initial_frames_emits_canonical_sequence() -> None: - svc = VercelStreamingService() - frames = list(iter_initial_frames(svc, turn_id="42:1700000000000")) - # Exactly 4 frames: message_start, start_step, turn-info (turn_id), turn-status (busy). - assert len(frames) == 4 - assert "42:1700000000000" in frames[2] - assert '"status":"busy"' in frames[3] or '"status": "busy"' in frames[3] - - -def test_iter_final_frames_emits_idle_then_finish_done() -> None: - svc = VercelStreamingService() - frames = list(iter_final_frames(svc)) - assert len(frames) == 4 - assert '"status":"idle"' in frames[0] or '"status": "idle"' in frames[0] - - -# ----------------------------------------------------------- token usage frame - - -class _FakeAccumulator: - """Minimal stand-in covering only the fields ``iter_token_usage_frame`` reads.""" - - def __init__(self, summary: Any = None) -> None: - self._summary = summary - self.calls = [1, 2, 3] - self.grand_total = 100 - self.total_cost_micros = 50_000 - self.total_prompt_tokens = 60 - self.total_completion_tokens = 40 - - def per_message_summary(self) -> Any: - return self._summary - - def serialized_calls(self) -> list[Any]: - return list(self.calls) - - -def test_token_usage_frame_skipped_when_no_summary() -> None: - svc = VercelStreamingService() - frames = list( - iter_token_usage_frame( - svc, - accumulator=_FakeAccumulator(summary=None), # type: ignore[arg-type] - log_label="parity-empty", - ) - ) - assert frames == [] - - -def test_token_usage_frame_emitted_when_summary_present() -> None: - svc = VercelStreamingService() - frames = list( - iter_token_usage_frame( - svc, - accumulator=_FakeAccumulator(summary=[{"m": "x", "t": 100}]), # type: ignore[arg-type] - log_label="parity-populated", - ) - ) - assert len(frames) == 1 - # Field shape on the wire is fixed by the FE; assert each surfaces. - payload = frames[0] - for key in ( - '"prompt_tokens":60', - '"completion_tokens":40', - '"total_tokens":100', - '"cost_micros":50000', - ): - assert key in payload.replace(" ", "") - - -# ------------------------------------------------------------------ llm_bundle - - -def test_load_llm_bundle_routes_negative_id_to_yaml_loader() -> None: - async def _run() -> tuple[Any, Any, str | None]: - with ( - patch( - "app.tasks.chat.streaming.flows.shared.llm_bundle.load_global_llm_config_by_id", - return_value=None, - ), - ): - return await load_llm_bundle( - session=AsyncMock(), # type: ignore[arg-type] - config_id=-1, - search_space_id=7, - ) - - llm, agent_config, error = asyncio.run(_run()) - assert llm is None - assert agent_config is None - assert error is not None and "id -1" in error - - -def test_load_llm_bundle_routes_nonnegative_id_to_db_loader() -> None: - async def _run() -> tuple[Any, Any, str | None]: - with ( - patch( - "app.tasks.chat.streaming.flows.shared.llm_bundle.load_agent_config", - new=AsyncMock(return_value=None), - ), - ): - return await load_llm_bundle( - session=AsyncMock(), # type: ignore[arg-type] - config_id=12, - search_space_id=7, - ) - - llm, agent_config, error = asyncio.run(_run()) - assert llm is None - assert agent_config is None - assert error is not None and "id 12" in error - - -# ----------------------------------------------------------------- premium quota - - -def test_needs_premium_quota_requires_user_and_premium_flag() -> None: - class _AgentConfig: - is_premium = True - - class _NonPremium: - is_premium = False - - assert needs_premium_quota(_AgentConfig(), "user-1") is True # type: ignore[arg-type] - assert needs_premium_quota(_AgentConfig(), None) is False # type: ignore[arg-type] - assert needs_premium_quota(_NonPremium(), "user-1") is False # type: ignore[arg-type] - assert needs_premium_quota(None, "user-1") is False - - -def test_premium_reservation_dataclass_shape() -> None: - # Sanity: the dataclass exists and carries the fields the orchestrator uses. - r = PremiumReservation(request_id="abc", reserved_micros=100, allowed=True) - assert r.request_id == "abc" - assert r.reserved_micros == 100 - assert r.allowed is True - - -# ----------------------------------------------------------- rate-limit guard - - -@pytest.mark.parametrize( - "first_event_seen, recovered, requested_id, current_id, expected", - [ - (False, False, 0, -1, True), - # Already recovered: no second pass. - (False, True, 0, -1, False), - # User explicitly picked a config: don't silently switch. - (False, False, 5, -1, False), - # Already on a database-backed (positive) id. - (False, False, 0, 7, False), - # User has already seen output: silent rebuild not possible. - (True, False, 0, -1, False), - ], -) -def test_can_recover_provider_rate_limit_truth_table( - first_event_seen: bool, - recovered: bool, - requested_id: int, - current_id: int, - expected: bool, -) -> None: - # Use a known rate-limit-shaped exception so the helper's last condition - # is satisfied; the guard only short-circuits to False when one of the - # *other* preconditions fails. - exc = Exception('{"error":{"type":"rate_limit_error","message":"slow"}}') - assert ( - can_recover_provider_rate_limit( - exc, - first_event_seen=first_event_seen, - runtime_rate_limit_recovered=recovered, - requested_llm_config_id=requested_id, - current_llm_config_id=current_id, - ) - is expected - ) - - -def test_can_recover_provider_rate_limit_rejects_non_rate_limit_exception() -> None: - assert ( - can_recover_provider_rate_limit( - ValueError("not a rate limit"), - first_event_seen=False, - runtime_rate_limit_recovered=False, - requested_llm_config_id=0, - current_llm_config_id=-1, - ) - is False - ) - - -# --------------------------------------------------------- persistence spawn - - -def test_spawn_set_ai_responding_bg_noop_without_user_id() -> None: - async def _run() -> set[asyncio.Task]: - background: set[asyncio.Task] = set() - spawn_set_ai_responding_bg(chat_id=1, user_id=None, background_tasks=background) - return background - - bg = asyncio.run(_run()) - assert bg == set() - - -def test_spawn_persist_user_task_registers_and_self_unregisters() -> None: - async def _run() -> tuple[int, int]: - background: set[asyncio.Task] = set() - with patch( - "app.tasks.chat.streaming.flows.new_chat.persistence_spawn.persist_user_turn", - new=AsyncMock(return_value=99), - ): - task = spawn_persist_user_task( - chat_id=1, - user_id="u", - turn_id="t", - user_query="hi", - user_image_data_urls=None, - mentioned_documents=None, - background_tasks=background, - ) - size_before_await = len(background) - result = await asyncio.shield(task) - # Give the done-callback one event-loop tick to run. - await asyncio.sleep(0) - return size_before_await, result # type: ignore[return-value] - - size_before, result = asyncio.run(_run()) - assert size_before == 1 - assert result == 99 - - -def test_spawn_persist_assistant_shell_task_registers() -> None: - async def _run() -> int | None: - background: set[asyncio.Task] = set() - with patch( - "app.tasks.chat.streaming.flows.new_chat.persistence_spawn.persist_assistant_shell", - new=AsyncMock(return_value=42), - ): - task = spawn_persist_assistant_shell_task( - chat_id=1, - user_id="u", - turn_id="t", - background_tasks=background, - ) - return await asyncio.shield(task) - - assert asyncio.run(_run()) == 42 - - -def test_await_persist_task_returns_none_on_failure() -> None: - async def _run() -> int | None: - async def _boom() -> int: - raise RuntimeError("DB down") - - task = asyncio.create_task(_boom()) - return await await_persist_task( - task, - chat_id=1, - turn_id="t", - log_label="parity-failure", - ) - - assert asyncio.run(_run()) is None - - -def test_await_persist_task_returns_none_for_none_input() -> None: - async def _run() -> int | None: - return await await_persist_task( - None, - chat_id=1, - turn_id="t", - log_label="parity-none", - ) - - assert asyncio.run(_run()) is None diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_1_parity.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_1_parity.py deleted file mode 100644 index 8fde773e3..000000000 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_1_parity.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Pin Stage 1 extractions as faithful copies of the old helpers. - -Extractions under ``app.tasks.chat.streaming`` are compared to -``app.tasks.chat.stream_new_chat`` helpers. -For each Stage 1 extraction we assert the new function returns the same -output as the old one for a representative input set. The moment the -two diverge - intentionally or otherwise - this file fails loudly so -the divergence is reviewed rather than shipped silently. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass, field -from typing import Any - -import pytest - -from app.agents.new_chat.errors import BusyError -from app.agents.new_chat.middleware.busy_mutex import request_cancel, reset_cancel -from app.tasks.chat.stream_new_chat import ( - _classify_stream_exception as old_classify, - _emit_stream_terminal_error as old_emit_terminal_error, - _extract_chunk_parts as old_extract_chunk_parts, - _extract_resolved_file_path as old_extract_resolved_file_path, - _tool_output_has_error as old_tool_output_has_error, - _tool_output_to_text as old_tool_output_to_text, -) -from app.tasks.chat.streaming.errors.classifier import ( - classify_stream_exception as new_classify, -) -from app.tasks.chat.streaming.errors.emitter import ( - emit_stream_terminal_error as new_emit_terminal_error, -) -from app.tasks.chat.streaming.helpers.chunk_parts import ( - extract_chunk_parts as new_extract_chunk_parts, -) -from app.tasks.chat.streaming.helpers.tool_output import ( - extract_resolved_file_path as new_extract_resolved_file_path, - tool_output_has_error as new_tool_output_has_error, - tool_output_to_text as new_tool_output_to_text, -) - -pytestmark = pytest.mark.unit - - -# ---------------------------------------------------------------- chunk parts - - -@dataclass -class _Chunk: - content: Any = "" - additional_kwargs: dict[str, Any] = field(default_factory=dict) - tool_call_chunks: list[dict[str, Any]] = field(default_factory=list) - - -_CHUNK_CASES: list[Any] = [ - None, - _Chunk(content=""), - _Chunk(content="hello"), - _Chunk(content=42), # invalid type, defensively coerced to empty - _Chunk( - content=[ - {"type": "text", "text": "Hello "}, - {"type": "text", "text": "world"}, - ] - ), - _Chunk( - content=[ - {"type": "reasoning", "reasoning": "hmm "}, - {"type": "reasoning", "text": "still"}, - {"type": "text", "text": "answer"}, - ] - ), - _Chunk( - content=[ - {"type": "tool_call_chunk", "id": "c1", "name": "x", "args": "{"}, - {"type": "tool_use", "id": "c2", "name": "y"}, - {"type": "image_url", "url": "ignored"}, - ] - ), - _Chunk( - content="visible", - additional_kwargs={"reasoning_content": "private"}, - ), - _Chunk( - tool_call_chunks=[ - {"id": None, "name": None, "args": '{"a":1}', "index": 0}, - {"id": "c", "name": "n", "args": "}", "index": 0}, - ] - ), - _Chunk( - content=[{"type": "tool_call_chunk", "id": "from-block", "name": "x"}], - tool_call_chunks=[{"id": "from-attr", "name": "y"}], - ), -] - - -@pytest.mark.parametrize("chunk", _CHUNK_CASES) -def test_extract_chunk_parts_matches_old_implementation(chunk: Any) -> None: - assert new_extract_chunk_parts(chunk) == old_extract_chunk_parts(chunk) - - -# ----------------------------------------------------------- error classifier - - -def _classify_cases() -> list[Exception]: - """Inputs that the FE depends on being mapped to specific error codes.""" - return [ - Exception("totally generic error"), - Exception('{"error":{"type":"rate_limit_error","message":"slow down"}}'), - Exception( - 'OpenrouterException - {"error":{"message":"Provider returned error",' - '"code":429}}' - ), - BusyError(request_id="thread-busy-parity"), - Exception("Thread is busy with another request"), - ] - - -@pytest.mark.parametrize("exc", _classify_cases()) -def test_classify_stream_exception_matches_old_implementation( - exc: Exception, -) -> None: - new = new_classify(exc, flow_label="parity-test") - old = old_classify(exc, flow_label="parity-test") - # Strip the wall-clock retry timestamp before comparing — both - # implementations call ``time.time()`` independently and the call - # order is enough to differ by 1 ms in practice. Every other field - # in the tuple must match exactly. - new_extra = dict(new[5]) if isinstance(new[5], dict) else new[5] - old_extra = dict(old[5]) if isinstance(old[5], dict) else old[5] - if isinstance(new_extra, dict) and isinstance(old_extra, dict): - new_extra.pop("retry_after_at", None) - old_extra.pop("retry_after_at", None) - assert new[:5] == old[:5] - assert new_extra == old_extra - - -def test_classify_turn_cancelling_branch_parity() -> None: - """The TURN_CANCELLING branch reads cancel state for the busy thread id; - both implementations must agree on retry-window semantics, not just the - plain THREAD_BUSY code.""" - thread_id = "parity-cancelling-thread" - reset_cancel(thread_id) - request_cancel(thread_id) - exc = BusyError(request_id=thread_id) - new = new_classify(exc, flow_label="parity-test") - old = old_classify(exc, flow_label="parity-test") - assert new[0] == old[0] == "thread_busy" - assert new[1] == old[1] == "TURN_CANCELLING" - assert isinstance(new[5], dict) and isinstance(old[5], dict) - assert new[5]["retry_after_ms"] == old[5]["retry_after_ms"] - - -# ------------------------------------------------------------ terminal emitter - - -class _FakeStreamingService: - """Duck-types ``format_error`` for both old and new emitters.""" - - def __init__(self) -> None: - self.calls: list[dict[str, Any]] = [] - - def format_error( - self, message: str, *, error_code: str, extra: dict[str, Any] | None = None - ) -> str: - self.calls.append( - {"message": message, "error_code": error_code, "extra": extra} - ) - return f'data: {{"type":"error","errorText":"{message}"}}\n\n' - - -def test_emit_stream_terminal_error_matches_old_output_and_logs(caplog) -> None: - """The new emitter must produce the same SSE frame and log the same - structured payload as the old one for the same arguments.""" - args: dict[str, Any] = { - "flow": "new", - "request_id": "req-parity", - "thread_id": 7, - "search_space_id": 9, - "user_id": "user-parity", - "message": "boom", - "error_kind": "server_error", - "error_code": "SERVER_ERROR", - "severity": "error", - "is_expected": False, - "extra": {"foo": "bar"}, - } - - new_svc = _FakeStreamingService() - old_svc = _FakeStreamingService() - - with caplog.at_level(logging.ERROR): - new_frame = new_emit_terminal_error(streaming_service=new_svc, **args) - old_frame = old_emit_terminal_error(streaming_service=old_svc, **args) - - assert new_frame == old_frame - assert new_svc.calls == old_svc.calls - chat_error_records = [ - r for r in caplog.records if "[chat_stream_error]" in r.message - ] - # One log line per emit call (two emits -> two records). - assert len(chat_error_records) == 2 - - -# ---------------------------------------------------------------- tool output - - -def test_tool_output_helpers_match_old_implementation() -> None: - samples: list[Any] = [ - {"result": "ok"}, - {"error": "bad"}, - {"result": "Error: x"}, - "Error: plain", - "fine", - {"nested": {"a": 1}}, - ] - for s in samples: - assert new_tool_output_to_text(s) == old_tool_output_to_text(s) - assert new_tool_output_has_error(s) == old_tool_output_has_error(s) - - assert new_extract_resolved_file_path( - tool_name="write_file", - tool_output={"path": " /tmp/x "}, - tool_input=None, - ) == old_extract_resolved_file_path( - tool_name="write_file", - tool_output={"path": " /tmp/x "}, - tool_input=None, - ) - assert new_extract_resolved_file_path( - tool_name="write_file", - tool_output={}, - tool_input={"file_path": " /fallback "}, - ) == old_extract_resolved_file_path( - tool_name="write_file", - tool_output={}, - tool_input={"file_path": " /fallback "}, - ) diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_2_parity.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_2_parity.py deleted file mode 100644 index 3ee1ab622..000000000 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_stage_2_parity.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Parity tests for Stage 2 extractions (tool matching, thinking step, custom events).""" - -from __future__ import annotations - -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from app.tasks.chat.stream_new_chat import _legacy_match_lc_id as old_legacy_match -from app.tasks.chat.streaming.handlers.custom_events import ( - handle_action_log, - handle_action_log_updated, - handle_document_created, - handle_report_progress, -) -from app.tasks.chat.streaming.helpers.tool_call_matching import ( - match_buffered_langchain_tool_call_id as new_legacy_match, -) -from app.tasks.chat.streaming.relay.state import AgentEventRelayState -from app.tasks.chat.streaming.relay.thinking_step_completion import ( - complete_active_thinking_step, -) -from app.tasks.chat.streaming.relay.thinking_step_sse import emit_thinking_step_frame - -pytestmark = pytest.mark.unit - - -def _copy_chunk_buffer(raw: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [dict(x) for x in raw] - - -def test_legacy_tool_call_match_matches_old_implementation() -> None: - cases: list[tuple[list[dict[str, Any]], str, str, dict[str, str]]] = [ - ( - [ - {"name": "write_file", "id": "lc-a"}, - {"name": "other", "id": "lc-b"}, - ], - "write_file", - "run-1", - {}, - ), - ( - [{"name": "x", "id": None}, {"name": "y", "id": "lc-fallback"}], - "write_file", - "run-2", - {}, - ), - ([{"name": "no_id"}], "write_file", "run-3", {}), - ] - for chunks_template, tool_name, run_id, lc_map_seed in cases: - old_chunks = _copy_chunk_buffer(chunks_template) - new_chunks = _copy_chunk_buffer(chunks_template) - old_map = dict(lc_map_seed) - new_map = dict(lc_map_seed) - old_out = old_legacy_match(old_chunks, tool_name, run_id, old_map) - new_out = new_legacy_match(new_chunks, tool_name, run_id, new_map) - assert new_out == old_out - assert new_chunks == old_chunks - assert new_map == old_map - - -def test_emit_thinking_step_frame_invokes_builder_before_service() -> None: - order: list[str] = [] - builder = MagicMock() - - def on_ts(*args: Any, **kwargs: Any) -> None: - order.append("builder") - - builder.on_thinking_step.side_effect = on_ts - - svc = MagicMock() - - def fmt(**kwargs: Any) -> str: - order.append("service") - return "frame" - - svc.format_thinking_step.side_effect = fmt - - out = emit_thinking_step_frame( - streaming_service=svc, - content_builder=builder, - step_id="thinking-1", - title="Working", - status="in_progress", - items=["a"], - ) - assert out == "frame" - assert order == ["builder", "service"] - builder.on_thinking_step.assert_called_once() - svc.format_thinking_step.assert_called_once() - - -def test_emit_thinking_step_frame_skips_builder_when_none() -> None: - svc = MagicMock(return_value="x") - svc.format_thinking_step.return_value = "frame" - assert ( - emit_thinking_step_frame( - streaming_service=svc, - content_builder=None, - step_id="s", - title="t", - ) - == "frame" - ) - svc.format_thinking_step.assert_called_once() - - -def test_complete_active_thinking_step_mirrors_closure_semantics() -> None: - svc = MagicMock() - svc.format_thinking_step.return_value = "done-frame" - completed: set[str] = set() - relay_state = AgentEventRelayState.for_invocation() - - frame, new_id = complete_active_thinking_step( - state=relay_state, - streaming_service=svc, - content_builder=None, - last_active_step_id="thinking-1", - last_active_step_title="T", - last_active_step_items=["x"], - completed_step_ids=completed, - ) - assert frame == "done-frame" - assert new_id is None - assert "thinking-1" in completed - - frame2, id2 = complete_active_thinking_step( - state=relay_state, - streaming_service=svc, - content_builder=None, - last_active_step_id="thinking-1", - last_active_step_title="T", - last_active_step_items=[], - completed_step_ids=completed, - ) - assert frame2 is None - assert id2 == "thinking-1" - - -def test_agent_event_relay_state_factory_matches_counter_rule() -> None: - s0 = AgentEventRelayState.for_invocation() - assert s0.thinking_step_counter == 0 - assert s0.last_active_step_id is None - - s1 = AgentEventRelayState.for_invocation( - initial_step_id="thinking-resume-1", - initial_step_title="Inherited", - initial_step_items=["Topic: X"], - ) - assert s1.thinking_step_counter == 1 - assert s1.last_active_step_id == "thinking-resume-1" - assert s1.next_thinking_step_id("thinking") == "thinking-2" - - -@pytest.mark.parametrize( - ("phase", "message", "start_items", "expected_tail"), - [ - ( - "revising_section", - "progress line", - ["Topic: Foo", "Modifying bar", "stale..."], - ["Topic: Foo", "Modifying bar", "progress line"], - ), - ( - "other", - "phase msg", - ["Topic: Foo", "old line"], - ["Topic: Foo", "phase msg"], - ), - ], -) -def test_report_progress_items_match_reference( - phase: str, - message: str, - start_items: list[str], - expected_tail: list[str], -) -> None: - svc = MagicMock() - svc.format_thinking_step.return_value = "sse" - - items = list(start_items) - frame, new_items = handle_report_progress( - {"message": message, "phase": phase}, - last_active_step_id="step-1", - last_active_step_title="Report", - last_active_step_items=items, - streaming_service=svc, - content_builder=None, - ) - assert frame == "sse" - assert new_items == expected_tail - kwargs = svc.format_thinking_step.call_args.kwargs - assert kwargs["items"] == expected_tail - - -def test_report_progress_noop_when_missing_message_or_step() -> None: - svc = MagicMock() - items = ["Topic: A"] - f1, i1 = handle_report_progress( - {"message": "", "phase": "x"}, - last_active_step_id="s", - last_active_step_title="t", - last_active_step_items=items, - streaming_service=svc, - content_builder=None, - ) - assert f1 is None and i1 is items - - f2, i2 = handle_report_progress( - {"message": "m", "phase": "x"}, - last_active_step_id=None, - last_active_step_title="t", - last_active_step_items=items, - streaming_service=svc, - content_builder=None, - ) - assert f2 is None and i2 is items - - -def test_document_action_handlers_match_format_data_guards() -> None: - svc = MagicMock() - svc.format_data.return_value = "data-frame" - - assert handle_document_created({}, streaming_service=svc) is None - assert handle_document_created({"id": 0}, streaming_service=svc) is None - handle_document_created({"id": 42, "title": "x"}, streaming_service=svc) - svc.format_data.assert_called_with( - "documents-updated", {"action": "created", "document": {"id": 42, "title": "x"}} - ) - - svc.reset_mock() - assert handle_action_log({"id": None}, streaming_service=svc) is None - handle_action_log({"id": 1}, streaming_service=svc) - svc.format_data.assert_called_once_with("action-log", {"id": 1}) - - svc.reset_mock() - assert handle_action_log_updated({"id": None}, streaming_service=svc) is None - handle_action_log_updated({"id": 2}, streaming_service=svc) - svc.format_data.assert_called_once_with("action-log-updated", {"id": 2}) diff --git a/surfsense_backend/tests/unit/tasks/chat/test_extract_chunk_parts.py b/surfsense_backend/tests/unit/tasks/chat/test_extract_chunk_parts.py index 1263a5fe1..0f154f1dc 100644 --- a/surfsense_backend/tests/unit/tasks/chat/test_extract_chunk_parts.py +++ b/surfsense_backend/tests/unit/tasks/chat/test_extract_chunk_parts.py @@ -1,4 +1,4 @@ -"""Unit tests for ``stream_new_chat._extract_chunk_parts``. +"""Unit tests for ``streaming.helpers.chunk_parts.extract_chunk_parts``. Earlier versions only handled ``isinstance(chunk.content, str)`` and silently dropped every other shape (Anthropic typed-block lists, @@ -14,7 +14,9 @@ from typing import Any import pytest -from app.tasks.chat.stream_new_chat import _extract_chunk_parts +from app.tasks.chat.streaming.helpers.chunk_parts import ( + extract_chunk_parts as _extract_chunk_parts, +) @dataclass diff --git a/surfsense_backend/tests/unit/tasks/chat/test_thinking_step_id_uniqueness.py b/surfsense_backend/tests/unit/tasks/chat/test_thinking_step_id_uniqueness.py index 50f7b8070..7f8285e98 100644 --- a/surfsense_backend/tests/unit/tasks/chat/test_thinking_step_id_uniqueness.py +++ b/surfsense_backend/tests/unit/tasks/chat/test_thinking_step_id_uniqueness.py @@ -7,7 +7,7 @@ constructs a fresh :class:`AgentEventRelayState` with ``thinking_step_counter=0``), React renders sibling timeline rows with the same key — the warning the user reported in production. -The contract this module pins: each ``_stream_agent_events`` invocation must +The contract this module pins: each ``stream_agent_events`` invocation must receive a ``step_prefix`` that is unique within the thread (we salt with the per-turn ``turn_id``), so the resulting step IDs across consecutive turns are always disjoint. @@ -23,10 +23,12 @@ from typing import Any import pytest from app.services.new_streaming_service import VercelStreamingService -from app.tasks.chat.stream_new_chat import ( - StreamResult, - _resume_step_prefix, - _stream_agent_events, +from app.tasks.chat.streaming.agent.event_loop import ( + stream_agent_events as _stream_agent_events, +) +from app.tasks.chat.streaming.shared.stream_result import StreamResult +from app.tasks.chat.streaming.shared.utils import ( + resume_step_prefix as _resume_step_prefix, ) pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/tasks/chat/test_tool_input_streaming.py b/surfsense_backend/tests/unit/tasks/chat/test_tool_input_streaming.py index ada32d168..f3f28eb1c 100644 --- a/surfsense_backend/tests/unit/tasks/chat/test_tool_input_streaming.py +++ b/surfsense_backend/tests/unit/tasks/chat/test_tool_input_streaming.py @@ -1,6 +1,6 @@ """Unit tests for live tool-call argument streaming. -Pins the wire format that ``_stream_agent_events`` emits: +Pins the wire format that ``stream_agent_events`` emits: ``tool-input-start`` → ``tool-input-delta``... → ``tool-input-available`` → ``tool-output-available``, keyed consistently with LangChain ``tool_call.id`` when the model streams indexed chunks. @@ -20,11 +20,13 @@ from typing import Any import pytest from app.services.new_streaming_service import VercelStreamingService -from app.tasks.chat.stream_new_chat import ( - StreamResult, - _legacy_match_lc_id, - _stream_agent_events, +from app.tasks.chat.streaming.agent.event_loop import ( + stream_agent_events as _stream_agent_events, ) +from app.tasks.chat.streaming.helpers.tool_call_matching import ( + match_buffered_langchain_tool_call_id as _legacy_match_lc_id, +) +from app.tasks.chat.streaming.shared.stream_result import StreamResult pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/test_stream_new_chat_contract.py b/surfsense_backend/tests/unit/test_stream_new_chat_contract.py deleted file mode 100644 index 19b06201f..000000000 --- a/surfsense_backend/tests/unit/test_stream_new_chat_contract.py +++ /dev/null @@ -1,474 +0,0 @@ -import inspect -import json -import logging -import re -from pathlib import Path - -import pytest - -import app.tasks.chat.stream_new_chat as stream_new_chat_module -from app.agents.new_chat.errors import BusyError -from app.agents.new_chat.middleware.busy_mutex import request_cancel, reset_cancel -from app.tasks.chat.stream_new_chat import ( - StreamResult, - _classify_stream_exception, - _contract_enforcement_active, - _evaluate_file_contract_outcome, - _extract_resolved_file_path, - _log_chat_stream_error, - _tool_output_has_error, -) - -pytestmark = pytest.mark.unit - - -def test_tool_output_error_detection(): - assert _tool_output_has_error("Error: failed to write file") - assert _tool_output_has_error({"error": "boom"}) - assert _tool_output_has_error({"result": "Error: disk is full"}) - assert not _tool_output_has_error({"result": "Updated file /notes.md"}) - - -def test_extract_resolved_file_path_prefers_structured_path(): - assert ( - _extract_resolved_file_path( - tool_name="write_file", - tool_output={"status": "completed", "path": "/docs/note.md"}, - tool_input=None, - ) - == "/docs/note.md" - ) - - -def test_extract_resolved_file_path_falls_back_to_tool_input(): - assert ( - _extract_resolved_file_path( - tool_name="edit_file", - tool_output={"status": "completed", "result": "updated"}, - tool_input={"file_path": "/docs/edited.md"}, - ) - == "/docs/edited.md" - ) - - -def test_extract_resolved_file_path_does_not_parse_result_text(): - assert ( - _extract_resolved_file_path( - tool_name="write_file", - tool_output={"result": "Updated file /docs/from-text.md"}, - tool_input=None, - ) - is None - ) - - -def test_file_write_contract_outcome_reasons(): - result = StreamResult(intent_detected="file_write") - passed, reason = _evaluate_file_contract_outcome(result) - assert not passed - assert reason == "no_write_attempt" - - result.write_attempted = True - passed, reason = _evaluate_file_contract_outcome(result) - assert not passed - assert reason == "write_failed" - - result.write_succeeded = True - passed, reason = _evaluate_file_contract_outcome(result) - assert not passed - assert reason == "verification_failed" - - result.verification_succeeded = True - passed, reason = _evaluate_file_contract_outcome(result) - assert passed - assert reason == "" - - -def test_contract_enforcement_local_only(): - result = StreamResult(filesystem_mode="desktop_local_folder") - assert _contract_enforcement_active(result) - - result.filesystem_mode = "cloud" - assert not _contract_enforcement_active(result) - - -def _extract_chat_stream_payload(record_message: str) -> dict: - prefix = "[chat_stream_error] " - assert record_message.startswith(prefix) - return json.loads(record_message[len(prefix) :]) - - -def test_unified_chat_stream_error_log_schema(caplog): - with caplog.at_level(logging.INFO, logger="app.tasks.chat.stream_new_chat"): - _log_chat_stream_error( - flow="new", - error_kind="server_error", - error_code="SERVER_ERROR", - severity="warn", - is_expected=False, - request_id="req-123", - thread_id=101, - search_space_id=202, - user_id="user-1", - message="Error during chat: boom", - ) - - record = next(r for r in caplog.records if "[chat_stream_error]" in r.message) - payload = _extract_chat_stream_payload(record.message) - - required_keys = { - "event", - "flow", - "error_kind", - "error_code", - "severity", - "is_expected", - "request_id", - "thread_id", - "search_space_id", - "user_id", - "message", - } - assert required_keys.issubset(payload.keys()) - assert payload["event"] == "chat_stream_error" - assert payload["flow"] == "new" - assert payload["error_code"] == "SERVER_ERROR" - - -def test_premium_quota_uses_unified_chat_stream_log_shape(caplog): - with caplog.at_level(logging.INFO, logger="app.tasks.chat.stream_new_chat"): - _log_chat_stream_error( - flow="resume", - error_kind="premium_quota_exhausted", - error_code="PREMIUM_QUOTA_EXHAUSTED", - severity="info", - is_expected=True, - request_id="req-premium", - thread_id=303, - search_space_id=404, - user_id="user-2", - message="Buy more tokens to continue with this model, or switch to a free model", - extra={"auto_fallback": False}, - ) - - record = next(r for r in caplog.records if "[chat_stream_error]" in r.message) - payload = _extract_chat_stream_payload(record.message) - assert payload["event"] == "chat_stream_error" - assert payload["error_kind"] == "premium_quota_exhausted" - assert payload["error_code"] == "PREMIUM_QUOTA_EXHAUSTED" - assert payload["flow"] == "resume" - assert payload["is_expected"] is True - assert payload["auto_fallback"] is False - - -def test_stream_error_emission_keeps_machine_error_codes(): - source = inspect.getsource(stream_new_chat_module) - format_error_calls = re.findall(r"format_error\(", source) - emitted_error_codes = set(re.findall(r'error_code="([A-Z_]+)"', source)) - - # All stream paths should route through one shared terminal error emitter. - assert len(format_error_calls) == 1 - assert { - "PREMIUM_QUOTA_EXHAUSTED", - "SERVER_ERROR", - }.issubset(emitted_error_codes) - assert 'flow: Literal["new", "regenerate"] = "new"' in source - assert "_emit_stream_terminal_error" in source - assert "flow=flow" in source - assert 'flow="resume"' in source - - -def test_stream_exception_classifies_rate_limited(): - exc = Exception( - '{"error":{"type":"rate_limit_error","message":"Rate limited. Please try again later."}}' - ) - kind, code, severity, is_expected, user_message, extra = _classify_stream_exception( - exc, flow_label="chat" - ) - assert kind == "rate_limited" - assert code == "RATE_LIMITED" - assert severity == "warn" - assert is_expected is True - assert "temporarily rate-limited" in user_message - assert extra is None - - -def test_stream_exception_classifies_openrouter_429_payload(): - exc = Exception( - 'OpenrouterException - {"error":{"message":"Provider returned error","code":429,' - '"metadata":{"raw":"foo is temporarily rate-limited upstream"}}}' - ) - kind, code, severity, is_expected, user_message, extra = _classify_stream_exception( - exc, flow_label="chat" - ) - assert kind == "rate_limited" - assert code == "RATE_LIMITED" - assert severity == "warn" - assert is_expected is True - assert "temporarily rate-limited" in user_message - assert extra is None - - -def test_stream_exception_classifies_thread_busy(): - exc = BusyError(request_id="thread-123") - kind, code, severity, is_expected, user_message, extra = _classify_stream_exception( - exc, flow_label="chat" - ) - assert kind == "thread_busy" - assert code == "THREAD_BUSY" - assert severity == "warn" - assert is_expected is True - assert "still finishing for this thread" in user_message - assert extra is None - - -def test_stream_exception_classifies_thread_busy_from_message(): - exc = Exception("Thread is busy with another request") - kind, code, severity, is_expected, user_message, extra = _classify_stream_exception( - exc, flow_label="chat" - ) - assert kind == "thread_busy" - assert code == "THREAD_BUSY" - assert severity == "warn" - assert is_expected is True - assert "still finishing for this thread" in user_message - assert extra is None - - -def test_stream_exception_classifies_turn_cancelling_when_cancel_requested(): - thread_id = "thread-cancelling-1" - reset_cancel(thread_id) - request_cancel(thread_id) - exc = BusyError(request_id=thread_id) - kind, code, severity, is_expected, user_message, extra = _classify_stream_exception( - exc, flow_label="chat" - ) - assert kind == "thread_busy" - assert code == "TURN_CANCELLING" - assert severity == "info" - assert is_expected is True - assert "stopping" in user_message - assert isinstance(extra, dict) - assert "retry_after_ms" in extra - - -def test_premium_classification_is_error_code_driven(): - classifier_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/lib/chat/chat-error-classifier.ts" - ) - source = classifier_path.read_text(encoding="utf-8") - - assert "PREMIUM_KEYWORDS" not in source - assert "RATE_LIMIT_KEYWORDS" not in source - assert "normalized.includes(" not in source - assert 'if (errorCode === "PREMIUM_QUOTA_EXHAUSTED") {' in source - - -def test_stream_terminal_error_handler_has_pre_accept_soft_rollback_hook(): - page_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx" - ) - source = page_path.read_text(encoding="utf-8") - - assert "onPreAcceptFailure?: () => Promise;" in source - assert "if (!accepted) {" in source - assert "await onPreAcceptFailure?.();" in source - assert "await onAcceptedStreamError?.();" in source - assert "setMessages((prev) => prev.filter((m) => m.id !== userMsgId));" in source - assert "setMessageDocumentsMap((prev) => {" in source - - -def test_toast_only_pre_accept_policy_has_no_inline_failed_marker(): - user_message_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/components/assistant-ui/user-message.tsx" - ) - source = user_message_path.read_text(encoding="utf-8") - - assert "Not sent. Edit and retry." not in source - assert "failed_pre_accept" not in source - - -def test_network_send_failures_use_unified_retry_toast_message(): - classifier_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/lib/chat/chat-error-classifier.ts" - ) - classifier_source = classifier_path.read_text(encoding="utf-8") - request_errors_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/lib/chat/chat-request-errors.ts" - ) - request_errors_source = request_errors_path.read_text(encoding="utf-8") - - assert '"send_failed_pre_accept"' in classifier_source - assert 'errorCode === "SEND_FAILED_PRE_ACCEPT"' in classifier_source - assert 'errorCode === "TURN_CANCELLING"' in classifier_source - assert "if (withCode.code) return withCode.code;" in classifier_source - assert 'userMessage: "Message not sent. Please retry."' in classifier_source - assert 'userMessage: "Connection issue. Please try again."' in classifier_source - assert "const passthroughCodes = new Set([" in request_errors_source - assert '"PREMIUM_QUOTA_EXHAUSTED"' in request_errors_source - assert '"THREAD_BUSY"' in request_errors_source - assert '"TURN_CANCELLING"' in request_errors_source - assert '"AUTH_EXPIRED"' in request_errors_source - assert '"UNAUTHORIZED"' in request_errors_source - assert '"RATE_LIMITED"' in request_errors_source - assert '"NETWORK_ERROR"' in request_errors_source - assert '"STREAM_PARSE_ERROR"' in request_errors_source - assert '"TOOL_EXECUTION_ERROR"' in request_errors_source - assert '"PERSIST_MESSAGE_FAILED"' in request_errors_source - assert '"SERVER_ERROR"' in request_errors_source - assert "passthroughCodes.has(existingCode)" in request_errors_source - assert 'errorCode: "SEND_FAILED_PRE_ACCEPT"' in request_errors_source - assert 'errorCode: "NETWORK_ERROR"' not in request_errors_source - assert "Failed to start chat. Please try again." not in classifier_source - - -def test_pre_post_accept_abort_contract_exists_for_new_resume_regenerate_flows(): - page_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx" - ) - source = page_path.read_text(encoding="utf-8") - - # Each flow tracks accepted boundary and passes it into shared terminal handling. - # The acceptance boundary is still meaningful post-refactor: it gates - # local-state cleanup (onPreAcceptFailure path) and lets the shared - # terminal handler distinguish pre-accept aborts from in-stream errors. - assert "let newAccepted = false;" in source - assert "let resumeAccepted = false;" in source - assert "let regenerateAccepted = false;" in source - assert "accepted: newAccepted," in source - assert "accepted: resumeAccepted," in source - assert "accepted: regenerateAccepted," in source - - # NOTE: The FE-side persistence guards previously asserted here - # ("if (!resumeAccepted) return;", "if (!regenerateAccepted) return;", - # "if (newAccepted && !userPersisted) {") have been intentionally - # removed by the SSE-based message-id handshake refactor. Persistence - # is now server-authoritative: persist_user_turn / persist_assistant_shell - # run inside stream_new_chat / stream_resume_chat unconditionally and - # the FE consumes data-user-message-id / data-assistant-message-id - # SSE events to learn the canonical primary keys. There is therefore - # no FE call-site to guard, and the shared terminal handler relies - # purely on the `accepted` field above (forwarded to onAbort / - # onAcceptedStreamError) to drive UI cleanup. See - # tests/integration/chat/test_message_id_sse.py for the new - # cross-tier ID coherence guarantees. - - # The TURN_CANCELLING / THREAD_BUSY retry plumbing is independent - # of the persistence refactor and must still exist on every - # start-stream fetch. - assert "const fetchWithTurnCancellingRetry = useCallback(" in source - assert "computeFallbackTurnCancellingRetryDelay" in source - assert 'withMeta.errorCode === "TURN_CANCELLING"' in source - assert 'withMeta.errorCode === "THREAD_BUSY"' in source - assert "await fetchWithTurnCancellingRetry(() =>" in source - - -def test_cancel_active_turn_route_contract_exists(): - routes_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_backend/app/routes/new_chat_routes.py" - ) - source = routes_path.read_text(encoding="utf-8") - - assert '@router.post(\n "/threads/{thread_id}/cancel-active-turn",' in source - assert "response_model=CancelActiveTurnResponse" in source - assert 'status="cancelling",' in source - assert 'error_code="TURN_CANCELLING",' in source - assert "retry_after_ms=retry_after_ms if retry_after_ms > 0 else None," in source - assert "retry_after_at=" in source - assert 'status="idle",' in source - assert 'error_code="NO_ACTIVE_TURN",' in source - - -def test_turn_status_route_contract_exists(): - routes_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_backend/app/routes/new_chat_routes.py" - ) - source = routes_path.read_text(encoding="utf-8") - - assert '@router.get(\n "/threads/{thread_id}/turn-status",' in source - assert "response_model=TurnStatusResponse" in source - assert "_build_turn_status_payload(thread_id)" in source - assert "Permission.CHATS_READ.value" in source - assert "_raise_if_thread_busy_for_start(" in source - - -def test_turn_cancelling_retry_policy_contract_exists(): - routes_path = ( - Path(__file__).resolve().parents[3] - / "surfsense_backend/app/routes/new_chat_routes.py" - ) - source = routes_path.read_text(encoding="utf-8") - - assert "TURN_CANCELLING_INITIAL_DELAY_MS = 200" in source - assert "TURN_CANCELLING_BACKOFF_FACTOR = 2" in source - assert "TURN_CANCELLING_MAX_DELAY_MS = 1500" in source - assert "def _compute_turn_cancelling_retry_delay(" in source - assert "retry-after-ms" in source - assert '"Retry-After"' in source - assert '"errorCode": "TURN_CANCELLING"' in source - - -def test_turn_status_sse_contract_exists(): - stream_source = ( - Path(__file__).resolve().parents[3] - / "surfsense_backend/app/tasks/chat/stream_new_chat.py" - ).read_text(encoding="utf-8") - state_source = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/lib/chat/streaming-state.ts" - ).read_text(encoding="utf-8") - pipeline_source = ( - Path(__file__).resolve().parents[3] - / "surfsense_web/lib/chat/stream-pipeline.ts" - ).read_text(encoding="utf-8") - - assert '"turn-status"' in stream_source - assert '"status": "busy"' in stream_source - assert '"status": "idle"' in stream_source - assert 'type: "data-turn-status"' in state_source - assert 'case "data-turn-status":' in pipeline_source - assert "end_turn(str(chat_id))" in stream_source - - -def test_chat_deepagent_forwards_resolved_model_name_to_both_builders(): - """Regression guard: both system-prompt builders in chat_deepagent.py - must receive ``model_name=_resolve_prompt_model_name(...)`` so the - provider-variant dispatch can render the right ```` - block. Without this the prompt silently falls back to the empty - ``"default"`` variant — the original bug being fixed. - - This test mirrors :func:`test_stream_error_emission_keeps_machine_error_codes` - in style: it inspects module source text + a regex to enforce the - call-site shape, not just the wrapper layer (the wrappers already - forward ``model_name`` correctly, so testing them would not catch - the actual missed plumbing). - """ - import app.agents.new_chat.chat_deepagent as chat_deepagent_module - - source = inspect.getsource(chat_deepagent_module) - - # Helper itself must be defined. - assert "def _resolve_prompt_model_name(" in source - - # Both builder calls must forward the resolved model name. Match - # across newlines + whitespace because the kwargs are split over - # multiple lines. - pattern = re.compile( - r"build_(?:surfsense|configurable)_system_prompt\([^)]*" - r"model_name=_resolve_prompt_model_name\(", - re.DOTALL, - ) - matches = pattern.findall(source) - assert len(matches) == 2, ( - "Expected both system-prompt builder call sites to forward " - "`model_name=_resolve_prompt_model_name(...)`, found " - f"{len(matches)}" - ) diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index eae54b1d4..a927a928d 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -2,18 +2,581 @@ version = 1 revision = 1 requires-python = ">=3.12" resolution-markers = [ - "python_full_version >= '3.14' and python_full_version < '4' and sys_platform == 'win32'", - "python_full_version >= '3.14' and python_full_version < '4' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and python_full_version < '4' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '4' and sys_platform == 'win32'", - "python_full_version >= '4' and sys_platform == 'emscripten'", - "python_full_version >= '4' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'win32'", - "python_full_version == '3.13.*' and sys_platform == 'emscripten'", - "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.13' and sys_platform != 'win32'", - "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] +conflicts = [[ + { package = "surf-new-backend", extra = "cpu" }, + { package = "surf-new-backend", extra = "cu126" }, + { package = "surf-new-backend", extra = "cu128" }, +]] [[package]] name = "accelerate" @@ -26,7 +589,10 @@ dependencies = [ { name = "psutil" }, { name = "pyyaml" }, { name = "safetensors" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ca/14/787e5498cd062640f0f3d92ef4ae4063174f76f9afd29d13fc52a319daae/accelerate-1.13.0.tar.gz", hash = "sha256:d631b4e0f5b3de4aff2d7e9e6857d164810dfc3237d54d017f075122d057b236", size = 402835 } wheels = [ @@ -172,7 +738,7 @@ version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007 } wheels = [ @@ -263,13 +829,22 @@ version = "4.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622 } wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353 }, ] +[[package]] +name = "apify-fingerprint-datapoints" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/f1/b74f95767581372ab849c8b13e384b62f60d034584892c60c4a3442d9312/apify_fingerprint_datapoints-0.13.0.tar.gz", hash = "sha256:263141c19e9bc90a821e6b4e2b845925f17e0b8fbd53a897fc71546bd50df7f1", size = 934827 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/58/8402442bf6af5a3a8068fe5431c42ea4f73c1eb18f621f9bf7c5de80caf5/apify_fingerprint_datapoints-0.13.0-py3-none-any.whl", hash = "sha256:0213d42297be19e8035202b41fb2e840a1e5d79874c99c882a5027a7d0b1a0eb", size = 761652 }, +] + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -506,6 +1081,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138 }, ] +[[package]] +name = "azure-storage-blob" +version = "12.29.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "isodate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/25/fdcf1e381922dbab8ba23d6fd78d397fe6cbac6b480310218834b7bc91fe/azure_storage_blob-12.29.0.tar.gz", hash = "sha256:2824ddd7ebc9056034ebc76b17971a38e9aa5835abb0d565b9700493f2a6c657", size = 611359 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2c/6ddee6a3e42d0236ba9259e4df7fa97fdc415ff0802b736c634baaf4b285/azure_storage_blob-12.29.0-py3-none-any.whl", hash = "sha256:ccf8a1bcd5e49df83ab85aab793b579e5ba2eeea2ad8900b2f62ca3a37dc391f", size = 434823 }, +] + [[package]] name = "babel" version = "2.18.0" @@ -689,12 +1279,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/2a/9186535ce58db529927f6cf5990a849aa9e052eea3e2cfefe20b9e1802da/bracex-2.6-py3-none-any.whl", hash = "sha256:0b0049264e7340b3ec782b5cb99beb325f36c3782a32e36e876452fd49a09952", size = 11508 }, ] +[[package]] +name = "browserforge" +version = "1.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apify-fingerprint-datapoints" }, + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/6f/8975af88d203efd70cc69477ebac702babef38201d04621c9583f2508f25/browserforge-1.2.4.tar.gz", hash = "sha256:05686473793769856ebd3528c69071f5be0e511260993e8b2ba839863711a0c4", size = 36700 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/35/ce962f738ae28ffce6293e7607b129075633e6bb185a5ab87e49246eedc2/browserforge-1.2.4-py3-none-any.whl", hash = "sha256:fb1c14e62ac09de221dcfc73074200269f697596c642cb200ceaab1127a17542", size = 37890 }, +] + [[package]] name = "build" version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "os_name == 'nt'" }, + { name = "colorama", marker = "(os_name == 'nt' and sys_platform != 'linux') or (os_name != 'nt' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (os_name != 'nt' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (os_name != 'nt' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (os_name == 'nt' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "packaging" }, { name = "pyproject-hooks" }, ] @@ -793,7 +1396,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588 } wheels = [ @@ -964,7 +1567,10 @@ all = [ { name = "tabulate" }, { name = "tiktoken" }, { name = "tokenizers" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "transformers" }, { name = "tree-sitter" }, { name = "tree-sitter-language-pack" }, @@ -1046,14 +1652,14 @@ wheels = [ [[package]] name = "click" -version = "8.1.8" +version = "8.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } +sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, + { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639 }, ] [[package]] @@ -1135,7 +1741,7 @@ name = "colorlog" version = "6.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/61/f083b5ac52e505dfc1c624eafbf8c7589a0d7f32daa398d2e7590efa5fda/colorlog-6.10.1.tar.gz", hash = "sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321", size = 17162 } wheels = [ @@ -1190,7 +1796,7 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174 } wheels = [ @@ -1282,7 +1888,7 @@ name = "cryptography" version = "46.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a4/ba/04b1bd4218cbc58dc90ce967106d51582371b898690f3ae0402876cc4f34/cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759", size = 750542 } wheels = [ @@ -1330,6 +1936,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/82/ca4893968aeb2709aacfb57a30dec6fa2ab25b10fa9f064b8882ce33f599/cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f", size = 3471160 }, ] +[[package]] +name = "cssselect" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/2e/cdfd8b01c37cbf4f9482eefd455853a3cf9c995029a46acd31dfaa9c1dd6/cssselect-1.4.0.tar.gz", hash = "sha256:fdaf0a1425e17dfe8c5cf66191d211b357cf7872ae8afc4c6762ddd8ac47fc92", size = 40589 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/0c/7bb51e3acfafd16c48875bf3db03607674df16f5b6ef8d056586af7e2b8b/cssselect-1.4.0-py3-none-any.whl", hash = "sha256:c0ec5c0191c8ee39fcc8afc1540331d8b55b0183478c50e9c8a79d44dbceb1d8", size = 18540 }, +] + [[package]] name = "csvw" version = "1.11.0" @@ -1378,22 +1993,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/5c/9fa0ad6462b62efd0fb5ac1100eee47bc96ecc198ff4e237c731e5473616/ctranslate2-4.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:dfb7657bdb7b8211c8f9ecb6f3b70bc0db0e0384d01a8b1808cb66fe7199df59", size = 19123451 }, ] +[[package]] +name = "cuda-bindings" +version = "12.9.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cuda-pathfinder", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/45/557d4ed1fa54f0c7db8aee083229f624990d69f7d00f55477eed5c7e169a/cuda_bindings-12.9.7-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0666d3c082ef8f4b2d670950589373550e9f3bf564d635dd883f24a0b40402ff", size = 7071026 }, + { url = "https://files.pythonhosted.org/packages/91/97/e3c6e58ece26a053419ba0a18444b5443cfc64451bbf37f84e8143b8bdca/cuda_bindings-12.9.7-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c7ef48c5e13ae90f3b2ecfb72f8e99ac43c8f4c43e67e1325b8aae331453687", size = 7611059 }, + { url = "https://files.pythonhosted.org/packages/6d/39/afaa3de4d491a55af8961081e0b69c08d51bfbe471c359a7bddb4a28ca41/cuda_bindings-12.9.7-cp312-cp312-win_amd64.whl", hash = "sha256:3c089aaf4f5f570ec50244c68f5a2b00a2c9a8e01e04219fd2e36e340be0d88b", size = 7400841 }, + { url = "https://files.pythonhosted.org/packages/eb/7b/f1575e41e1a17dc2f2a408b2e8e864c9324e41e3e23f6401e5efc54c152a/cuda_bindings-12.9.7-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:266379e4942051f544a8e7ea1a30ead8d7e8199b6b30fcdc8917cae2bf614e61", size = 6978549 }, + { url = "https://files.pythonhosted.org/packages/9d/dc/62d62eb4f91eb721bcf46da51b13e9872ccd8fa7e60eb8ba7b7baeac72c6/cuda_bindings-12.9.7-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59cf4a37b0d662ba15037c9ceebe1a306ebf2c01a8235a09be13cd07094fdb74", size = 7457675 }, + { url = "https://files.pythonhosted.org/packages/43/b2/753fe88151001d0dc23f56a8e119fe06b991b0d1a885fa02f9852b12f523/cuda_bindings-12.9.7-cp313-cp313-win_amd64.whl", hash = "sha256:5bd89dcb78475a6d8a4620ea94b74edf0cbbeacee6d1622d8f94452c1e8d3f15", size = 7360097 }, + { url = "https://files.pythonhosted.org/packages/f9/77/94d9b85f26add6fe9c9cb7c4ec3b96bc598f7ea5cfbd7490cc0a36adf5be/cuda_bindings-12.9.7-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2dbcd4801954eb3508f4dc2fa0d0c8eb93eb3f45326fd61be2731418c371e7a0", size = 6870886 }, + { url = "https://files.pythonhosted.org/packages/04/dd/3ec34b569e1b990b11276feba306bf8f446656cc38e8ed0f49b5facfeffa/cuda_bindings-12.9.7-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3747ea132642416786a8e31bf229032df3a7856911ae5426a7be53d032df183d", size = 7345663 }, + { url = "https://files.pythonhosted.org/packages/b8/c8/d79a20ba396e7ab2dfdd4b72b62356972b25b88aee2ded49a70c797ddea1/cuda_bindings-12.9.7-cp313-cp313t-win_amd64.whl", hash = "sha256:64f7ade7a7a3b69001489753acc21706d9dbda32db8deb68a767a0a0aab30b68", size = 7780136 }, + { url = "https://files.pythonhosted.org/packages/68/e4/075052d42872cf8162da53f14447a4b8abc004c3750e4b724ee502428da0/cuda_bindings-12.9.7-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:775960ac9e530717f3b48e165cc6f68684fa9a4141764fd923e4c1a9820acc73", size = 7060090 }, + { url = "https://files.pythonhosted.org/packages/ec/cd/3289c810a4d45e5364a3387a74b4c9b6f6f57ee96ae0e5b537cc61dec242/cuda_bindings-12.9.7-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c47ec1a7a441d91aab32339951df7a1be53451121a12c094bba51467717a35a", size = 7504419 }, + { url = "https://files.pythonhosted.org/packages/5b/a0/c429fdcfa5aae181415504c5085ea5944f782b417dd16a7f2a14be0da80d/cuda_bindings-12.9.7-cp314-cp314-win_amd64.whl", hash = "sha256:1e2a4f2ec5b67408c04bb4fbed45d214b66de1f00ee2e972865cacb8708d4e1e", size = 7493876 }, + { url = "https://files.pythonhosted.org/packages/11/43/472a6281c3d94e71687e27c657a8f60718d3579b4d94c41deea503165f8a/cuda_bindings-12.9.7-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:00a833d399b31071fab4cf3de2929840ae462dc4848116eeff033d09219e7116", size = 6899146 }, + { url = "https://files.pythonhosted.org/packages/2b/13/10c1d0b32a9da65142d213e0733d748457fb3fd066aee4317335266f15c6/cuda_bindings-12.9.7-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11aeafa2b33995f890086b3fb0f062075176d956e9b6a6fe1a699dddc413f6ad", size = 7369087 }, + { url = "https://files.pythonhosted.org/packages/33/10/c71a07cd2a1d4db119bada1848b4752a874ccfe4927d419bfdd05f250920/cuda_bindings-12.9.7-cp314-cp314t-win_amd64.whl", hash = "sha256:ece8dfbc22e6de96a26940ab9887eb3cfe1fc1bc3966169391cdb866bb82bb64", size = 8208198 }, +] + [[package]] name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'win32'", +] dependencies = [ - { name = "cuda-pathfinder", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cuda-pathfinder", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404 }, { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619 }, + { url = "https://files.pythonhosted.org/packages/bb/a5/d7f01a415e134546248cef612adad8153c9f1eb10ec79505a7cd8294370b/cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d", size = 5840830 }, { url = "https://files.pythonhosted.org/packages/df/93/eef988860a3ca985f82c4f3174fc0cdd94e07331ba9a92e8e064c260337f/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0", size = 5614610 }, { url = "https://files.pythonhosted.org/packages/18/23/6db3aba46864aee357ab2415135b3fe3da7e9f1fa0221fa2a86a5968099c/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d", size = 6149914 }, + { url = "https://files.pythonhosted.org/packages/c4/84/d3b6220b51cbc02ca14db7387e97445126b4ff5125aaa6c5dd7dcb75e679/cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8", size = 5796512 }, { url = "https://files.pythonhosted.org/packages/c0/87/87a014f045b77c6de5c8527b0757fe644417b184e5367db977236a141602/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e", size = 5685673 }, { url = "https://files.pythonhosted.org/packages/ee/5e/c0fe77a73aaefd3fff25ffaccaac69c5a63eafdf8b9a4c476626ef0ac703/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626", size = 6191386 }, + { url = "https://files.pythonhosted.org/packages/e3/73/98bcb069778fe420226db75aff54b5dd6c3ecfd0912edabab723326e80b7/cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2", size = 5938605 }, { url = "https://files.pythonhosted.org/packages/5f/58/ed2c3b39c8dd5f96aa7a4abef0d47a73932c7a988e30f5fa428f00ed0da1/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771", size = 5507469 }, { url = "https://files.pythonhosted.org/packages/1f/01/0c941b112ceeb21439b05895eace78ca1aa2eaaf695c8521a068fd9b4c00/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b", size = 6059693 }, + { url = "https://files.pythonhosted.org/packages/52/49/4e01cc06447d39476e138d1b1adec8d35c0d04eccd2c8d69befc08cd66e8/cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7", size = 6662637 }, ] [[package]] @@ -1404,47 +2058,148 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/93/66/0c02bd330e7d976f83fa68583d6198d76f23581bcbb5c0e98a6148f326e5/cuda_pathfinder-1.5.0-py3-none-any.whl", hash = "sha256:498f90a9e9de36044a7924742aecce11c50c49f735f1bc53e05aa46de9ea4110", size = 49739 }, ] +[[package]] +name = "cuda-toolkit" +version = "12.6.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/88/2dbc37975fffb874418b14380418a1b99cb36f2101fd1d08c54e06ee8c95/cuda_toolkit-12.6.3-py2.py3-none-any.whl", hash = "sha256:79d8605baeb6c2f695761e0efb54bc62dbc3c9e32eb0742df7669c07befaa8f7", size = 2288 }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas-cu12", version = "12.6.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cudart = [ + { name = "nvidia-cuda-runtime-cu12", version = "12.6.77", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cufft = [ + { name = "nvidia-cufft-cu12", version = "11.3.0.4", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cufile = [ + { name = "nvidia-cufile-cu12", version = "1.11.1.6", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cupti = [ + { name = "nvidia-cuda-cupti-cu12", version = "12.6.80", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +curand = [ + { name = "nvidia-curand-cu12", version = "10.3.7.77", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cusolver = [ + { name = "nvidia-cusolver-cu12", version = "11.7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cusparse = [ + { name = "nvidia-cusparse-cu12", version = "12.5.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink-cu12", version = "12.6.85", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc-cu12", version = "12.6.85", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvtx = [ + { name = "nvidia-nvtx-cu12", version = "12.6.77", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] + +[[package]] +name = "cuda-toolkit" +version = "12.8.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/c8/7dce3a0b15b42a3b58e7d96eb22a687d3bf2c44e01d149a6874629cd9938/cuda_toolkit-12.8.1-py2.py3-none-any.whl", hash = "sha256:adc7906af4ecbf9a352f9dca5734eceb21daec281ccfcf5675e1d2f724fc2cba", size = 2283 }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cudart = [ + { name = "nvidia-cuda-runtime-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cufft = [ + { name = "nvidia-cufft-cu12", version = "11.3.3.83", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cufile = [ + { name = "nvidia-cufile-cu12", version = "1.13.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cupti = [ + { name = "nvidia-cuda-cupti-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +curand = [ + { name = "nvidia-curand-cu12", version = "10.3.9.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cusolver = [ + { name = "nvidia-cusolver-cu12", version = "11.7.3.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +cusparse = [ + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +nvtx = [ + { name = "nvidia-nvtx-cu12", version = "12.8.90", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] + [[package]] name = "cuda-toolkit" version = "13.0.2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'win32'", +] wheels = [ { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364 }, ] [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufft", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux'" }, + { name = "nvidia-curand", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusolver", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvtx", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] [[package]] @@ -1468,13 +2223,49 @@ name = "curated-transformers" version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/70/06/6c12c149a7f737dacc76b4c3949dbc7ff87d622567b86996896ae4d104aa/curated-transformers-0.1.1.tar.gz", hash = "sha256:4671f03314df30efda2ec2b59bc7692ea34fcea44cb65382342c16684e8a2119", size = 16313 } wheels = [ { url = "https://files.pythonhosted.org/packages/d6/67/3b72b3fdfcadab61bc8f59c17e63770e526ffabd583ed32f174a7c01af85/curated_transformers-0.1.1-py2.py3-none-any.whl", hash = "sha256:d716063d73d803c6925d2dab56fde9b9ab8e89e663c2c0587804944ba488ff01", size = 25972 }, ] +[[package]] +name = "curl-cffi" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "cffi" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/5b/89fcfebd3e5e85134147ac99e9f2b2271165fd4d71984fc65da5f17819b7/curl_cffi-0.15.0.tar.gz", hash = "sha256:ea0c67652bf6893d34ee0f82c944f37e488f6147e9421bef1771cc6545b02ded", size = 196437 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/42/54ddd442c795f30ce5dd4e49f87ce77505958d3777cd96a91567a3975d2a/curl_cffi-0.15.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:bda66404010e9ed743b1b83c20c86f24fe21a9a6873e17479d6e67e29d8ded28", size = 2795267 }, + { url = "https://files.pythonhosted.org/packages/83/2d/3915e238579b3c5a92cead5c79130c3b8d20caaba7616cc4d894650e1d6b/curl_cffi-0.15.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a25620d9bf989c9c029a7d1642999c4c265abb0bad811deb2f77b0b5b2b12e5b", size = 2573544 }, + { url = "https://files.pythonhosted.org/packages/2a/b3/9d2f1057749a1b07ba1989db3c1503ce8bed998310bae9aea2c43aa64f20/curl_cffi-0.15.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:582e570aa2586b96ed47cf4a17586b9a3c462cbe43f780487c3dc245c6ef1527", size = 10515369 }, + { url = "https://files.pythonhosted.org/packages/b5/1d/6d10dded5ce3fd8157e558ebd97d09e551b77a62cdc1c31e93d0a633cee5/curl_cffi-0.15.0-cp310-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:838e48212447d9c81364b04707a5c861daf08f8320f9ecb3406a8919d1d5c3b3", size = 10160045 }, + { url = "https://files.pythonhosted.org/packages/5c/12/c70b835487ace3b9ba1502631912e3440082b8ae3a162f60b59cb0b6444d/curl_cffi-0.15.0-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b6c847d86283b07ae69bb72c82eb8a59242277142aa35b89850f89e792a02fc", size = 11090433 }, + { url = "https://files.pythonhosted.org/packages/ea/0d/78edcc4f71934225db99df68197a107386d59080742fc7bf6bb4d007924f/curl_cffi-0.15.0-cp310-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e5e69eee735f659287e2c84444319d68a1fa68dd37abf228943a4074864283a", size = 10479178 }, + { url = "https://files.pythonhosted.org/packages/5b/84/1e101c1acb1ea2f0b4992f5c3024f596d8e21db0d53540b9d583f673c4e7/curl_cffi-0.15.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa1323950224db24f4c510d010b3affa02196ca853fb424191fa917a513d3f4b", size = 10317051 }, + { url = "https://files.pythonhosted.org/packages/28/42/8ef236b22a6c23d096c85a1dc507efe37bfdfc7a2f8a4b34efb590197369/curl_cffi-0.15.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:41f80170ba844009273b2660da1964ec31e99e5719d16b3422ada87177e32e13", size = 11299660 }, + { url = "https://files.pythonhosted.org/packages/1d/01/56aeb055d962da87a1be0d74c6c644e251c7e88129b5471dc44ac724e678/curl_cffi-0.15.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1977e1e12cfb5c11352cbb74acef1bed24eb7d226dab61ca57c168c21acd4d61", size = 11945049 }, + { url = "https://files.pythonhosted.org/packages/d8/8c/2abf99a38d6340d66cf0557e0c750ef3f8883dfc5d450087e01c85861343/curl_cffi-0.15.0-cp310-abi3-win_amd64.whl", hash = "sha256:5a0c1896a0d5a5ac1eb89cd24b008d2b718dd1df6fd2f75451b59ca66e49e572", size = 1661649 }, + { url = "https://files.pythonhosted.org/packages/3d/39/dfd54f2240d3a9b96d77bacc62b97813b35e2aa8ecf5cd5013c683f1ba96/curl_cffi-0.15.0-cp310-abi3-win_arm64.whl", hash = "sha256:a6d57f8389273a3a1f94370473c74897467bcc36af0a17336989780c507fa43d", size = 1410741 }, + { url = "https://files.pythonhosted.org/packages/19/6a/c24df8a4fc22fa84070dcd94abeba43c15e08cc09e35869565c0bad196fd/curl_cffi-0.15.0-cp313-abi3-android_24_arm64_v8a.whl", hash = "sha256:4682dc38d4336e0eb0b185374db90a760efde63cbea994b4e63f3521d44c4c92", size = 7190427 }, + { url = "https://files.pythonhosted.org/packages/11/56/132225cb3491d07cc6adcce5fe395e059bde87c68cff1ef87a31c88c7819/curl_cffi-0.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:967ad7355bd8e9586f8c2d02eaa99953747549e7ea4a9b25cd53353e6b67fe6d", size = 2795723 }, + { url = "https://files.pythonhosted.org/packages/07/8f/f4f83cd303bef7e8f1749512e5dd157e7e5d08b0a36c8211f9640a2757bf/curl_cffi-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7e63539d0d839d0a8c5eacf86229bc68c57803547f35e0db7ee0986328b478c3", size = 2573739 }, + { url = "https://files.pythonhosted.org/packages/e8/5c/643d65c7fc9acd742876aa55c2d7823c438cb7665810acd2e66c9976c4d9/curl_cffi-0.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:08c799b89740b9bc49c09fbc3d5907f13ac1f845ca52620507ef9466d4639dd5", size = 10521046 }, + { url = "https://files.pythonhosted.org/packages/7f/0b/9b8037113c93f4c5323096163471fa7c35c7676c3f608eeaf1287cd99d58/curl_cffi-0.15.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b7a92767a888ee90147e18964b396d8435ff42737030d6fb00824ffd6094805", size = 11096115 }, + { url = "https://files.pythonhosted.org/packages/5f/96/fff2fcbd924ef4042e0d67379f751a8a4e3186a91e75e35a4cf218b306ee/curl_cffi-0.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:829cc357061ecb99cc2d406301f609a039e05665322f5c025ec67c38b0dc49ce", size = 11305346 }, + { url = "https://files.pythonhosted.org/packages/53/1b/304b253a45ab28691c8c5e8cca1e6cbb9cf8e46dfceae4648dd536f75e73/curl_cffi-0.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:408d6f14e346841cd889c2e0962832bb235ba3b6749ebf609f347f747da5e60f", size = 11949834 }, + { url = "https://files.pythonhosted.org/packages/5a/ff/4723d92f08259c707a974aba27a08d0a822b9555e35ca581bf18d055a364/curl_cffi-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b624c7ce087bfda967a013ed0a64702a525444e5b6e97d23534d567ccc6525aa", size = 1702771 }, + { url = "https://files.pythonhosted.org/packages/59/8c/36bbe06d66fa2b765e4a07199f643a59a9cd1a754207a96335402a9520f4/curl_cffi-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0b6c0543b993996670e9e4b78e305a2d60809d5681903ffb5568e21a387434d3", size = 1466312 }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -1750,7 +2541,7 @@ version = "2.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, - { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/57/9a2d9abdabdc9db8ef28ce0cf4129669e1c8717ba28d607b5ba357c4de3b/discord_py-2.7.1.tar.gz", hash = "sha256:24d5e6a45535152e4b98148a9dd6b550d25dc2c9fb41b6d670319411641249da", size = 1106326 } wheels = [ @@ -1800,7 +2591,7 @@ dependencies = [ { name = "huggingface-hub" }, { name = "lxml" }, { name = "marko" }, - { name = "ocrmac", marker = "sys_platform == 'darwin'" }, + { name = "ocrmac", marker = "sys_platform == 'darwin' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "openpyxl" }, { name = "pandas" }, { name = "pillow" }, @@ -1816,8 +2607,14 @@ dependencies = [ { name = "requests" }, { name = "rtree" }, { name = "scipy" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "tqdm" }, { name = "typer" }, ] @@ -1873,8 +2670,14 @@ dependencies = [ { name = "pydantic" }, { name = "rtree" }, { name = "safetensors", extra = ["torch"] }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "tqdm" }, { name = "transformers" }, ] @@ -1891,7 +2694,7 @@ dependencies = [ { name = "docling-core" }, { name = "pillow" }, { name = "pydantic" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "tabulate" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/34/f951e261d20cc71bc55703a3f4f51b13d8dc98ed634995905ecc41e5650a/docling_parse-5.6.1.tar.gz", hash = "sha256:e47d40a7c268a775c41a8cc7773555a5856a1bcfd5a05ee97ee0a162270ad7f9", size = 61127846 } @@ -2039,7 +2842,7 @@ name = "faker" version = "40.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "tzdata", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/e5/b16bf568a2f20fe7423282db4a4059dbcadef70e9029c1c106836f8edd84/faker-40.11.1.tar.gz", hash = "sha256:61965046e79e8cfde4337d243eac04c0d31481a7c010033141103b43f603100c", size = 1957415 } wheels = [ @@ -2966,7 +3769,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "packaging" }, { name = "pyyaml" }, { name = "requests" }, @@ -3302,9 +4105,9 @@ dependencies = [ { name = "jaraco-classes" }, { name = "jaraco-context" }, { name = "jaraco-functools" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, + { name = "jeepney", marker = "sys_platform == 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "secretstorage", marker = "sys_platform == 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516 } wheels = [ @@ -3402,7 +4205,10 @@ dependencies = [ { name = "loguru" }, { name = "misaki", extra = ["en"] }, { name = "numpy" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "transformers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e8/48/88b8cdf28b068d070195c2817175549dee48e7682e3ab8994bee5f69217e/kokoro-0.9.4.tar.gz", hash = "sha256:fbf633262797f8cf46fdac3315cf9cade67dc8b762c0feccf334892772fb9ac4", size = 26215928 } @@ -3692,7 +4498,7 @@ version = "0.7.22" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, - { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, + { name = "orjson", marker = "platform_python_implementation != 'PyPy' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "packaging" }, { name = "pydantic" }, { name = "requests" }, @@ -3744,7 +4550,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.83.14" +version = "1.88.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -3760,9 +4566,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8d/7c/c095649380adc96c8630273c1768c2ad1e74aa2ee1dd8dd05d218a60569f/litellm-1.83.14.tar.gz", hash = "sha256:24aef9b47cdc424c833e32f3727f411741c690832cd1fe4405e0077144fe09c9", size = 14836599 } +sdist = { url = "https://files.pythonhosted.org/packages/16/ea/f99ececb7f22703fe120f1d8be9ffb749ec9453fbbbbbebc0d6a6b4d7864/litellm-1.88.1.tar.gz", hash = "sha256:89c6b74cc7912d6365793006ff951c0450fe847625008dfe49de8a7dc4529aa5", size = 13885969 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/5c/1b5691575420135e90578543b2bf219497caa33cfd0af64cb38f30288450/litellm-1.83.14-py3-none-any.whl", hash = "sha256:92b11ba2a32cf80707ddf388d18526696c7999a21b418c5e3b6eda1243d2cfdb", size = 16457054 }, + { url = "https://files.pythonhosted.org/packages/42/9a/8f8909201b4bebaf96498c09226f6baa8540086a4c4188ad57d7dfbd97c1/litellm-1.88.1-py3-none-any.whl", hash = "sha256:369b84e57d9426582ddc35e731956ddb6618cda97cc44e4e4d2dfa75982a6e3a", size = 15276206 }, ] [[package]] @@ -3889,8 +4695,8 @@ name = "loguru" version = "0.7.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "win32-setctime", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "win32-setctime", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559 } wheels = [ @@ -3899,82 +4705,82 @@ wheels = [ [[package]] name = "lxml" -version = "6.0.2" +version = "6.1.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426 } +sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887 }, - { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818 }, - { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807 }, - { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179 }, - { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044 }, - { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685 }, - { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127 }, - { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958 }, - { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541 }, - { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426 }, - { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917 }, - { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795 }, - { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759 }, - { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666 }, - { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989 }, - { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456 }, - { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793 }, - { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836 }, - { url = "https://files.pythonhosted.org/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494 }, - { url = "https://files.pythonhosted.org/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146 }, - { url = "https://files.pythonhosted.org/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932 }, - { url = "https://files.pythonhosted.org/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060 }, - { url = "https://files.pythonhosted.org/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000 }, - { url = "https://files.pythonhosted.org/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496 }, - { url = "https://files.pythonhosted.org/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779 }, - { url = "https://files.pythonhosted.org/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072 }, - { url = "https://files.pythonhosted.org/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675 }, - { url = "https://files.pythonhosted.org/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171 }, - { url = "https://files.pythonhosted.org/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175 }, - { url = "https://files.pythonhosted.org/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688 }, - { url = "https://files.pythonhosted.org/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655 }, - { url = "https://files.pythonhosted.org/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695 }, - { url = "https://files.pythonhosted.org/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841 }, - { url = "https://files.pythonhosted.org/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700 }, - { url = "https://files.pythonhosted.org/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347 }, - { url = "https://files.pythonhosted.org/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248 }, - { url = "https://files.pythonhosted.org/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801 }, - { url = "https://files.pythonhosted.org/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403 }, - { url = "https://files.pythonhosted.org/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974 }, - { url = "https://files.pythonhosted.org/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953 }, - { url = "https://files.pythonhosted.org/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054 }, - { url = "https://files.pythonhosted.org/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421 }, - { url = "https://files.pythonhosted.org/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684 }, - { url = "https://files.pythonhosted.org/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463 }, - { url = "https://files.pythonhosted.org/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437 }, - { url = "https://files.pythonhosted.org/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890 }, - { url = "https://files.pythonhosted.org/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185 }, - { url = "https://files.pythonhosted.org/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895 }, - { url = "https://files.pythonhosted.org/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246 }, - { url = "https://files.pythonhosted.org/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797 }, - { url = "https://files.pythonhosted.org/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404 }, - { url = "https://files.pythonhosted.org/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072 }, - { url = "https://files.pythonhosted.org/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617 }, - { url = "https://files.pythonhosted.org/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930 }, - { url = "https://files.pythonhosted.org/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380 }, - { url = "https://files.pythonhosted.org/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632 }, - { url = "https://files.pythonhosted.org/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171 }, - { url = "https://files.pythonhosted.org/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109 }, - { url = "https://files.pythonhosted.org/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061 }, - { url = "https://files.pythonhosted.org/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233 }, - { url = "https://files.pythonhosted.org/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739 }, - { url = "https://files.pythonhosted.org/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119 }, - { url = "https://files.pythonhosted.org/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665 }, - { url = "https://files.pythonhosted.org/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997 }, - { url = "https://files.pythonhosted.org/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957 }, - { url = "https://files.pythonhosted.org/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372 }, - { url = "https://files.pythonhosted.org/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653 }, - { url = "https://files.pythonhosted.org/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795 }, - { url = "https://files.pythonhosted.org/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023 }, - { url = "https://files.pythonhosted.org/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420 }, - { url = "https://files.pythonhosted.org/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837 }, - { url = "https://files.pythonhosted.org/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205 }, + { url = "https://files.pythonhosted.org/packages/6a/6e/c4add832b6fc1e887125b96f880d7b9b70aae5248718e046b1704bcac4b9/lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7", size = 8570821 }, + { url = "https://files.pythonhosted.org/packages/22/00/ff3009c88e65de8011630acf8ab5a09cb2becd2aaf47fba2f3449f6224e9/lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1", size = 4624252 }, + { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746 }, + { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723 }, + { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557 }, + { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036 }, + { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367 }, + { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171 }, + { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874 }, + { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492 }, + { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232 }, + { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023 }, + { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773 }, + { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088 }, + { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995 }, + { url = "https://files.pythonhosted.org/packages/69/8b/6772e1a4b513fc50a8d931f19edde0e13ae6918510a1e13ff67864f3e5ed/lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a", size = 3596382 }, + { url = "https://files.pythonhosted.org/packages/1b/89/45198e9624762af2dfd2cb8782598477ceb29f6e59caab560388ae1f4ec1/lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77", size = 3997255 }, + { url = "https://files.pythonhosted.org/packages/90/a9/7a54b6834088d9ae528a7b780584ba6a39a9457b0ac330479f20ffbc9449/lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f", size = 3659610 }, + { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780 }, + { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006 }, + { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139 }, + { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329 }, + { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564 }, + { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467 }, + { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304 }, + { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607 }, + { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168 }, + { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487 }, + { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231 }, + { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450 }, + { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874 }, + { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987 }, + { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276 }, + { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903 }, + { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869 }, + { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490 }, + { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146 }, + { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866 }, + { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022 }, + { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695 }, + { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642 }, + { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338 }, + { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528 }, + { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730 }, + { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530 }, + { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670 }, + { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485 }, + { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635 }, + { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681 }, + { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229 }, + { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191 }, + { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202 }, + { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497 }, + { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991 }, + { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545 }, + { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736 }, + { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291 }, + { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822 }, + { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923 }, + { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843 }, + { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515 }, + { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511 }, + { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206 }, + { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404 }, + { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769 }, + { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936 }, + { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296 }, + { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598 }, + { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845 }, + { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345 }, + { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350 }, + { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223 }, ] [package.optional-dependencies] @@ -4154,15 +4960,15 @@ name = "matplotlib" version = "3.10.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "cycler", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "fonttools", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "kiwisolver", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "packaging", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pillow", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pyparsing", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "python-dateutil", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "contourpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "cycler", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "fonttools", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "kiwisolver", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "packaging", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pillow", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pyparsing", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "python-dateutil", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/76/d3c6e3a13fe484ebe7718d14e269c9569c4eb0020a968a327acb3b9a8fe6/matplotlib-3.10.8.tar.gz", hash = "sha256:2299372c19d56bcd35cf05a2738308758d32b9eaed2371898d8f5bd33f084aa3", size = 34806269 } wheels = [ @@ -4216,12 +5022,12 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pyjwt", extra = ["crypto"] }, { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "sse-starlette" }, { name = "starlette" }, { name = "typing-extensions" }, { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005 } wheels = [ @@ -4273,7 +5079,7 @@ name = "ml-dtypes" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314 } wheels = [ @@ -4420,7 +5226,7 @@ version = "2.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pygments" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3a/93/80ac75c20ce54c785648b4ed363c88f148bf22637e10c9863db4fbe73e74/mpire-2.10.2.tar.gz", hash = "sha256:f66a321e93fadff34585a4bfa05e95bd946cf714b442f51c529038eb45773d97", size = 271270 } @@ -4468,6 +5274,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583 }, ] +[[package]] +name = "msgspec" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/cf/317224852c00248c620a9bcf4b26e2e4ab8afd752f18d2a6ef73ebd423b6/msgspec-0.21.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4248cf0b6129b7d230eacd493c17cc2d4f3989f3bb7f633a928a85b7dcfa251", size = 196188 }, + { url = "https://files.pythonhosted.org/packages/6d/81/074612945c0666078f7366f40000013de9f6ba687491d450df699bceebc9/msgspec-0.21.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5102c7e9b3acff82178449b85006d96310e690291bb1ea0142f1b24bcb8aabcb", size = 188473 }, + { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871 }, + { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025 }, + { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672 }, + { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303 }, + { url = "https://files.pythonhosted.org/packages/cc/4d/619866af2840875be408047bf9e70ceafbae6ab50660de7134ed1b25eb86/msgspec-0.21.1-cp312-cp312-win_amd64.whl", hash = "sha256:d4ab834a054c6f0cbeef6df9e7e1b33d5f1bc7b86dea1d2fd7cad003873e783d", size = 190017 }, + { url = "https://files.pythonhosted.org/packages/5e/2e/a8f9eca8fd00e097d7a9e99ba8a4685db994494448e3d4f0b7f6e9a3c0f7/msgspec-0.21.1-cp312-cp312-win_arm64.whl", hash = "sha256:628aaa35c74950a8c59da330d7e98917e1c7188f983745782027748ee4ca573e", size = 175345 }, + { url = "https://files.pythonhosted.org/packages/7e/74/f11ede02839b19ff459f88e3145df5d711626ca84da4e23520cebf819367/msgspec-0.21.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:764173717a01743f007e9f74520ed281f24672c604514f7d76c1c3a10e8edb66", size = 196176 }, + { url = "https://files.pythonhosted.org/packages/bb/40/4476c1bd341418a046c4955aff632ec769315d1e3cb94e6acf86d461f9ed/msgspec-0.21.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:344c7cd0eaed1fb81d7959f99100ef71ec9b536881a376f11b9a6c4803365697", size = 188524 }, + { url = "https://files.pythonhosted.org/packages/ca/d9/9e9d7d7e5061b47540d03d640fab9b3965ba7ae49c1b2154861c8f007518/msgspec-0.21.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48943e278b3854c2f89f955ddc6f9f430d3f0784b16e47d10604ee0463cd21f5", size = 218880 }, + { url = "https://files.pythonhosted.org/packages/74/66/2bb344f34abb4b57e60c7c9c761994e0417b9718ec1460bf00c296f2a7ea/msgspec-0.21.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9aa659ebb0101b1cbc31461212b87e341d961f0ab0772aaf068a99e001ec4aa", size = 225050 }, + { url = "https://files.pythonhosted.org/packages/1a/84/7c1e412f76092277bf760cef12b7979d03314d259ab5b5cafde5d0c1722d/msgspec-0.21.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7b27d1a8ead2b6f5b0c4f2d07b8be1ccfcc041c8a0e704781edebe3ae13c484", size = 222713 }, + { url = "https://files.pythonhosted.org/packages/4e/27/0bba04b2b4ef05f3d068429410bc71d2cea925f1596a8f41152cccd5edb8/msgspec-0.21.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:38fe93e86b61328fe544cb7fd871fad5a27c8734bfda90f65e5dbe288ae50f61", size = 227259 }, + { url = "https://files.pythonhosted.org/packages/b0/2d/09574b0eea02fed2c2c1383dbaae2c7f79dc16dcd6487a886000afb5d7c4/msgspec-0.21.1-cp313-cp313-win_amd64.whl", hash = "sha256:8bc666331c35fcce05a7cd2d6221adbe0f6058f8e750711413d22793c080ac6a", size = 189857 }, + { url = "https://files.pythonhosted.org/packages/46/34/105b1576ad182879914f0c821f17ee1d13abb165cb060448f96fe2aff078/msgspec-0.21.1-cp313-cp313-win_arm64.whl", hash = "sha256:42bb1241e0750c1a4346f2aa84db26c5ffd99a4eb3a954927d9f149ff2f42898", size = 175403 }, + { url = "https://files.pythonhosted.org/packages/5a/ad/86954e987d1d6a5c579e2c2e7832b65e0fff194179fdac4f581536086024/msgspec-0.21.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fab48eb45fdbfbdb2c0edfec00ffc53b6b6085beefc6b50b61e01659f9f8757f", size = 196261 }, + { url = "https://files.pythonhosted.org/packages/d1/a1/c5e46c3e42b866199365e35d11dddfd1fbd8bba4fdb3c52f965b1607ce94/msgspec-0.21.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3cb779ea0c35bc807ff941d415875c1f69ca0be91a2e907ab99a171811d86a9a", size = 188729 }, + { url = "https://files.pythonhosted.org/packages/85/7d/1e29a319d678d6cb962ae5bdf32a6858ebdf38f73bc654c0e9c742a0c2c8/msgspec-0.21.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68604db36b3b4dd9bf160e436e12798a4738848144cea1aca1cb984011eb160f", size = 219866 }, + { url = "https://files.pythonhosted.org/packages/25/1f/cca084ca2572810fff12ea9dbdcbe39eac048f40daf4a9077b49fcbe8cee/msgspec-0.21.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d6b9dc50948eaf65df54d2fd0ff66e6d8c32f116037209ee861810eb9b676cb", size = 224993 }, + { url = "https://files.pythonhosted.org/packages/71/94/d2120fc9d419a89a3a7c13e5b7078798c4b392a96a02a6e2b3ce43a8766c/msgspec-0.21.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:52c5e21930942302394429c5a582ce7e6b62c7f983b3760834c2ce107e0dd6df", size = 223535 }, + { url = "https://files.pythonhosted.org/packages/75/17/42418b66a3ad972a89bab73dd78b79cc6282bb488a25e73c853cee7443b9/msgspec-0.21.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:abbb39d65681fa24ed394e01af3d59d869068324f900c61d06062b7fb9980f2f", size = 227222 }, + { url = "https://files.pythonhosted.org/packages/c4/33/265c894268cca88ff67b144ca2b4c522fc8b9a6f1966a3640c70516e78e1/msgspec-0.21.1-cp314-cp314-win_amd64.whl", hash = "sha256:5666b1b560b97b6ec2eb3fca8a502298ebac56e13bbca1f88523538ce83d01ea", size = 193810 }, + { url = "https://files.pythonhosted.org/packages/3b/8f/a6d35f25bf1fc63c492fdd88fdce01ba0875ead48c2b91f90f33653b4131/msgspec-0.21.1-cp314-cp314-win_arm64.whl", hash = "sha256:d8b8578e4c83b14ceea4cef0d0b747e31d9330fe4b03b2b2ad4063866a178f93", size = 179125 }, + { url = "https://files.pythonhosted.org/packages/c6/39/74839641e64b99d87da55af0fc472854d42b46e2183b9e2a67fe1bb2a512/msgspec-0.21.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:15f523d51c00ebad412213bfe9f06f0a50ec2b93e0c19e824a2d267cabb48ea2", size = 200171 }, + { url = "https://files.pythonhosted.org/packages/70/9b/ce0cca6d2d87fcd4b6ff97600790494e64f26a2c55d61507cd2755c16193/msgspec-0.21.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4e47390360583ba3d5c6cb44cf0a9f61b0a06a899d3c2c00627cedebb2e2884b", size = 192879 }, + { url = "https://files.pythonhosted.org/packages/a7/08/673a7bb05e5702dc787ddd3011195b509f9867927970da59052211929987/msgspec-0.21.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f60800e6299b798142dc40b0644da77ceac5ea0568be58228417eae14135c847", size = 226281 }, + { url = "https://files.pythonhosted.org/packages/7d/45/86508cf57283e9070b3c447e3ab25b792a7a0855a3ea4e0c6d111ac34c97/msgspec-0.21.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f8e9dfcd98419cf7568808470c4317a3fb30bef0e3715b568730a2b272a20d7", size = 229863 }, + { url = "https://files.pythonhosted.org/packages/2c/62/e7c9367cd08d590559faacd711edbae36840342843e669440363f33c7d36/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92d89dfad13bd1ea640dc3e37e724ed380da1030b272bdf5ecafb983c3ad7c75", size = 230445 }, + { url = "https://files.pythonhosted.org/packages/42/b4/c0f54632103846b658a10930025f4de41c8724b5e4805a5f3b395586cb7e/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d03867786e5d7ba25d666df4b11320c27170f4aeafcb8e3a8b0a50a4fb742ca", size = 231822 }, + { url = "https://files.pythonhosted.org/packages/ea/1d/0d85cc79d0ccf5508e9c846cc66552a6a16bf92abd1dbd8362617f7b35cd/msgspec-0.21.1-cp314-cp314t-win_amd64.whl", hash = "sha256:740fbf1c9d59992ca3537d6fbe9ebbf9eaf726a65fbf31448e0ecbc710697a63", size = 206650 }, + { url = "https://files.pythonhosted.org/packages/90/91/56c5d560f20e6c20e9e4f55bd0e458f7f162aa689ee350346c04c48eac0b/msgspec-0.21.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0d2cc73df6058d811a126ac3a8ad63a4dfa210c82f9cf5a004802eaf4712de90", size = 183149 }, +] + [[package]] name = "msoffcrypto-tool" version = "6.0.0" @@ -4849,6 +5695,37 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226 }, { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236 }, + { url = "https://files.pythonhosted.org/packages/10/f5/f50bc3f5c2bb57ab8f5b4d78bc1146b57810d42cb8fcb28cbe2e14050376/nvidia_cublas-13.1.0.3-py3-none-win_amd64.whl", hash = "sha256:2a3b94a37def342471c59fad7856caee4926809a72dd5270155d6a31b5b277be", size = 404355960 }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.6.4.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/eb/ff4b8c503fa1f1796679dce648854d58751982426e4e4b37d6fce49d259c/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb", size = 393138322 }, + { url = "https://files.pythonhosted.org/packages/97/0d/f1f0cadbf69d5b9ef2e4f744c9466cb0a850741d08350736dfdb4aa89569/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668", size = 390794615 }, + { url = "https://files.pythonhosted.org/packages/84/f7/985e9bdbe3e0ac9298fcc8cfa51a392862a46a0ffaccbbd56939b62a9c83/nvidia_cublas_cu12-12.6.4.1-py3-none-win_amd64.whl", hash = "sha256:9e4fa264f4d8a4eb0cdbd34beadc029f453b3bafae02401e999cf3d5a5af75f8", size = 434535301 }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124 }, + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921 }, + { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208 }, ] [[package]] @@ -4858,6 +5735,39 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827 }, { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597 }, + { url = "https://files.pythonhosted.org/packages/ad/df/b74b10025c1205695c5676373f2edd3e87a7202cc62ead0dfbc373b0f6ea/nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00", size = 7736776 }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.6.80" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/8b/2f6230cb715646c3a9425636e513227ce5c93c4d65823a734f4bb86d43c3/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc", size = 8236764 }, + { url = "https://files.pythonhosted.org/packages/25/0f/acb326ac8fd26e13c799e0b4f3b2751543e1834f04d62e729485872198d4/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.whl", hash = "sha256:358b4a1d35370353d52e12f0a7d1769fc01ff74a191689d3870b2123156184c4", size = 8236756 }, + { url = "https://files.pythonhosted.org/packages/49/60/7b6497946d74bcf1de852a21824d63baad12cd417db4195fc1bfe59db953/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132", size = 8917980 }, + { url = "https://files.pythonhosted.org/packages/a5/24/120ee57b218d9952c379d1e026c4479c9ece9997a4fb46303611ee48f038/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73", size = 8917972 }, + { url = "https://files.pythonhosted.org/packages/1c/81/7796f096afaf726796b1b648f3bc80cafc61fe7f77f44a483c89e6c5ef34/nvidia_cuda_cupti_cu12-12.6.80-py3-none-win_amd64.whl", hash = "sha256:bbe6ae76e83ce5251b56e8c8e61a964f757175682bbad058b170b136266ab00a", size = 5724175 }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318 }, + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621 }, + { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759 }, ] [[package]] @@ -4867,6 +5777,37 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200 }, { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449 }, + { url = "https://files.pythonhosted.org/packages/4a/af/345fedb9f4c76c84ab4fa445b36bd4048a4d9db60e6bc76b4f913ff4b852/nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872", size = 76807835 }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.6.85" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/31/ffb400c5ae99daf09687aa6c42831c5d824f71c4851363ed2a4a1ac52bab/nvidia_cuda_nvrtc_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:800927308ccc5dd6246d3f61f7fcef2ed7ec4e59e199090d360d3293f78bd5a2", size = 23649944 }, + { url = "https://files.pythonhosted.org/packages/48/ab/476146f59ff5ef5bd6e62c187097d859ea78b5752d19c6c3f9be5f90dafc/nvidia_cuda_nvrtc_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f3134f50963882373063901657554f230bedf6039d30b09f6be55c64c993a37", size = 23162872 }, + { url = "https://files.pythonhosted.org/packages/0c/f7/472414aee887d626373d0b2140a59ac4308e3eaed815060e5410fc83305a/nvidia_cuda_nvrtc_cu12-12.6.85-py3-none-win_amd64.whl", hash = "sha256:a419e2c95e75b88b602f8bb66f82a6c5651e8475a509841c958486b1b71510bf", size = 39026436 }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029 }, + { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076 }, + { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838 }, ] [[package]] @@ -4876,6 +5817,75 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060 }, { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632 }, + { url = "https://files.pythonhosted.org/packages/b7/94/6b867483bec07da24ffa32736c79fabb94ef3a7af4d787a9d4a974868576/nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492", size = 2927037 }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/ea/590b2ac00d772a8abd1c387a92b46486d2679ca6622fd25c18ff76265663/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd", size = 908052 }, + { url = "https://files.pythonhosted.org/packages/b7/3d/159023799677126e20c8fd580cca09eeb28d5c5a624adc7f793b9aa8bbfa/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e", size = 908040 }, + { url = "https://files.pythonhosted.org/packages/e1/23/e717c5ac26d26cf39a27fbc076240fad2e3b817e5889d671b67f4f9f49c5/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7", size = 897690 }, + { url = "https://files.pythonhosted.org/packages/f0/62/65c05e161eeddbafeca24dc461f47de550d9fa8a7e04eb213e32b55cfd99/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8", size = 897678 }, + { url = "https://files.pythonhosted.org/packages/fa/76/4c80fa138333cc975743fd0687a745fccb30d167f906f13c1c7f9a85e5ea/nvidia_cuda_runtime_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:86c58044c824bf3c173c49a2dbc7a6c8b53cb4e4dca50068be0bf64e9dab3f7f", size = 891773 }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265 }, + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765 }, + { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318 }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.6.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878 }, + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467 }, + { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268 }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/b8/277c51962ee46fa3e5b203ac5f76107c650f781d6891e681e28e6f3e9fe6/nvidia_cudnn_cu12-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:08caaf27fe556aca82a3ee3b5aa49a77e7de0cfcb7ff4e5c29da426387a8267e", size = 656910700 }, + { url = "https://files.pythonhosted.org/packages/c5/41/65225d42fba06fb3dd3972485ea258e7dd07a40d6e01c95da6766ad87354/nvidia_cudnn_cu12-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ac6ad90a075bb33a94f2b4cf4622eac13dd4dc65cf6dd9c7572a318516a36625", size = 657906812 }, + { url = "https://files.pythonhosted.org/packages/a7/a5/48f07449fc9c6cc146dcafe6149fa5d69630137d2ec5b7d9e09f255fadd7/nvidia_cudnn_cu12-9.19.0.56-py3-none-win_amd64.whl", hash = "sha256:cec70596b9ce878fab83810c3f5a2e606d35f510e5fee579759e4cbc68a23750", size = 644003014 }, ] [[package]] @@ -4883,11 +5893,12 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201 }, { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321 }, + { url = "https://files.pythonhosted.org/packages/91/a2/f020386683ee9ab2c9a9f7f79290d9b0d07f7241de54dc746af2abd188d2/nvidia_cudnn_cu13-9.19.0.56-py3-none-win_amd64.whl", hash = "sha256:40d8c375005bcb01495f8edf375230b203a411a0c05fb6dc92a3781edcb23eac", size = 350547366 }, ] [[package]] @@ -4895,11 +5906,50 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554 }, { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489 }, + { url = "https://files.pythonhosted.org/packages/85/b2/f8af21a2ed1beed337a6a02c5a28aeb85441f4d578ec3d529543c775ea4b/nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb", size = 213342123 }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.6.85", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/37/c50d2b2f2c07e146776389e3080f4faf70bcc4fa6e19d65bb54ca174ebc3/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6", size = 200164144 }, + { url = "https://files.pythonhosted.org/packages/ce/f5/188566814b7339e893f8d210d3a5332352b1409815908dad6a363dcceac1/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8510990de9f96c803a051822618d42bf6cb8f069ff3f48d93a8486efdacb48fb", size = 200164135 }, + { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632 }, + { url = "https://files.pythonhosted.org/packages/60/de/99ec247a07ea40c969d904fc14f3a356b3e2a704121675b75c366b694ee1/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca", size = 200221622 }, + { url = "https://files.pythonhosted.org/packages/b4/38/36fd800cec8f6e89b7c1576edaaf8076e69ec631644cdbc1b5f2e2b5a9df/nvidia_cufft_cu12-11.3.0.4-py3-none-win_amd64.whl", hash = "sha256:6048ebddfb90d09d2707efb1fd78d4e3a77cb3ae4dc60e19aab6be0ece2ae464", size = 199356881 }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211 }, + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695 }, + { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559 }, ] [[package]] @@ -4911,6 +5961,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992 }, ] +[[package]] +name = "nvidia-cufile-cu12" +version = "1.11.1.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/66/cc9876340ac68ae71b15c743ddb13f8b30d5244af344ec8322b449e35426/nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159", size = 1142103 }, + { url = "https://files.pythonhosted.org/packages/17/bf/cc834147263b929229ce4aadd62869f0b195e98569d4c28b23edc72b85d9/nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db", size = 1066155 }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834 }, + { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705 }, +] + [[package]] name = "nvidia-curand" version = "10.4.0.35" @@ -4918,6 +5996,39 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106 }, { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258 }, + { url = "https://files.pythonhosted.org/packages/99/27/72103153b1ffc00e09fdc40ac970235343dcd1ea8bd762e84d2d73219ffa/nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f", size = 55242481 }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.7.77" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/ac/36543605358a355632f1a6faa3e2d5dfb91eab1e4bc7d552040e0383c335/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8", size = 56289881 }, + { url = "https://files.pythonhosted.org/packages/73/1b/44a01c4e70933637c93e6e1a8063d1e998b50213a6b65ac5a9169c47e98e/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf", size = 56279010 }, + { url = "https://files.pythonhosted.org/packages/4a/aa/2c7ff0b5ee02eaef890c0ce7d4f74bc30901871c5e45dee1ae6d0083cd80/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117", size = 56279000 }, + { url = "https://files.pythonhosted.org/packages/a6/02/5362a9396f23f7de1dd8a64369e87c85ffff8216fc8194ace0fa45ba27a5/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7b2ed8e95595c3591d984ea3603dd66fe6ce6812b886d59049988a712ed06b6e", size = 56289882 }, + { url = "https://files.pythonhosted.org/packages/a9/a8/0cd0cec757bd4b4b4ef150fca62ec064db7d08a291dced835a0be7d2c147/nvidia_curand_cu12-10.3.7.77-py3-none-win_amd64.whl", hash = "sha256:6d6d935ffba0f3d439b7cd968192ff068fafd9018dbf1b85b37261b13cfc9905", size = 55783873 }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754 }, + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976 }, + { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309 }, ] [[package]] @@ -4925,13 +6036,56 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparse", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760 }, { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980 }, + { url = "https://files.pythonhosted.org/packages/99/ef/332a0101260ca78a1daef046bf0b06199e8ed4dac1d2aa698289c358169c/nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65", size = 193551444 }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.1.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.6.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.6.85", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/17/dbe1aa865e4fdc7b6d4d0dd308fdd5aaab60f939abfc0ea1954eac4fb113/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0", size = 157833628 }, + { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790 }, + { url = "https://files.pythonhosted.org/packages/9f/81/baba53585da791d043c10084cf9553e074548408e04ae884cfe9193bd484/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6", size = 158229780 }, + { url = "https://files.pythonhosted.org/packages/7c/5f/07d0ba3b7f19be5a5ec32a8679fc9384cfd9fc6c869825e93be9f28d6690/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dbbe4fc38ec1289c7e5230e16248365e375c3673c9c8bac5796e2e20db07f56e", size = 157833630 }, + { url = "https://files.pythonhosted.org/packages/d4/53/fff50a0808df7113d77e3bbc7c2b7eaed6f57d5eb80fbe93ead2aea1e09a/nvidia_cusolver_cu12-11.7.1.2-py3-none-win_amd64.whl", hash = "sha256:6813f9d8073f555444a8705f3ab0296d3e1cb37a16d694c5fc8b862a0d8706d7", size = 149287877 }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-cublas-cu12", version = "12.8.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparse-cu12", version = "12.5.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841 }, + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905 }, + { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438 }, ] [[package]] @@ -4939,11 +6093,60 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568 }, { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937 }, + { url = "https://files.pythonhosted.org/packages/02/b0/b043d6f3480f102f885cf87fc3ffd3edcb5e23b855025a50e2ef4d059185/nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79", size = 143783033 }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.6.85", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/eb/6681efd0aa7df96b4f8067b3ce7246833dd36830bb4cec8896182773db7d/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887", size = 216451147 }, + { url = "https://files.pythonhosted.org/packages/d3/56/3af21e43014eb40134dea004e8d0f1ef19d9596a39e4d497d5a7de01669f/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1", size = 216451135 }, + { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367 }, + { url = "https://files.pythonhosted.org/packages/43/ac/64c4316ba163e8217a99680c7605f779accffc6a4bcd0c778c12948d3707/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f", size = 216561357 }, + { url = "https://files.pythonhosted.org/packages/45/ef/876ad8e4260e1128e6d4aac803d9d51baf3791ebdb4a9b8d9b8db032b4b0/nvidia_cusparse_cu12-12.5.4.2-py3-none-win_amd64.whl", hash = "sha256:4acb8c08855a26d737398cba8fb6f8f5045d93f82612b4cfd84645a2332ccf20", size = 213712630 }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "nvidia-nvjitlink-cu12", version = "12.8.93", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129 }, + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466 }, + { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404 }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557 }, + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691 }, + { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911 }, ] [[package]] @@ -4953,6 +6156,16 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277 }, { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119 }, + { url = "https://files.pythonhosted.org/packages/57/de/8f0578928b9b1246d7b1324db0528e6b9f9fb54496a49f40bf71f09f1a27/nvidia_cusparselt_cu13-0.8.0-py3-none-win_amd64.whl", hash = "sha256:e80212ed7b1afc97102fbb2b5c82487aa73f6a0edfa6d26c5a152593e520bb8f", size = 156459710 }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/c4/120d2dfd92dff2c776d68f361ff8705fdea2ca64e20b612fab0fd3f581ac/nvidia_nccl_cu12-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:50a36e01c4a090b9f9c47d92cec54964de6b9fcb3362d0e19b8ffc6323c21b60", size = 296766525 }, + { url = "https://files.pythonhosted.org/packages/4a/4e/44dbb46b3d1b0ec61afda8e84837870f2f9ace33c564317d59b70bc19d3e/nvidia_nccl_cu12-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:485776daa8447da5da39681af455aa3b2c2586ddcf4af8772495e7c532c7e5ab", size = 296782137 }, ] [[package]] @@ -4971,6 +6184,46 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933 }, { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748 }, + { url = "https://files.pythonhosted.org/packages/e4/01/07530b0e37546231052e30234540289c42eaffa486f1a34a87fed340157b/nvidia_nvjitlink-13.0.88-py3-none-win_amd64.whl", hash = "sha256:634e96e3da9ef845ae744097a1f289238ecf946ce0b82e93cdce14b9782e682f", size = 36035115 }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.6.85" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a", size = 19744971 }, + { url = "https://files.pythonhosted.org/packages/31/db/dc71113d441f208cdfe7ae10d4983884e13f464a6252450693365e166dcf/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41", size = 19270338 }, + { url = "https://files.pythonhosted.org/packages/89/76/93c1467b1387387440a4d25102d86b7794535449b689f8e2dc22c1c8ff7f/nvidia_nvjitlink_cu12-12.6.85-py3-none-win_amd64.whl", hash = "sha256:e61120e52ed675747825cdd16febc6a0730537451d867ee58bee3853b1b13d1c", size = 161908572 }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836 }, + { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204 }, + { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710 }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/6a/03aa43cc9bd3ad91553a88b5f6fb25ed6a3752ae86ce2180221962bc2aa5/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b48363fc6964dede448029434c6abed6c5e37f823cb43c3bcde7ecfc0457e15", size = 138936938 }, + { url = "https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:042f2500f24c021db8a06c5eec2539027d57460e1c1a762055a6554f72c369bd", size = 139103095 }, ] [[package]] @@ -4989,6 +6242,39 @@ source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047 }, { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878 }, + { url = "https://files.pythonhosted.org/packages/d2/50/0e2220f8620a177de994211186ffc5bfa9f2ce1e1282797f8f90096f9f88/nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519", size = 137066 }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/93/80f8a520375af9d7ee44571a6544653a176e53c2b8ccce85b97b83c2491b/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b", size = 90549 }, + { url = "https://files.pythonhosted.org/packages/2b/53/36e2fd6c7068997169b49ffc8c12d5af5e5ff209df6e1a2c4d373b3a638f/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:adcaabb9d436c9761fca2b13959a2d237c5f9fd406c8e4b723c695409ff88059", size = 90539 }, + { url = "https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2", size = 89276 }, + { url = "https://files.pythonhosted.org/packages/9e/4e/0d0c945463719429b7bd21dece907ad0bde437a2ff12b9b12fee94722ab0/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1", size = 89265 }, + { url = "https://files.pythonhosted.org/packages/f7/cd/98a447919d4ed14d407ac82b14b0a0c9c1dbfe81099934b1fc3bfd1e6316/nvidia_nvtx_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:2fb11a4af04a5e6c84073e6404d26588a34afd35379f0855a99797897efa75c0", size = 56434 }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161 }, + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954 }, + { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492 }, ] [[package]] @@ -5005,7 +6291,7 @@ name = "obstore" version = "0.8.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/8c/9ec984edd0f3b72226adfaa19b1c61b15823b35b52f311ca4af36d009d15/obstore-0.8.2.tar.gz", hash = "sha256:a467bc4e97169e2ba749981b4fd0936015428d9b8f3fb83a5528536b1b6f377f", size = 168852 } wheels = [ @@ -5054,9 +6340,9 @@ name = "ocrmac" version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pillow", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-vision", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "click", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pillow", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-vision", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317 } wheels = [ @@ -5090,10 +6376,10 @@ name = "onnx" version = "1.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ml-dtypes", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "protobuf", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "ml-dtypes", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "protobuf", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/8a/335c03a8683a88a32f9a6bb98899ea6df241a41df64b37b9696772414794/onnx-1.20.1.tar.gz", hash = "sha256:ded16de1df563d51fbc1ad885f2a426f814039d8b5f4feb77febe09c0295ad67", size = 12048980 } wheels = [ @@ -5482,55 +6768,55 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.7" +version = "3.11.9" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/45/b268004f745ede84e5798b48ee12b05129d19235d0e15267aa57dcdb400b/orjson-3.11.7.tar.gz", hash = "sha256:9b1a67243945819ce55d24a30b59d6a168e86220452d2c96f4d1f093e71c0c49", size = 6144992 } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163 } wheels = [ - { url = "https://files.pythonhosted.org/packages/80/bf/76f4f1665f6983385938f0e2a5d7efa12a58171b8456c252f3bae8a4cf75/orjson-3.11.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:bd03ea7606833655048dab1a00734a2875e3e86c276e1d772b2a02556f0d895f", size = 228545 }, - { url = "https://files.pythonhosted.org/packages/79/53/6c72c002cb13b5a978a068add59b25a8bdf2800ac1c9c8ecdb26d6d97064/orjson-3.11.7-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:89e440ebc74ce8ab5c7bc4ce6757b4a6b1041becb127df818f6997b5c71aa60b", size = 125224 }, - { url = "https://files.pythonhosted.org/packages/2c/83/10e48852865e5dd151bdfe652c06f7da484578ed02c5fca938e3632cb0b8/orjson-3.11.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ede977b5fe5ac91b1dffc0a517ca4542d2ec8a6a4ff7b2652d94f640796342a", size = 128154 }, - { url = "https://files.pythonhosted.org/packages/6e/52/a66e22a2b9abaa374b4a081d410edab6d1e30024707b87eab7c734afe28d/orjson-3.11.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b7b1dae39230a393df353827c855a5f176271c23434cfd2db74e0e424e693e10", size = 123548 }, - { url = "https://files.pythonhosted.org/packages/de/38/605d371417021359f4910c496f764c48ceb8997605f8c25bf1dfe58c0ebe/orjson-3.11.7-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed46f17096e28fb28d2975834836a639af7278aa87c84f68ab08fbe5b8bd75fa", size = 129000 }, - { url = "https://files.pythonhosted.org/packages/44/98/af32e842b0ffd2335c89714d48ca4e3917b42f5d6ee5537832e069a4b3ac/orjson-3.11.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3726be79e36e526e3d9c1aceaadbfb4a04ee80a72ab47b3f3c17fefb9812e7b8", size = 141686 }, - { url = "https://files.pythonhosted.org/packages/96/0b/fc793858dfa54be6feee940c1463370ece34b3c39c1ca0aa3845f5ba9892/orjson-3.11.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0724e265bc548af1dedebd9cb3d24b4e1c1e685a343be43e87ba922a5c5fff2f", size = 130812 }, - { url = "https://files.pythonhosted.org/packages/dc/91/98a52415059db3f374757d0b7f0f16e3b5cd5976c90d1c2b56acaea039e6/orjson-3.11.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7745312efa9e11c17fbd3cb3097262d079da26930ae9ae7ba28fb738367cbad", size = 133440 }, - { url = "https://files.pythonhosted.org/packages/dc/b6/cb540117bda61791f46381f8c26c8f93e802892830a6055748d3bb1925ab/orjson-3.11.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f904c24bdeabd4298f7a977ef14ca2a022ca921ed670b92ecd16ab6f3d01f867", size = 138386 }, - { url = "https://files.pythonhosted.org/packages/63/1a/50a3201c334a7f17c231eee5f841342190723794e3b06293f26e7cf87d31/orjson-3.11.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b9fc4d0f81f394689e0814617aadc4f2ea0e8025f38c226cbf22d3b5ddbf025d", size = 408853 }, - { url = "https://files.pythonhosted.org/packages/87/cd/8de1c67d0be44fdc22701e5989c0d015a2adf391498ad42c4dc589cd3013/orjson-3.11.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:849e38203e5be40b776ed2718e587faf204d184fc9a008ae441f9442320c0cab", size = 144130 }, - { url = "https://files.pythonhosted.org/packages/0f/fe/d605d700c35dd55f51710d159fc54516a280923cd1b7e47508982fbb387d/orjson-3.11.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4682d1db3bcebd2b64757e0ddf9e87ae5f00d29d16c5cdf3a62f561d08cc3dd2", size = 134818 }, - { url = "https://files.pythonhosted.org/packages/e4/e4/15ecc67edb3ddb3e2f46ae04475f2d294e8b60c1825fbe28a428b93b3fbd/orjson-3.11.7-cp312-cp312-win32.whl", hash = "sha256:f4f7c956b5215d949a1f65334cf9d7612dde38f20a95f2315deef167def91a6f", size = 127923 }, - { url = "https://files.pythonhosted.org/packages/34/70/2e0855361f76198a3965273048c8e50a9695d88cd75811a5b46444895845/orjson-3.11.7-cp312-cp312-win_amd64.whl", hash = "sha256:bf742e149121dc5648ba0a08ea0871e87b660467ef168a3a5e53bc1fbd64bb74", size = 125007 }, - { url = "https://files.pythonhosted.org/packages/68/40/c2051bd19fc467610fed469dc29e43ac65891571138f476834ca192bc290/orjson-3.11.7-cp312-cp312-win_arm64.whl", hash = "sha256:26c3b9132f783b7d7903bf1efb095fed8d4a3a85ec0d334ee8beff3d7a4749d5", size = 126089 }, - { url = "https://files.pythonhosted.org/packages/89/25/6e0e52cac5aab51d7b6dcd257e855e1dec1c2060f6b28566c509b4665f62/orjson-3.11.7-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1d98b30cc1313d52d4af17d9c3d307b08389752ec5f2e5febdfada70b0f8c733", size = 228390 }, - { url = "https://files.pythonhosted.org/packages/a5/29/a77f48d2fc8a05bbc529e5ff481fb43d914f9e383ea2469d4f3d51df3d00/orjson-3.11.7-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:d897e81f8d0cbd2abb82226d1860ad2e1ab3ff16d7b08c96ca00df9d45409ef4", size = 125189 }, - { url = "https://files.pythonhosted.org/packages/89/25/0a16e0729a0e6a1504f9d1a13cdd365f030068aab64cec6958396b9969d7/orjson-3.11.7-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:814be4b49b228cfc0b3c565acf642dd7d13538f966e3ccde61f4f55be3e20785", size = 128106 }, - { url = "https://files.pythonhosted.org/packages/66/da/a2e505469d60666a05ab373f1a6322eb671cb2ba3a0ccfc7d4bc97196787/orjson-3.11.7-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d06e5c5fed5caedd2e540d62e5b1c25e8c82431b9e577c33537e5fa4aa909539", size = 123363 }, - { url = "https://files.pythonhosted.org/packages/23/bf/ed73f88396ea35c71b38961734ea4a4746f7ca0768bf28fd551d37e48dd0/orjson-3.11.7-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:31c80ce534ac4ea3739c5ee751270646cbc46e45aea7576a38ffec040b4029a1", size = 129007 }, - { url = "https://files.pythonhosted.org/packages/73/3c/b05d80716f0225fc9008fbf8ab22841dcc268a626aa550561743714ce3bf/orjson-3.11.7-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f50979824bde13d32b4320eedd513431c921102796d86be3eee0b58e58a3ecd1", size = 141667 }, - { url = "https://files.pythonhosted.org/packages/61/e8/0be9b0addd9bf86abfc938e97441dcd0375d494594b1c8ad10fe57479617/orjson-3.11.7-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e54f3808e2b6b945078c41aa8d9b5834b28c50843846e97807e5adb75fa9705", size = 130832 }, - { url = "https://files.pythonhosted.org/packages/c9/ec/c68e3b9021a31d9ec15a94931db1410136af862955854ed5dd7e7e4f5bff/orjson-3.11.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12b80df61aab7b98b490fe9e4879925ba666fccdfcd175252ce4d9035865ace", size = 133373 }, - { url = "https://files.pythonhosted.org/packages/d2/45/f3466739aaafa570cc8e77c6dbb853c48bf56e3b43738020e2661e08b0ac/orjson-3.11.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:996b65230271f1a97026fd0e6a753f51fbc0c335d2ad0c6201f711b0da32693b", size = 138307 }, - { url = "https://files.pythonhosted.org/packages/e1/84/9f7f02288da1ffb31405c1be07657afd1eecbcb4b64ee2817b6fe0f785fa/orjson-3.11.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ab49d4b2a6a1d415ddb9f37a21e02e0d5dbfe10b7870b21bf779fc21e9156157", size = 408695 }, - { url = "https://files.pythonhosted.org/packages/18/07/9dd2f0c0104f1a0295ffbe912bc8d63307a539b900dd9e2c48ef7810d971/orjson-3.11.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:390a1dce0c055ddf8adb6aa94a73b45a4a7d7177b5c584b8d1c1947f2ba60fb3", size = 144099 }, - { url = "https://files.pythonhosted.org/packages/a5/66/857a8e4a3292e1f7b1b202883bcdeb43a91566cf59a93f97c53b44bd6801/orjson-3.11.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1eb80451a9c351a71dfaf5b7ccc13ad065405217726b59fdbeadbcc544f9d223", size = 134806 }, - { url = "https://files.pythonhosted.org/packages/0a/5b/6ebcf3defc1aab3a338ca777214966851e92efb1f30dc7fc8285216e6d1b/orjson-3.11.7-cp313-cp313-win32.whl", hash = "sha256:7477aa6a6ec6139c5cb1cc7b214643592169a5494d200397c7fc95d740d5fcf3", size = 127914 }, - { url = "https://files.pythonhosted.org/packages/00/04/c6f72daca5092e3117840a1b1e88dfc809cc1470cf0734890d0366b684a1/orjson-3.11.7-cp313-cp313-win_amd64.whl", hash = "sha256:b9f95dcdea9d4f805daa9ddf02617a89e484c6985fa03055459f90e87d7a0757", size = 124986 }, - { url = "https://files.pythonhosted.org/packages/03/ba/077a0f6f1085d6b806937246860fafbd5b17f3919c70ee3f3d8d9c713f38/orjson-3.11.7-cp313-cp313-win_arm64.whl", hash = "sha256:800988273a014a0541483dc81021247d7eacb0c845a9d1a34a422bc718f41539", size = 126045 }, - { url = "https://files.pythonhosted.org/packages/e9/1e/745565dca749813db9a093c5ebc4bac1a9475c64d54b95654336ac3ed961/orjson-3.11.7-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:de0a37f21d0d364954ad5de1970491d7fbd0fb1ef7417d4d56a36dc01ba0c0a0", size = 228391 }, - { url = "https://files.pythonhosted.org/packages/46/19/e40f6225da4d3aa0c8dc6e5219c5e87c2063a560fe0d72a88deb59776794/orjson-3.11.7-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:c2428d358d85e8da9d37cba18b8c4047c55222007a84f97156a5b22028dfbfc0", size = 125188 }, - { url = "https://files.pythonhosted.org/packages/9d/7e/c4de2babef2c0817fd1f048fd176aa48c37bec8aef53d2fa932983032cce/orjson-3.11.7-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c4bc6c6ac52cdaa267552544c73e486fecbd710b7ac09bc024d5a78555a22f6", size = 128097 }, - { url = "https://files.pythonhosted.org/packages/eb/74/233d360632bafd2197f217eee7fb9c9d0229eac0c18128aee5b35b0014fe/orjson-3.11.7-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd0d68edd7dfca1b2eca9361a44ac9f24b078de3481003159929a0573f21a6bf", size = 123364 }, - { url = "https://files.pythonhosted.org/packages/79/51/af79504981dd31efe20a9e360eb49c15f06df2b40e7f25a0a52d9ae888e8/orjson-3.11.7-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:623ad1b9548ef63886319c16fa317848e465a21513b31a6ad7b57443c3e0dcf5", size = 129076 }, - { url = "https://files.pythonhosted.org/packages/67/e2/da898eb68b72304f8de05ca6715870d09d603ee98d30a27e8a9629abc64b/orjson-3.11.7-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6e776b998ac37c0396093d10290e60283f59cfe0fc3fccbd0ccc4bd04dd19892", size = 141705 }, - { url = "https://files.pythonhosted.org/packages/c5/89/15364d92acb3d903b029e28d834edb8780c2b97404cbf7929aa6b9abdb24/orjson-3.11.7-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:652c6c3af76716f4a9c290371ba2e390ede06f6603edb277b481daf37f6f464e", size = 130855 }, - { url = "https://files.pythonhosted.org/packages/c2/8b/ecdad52d0b38d4b8f514be603e69ccd5eacf4e7241f972e37e79792212ec/orjson-3.11.7-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a56df3239294ea5964adf074c54bcc4f0ccd21636049a2cf3ca9cf03b5d03cf1", size = 133386 }, - { url = "https://files.pythonhosted.org/packages/b9/0e/45e1dcf10e17d0924b7c9162f87ec7b4ca79e28a0548acf6a71788d3e108/orjson-3.11.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bda117c4148e81f746655d5a3239ae9bd00cb7bc3ca178b5fc5a5997e9744183", size = 138295 }, - { url = "https://files.pythonhosted.org/packages/63/d7/4d2e8b03561257af0450f2845b91fbd111d7e526ccdf737267108075e0ba/orjson-3.11.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:23d6c20517a97a9daf1d48b580fcdc6f0516c6f4b5038823426033690b4d2650", size = 408720 }, - { url = "https://files.pythonhosted.org/packages/78/cf/d45343518282108b29c12a65892445fc51f9319dc3c552ceb51bb5905ed2/orjson-3.11.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8ff206156006da5b847c9304b6308a01e8cdbc8cce824e2779a5ba71c3def141", size = 144152 }, - { url = "https://files.pythonhosted.org/packages/a9/3a/d6001f51a7275aacd342e77b735c71fa04125a3f93c36fee4526bc8c654e/orjson-3.11.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:962d046ee1765f74a1da723f4b33e3b228fe3a48bd307acce5021dfefe0e29b2", size = 134814 }, - { url = "https://files.pythonhosted.org/packages/1d/d3/f19b47ce16820cc2c480f7f1723e17f6d411b3a295c60c8ad3aa9ff1c96a/orjson-3.11.7-cp314-cp314-win32.whl", hash = "sha256:89e13dd3f89f1c38a9c9eba5fbf7cdc2d1feca82f5f290864b4b7a6aac704576", size = 127997 }, - { url = "https://files.pythonhosted.org/packages/12/df/172771902943af54bf661a8d102bdf2e7f932127968080632bda6054b62c/orjson-3.11.7-cp314-cp314-win_amd64.whl", hash = "sha256:845c3e0d8ded9c9271cd79596b9b552448b885b97110f628fb687aee2eed11c1", size = 124985 }, - { url = "https://files.pythonhosted.org/packages/6f/1c/f2a8d8a1b17514660a614ce5f7aac74b934e69f5abc2700cc7ced882a009/orjson-3.11.7-cp314-cp314-win_arm64.whl", hash = "sha256:4a2e9c5be347b937a2e0203866f12bba36082e89b402ddb9e927d5822e43088d", size = 126038 }, + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515 }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409 }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106 }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864 }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213 }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994 }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744 }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014 }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509 }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127 }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025 }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943 }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606 }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101 }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736 }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458 }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368 }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070 }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892 }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217 }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980 }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738 }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033 }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492 }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087 }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031 }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915 }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613 }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086 }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696 }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465 }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364 }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063 }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356 }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592 }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491 }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309 }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030 }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482 }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178 }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089 }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921 }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638 }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078 }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687 }, ] [[package]] @@ -5597,7 +6883,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/0c/b28ed414f080ee0ad153f848586d61d1878f91689950f037f976ce15f6c8/pandas-3.0.1.tar.gz", hash = "sha256:4186a699674af418f655dbd420ed87f50d56b4cd6603784279d9eef6627823c8", size = 4641901 } wheels = [ @@ -5642,6 +6928,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/b0/34937815889fa982613775e4b97fddd13250f11012d769949c5465af2150/pandas-3.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:108dd1790337a494aa80e38def654ca3f0968cf4f362c85f44c15e471667102d", size = 9452085 }, ] +[[package]] +name = "patchright" +version = "1.60.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/26/c1e858fd1acc63e410b3d33243955f36d2a0814487b97a7aa604ad2baffd/patchright-1.60.1-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:e9492100d4e2a85ff92fc3a668dd16dee03f21df6e559c7b9f7c71e86ff48c6b", size = 43458936 }, + { url = "https://files.pythonhosted.org/packages/55/dd/2dd8e4e02489ec8fd57ad93dec9ef444b6f42adcc4fe95df30237c92841d/patchright-1.60.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:20bd806df2469b451ccd2ea10f5f944ceb0e0d83c716f5752b0c956c1ee59476", size = 42245629 }, + { url = "https://files.pythonhosted.org/packages/54/cc/0fa0bedec61045fd9068682e3695557b622c483f829d341093879ec8dbd9/patchright-1.60.1-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9fd15a64c0ca80740dc2a3f41cda336a06a2ed6068d0ab893172654290b06e6b", size = 43458935 }, + { url = "https://files.pythonhosted.org/packages/ce/c2/4b8f69de0a20d90792980c43c0e60b10b801e08cf0224ccc8a8266e1fffb/patchright-1.60.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:547e7bfb813102309789cc42933780e5fdf7c4727de59fb2791e64bd1298a7f3", size = 47451519 }, + { url = "https://files.pythonhosted.org/packages/db/fc/9fd6a70818cf0bc3b62574af483d762963cb65ca0a369826a0536faffe41/patchright-1.60.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:023945a2fd30219a284721ca36385bd44075ae7b53071dc1da38036b6dbe88ec", size = 47139153 }, + { url = "https://files.pythonhosted.org/packages/08/bc/81fb621e5ab4131e6f324c9ba6cb2a9f3b146c92f12484bec95efe8c8347/patchright-1.60.1-py3-none-win32.whl", hash = "sha256:05b98a6afdbe7e6645fe223009c47cc8e7859df55fd8ce9d8a9925b3389b0ee1", size = 37886460 }, + { url = "https://files.pythonhosted.org/packages/74/8e/fff80350ed2c2c1f62145d667070799c60ca9e9b28d54e4f751f0b4f8da6/patchright-1.60.1-py3-none-win_amd64.whl", hash = "sha256:51b306ed55cd58f1bca24641458f5c9f7e86a1f1727dcffdead669cfe4c0a485", size = 37886465 }, + { url = "https://files.pythonhosted.org/packages/ee/b8/b6d1bfe98a420c1ccfb2f23a7bb2b4cdb40170907bae638e7ae92abd287c/patchright-1.60.1-py3-none-win_arm64.whl", hash = "sha256:f795728c1e27fc226dbe203c1aec713a537f01be963474fe0f3691f5e6457f9f", size = 34022283 }, +] + [[package]] name = "pathspec" version = "1.0.4" @@ -5898,21 +7203,21 @@ wheels = [ [[package]] name = "playwright" -version = "1.58.0" +version = "1.60.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet" }, { name = "pyee" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098 }, - { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625 }, - { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098 }, - { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268 }, - { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214 }, - { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998 }, - { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005 }, - { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919 }, + { url = "https://files.pythonhosted.org/packages/21/f0/832bd9677194908da118064eef20082f2791e3d18215cc6d9391ee2c5a67/playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7", size = 43474635 }, + { url = "https://files.pythonhosted.org/packages/59/7b/e1d32ae8a3ed937ec2be3721c5f728b13d731a0b7c6442e0b3bec5094ac0/playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5", size = 42261327 }, + { url = "https://files.pythonhosted.org/packages/d7/bc/23de499ded6411c188a20c5a0dea6f0cd4ed5d2b3cc6042a5dbd3ed609aa/playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705", size = 43474636 }, + { url = "https://files.pythonhosted.org/packages/22/7b/1d679f4fced4ea94efadd17103856d8c565384f68382a1681264e46f5925/playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e", size = 47467220 }, + { url = "https://files.pythonhosted.org/packages/84/c2/1528d267d4442bd2c6b8eaeab819dd52c2030bf80e89293f0ba1f687473b/playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353", size = 47154856 }, + { url = "https://files.pythonhosted.org/packages/bb/4e/b008b6440a7a1624378041da94829956d4b8f7ab9ef5aad22d0dc3f2e26d/playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7", size = 37902157 }, + { url = "https://files.pythonhosted.org/packages/55/f0/0541524133104f9cc20bf900870ff4a736b76a23483f3a55295ddfa58409/playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02", size = 37902159 }, + { url = "https://files.pythonhosted.org/packages/80/c8/210f282d278e4709cdd71b12a31af45a30a22ab3207b387e29b37e478713/playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537", size = 34037981 }, ] [[package]] @@ -5942,7 +7247,7 @@ name = "portalocker" version = "3.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644 } wheels = [ @@ -6107,6 +7412,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305 }, ] +[[package]] +name = "protego" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/a7/955c422611d00a6e4a06d30b367ea9bb4fb09d48552e92aef1ba312493c7/protego-0.6.0.tar.gz", hash = "sha256:3466f41438421cf90008e98534d5fde47dc16a17482571d021143ac18b70ace9", size = 3137423 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/8c/f4dd590f48addf31398f78a78962eaa99eb4c87ac09c1927497032644731/protego-0.6.0-py3-none-any.whl", hash = "sha256:7210e6e06a8db839502baf1bfbcb810689a58e394d31408ef1ef9e4e3d79fc44", size = 10313 }, +] + [[package]] name = "proto-plus" version = "1.27.2" @@ -6167,8 +7481,8 @@ name = "psycopg" version = "3.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "tzdata", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d3/b6/379d0a960f8f435ec78720462fd94c4863e7a31237cf81bf76d0af5883bf/psycopg-3.3.3.tar.gz", hash = "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", size = 165624 } wheels = [ @@ -6177,7 +7491,7 @@ wheels = [ [package.optional-dependencies] binary = [ - { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, + { name = "psycopg-binary", marker = "implementation_name != 'pypy' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] pool = [ { name = "psycopg-pool" }, @@ -6719,7 +8033,7 @@ name = "pyobjc-framework-cocoa" version = "12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/a3/16ca9a15e77c061a9250afbae2eae26f2e1579eb8ca9462ae2d2c71e1169/pyobjc_framework_cocoa-12.1.tar.gz", hash = "sha256:5556c87db95711b985d5efdaaf01c917ddd41d148b1e52a0c66b1a2e2c5c1640", size = 2772191 } wheels = [ @@ -6735,8 +8049,8 @@ name = "pyobjc-framework-coreml" version = "12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/30/2d/baa9ea02cbb1c200683cb7273b69b4bee5070e86f2060b77e6a27c2a9d7e/pyobjc_framework_coreml-12.1.tar.gz", hash = "sha256:0d1a4216891a18775c9e0170d908714c18e4f53f9dc79fb0f5263b2aa81609ba", size = 40465 } wheels = [ @@ -6752,8 +8066,8 @@ name = "pyobjc-framework-quartz" version = "12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/94/18/cc59f3d4355c9456fc945eae7fe8797003c4da99212dd531ad1b0de8a0c6/pyobjc_framework_quartz-12.1.tar.gz", hash = "sha256:27f782f3513ac88ec9b6c82d9767eef95a5cf4175ce88a1e5a65875fee799608", size = 3159099 } wheels = [ @@ -6769,10 +8083,10 @@ name = "pyobjc-framework-vision" version = "12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-coreml", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "pyobjc-framework-quartz", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "pyobjc-core", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-cocoa", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-coreml", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "pyobjc-framework-quartz", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/5a/08bb3e278f870443d226c141af14205ff41c0274da1e053b72b11dfc9fb2/pyobjc_framework_vision-12.1.tar.gz", hash = "sha256:a30959100e85dcede3a786c544e621ad6eb65ff6abf85721f805822b8c5fe9b0", size = 59538 } wheels = [ @@ -6886,7 +8200,7 @@ name = "pytest" version = "9.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, @@ -6903,7 +8217,7 @@ version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087 } wheels = [ @@ -7025,6 +8339,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788 }, ] +[[package]] +name = "python-telegram-bot" +version = "22.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpcore", marker = "python_full_version >= '3.14' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/25/2258161b1069e66d6c39c0a602dbe57461d4767dc0012539970ea40bc9d6/python_telegram_bot-22.7.tar.gz", hash = "sha256:784b59ea3852fe4616ad63b4a0264c755637f5d725e87755ecdee28300febf61", size = 1516454 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/f7/0e2f89dd62f45d46d4ea0d8aec5893ce5b37389638db010c117f46f11450/python_telegram_bot-22.7-py3-none-any.whl", hash = "sha256:d72eed532cf763758cd9331b57a6d790aff0bb4d37d8f4e92149436fe21c6475", size = 745365 }, +] + [[package]] name = "pytz" version = "2026.1.post1" @@ -7237,7 +8564,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036 } wheels = [ @@ -7568,7 +8895,10 @@ wheels = [ torch = [ { name = "numpy" }, { name = "packaging" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, ] [[package]] @@ -7676,13 +9006,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165 }, ] +[[package]] +name = "scrapling" +version = "0.4.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cssselect" }, + { name = "lxml" }, + { name = "orjson" }, + { name = "tld" }, + { name = "typing-extensions" }, + { name = "w3lib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/34/61b72964fdeed6bcb9528262870b9870c936eac55541f39254ef876985ef/scrapling-0.4.9.tar.gz", hash = "sha256:e08afab736e5bd3337173e524fee99aea3073476e86d44d942af4bef9496a499", size = 156809 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/e6/03712d0b3240b3b6af6dc8aa19f0f24348fa54f4a4497b09c2d5c511552f/scrapling-0.4.9-py3-none-any.whl", hash = "sha256:00c7fae4641d948fb26486fd1da22143e5296dc86ef821b8426c0c2854c529ef", size = 158609 }, +] + +[package.optional-dependencies] +fetchers = [ + { name = "anyio" }, + { name = "apify-fingerprint-datapoints" }, + { name = "browserforge" }, + { name = "click" }, + { name = "curl-cffi" }, + { name = "msgspec" }, + { name = "patchright" }, + { name = "playwright" }, + { name = "protego" }, +] + [[package]] name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "jeepney", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cryptography", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or sys_platform == 'linux' or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "jeepney", marker = "(python_full_version < '3.13' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or sys_platform == 'linux' or (sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884 } wheels = [ @@ -7724,7 +9084,10 @@ dependencies = [ { name = "numpy" }, { name = "scikit-learn" }, { name = "scipy" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "tqdm" }, { name = "transformers" }, { name = "typing-extensions" }, @@ -7942,7 +9305,10 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "curated-tokenizers" }, { name = "curated-transformers" }, - { name = "torch" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d8/b3/a4fd3cf28008cbe1d95463b5c76a0d9c8da7b9ad4f06289c2be4aae62052/spacy_curated_transformers-0.3.1.tar.gz", hash = "sha256:7e53fccf64260e641b0a3f2b65b6d98381b86cef6eeb21ce279e8db849e8525d", size = 218990 } wheels = [ @@ -7972,7 +9338,7 @@ name = "sqlalchemy" version = "2.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/73/b4a9737255583b5fa858e0bb8e116eb94b88c910164ed2ed719147bde3de/sqlalchemy-2.0.48.tar.gz", hash = "sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7", size = 9886075 } @@ -8080,7 +9446,7 @@ version = "0.50.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ba/b8/73a0e6a6e079a9d9cfa64113d771e421640b6f679a52eeb9b32f72d871a1/starlette-0.50.0.tar.gz", hash = "sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca", size = 2646985 } wheels = [ @@ -8116,13 +9482,14 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.26" +version = "0.0.27" source = { editable = "." } dependencies = [ { name = "alembic" }, { name = "asyncpg" }, { name = "authlib" }, { name = "azure-ai-documentintelligence" }, + { name = "azure-storage-blob" }, { name = "boto3" }, { name = "celery", extra = ["redis"] }, { name = "chonkie", extra = ["all"] }, @@ -8175,7 +9542,6 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "opentelemetry-semantic-conventions" }, { name = "pgvector" }, - { name = "playwright" }, { name = "psycopg", extra = ["binary", "pool"] }, { name = "pyarrow" }, { name = "pyjwt" }, @@ -8183,8 +9549,10 @@ dependencies = [ { name = "pypandoc-binary" }, { name = "pypdf" }, { name = "python-ffmpeg" }, + { name = "python-telegram-bot" }, { name = "redis" }, { name = "rerankers", extra = ["flashrank"] }, + { name = "scrapling", extra = ["fetchers"] }, { name = "sentence-transformers" }, { name = "slack-sdk" }, { name = "slowapi" }, @@ -8205,6 +9573,26 @@ dependencies = [ { name = "youtube-transcript-api" }, ] +[package.optional-dependencies] +cpu = [ + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" }, +] +cu126 = [ + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "sys_platform == 'linux'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torchvision", version = "0.26.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "sys_platform == 'linux'" }, +] +cu128 = [ + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, + { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux'" }, +] + [package.dev-dependencies] dev = [ { name = "httpx" }, @@ -8220,6 +9608,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.30.0" }, { name = "authlib", specifier = ">=1.6.9" }, { name = "azure-ai-documentintelligence", specifier = ">=1.0.2" }, + { name = "azure-storage-blob", specifier = ">=12.23.0" }, { name = "boto3", specifier = ">=1.35.0" }, { name = "celery", extras = ["redis"], specifier = ">=5.5.3" }, { name = "chonkie", extras = ["all"], specifier = ">=1.5.0" }, @@ -8272,7 +9661,6 @@ requires-dist = [ { name = "opentelemetry-sdk", specifier = ">=1.40.0" }, { name = "opentelemetry-semantic-conventions", specifier = ">=0.61b0" }, { name = "pgvector", specifier = ">=0.3.6" }, - { name = "playwright", specifier = ">=1.50.0" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.2" }, { name = "pyarrow", specifier = ">=15.0.0,<19.0.0" }, { name = "pyjwt", specifier = ">=2.12.0" }, @@ -8280,8 +9668,10 @@ requires-dist = [ { name = "pypandoc-binary", specifier = ">=1.16.2" }, { name = "pypdf", specifier = ">=5.1.0" }, { name = "python-ffmpeg", specifier = ">=2.0.12" }, + { name = "python-telegram-bot", specifier = ">=22.7" }, { name = "redis", specifier = ">=5.2.1" }, { name = "rerankers", extras = ["flashrank"], specifier = ">=0.7.1" }, + { name = "scrapling", extras = ["fetchers"], specifier = ">=0.4.9" }, { name = "sentence-transformers", specifier = ">=3.4.1" }, { name = "slack-sdk", specifier = ">=3.34.0" }, { name = "slowapi", specifier = ">=0.1.9" }, @@ -8292,6 +9682,18 @@ requires-dist = [ { name = "static-ffmpeg", specifier = ">=2.13" }, { name = "stripe", specifier = ">=15.0.0" }, { name = "tavily-python", specifier = ">=0.3.2" }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'cpu'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cpu", conflict = { package = "surf-new-backend", extra = "cpu" } }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'cu126'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cu126", conflict = { package = "surf-new-backend", extra = "cu126" } }, + { name = "torch", marker = "sys_platform == 'linux' and extra == 'cu128'", specifier = "==2.11.0", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "surf-new-backend", extra = "cu128" } }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'cpu'", specifier = "==2.11.0" }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'cu126'", specifier = "==2.11.0" }, + { name = "torch", marker = "sys_platform != 'linux' and extra == 'cu128'", specifier = "==2.11.0" }, + { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'cpu'", specifier = "==0.26.0", index = "https://download.pytorch.org/whl/cpu", conflict = { package = "surf-new-backend", extra = "cpu" } }, + { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'cu126'", specifier = "==0.26.0", index = "https://download.pytorch.org/whl/cu126", conflict = { package = "surf-new-backend", extra = "cu126" } }, + { name = "torchvision", marker = "sys_platform == 'linux' and extra == 'cu128'", specifier = "==0.26.0", index = "https://download.pytorch.org/whl/cu128", conflict = { package = "surf-new-backend", extra = "cu128" } }, + { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'cpu'", specifier = "==0.26.0" }, + { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'cu126'", specifier = "==0.26.0" }, + { name = "torchvision", marker = "sys_platform != 'linux' and extra == 'cu128'", specifier = "==0.26.0" }, { name = "tornado", specifier = ">=6.5.5" }, { name = "trafilatura", specifier = ">=2.0.0" }, { name = "typst", specifier = ">=0.14.0" }, @@ -8301,6 +9703,7 @@ requires-dist = [ { name = "validators", specifier = ">=0.34.0" }, { name = "youtube-transcript-api", specifier = ">=1.0.3" }, ] +provides-extras = ["cpu", "cu126", "cu128"] [package.metadata.requires-dev] dev = [ @@ -8462,11 +9865,17 @@ name = "timm" version = "1.0.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pyyaml", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "safetensors", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "torch", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "torchvision", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "huggingface-hub", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pyyaml", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "safetensors", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and sys_platform == 'win32') or (sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and sys_platform == 'win32') or (sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torchvision", version = "0.26.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7b/1e/e924b3b2326a856aaf68586f9c52a5fc81ef45715eca408393b68c597e0e/timm-1.0.26.tar.gz", hash = "sha256:f66f082f2f381cf68431c22714c8b70f723837fa2a185b155961eab90f2d5b10", size = 2419859 } wheels = [ @@ -8530,21 +9939,55 @@ wheels = [ name = "torch" version = "2.11.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", +] dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "cuda-bindings", version = "13.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "cuda-toolkit", version = "13.0.2", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "filelock", marker = "sys_platform != 'linux'" }, + { name = "fsspec", marker = "sys_platform != 'linux'" }, + { name = "jinja2", marker = "sys_platform != 'linux'" }, + { name = "networkx", marker = "sys_platform != 'linux'" }, + { name = "nvidia-cudnn-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparselt-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nccl-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvshmem-cu13", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "setuptools", marker = "sys_platform != 'linux'" }, + { name = "sympy", marker = "sys_platform != 'linux'" }, + { name = "triton", marker = "(sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "typing-extensions", marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338 }, @@ -8569,14 +10012,175 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991 }, ] +[[package]] +name = "torch" +version = "2.11.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "filelock", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "fsspec", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "jinja2", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "networkx", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "setuptools", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "sympy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "typing-extensions", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp312-cp312-linux_s390x.whl", hash = "sha256:2db3ae5404e32cb42b5fcbd94f13607761eaec0cf1687fde95095289d1e26cfb" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:70ecb2659af6373b7c5336e692e665605b0201ea21ff51aaea47e1d75ea6b5aa" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f82e2ae20c1545bb03997d1cc3143d94e14b800038669ee1aca45808a9acc338" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:1abeaa46fa7532ed35ed79146f4de5d7a9d4b30462c98052ea4ddfe781ea3eca" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:d1eff25ccc454faf21c9666c81bfab8e405e87c12d300708d4559620bc191a36" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:48b3e21a311445acdd0b27f13830e21d93adef70d4721e051e9f059baeb9b8f9" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:45025d7752dbc6b4c784c03afaee9c5f19730ce084b2e43fc9a2fe1677d9ff86" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:ed70d4a4fc9f8b826c02fa1a9800a83820fb2fa6ae607680b53390f9ef394d85" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-linux_s390x.whl", hash = "sha256:65d427a196ab0abe359b93c5bffedd76ded02df2b1b1d2d9f11a2609b69f426a" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:8f13dc7075ae04ca5f876a9f40b4e47522a04c23e30824b4409f42a3f3e57aa4" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8713bb8679376ea0ec25742100b6cfb8447e0904c48bddefb9eb0ac1abbfa60a" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:62ec1f1694c185f601eab74eb7fc0e8e10c64c06ae82f13c3592774c231c4877" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:c9a14c367f470623b978e273a4e1915995b4ba7a0ae999178b06c273eea3536f" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:71676f6a9a84bbd385e010198b51fa1c2324fb8f3c512a32d2c81af65f68f4c9" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:f8481ea9088e4e5b81178a75aabdbb658bde8639bc1a15fd5d8f930abc966735" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:7575af4c9f7f7500ed62b1dafeb069aa0ba35b368a5f09793b3976b3d50f4fe4" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:825f1596878280a3a4c861441674888bc2d792e4ab7b045cb35feeab3f4f5dd7" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c8a0bdfb2fd915b6c2cd27c856f63f729c366a4917772eba6b2b02aa3bce70d5" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:768f22924a25cad2adeb9c6cbac5159e71067c8d4019b1511960d7435a5ca652" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:6db45e7b2526d996fbf47c3d08737807a60a4e17996a6d91a97027fe260832c8" }, +] + +[[package]] +name = "torch" +version = "2.11.0+cu126" +source = { registry = "https://download.pytorch.org/whl/cu126" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cuda-bindings", version = "12.9.7", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "cuda-toolkit", version = "12.6.3", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "filelock", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "fsspec", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "jinja2", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "networkx", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cudnn-cu12", version = "9.10.2.21", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparselt-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nccl-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvshmem-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "setuptools", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "sympy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "triton", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "typing-extensions", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:dc05f49d97c04a32d139df19d77209ceca3c484083f0db1dc332f168a19ef8d8" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9235c0c2f5032d1f8af75dd97493f3ad12aaedf81c0f3fab2d13c1d90bc54ff9" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp312-cp312-win_amd64.whl", hash = "sha256:1b7d728ddcf84b12f64ca10ec5f6ce91dd9edcc5e62ed5f0a7260ca00f029e8b" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:3bbbfc7ed16d878aef9b91a8e78c033bf131ffb8357e23ff80911210683a31b3" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5f6e2a5709876dd25a5fe0d4f0ab1ef33b33bb60431fcbfe768ab6468dcfe88a" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313-win_amd64.whl", hash = "sha256:12f7f1b73ba522576ea2050fbe2dc720a30cf681c73b8ea2d4f5480d66fad8f3" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7945e39fe82b85d7d018f68085ffaa6c5305a1634adb0bc792d835953b36fc9b" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:47adcccf4de0fdb4a1550f477d919daec74f1007d15d939acfcef3080d26f9ec" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp313-cp313t-win_amd64.whl", hash = "sha256:478ba5a57cba329f6878cf4c9bf6baa21c17c2e2b39784e0f7d3ee56b10a85a7" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9bef6e2c04c24163bf0cafab82ee2aecdf503b1817f936e1e6f5c652231b06f1" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:09083ecca5f6f6214b021d12c971763267cd1270df9ffcbb6291c815c483b1a0" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314-win_amd64.whl", hash = "sha256:146a8016c5abba673ea8af7d78e45e297b23e1f53c2a6749418a7c94f5d8d9ed" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:561ac0b9041f43c6acb927526f892cd64304eff8725536be0d86e27992588ab3" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:e0e31c656a407c11164d2d9257e31773562ca9ad82f755851dac511d48e9fc4d" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torch-2.11.0%2Bcu126-cp314-cp314t-win_amd64.whl", hash = "sha256:3b4deea8cb01bf8f336ac8f87298f67a44835044f24deea8e0dc2d33f5d3f68d" }, +] + +[[package]] +name = "torch" +version = "2.11.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "cuda-bindings", version = "12.9.7", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "cuda-toolkit", version = "12.8.1", source = { registry = "https://pypi.org/simple" }, extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "filelock", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "fsspec", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "jinja2", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "networkx", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cudnn-cu12", version = "9.19.0.56", source = { registry = "https://pypi.org/simple" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-cusparselt-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nccl-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "nvidia-nvshmem-cu12", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "setuptools", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "sympy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "triton", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "typing-extensions", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9c8f38efee365cb9d334de8a83ce52fc7e5fc9e5a7b0853285efa1b69e00b0f2" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d252cf975fb18c94a85336323ad425f473df56dab35a44b00399bd70c7a3b997" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:7c78215c3af4f62e63f2b2e360f1722fc719b0853c7ac22666483d9810613a4c" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7db3580106bba044da5b8950f3fb8fe5f31999eaab3f6a3aa2ac5d202c3684d2" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:db964b33c55035a72ab3e2162287af8f1cc276039c65d015740cc88c26dcedf7" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313-win_amd64.whl", hash = "sha256:6f367e62fd81b75cdf23ca4b75ced834d2db2cf98d1588ac935bde345de9de23" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd1cf1005c5fe419194ee294b7b584ba5ad0f2fb1778b3fe5a7b9c3f4617ddbc" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:74b628dbc71603977b09f4e140792c6e997081a35ef3421555f3f6e201b81210" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp313-cp313t-win_amd64.whl", hash = "sha256:c2a5984deba8e001d166bf9cb83b8351f63a28b009e1a2fa0e4bbf08c90b259b" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:baa52f7b8a53cab16587b10f1c27d1000ca033f97236878b685b75d5a1b92408" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d389a850677f0d24dafae1573644034428d8d3b9c80b51d55ba62fed7e6c8777" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314-win_amd64.whl", hash = "sha256:d6c21797ff75271b4fbdd905e2d703be4ecea5ea5bbdde4d1c201e9c71bc411d" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:06849e9311dbb0617c97557d9c26c99a9e1c4f2ac9cb8e9b6d9b420d522acb91" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:169a9987e1f84f0c5eee07544b3a34827a163ac9180e23abf0c3548f1335762c" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.11.0%2Bcu128-cp314-cp314t-win_amd64.whl", hash = "sha256:d86c125d720c2c368c53bd1a4ef062916d91fa965c10448c74c78b5d039faf2d" }, +] + [[package]] name = "torchvision" version = "0.26.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", + "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", +] dependencies = [ - { name = "numpy" }, - { name = "pillow" }, - { name = "torch" }, + { name = "numpy", marker = "sys_platform != 'linux'" }, + { name = "pillow", marker = "sys_platform != 'linux'" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502 }, @@ -8601,6 +10205,102 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/6a/09f3844c10643f6c0de5d95abc863420cfaf194c88c7dffd0ac523e2015f/torchvision-0.26.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e9d0e022c19a78552fb055d0414d47fecb4a649309b9968573daea160ba6869c", size = 4454275 }, ] +[[package]] +name = "torchvision" +version = "0.26.0+cpu" +source = { registry = "https://download.pytorch.org/whl/cpu" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pillow", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:17f0b542331fc94230b4214c6d123f038af7330fd81019608c0d2402f3bc3079" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:cf547dc0975eb40bc3249be4ccbeb736597d2c3ece305b1c4e5b7a5dd7363567" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:52aa8401850a9792e71a8a1e65ac004e2b23622a6b6fd278cd11179efbefc65b" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2e932af123a39137815dfd152c64cc683fa7cbd327c965e807c9728c7aa4971a" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:16c4f11eda096dc377e82238c8ebb26c7013622c0f1b2c4dcf85fc70f96c0ea7" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:34ac55a1f614baca2e0f5cef20ddb36184ee3503423871260e1ddd72caf9cb5f" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3d30ce3444698807d4b18b199645cd7a95e0b16a4cd0909b8aab47c562a7673a" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:870a97101168d4da68039d3d51f0c781047065e82dc4c19b2eb0ddff08486180" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:050aaf28cff9c2981ec72dc3f9b4ef77bcf9c9c99330ce426cb06c5bb9e6e726" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:78576c8d5a8665de6caaa6e7c3a3fb7caa5dc112032ba60e129a9e78a446a03b" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:78e88d0a57bfadcd17042aa92fe4dd1059e48fcaa2e54a10ac7f438c2eca10d5" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:93144d0997c51b27996c8305df4d9104efb0d38c9a9b6b05c8bc20ebdf7193b5" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:93a11b159613ad920b1d42c4eb4e585f48e5dff895f3e08f517ef482fe84e130" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:99f86ec0a83b9e4b5428a452bf667f99a9ae27d4c32bd4b2081fe917303e7710" }, + { url = "https://download-r2.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:6139108231a29ffb607931360ee24594553a939467c65530f734a2ed9918f011" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0+cu126" +source = { registry = "https://download.pytorch.org/whl/cu126" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pillow", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b9422be894e6969bcd33f20e0a1599c5e7d43b39ef548f35d3f83557053fd1a4" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a127341237ec2e70a54d031d2bae7be395f5d24c5a7c3e8b8b03cdd1fcebcb53" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp312-cp312-win_amd64.whl", hash = "sha256:10cc643e8c01634728e4f96b725a6096ea8cec423f2b8ba234ad139947b2ae17" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e2b05253a352b32cd8bb0a5d4ffa4d2c479b011c235bb73fefb052b1c18ac87f" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9d155c55b6f3516b60de24e32f194f79607d0ab9c6df6ba89ca20e181a352ca2" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313-win_amd64.whl", hash = "sha256:67f460b5cf6d8b6ede4cdca0c73987e51a00da434a4f5fa6aa325a264e7926f6" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5b7d96484a12491cb3f293464efa516154ccf9832862f32b601bb1a93645221" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:c22b31747f764ebb6d943f488f43a4de1d7ee4084ab0b945a6ca30d14407f9e8" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp313-cp313t-win_amd64.whl", hash = "sha256:bc331411de5b686012cdbf347169abc3973a6bdc333078a4c89c62431202d6a5" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:5730139e4d03e2121559a1f45625b14d149f1b7ce080fca824da7f9d2b6c7fc5" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:45b1aa89b7dd1110881675dbbe7e2409789b29494c495b004860570273aa5c73" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314-win_amd64.whl", hash = "sha256:4ff60e8041277de2f6c1b13d226063878650898afac7f72281af03158da70a3f" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:027ccbec15df76818557b6e0ae5afdb2cc8e389094e5ada19f75878e7def4aff" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:40cfd2c75ff3994b50c70f93220a6b5f2e77eb2acd08b4ad8340cebff89aeba2" }, + { url = "https://download-r2.pytorch.org/whl/cu126/torchvision-0.26.0%2Bcu126-cp314-cp314t-win_amd64.whl", hash = "sha256:56ebdb391066abcb28d748f8b0c67581a8cb03d2359d0cf3e86832e03c4fb1ee" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and sys_platform == 'linux'", + "python_full_version < '3.13' and sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pillow", marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:63e35234aed13b6edda37056f417b5c281249669db631e706811917af36b21d7" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ccf26b4b659cfce6f2208cb8326071d51c70219a34856dfdf468d1e19af52c0d" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:8c0d1c4fbb2c9a4d5d41d0aaa87da20e525bcb2a154ce405725b0be59456804b" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c4a9cacd521f2a4df0bcd9d8e96704771b928f478f1f3067e4085bb53a1da298" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cb1f6184a7ba30fba40580e1a01a6604a86c55e79fdda187f40116ee680441ec" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313-win_amd64.whl", hash = "sha256:0232cb219927a52d6c98ff202f32d1cdf4802c2195a85fc1f1a0c1b0b4983a4d" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e594732552a8c2fee2ace9c6475c6c6904fc44ccca622ee6765a89a045416a44" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6168abc019803ac9e97efce27eafd2fdb33db04dcc54a86039537729e5047b29" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp313-cp313t-win_amd64.whl", hash = "sha256:367d42ea703844ecdb516e9d5eb09929012a58705d2622cf4e9e3c37f278cb85" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:b3865fa227661dd75b7b28c96d3d14e739bd08bf0614132758922fe0e7206f91" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:aac647c9130f1f25f5c8f5bca3d95cfd96bdfac93ab54529690b088e64e4fa64" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314-win_amd64.whl", hash = "sha256:6319e1ba49c6f62ac9902f73d0eab207b8a4dc6b4d3392fe9edd9903fff1be0a" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:e2ee9e16ee4518292694537fcbd20d2d27044e381d92b864f637e82795796a84" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b5772c55bfda4377df8f1930d43c4e0231ef231b0228eade4b227c8d3ba6e34e" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.26.0%2Bcu128-cp314-cp314t-win_amd64.whl", hash = "sha256:f160dc552a086244f7102c898f7be8ef46a41b36bce5ea80a4f2493cb30ca1fc" }, +] + [[package]] name = "tornado" version = "6.5.5" @@ -8623,7 +10323,7 @@ name = "tqdm" version = "4.67.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598 } wheels = [ @@ -8817,7 +10517,7 @@ version = "6.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "id" }, - { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x'" }, + { name = "keyring", marker = "(platform_machine != 'ppc64le' and platform_machine != 's390x') or (platform_machine == 'ppc64le' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (platform_machine == 'ppc64le' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (platform_machine == 'ppc64le' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (platform_machine == 's390x' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (platform_machine == 's390x' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (platform_machine == 's390x' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "packaging" }, { name = "readme-renderer" }, { name = "requests" }, @@ -8930,7 +10630,7 @@ name = "tzlocal" version = "5.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "tzdata", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761 } wheels = [ @@ -8983,11 +10683,11 @@ all-docs = [ { name = "pdfminer-six" }, { name = "pi-heif" }, { name = "pikepdf" }, - { name = "pypandoc-binary", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "pypandoc-binary", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "pypdf" }, { name = "python-docx" }, { name = "python-pptx" }, - { name = "unstructured-inference", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "unstructured-inference", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "unstructured-pytesseract" }, { name = "xlrd" }, ] @@ -9016,22 +10716,25 @@ name = "unstructured-inference" version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "accelerate", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "huggingface-hub", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "matplotlib", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "onnx", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "onnxruntime", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "opencv-python", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pandas", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pdfminer-six", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "pypdfium2", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "python-multipart", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "rapidfuzz", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "scipy", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "timm", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "torch", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, - { name = "transformers", marker = "python_full_version < '3.13' or sys_platform != 'win32'" }, + { name = "accelerate", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "huggingface-hub", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "matplotlib", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "numpy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "onnx", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "onnxruntime", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "opencv-python", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pandas", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pdfminer-six", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "pypdfium2", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "python-multipart", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "rapidfuzz", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "scipy", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "timm", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and sys_platform == 'win32') or (sys_platform != 'linux' and sys_platform != 'win32') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu126", source = { registry = "https://download.pytorch.org/whl/cu126" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128')" }, + { name = "transformers", marker = "python_full_version < '3.13' or sys_platform != 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/10/8f3bccfa9f1e0101a402ae1f529e07876541c6b18004747f0e793ed41f9e/unstructured_inference-1.2.0.tar.gz", hash = "sha256:19ca28512f3649c70a759cf2a4e98663e942a1b83c1acdb9506b0445f4862f23", size = 45732 } wheels = [ @@ -9106,11 +10809,11 @@ wheels = [ [package.optional-dependencies] standard = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "httptools" }, { name = "python-dotenv" }, { name = "pyyaml" }, - { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "uvloop", marker = "(platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32') or (platform_python_implementation == 'PyPy' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (platform_python_implementation == 'PyPy' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (platform_python_implementation == 'PyPy' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'cygwin' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'cygwin' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'cygwin' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "watchfiles" }, { name = "websockets" }, ] @@ -9187,7 +10890,7 @@ dependencies = [ { name = "aiolimiter" }, { name = "ffmpeg-python" }, { name = "langchain-text-splitters" }, - { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "numpy", marker = "python_full_version < '3.14' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, { name = "pillow" }, { name = "pydantic" }, { name = "requests" }, @@ -9199,12 +10902,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/64/89f6325666d6836979f94ac88b96fefc7527e02e61abc81359843585e088/voyageai-0.3.7-py3-none-any.whl", hash = "sha256:909f6c033001e5a3b3caf970525bf3614a1bfef9003cf3c3b68207dfdb53e86d", size = 34691 }, ] +[[package]] +name = "w3lib" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/91/b2eb59c2cf243de5de1e91c963655df78c015509f51297685a8c86a27b8c/w3lib-2.4.1.tar.gz", hash = "sha256:8dd69ee39ff6398d708c793abc779c334a69bac7cee1cdf71736c669ed6be864", size = 48494 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/c3/f8b216cbd742e5b84c40f045204c764ccb7524d2aeab021054ec69446b0a/w3lib-2.4.1-py3-none-any.whl", hash = "sha256:40930132907e68de906a5b89331ab8c8ff4f01bd35b5539ef7896017d814138d", size = 21695 }, +] + [[package]] name = "wasabi" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126') or (extra == 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu128') or (extra == 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/f9/054e6e2f1071e963b5e746b48d1e3727470b2a490834d18ad92364929db3/wasabi-1.1.3.tar.gz", hash = "sha256:4bb3008f003809db0c3e28b4daf20906ea871a2bb43f9914197d540f4f2e0878", size = 30391 } wheels = [ diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index 13cd31b80..959e0b395 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.26", + "version": "0.0.27", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 1f0e6dafc..147d8d16b 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,7 +1,7 @@ { "name": "surfsense-desktop", "productName": "SurfSense", - "version": "0.0.26", + "version": "0.0.27", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_evals/README.md b/surfsense_evals/README.md index c6314af80..c755c4de6 100644 --- a/surfsense_evals/README.md +++ b/surfsense_evals/README.md @@ -137,15 +137,14 @@ Notes: - `--skip-unanswerable` (run) — drop unanswerable questions - `--docs ,` (run) — scope to specific docs -## Ingestion knobs (vision LLM, processing mode, summarize) +## Ingestion knobs (vision LLM, processing mode) -The harness exposes `POST /api/v1/documents/fileupload`'s three knobs on every `ingest` subcommand: +The harness exposes `POST /api/v1/documents/fileupload`'s ingest knobs on every `ingest` subcommand: | Flag pair | Effect | |--------------------------------------------|-----------------------------------------------------------------------------------------| | `--use-vision-llm` / `--no-vision-llm` | Walk every embedded image in the PDF and inline image-derived text at the image's position (see below). | | `--processing-mode {basic,premium}` | `premium` carries a 10× page multiplier and routes to a stronger ETL (e.g. LlamaCloud). | -| `--should-summarize` / `--no-summarize` | Generate a per-document summary at ingest. | The "Default ingest" column in the benchmarks table is what runs if you don't pass any flag. Whatever was actually used is recorded as a `__settings__` header in the doc map (`data//maps/_*_map.jsonl`) and as `extra.ingest_settings` in `run_artifact.json`, then surfaced in the report — no need to hunt through CLI history. diff --git a/surfsense_evals/src/surfsense_evals/core/clients/documents.py b/surfsense_evals/src/surfsense_evals/core/clients/documents.py index 02bcf74da..362aae53b 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py @@ -110,7 +110,6 @@ class DocumentsClient: files: Iterable[Path], *, search_space_id: int, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> FileUploadResult: @@ -149,7 +148,6 @@ class DocumentsClient: f"{self._base}/api/v1/documents/fileupload", data={ "search_space_id": str(search_space_id), - "should_summarize": "true" if should_summarize else "false", "use_vision_llm": "true" if use_vision_llm else "false", "processing_mode": processing_mode, }, diff --git a/surfsense_evals/src/surfsense_evals/core/clients/search_space.py b/surfsense_evals/src/surfsense_evals/core/clients/search_space.py index 37fa69f80..e2d37694d 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/search_space.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/search_space.py @@ -83,7 +83,6 @@ class LlmPreferences: """ agent_llm_id: int | None - document_summary_llm_id: int | None image_generation_config_id: int | None vision_llm_config_id: int | None agent_llm: dict[str, Any] | None @@ -93,7 +92,6 @@ class LlmPreferences: def from_payload(cls, payload: dict[str, Any]) -> LlmPreferences: return cls( agent_llm_id=payload.get("agent_llm_id"), - document_summary_llm_id=payload.get("document_summary_llm_id"), image_generation_config_id=payload.get("image_generation_config_id"), vision_llm_config_id=payload.get("vision_llm_config_id"), agent_llm=payload.get("agent_llm"), @@ -154,7 +152,6 @@ class SearchSpaceClient: search_space_id: int, *, agent_llm_id: int | None = None, - document_summary_llm_id: int | None = None, image_generation_config_id: int | None = None, vision_llm_config_id: int | None = None, ) -> LlmPreferences: @@ -167,8 +164,6 @@ class SearchSpaceClient: body: dict[str, Any] = {} if agent_llm_id is not None: body["agent_llm_id"] = agent_llm_id - if document_summary_llm_id is not None: - body["document_summary_llm_id"] = document_summary_llm_id if image_generation_config_id is not None: body["image_generation_config_id"] = image_generation_config_id if vision_llm_config_id is not None: diff --git a/surfsense_evals/src/surfsense_evals/core/ingest_settings.py b/surfsense_evals/src/surfsense_evals/core/ingest_settings.py index 5cdece577..8328e0d46 100644 --- a/surfsense_evals/src/surfsense_evals/core/ingest_settings.py +++ b/surfsense_evals/src/surfsense_evals/core/ingest_settings.py @@ -8,15 +8,13 @@ exactly three knobs (verified at * ``processing_mode`` — ``"basic"`` (default) | ``"premium"`` * ``use_vision_llm`` — ``bool`` (run vision LLM during ingest to extract image content / captions / tables) -* ``should_summarize`` — ``bool`` (generate document summary) This module gives every benchmark a uniform way to: 1. Receive sensible per-benchmark defaults (text-only benchmarks default vision off; image-bearing benchmarks default vision on). 2. Accept CLI overrides (``--use-vision-llm`` / ``--no-vision-llm``, - ``--processing-mode {basic,premium}``, - ``--should-summarize`` / ``--no-summarize``). + ``--processing-mode {basic,premium}``). 3. Persist the *actual* settings used into the doc-map manifest and the run artifact so reports can show "vision=ON, mode=premium → 65% accuracy" head-to-head with "vision=OFF, mode=basic → 52%". @@ -71,13 +69,11 @@ class IngestSettings: use_vision_llm: bool = False processing_mode: str = "basic" - should_summarize: bool = False def to_dict(self) -> dict[str, Any]: return { "use_vision_llm": self.use_vision_llm, "processing_mode": self.processing_mode, - "should_summarize": self.should_summarize, } @classmethod @@ -87,14 +83,13 @@ class IngestSettings: ``opts`` is the kwargs dict built by ``core.cli`` from the argparse namespace (see ``_cmd_ingest`` / ``_cmd_run``). Keys we look for: ``use_vision_llm`` (bool or None), ``processing_mode`` - (str or None), ``should_summarize`` (bool or None). Anything + (str or None). Anything else is ignored so benchmarks can pass through their own opts. """ return cls( use_vision_llm=_coerce_bool(opts.get("use_vision_llm"), defaults.use_vision_llm), processing_mode=_coerce_mode(opts.get("processing_mode"), defaults.processing_mode), - should_summarize=_coerce_bool(opts.get("should_summarize"), defaults.should_summarize), ) def render_label(self) -> str: @@ -102,8 +97,7 @@ class IngestSettings: return ( f"vision={'on' if self.use_vision_llm else 'off'}, " - f"mode={self.processing_mode}, " - f"summarize={'on' if self.should_summarize else 'off'}" + f"mode={self.processing_mode}" ) @@ -179,14 +173,14 @@ def add_ingest_settings_args( *, defaults: IngestSettings, ) -> None: - """Attach the three ingest-settings flag pairs to ``parser``. + """Attach ingest-settings flags to ``parser``. - Each bool exposes a mutually exclusive ``--foo`` / ``--no-foo`` - pair so an operator can flip either direction without restating - every flag. Default is ``None`` so that "operator didn't pass the - flag" is distinguishable from "operator explicitly passed false" - — ``IngestSettings.merge`` then folds in the benchmark default - only when the operator was silent. + The vision bool exposes a mutually exclusive ``--foo`` / ``--no-foo`` + pair so an operator can flip either direction without restating every + flag. Default is ``None`` so that "operator didn't pass the flag" is + distinguishable from "operator explicitly passed false" — + ``IngestSettings.merge`` then folds in the benchmark default only when + the operator was silent. """ settings_group = parser.add_argument_group( @@ -217,18 +211,6 @@ def add_ingest_settings_args( f"Default for this benchmark: {defaults.processing_mode!r}." ), ) - _add_bool_pair( - settings_group, - dest="should_summarize", - on_flag="--should-summarize", - off_flag="--no-summarize", - on_help=( - "Have SurfSense generate a document summary at ingest " - f"(default for this benchmark: " - f"{'on' if defaults.should_summarize else 'off'})." - ), - off_help="Skip per-document summary generation.", - ) # --------------------------------------------------------------------------- @@ -292,10 +274,9 @@ def format_ingest_settings_md(settings: Any) -> str: return "- SurfSense ingest settings: (not recorded — re-ingest to capture)" vision = "on" if settings.get("use_vision_llm") else "off" mode = settings.get("processing_mode") or "basic" - summarize = "on" if settings.get("should_summarize") else "off" return ( f"- SurfSense ingest settings: vision_llm=`{vision}`, " - f"processing_mode=`{mode}`, summarize=`{summarize}`" + f"processing_mode=`{mode}`" ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py index 6eca8810c..275e28ce5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py @@ -160,8 +160,7 @@ async def run_ingest( upload_result = await docs_client.upload( files=[b.path for b in batches], search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, - use_vision_llm=settings.use_vision_llm, + use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) new_doc_ids = list(upload_result.document_ids) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py index 416912b14..041e0e8b5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py @@ -63,7 +63,6 @@ _DESCRIPTION = "CUREv1 retrieval (single-arm SurfSense): Recall@k / MRR / nDCG@1 _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py index 5293e116f..ff43c7049 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py @@ -208,7 +208,6 @@ async def _upload_pdfs( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py index 75646ef32..e1a830138 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py @@ -169,7 +169,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=True, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py index 9769d078b..59006b6c0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py @@ -480,7 +480,6 @@ async def run_ingest( upload_result = await docs_client.upload( files=[b.path for b in batches], search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py index 0f336c0d5..b01b645a9 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py @@ -48,7 +48,6 @@ _DESCRIPTION = "MIRAGE (7,663 medical MCQs) — single-arm SurfSense per-task ac _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py index cf0572df8..15cdbeb77 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py @@ -225,7 +225,6 @@ async def _upload_pdfs( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py index 0e352d7ae..95a1e15eb 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py @@ -178,7 +178,6 @@ _TEXT_ONLY_HINTS = ("gpt-5.4-mini", "gpt-3.5", "text-only", "instruct-") _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=True, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py index aad6a70bf..4e0c2bdc5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py @@ -189,7 +189,6 @@ async def _upload_pages( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) @@ -306,8 +305,7 @@ async def run_ingest( settings = settings or IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, - ) + ) bench_dir = ctx.benchmark_data_dir() pages_dir = bench_dir / "pages" raw_cache = bench_dir / ".raw_cache" diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py index 710f76744..8b759e0d8 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py @@ -177,7 +177,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py index 9780be4ed..98e035f28 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py @@ -136,7 +136,6 @@ async def _upload_markdowns( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) @@ -240,8 +239,7 @@ async def run_ingest( settings = settings or IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, - ) + ) bench_dir = ctx.benchmark_data_dir() wiki_cache = bench_dir / "wiki" wiki_cache.mkdir(parents=True, exist_ok=True) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py index a8dde0dd2..9c0e16b00 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py @@ -153,7 +153,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/tests/core/test_clients.py b/surfsense_evals/tests/core/test_clients.py index 9e2c4ad75..611408703 100644 --- a/surfsense_evals/tests/core/test_clients.py +++ b/surfsense_evals/tests/core/test_clients.py @@ -69,7 +69,7 @@ async def test_set_llm_preferences_partial_update(respx_mock, http): 200, json={ "agent_llm_id": -10042, - "document_summary_llm_id": None, + "agent_llm_id": None, "image_generation_config_id": None, "vision_llm_config_id": None, "agent_llm": { diff --git a/surfsense_evals/tests/core/test_ingest_settings.py b/surfsense_evals/tests/core/test_ingest_settings.py index acfac57a6..fd7e7818a 100644 --- a/surfsense_evals/tests/core/test_ingest_settings.py +++ b/surfsense_evals/tests/core/test_ingest_settings.py @@ -4,7 +4,7 @@ Covers: * ``IngestSettings.merge`` honours operator overrides and falls back to per-benchmark defaults when the operator is silent. -* ``add_ingest_settings_args`` exposes the three flag pairs and +* ``add_ingest_settings_args`` exposes ingest settings flags and argparse defaults of ``None`` correctly distinguish "not passed" from "explicitly false". * ``settings_header_line`` / ``read_settings_header`` round-trip @@ -40,7 +40,7 @@ from surfsense_evals.core.ingest_settings import ( class TestMerge: def test_silent_operator_uses_defaults(self) -> None: - defaults = IngestSettings(use_vision_llm=True, processing_mode="basic", should_summarize=True) + defaults = IngestSettings(use_vision_llm=True, processing_mode="basic") merged = IngestSettings.merge(defaults, {}) assert merged == defaults @@ -111,17 +111,16 @@ class TestMerge: assert merged.processing_mode == "basic" def test_to_dict_round_trips(self) -> None: - s = IngestSettings(use_vision_llm=True, processing_mode="premium", should_summarize=False) + s = IngestSettings(use_vision_llm=True, processing_mode="premium") d = s.to_dict() assert d == { "use_vision_llm": True, "processing_mode": "premium", - "should_summarize": False, } def test_render_label_format(self) -> None: - s = IngestSettings(use_vision_llm=True, processing_mode="premium", should_summarize=True) - assert s.render_label() == "vision=on, mode=premium, summarize=on" + s = IngestSettings(use_vision_llm=True, processing_mode="premium") + assert s.render_label() == "vision=on, mode=premium" # --------------------------------------------------------------------------- @@ -136,7 +135,7 @@ class TestAddArgs: add_ingest_settings_args( p, defaults=IngestSettings( - use_vision_llm=False, processing_mode="basic", should_summarize=False + use_vision_llm=False, processing_mode="basic" ), ) return p @@ -145,7 +144,6 @@ class TestAddArgs: args = parser.parse_args([]) assert args.use_vision_llm is None assert args.processing_mode is None - assert args.should_summarize is None def test_use_vision_llm_flag(self, parser: argparse.ArgumentParser) -> None: args = parser.parse_args(["--use-vision-llm"]) @@ -166,12 +164,6 @@ class TestAddArgs: with pytest.raises(SystemExit): parser.parse_args(["--processing-mode", "exotic"]) - def test_summarize_flag_pair(self, parser: argparse.ArgumentParser) -> None: - on = parser.parse_args(["--should-summarize"]) - assert on.should_summarize is True - off = parser.parse_args(["--no-summarize"]) - assert off.should_summarize is False - def test_vision_flags_mutually_exclusive( self, parser: argparse.ArgumentParser ) -> None: @@ -185,11 +177,11 @@ class TestAddArgs: ["--use-vision-llm", "--processing-mode", "premium"] ) defaults = IngestSettings( - use_vision_llm=False, processing_mode="basic", should_summarize=False + use_vision_llm=False, processing_mode="basic" ) merged = IngestSettings.merge(defaults, vars(args)) assert merged == IngestSettings( - use_vision_llm=True, processing_mode="premium", should_summarize=False + use_vision_llm=True, processing_mode="premium" ) @@ -249,19 +241,17 @@ class TestHeader: class TestFormatMd: def test_full_settings(self) -> None: out = format_ingest_settings_md( - {"use_vision_llm": True, "processing_mode": "premium", "should_summarize": True} + {"use_vision_llm": True, "processing_mode": "premium"} ) assert "vision_llm=`on`" in out assert "processing_mode=`premium`" in out - assert "summarize=`on`" in out def test_default_off(self) -> None: out = format_ingest_settings_md( - {"use_vision_llm": False, "processing_mode": "basic", "should_summarize": False} + {"use_vision_llm": False, "processing_mode": "basic"} ) assert "vision_llm=`off`" in out assert "processing_mode=`basic`" in out - assert "summarize=`off`" in out def test_missing_returns_re_ingest_hint(self) -> None: # Empty dict + None + non-mapping should all degrade gracefully. diff --git a/surfsense_web/app/(home)/blog/[slug]/loading.tsx b/surfsense_web/app/(home)/blog/[slug]/loading.tsx index 0cce7f80b..fb90819d1 100644 --- a/surfsense_web/app/(home)/blog/[slug]/loading.tsx +++ b/surfsense_web/app/(home)/blog/[slug]/loading.tsx @@ -4,15 +4,6 @@ export default function BlogPostLoading() { return (
- {/* Breadcrumb */} -
- - - - - -
- {/* Tags */}
diff --git a/surfsense_web/app/(home)/blog/[slug]/page.tsx b/surfsense_web/app/(home)/blog/[slug]/page.tsx index 1f4d62968..96398a252 100644 --- a/surfsense_web/app/(home)/blog/[slug]/page.tsx +++ b/surfsense_web/app/(home)/blog/[slug]/page.tsx @@ -3,7 +3,6 @@ import type { Metadata } from "next"; import Image from "next/image"; import { notFound } from "next/navigation"; import { blog } from "@/.source/server"; -import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; import { ArticleJsonLd, FAQJsonLd } from "@/components/seo/json-ld"; import { extractFaqFromBlogPost } from "@/lib/blog-faq"; import { formatDate } from "@/lib/utils"; @@ -99,15 +98,6 @@ export default async function BlogPostPage(props: { params: Promise<{ slug: stri /> {faqEntries.length > 0 && }
- - {page.data.image && (
setSearch(e.target.value)} - placeholder="Search by title or topic…" + placeholder="Search" className="w-full rounded-full bg-white py-3 pr-4 pl-12 text-sm text-neutral-800 shadow-sm ring-1 shadow-black/10 ring-black/10 transition outline-none placeholder:text-neutral-400 focus:border-neutral-400 focus:ring-2 focus:ring-neutral-200/80 dark:bg-neutral-900 dark:text-neutral-100 dark:placeholder:text-neutral-500 dark:focus:border-neutral-500 dark:focus:ring-neutral-700/50" /> diff --git a/surfsense_web/app/(home)/changelog/loading.tsx b/surfsense_web/app/(home)/changelog/loading.tsx index 648f5a5e6..8128d1e49 100644 --- a/surfsense_web/app/(home)/changelog/loading.tsx +++ b/surfsense_web/app/(home)/changelog/loading.tsx @@ -8,12 +8,6 @@ export default function ChangelogLoading() {
- {/* Breadcrumb */} -
- - - -
diff --git a/surfsense_web/app/(home)/changelog/page.tsx b/surfsense_web/app/(home)/changelog/page.tsx index f3ded03fa..42bac512a 100644 --- a/surfsense_web/app/(home)/changelog/page.tsx +++ b/surfsense_web/app/(home)/changelog/page.tsx @@ -1,7 +1,12 @@ import { loader } from "fumadocs-core/source"; +import type { MDXComponents } from "mdx/types"; import type { Metadata } from "next"; +import type { ComponentType } from "react"; import { changelog } from "@/.source/server"; -import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; +import { + ChangelogTimeline, + type ChangelogTimelineEntry, +} from "@/components/ui/changelog-timeline"; import { formatDate } from "@/lib/utils"; import { getMDXComponents } from "@/mdx-components"; @@ -19,11 +24,9 @@ const source = loader({ }); interface ChangelogData { - title: string; date: string; version?: string; - tags?: string[]; - body: React.ComponentType<{ components?: Record }>; + body: ComponentType<{ components?: MDXComponents }>; } interface ChangelogPageItem { @@ -38,96 +41,25 @@ export default async function ChangelogPage() { const dateB = new Date(b.data.date).getTime(); return dateB - dateA; }); + const entries: ChangelogTimelineEntry[] = sortedChangelogs.map((changelog) => { + const MDX = changelog.data.body; + const date = new Date(changelog.data.date); + + return { + version: changelog.data.version ? `Version ${changelog.data.version}` : "Release", + date: formatDate(date), + content: , + }; + }); return (
- {/* Header */} -
-
-
-
- -

- Changelog -

-

- Stay up to date with the latest updates and improvements to SurfSense. -

-
-
-
-
- - {/* Timeline */} -
-
- {sortedChangelogs.map((changelog) => { - const MDX = changelog.data.body; - const date = new Date(changelog.data.date); - const formattedDate = formatDate(date); - - return ( -
-
-
-
- - - {changelog.data.version && ( -
- {changelog.data.version} -
- )} -
-
- - {/* Right side - Content */} -
- {/* Vertical timeline line */} -
- {/* Timeline dot */} -
-
- -
-
-

- {changelog.data.title} -

- - {/* Tags */} - {changelog.data.tags && changelog.data.tags.length > 0 && ( -
- {changelog.data.tags.map((tag: string) => ( - - {tag} - - ))} -
- )} -
-
- -
-
-
-
-
- ); - })} -
-
+
); } diff --git a/surfsense_web/app/(home)/free/[model_slug]/loading.tsx b/surfsense_web/app/(home)/free/[model_slug]/loading.tsx index 97660188d..a62fdefa3 100644 --- a/surfsense_web/app/(home)/free/[model_slug]/loading.tsx +++ b/surfsense_web/app/(home)/free/[model_slug]/loading.tsx @@ -32,15 +32,6 @@ export default function FreeModelLoading() { {/* SEO section skeleton */}
- {/* Breadcrumb */} -
- - - - - -
- diff --git a/surfsense_web/app/(home)/free/[model_slug]/page.tsx b/surfsense_web/app/(home)/free/[model_slug]/page.tsx index 35c5c0512..e72c3d6e3 100644 --- a/surfsense_web/app/(home)/free/[model_slug]/page.tsx +++ b/surfsense_web/app/(home)/free/[model_slug]/page.tsx @@ -3,7 +3,6 @@ import type { Metadata } from "next"; import Link from "next/link"; import { notFound } from "next/navigation"; import { FreeChatPage } from "@/components/free-chat/free-chat-page"; -import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; @@ -192,15 +191,7 @@ export default async function FreeModelPage({ params }: PageProps) { {/* SEO content: in DOM for crawlers, clipped by parent overflow-hidden */}
- - -
+

Chat with {model.name} Free, No Login

Use {model.name} free online without login or sign-up. No account, no diff --git a/surfsense_web/app/(home)/free/loading.tsx b/surfsense_web/app/(home)/free/loading.tsx index 08a4ed6b6..4548aa35f 100644 --- a/surfsense_web/app/(home)/free/loading.tsx +++ b/surfsense_web/app/(home)/free/loading.tsx @@ -4,13 +4,6 @@ export default function FreeChatLoading() { return (

- {/* Breadcrumb */} -
- - - -
- {/* Hero section */}
diff --git a/surfsense_web/app/(home)/free/page.tsx b/surfsense_web/app/(home)/free/page.tsx index 89a4735ae..0092ca2d5 100644 --- a/surfsense_web/app/(home)/free/page.tsx +++ b/surfsense_web/app/(home)/free/page.tsx @@ -3,7 +3,6 @@ import type { Metadata } from "next"; import Link from "next/link"; import { AdUnit } from "@/components/ads/ad-unit"; import { ADSENSE_SLOTS } from "@/components/ads/adsense-config"; -import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; import { FAQJsonLd, JsonLd } from "@/components/seo/json-ld"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -183,13 +182,6 @@ export default async function FreeHubPage() {
- - {/* Hero */}

diff --git a/surfsense_web/app/(home)/pricing/page.tsx b/surfsense_web/app/(home)/pricing/page.tsx index 5b0f98905..0b45deff4 100644 --- a/surfsense_web/app/(home)/pricing/page.tsx +++ b/surfsense_web/app/(home)/pricing/page.tsx @@ -1,6 +1,5 @@ import type { Metadata } from "next"; import PricingBasic from "@/components/pricing/pricing-section"; -import { BreadcrumbNav } from "@/components/seo/breadcrumb-nav"; export const metadata: Metadata = { title: "Pricing | SurfSense - Free AI Workspace, Automations & Agents", @@ -14,14 +13,6 @@ export const metadata: Metadata = { const page = () => { return (
-
- -
); diff --git a/surfsense_web/app/api/zero/query/route.ts b/surfsense_web/app/api/zero/query/route.ts index bff86c25e..35ef51fb5 100644 --- a/surfsense_web/app/api/zero/query/route.ts +++ b/surfsense_web/app/api/zero/query/route.ts @@ -13,7 +13,7 @@ import { schema } from "@/zero/schema"; // container and would make every authenticated Zero query fail with a 503. const backendURL = ( process.env.FASTAPI_BACKEND_INTERNAL_URL || - BACKEND_URL || + process.env.BACKEND_URL || "http://localhost:8000" ).replace(/\/$/, ""); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx index 4ff9b8b8c..ab6168305 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx @@ -1,6 +1,8 @@ "use client"; -import { ListOrdered, Settings2, Tag, Target } from "lucide-react"; +import { Dot } from "lucide-react"; +import { useState } from "react"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import type { AutomationDefinition } from "@/contracts/types/automation.types"; import { ExecutionSummary } from "./execution-summary"; import { InputsSchemaPreview } from "./inputs-schema-preview"; @@ -11,34 +13,30 @@ interface AutomationDefinitionSectionProps { } /** - * The Definition card. Read view; editing happens on the sibling /edit - * route (Edit button in the header). Layout is top-down: - * goal → tags → execution defaults → inputs schema (if any) → plan - * - * The schema_version is rendered as a small badge next to the section - * title so it's discoverable but doesn't fight for attention. + * User-facing read view of the saved automation definition. Editing happens on + * the sibling /edit route; this card should summarize behavior, not expose the + * raw persisted schema. */ export function AutomationDefinitionSection({ definition }: AutomationDefinitionSectionProps) { const hasTags = definition.metadata.tags.length > 0; const hasInputs = !!definition.inputs; + const [advancedOpen, setAdvancedOpen] = useState(false); + const stepCount = `${definition.plan.length} step${definition.plan.length === 1 ? "" : "s"}`; return ( - - Definition - - v{definition.schema_version} - + + Automation details {definition.goal && ( - +

{definition.goal}

)} {hasTags && ( - +
{definition.metadata.tags.map((tag) => ( )} - - - - {hasInputs && ( - + {definition.inputs && } )} + Plan + + {stepCount} + + } >
{definition.plan.map((step, idx) => ( ))}
+ + + {advancedOpen ? "Hide advanced options" : "Advanced options"} + + +
+
+ Execution defaults +
+ +
+
+
); } -function Field({ - icon: Icon, - label, - children, -}: { - icon: typeof Target; - label: string; - children: React.ReactNode; -}) { +function Field({ label, children }: { label: React.ReactNode; children: React.ReactNode }) { return (
-
- - {label} -
+
{label}
{children}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx index 0bce3fa2d..71730baeb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx @@ -8,7 +8,6 @@ import { updateAutomationMutationAtom } from "@/atoms/automations/automations-mu import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import type { Automation } from "@/contracts/types/automation.types"; -import { AutomationStatusBadge } from "../../components/automation-status-badge"; import { DeleteAutomationDialog } from "../../components/delete-automation-dialog"; interface AutomationDetailHeaderProps { @@ -70,12 +69,9 @@ export function AutomationDetailHeader({
-
-

- {automation.name} -

- -
+

+ {automation.name} +

{automation.description && (

{automation.description}

)} @@ -83,9 +79,15 @@ export function AutomationDetailHeader({
{canUpdate && ( - @@ -93,28 +95,39 @@ export function AutomationDetailHeader({ {canToggle && ( )} {canDelete && ( )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx index d31bd696d..bd683fe57 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx @@ -27,7 +27,6 @@ export function AutomationRunsSection({ automationId }: AutomationRunsSectionPro
- Recent runs

diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx index 558a089ac..abe739dcc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx @@ -26,9 +26,7 @@ export function AutomationTriggersSection({ Triggers -

- When this automation fires. v1 supports scheduled triggers only. -

+

When this automation runs

{triggers.length === 0 ? ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx index 5c4dc381c..82abce173 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx @@ -15,8 +15,8 @@ export function ExecutionSummary({ execution }: ExecutionSummaryProps) {
- - + + {execution.on_failure.length > 0 && ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx index 29d79d99b..dce6ac4a7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx @@ -1,5 +1,4 @@ "use client"; -import { JsonView } from "@/components/json-view"; import type { Inputs } from "@/contracts/types/automation.types"; interface InputsSchemaPreviewProps { @@ -13,9 +12,63 @@ interface InputsSchemaPreviewProps { * is null. */ export function InputsSchemaPreview({ inputs }: InputsSchemaPreviewProps) { + const fields = getInputFields(inputs.schema); + + if (fields.length === 0) { + return

No extra inputs are required.

; + } + return ( -
- +
+ {fields.map((field) => ( +
+
+
{field.name}
+ {field.description ? ( +
{field.description}
+ ) : null} +
+
+ {field.type} + {field.required ? " · required" : ""} +
+
+ ))}
); } + +function getInputFields(schema: Record): { + name: string; + type: string; + description?: string; + required: boolean; +}[] { + const properties = schema.properties; + if (!properties || typeof properties !== "object" || Array.isArray(properties)) { + return []; + } + + const required = new Set(Array.isArray(schema.required) ? schema.required : []); + return Object.entries(properties as Record).map(([name, value]) => { + const field = value && typeof value === "object" && !Array.isArray(value) ? value : {}; + return { + name, + type: formatType((field as Record).type), + description: + typeof (field as Record).description === "string" + ? ((field as Record).description as string) + : undefined, + required: required.has(name), + }; + }); +} + +function formatType(value: unknown): string { + if (Array.isArray(value)) return value.join(" or "); + if (typeof value === "string") return value; + return "value"; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx index 27cecf3bf..7505ef49b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx @@ -1,6 +1,4 @@ "use client"; -import { ArrowRightCircle, GitCommitHorizontal } from "lucide-react"; -import { JsonView } from "@/components/json-view"; import type { PlanStep } from "@/contracts/types/automation.types"; interface PlanStepCardProps { @@ -9,62 +7,35 @@ interface PlanStepCardProps { } /** - * Read-only view of one plan step. Renders the step_id + action prominently, - * then a definition list of the per-step knobs, and finally the params as - * formatted JSON. Editable mode is out of scope here — definition edits live - * on the (future) raw-JSON path. + * Read-only view of one plan step. Keep this user-facing: summarize what the + * step does and only show advanced step controls when they are explicitly set. */ export function PlanStepCard({ step, index }: PlanStepCardProps) { + const title = getStepTitle(step); + const details = getStepDetails(step); + return ( -
-
- +
+
+ {index + 1} - {step.step_id} - - {step.action} -
- -
- {(step.when || - step.output_as || - step.max_retries != null || - step.timeout_seconds != null) && ( -
- {step.when && ( - {step.when}} /> - )} - {step.output_as && ( - {step.output_as}} - /> - )} - {step.max_retries != null && ( - - )} - {step.timeout_seconds != null && ( - - )} -
- )} - -
-
- - Params -
-
- -
+
+

{title}

+ {details.length > 0 ? ( +
+ {details.map((detail) => ( + + ))} +
+ ) : null}
); } -function DefRow({ label, value }: { label: string; value: React.ReactNode }) { +function DefRow({ label, value }: { label: string; value: string }) { return (
{label}:
@@ -72,3 +43,106 @@ function DefRow({ label, value }: { label: string; value: React.ReactNode }) {
); } + +function getStepTitle(step: PlanStep): string { + if (step.action === "agent_task") { + return readStringParam(step.params, "query") ?? "Run an agent task"; + } + return sentenceCase(formatAction(step.action)); +} + +function getStepDetails(step: PlanStep): { label: string; value: string }[] { + const details: { label: string; value: string }[] = []; + + if (step.action === "agent_task") { + if (typeof step.params.auto_approve_all === "boolean") { + details.push({ + label: "Approval", + value: step.params.auto_approve_all ? "Auto-approve agent actions" : "Ask before actions", + }); + } + + const mentionSummary = summarizeMentions(step.params); + if (mentionSummary) { + details.push({ label: "Scope", value: mentionSummary }); + } + } else { + const readableParams = Object.entries(step.params) + .filter(([, value]) => value !== null && value !== undefined && value !== "") + .map(([key, value]) => `${sentenceCase(formatKey(key))}: ${formatValue(value)}`); + if (readableParams.length > 0) { + details.push({ label: "Details", value: readableParams.join(" · ") }); + } + } + + if (step.when) details.push({ label: "Runs when", value: step.when }); + if (step.output_as) details.push({ label: "Saves output as", value: step.output_as }); + if (step.max_retries != null) + details.push({ label: "Max retries", value: String(step.max_retries) }); + if (step.timeout_seconds != null) + details.push({ label: "Timeout", value: `${step.timeout_seconds}s` }); + + return details; +} + +function readStringParam(params: Record, key: string): string | null { + const value = params[key]; + return typeof value === "string" && value.trim() ? value : null; +} + +function summarizeMentions(params: Record): string | null { + const parts: string[] = []; + addMentionTitles(parts, params.mentioned_documents, "Documents and folders"); + addMentionTitles(parts, params.mentioned_connectors, "Connectors"); + if (parts.length === 0) { + addCount(parts, params.mentioned_document_ids, "document"); + addCount(parts, params.mentioned_folder_ids, "folder"); + addCount(parts, params.mentioned_connector_ids, "connector"); + } + return parts.length > 0 ? parts.join(", ") : null; +} + +function addMentionTitles(parts: string[], value: unknown, label: string): void { + if (!Array.isArray(value) || value.length === 0) return; + const titles = value + .map((entry) => { + const record = asRecord(entry); + const title = typeof record.title === "string" ? record.title : null; + const accountName = typeof record.account_name === "string" ? record.account_name : null; + return title ?? accountName; + }) + .filter((title): title is string => !!title); + if (titles.length === 0) return; + parts.push(`${label}: ${titles.join(", ")}`); +} + +function addCount(parts: string[], value: unknown, singular: string): void { + if (!Array.isArray(value) || value.length === 0) return; + parts.push(`${value.length} ${singular}${value.length === 1 ? "" : "s"}`); +} + +function formatAction(action: string): string { + return formatKey(action); +} + +function formatKey(key: string): string { + return key.replace(/_/g, " "); +} + +function sentenceCase(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1); +} + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function formatValue(value: unknown): string { + if (typeof value === "boolean") return value ? "Yes" : "No"; + if (typeof value === "string" || typeof value === "number") return String(value); + if (Array.isArray(value)) return `${value.length} item${value.length === 1 ? "" : "s"}`; + if (value && typeof value === "object") return "Configured"; + return String(value); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx index 164f156e5..ab82589dc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx @@ -15,7 +15,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { ScrollArea } from "@/components/ui/scroll-area"; import { Separator } from "@/components/ui/separator"; import { Skeleton } from "@/components/ui/skeleton"; -import type { RunStepResult } from "@/contracts/types/automation.types"; +import type { RunStatus, RunStepResult } from "@/contracts/types/automation.types"; import { useAutomationRun } from "@/hooks/use-automation-runs"; import { cn } from "@/lib/utils"; import { RunStepResultCard } from "./run-step-result-card"; @@ -23,44 +23,50 @@ import { RunStepResultCard } from "./run-step-result-card"; interface RunDetailsPanelProps { automationId: number; runId: number; + /** Live step entries from Zero; rendered while the run is in-flight and + * also kept as the authoritative source once it finishes. */ + liveSteps: RunStepResult[]; + /** Live run status from Zero. Used to hide diagnostic sections that + * only make sense after the run reaches a terminal state. */ + liveStatus: RunStatus; } /** - * Expanded view of a single run. Fetches lazily — the parent only renders - * this once the row is opened, so the list view stays cheap. + * Expanded view of a single run. Steps render immediately from the live + * Zero row so the panel updates as the run progresses; the heavy REST + * payload (output, artifacts, resolved inputs, run-level error) is + * fetched lazily and merged in when it arrives. * - * We surface the run outcome readably: a run-level error first (when - * present), then per-step cards that render the agent's markdown - * ``final_message`` directly, and finally the structural artifacts/inputs. - * The full ``definition_snapshot`` is omitted because it usually mirrors the - * live definition — surfacing it would dominate the panel without informing + * Surfacing order is outcome-first: a run-level error (when present), + * then per-step cards that render the agent's markdown ``final_message`` + * directly, and finally the structural artifacts/inputs. The full + * ``definition_snapshot`` is omitted because it usually mirrors the live + * definition — surfacing it would dominate the panel without informing * what the user is trying to learn ("did this work? what did it do?"). */ -export function RunDetailsPanel({ automationId, runId }: RunDetailsPanelProps) { - const { data: run, isLoading, error } = useAutomationRun(automationId, runId); +export function RunDetailsPanel({ + automationId, + runId, + liveSteps, + liveStatus, +}: RunDetailsPanelProps) { + const isTerminal = liveStatus !== "pending" && liveStatus !== "running"; + // Defer the REST round-trip until the run can actually carry heavy + // fields — output/artifacts/error are only written at terminal mark. + const { + data: run, + isLoading, + error, + } = useAutomationRun(automationId, runId, { + enabled: isTerminal, + }); - if (isLoading) { - return ( -
- - -
- ); - } - - if (error || !run) { - return ( -
- Couldn't load run details{error?.message ? `: ${error.message}` : "."} -
- ); - } - - const runError = run.error && Object.keys(run.error).length > 0 ? run.error : null; - const hasOutput = run.output && Object.keys(run.output).length > 0; - const hasInputs = Object.keys(run.inputs ?? {}).length > 0; - const steps = run.step_results as RunStepResult[]; - const hasDiagnostics = run.artifacts.length > 0 || hasInputs; + const runError = run?.error && Object.keys(run.error).length > 0 ? run.error : null; + const hasOutput = !!run?.output && Object.keys(run.output).length > 0; + const hasInputs = !!run && Object.keys(run.inputs ?? {}).length > 0; + const hasDiagnostics = !!run && (run.artifacts.length > 0 || hasInputs); + const heavyLoading = isTerminal && isLoading && !run; + const heavyError = isTerminal && !!error; return (
@@ -72,30 +78,40 @@ export function RunDetailsPanel({ automationId, runId }: RunDetailsPanelProps) {

) : null} -
- {steps.length === 0 ? ( -

No steps recorded.

+
+ {liveSteps.length === 0 ? ( +

+ {isTerminal ? "No steps recorded." : "Waiting for first step…"} +

) : (
- {steps.map((step, index) => ( + {liveSteps.map((step, index) => ( ))}
)}
- {hasDiagnostics ? : null} - - {run.artifacts.length > 0 ? ( -
- -
- ) : null} - - {hasInputs ? ( -
- -
+ {heavyLoading ? ( + + ) : heavyError ? ( +

+ Couldn't load run details{error?.message ? `: ${error.message}` : "."} +

+ ) : hasDiagnostics ? ( + <> + + {run && run.artifacts.length > 0 ? ( +
+ +
+ ) : null} + {hasInputs ? ( +
+ +
+ ) : null} + ) : null}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx index 3f6a39c35..b48230e3f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-row.tsx @@ -1,21 +1,21 @@ "use client"; import { ChevronDown, ChevronRight, Hand } from "lucide-react"; import { useState } from "react"; -import type { RunSummary } from "@/contracts/types/automation.types"; +import type { LiveRunSummary } from "@/hooks/use-automation-runs"; import { formatDuration } from "@/lib/automations/run-duration"; import { formatRelativeDate } from "@/lib/format-date"; import { RunDetailsPanel } from "./run-details-panel"; import { RunStatusBadge } from "./run-status-badge"; interface RunRowProps { - run: RunSummary; + run: LiveRunSummary; automationId: number; } /** - * One run row. Click to expand → fetches the full run and shows the - * details panel inline. State is local to each row so multiple panels - * can be open at once (or none). + * One run row. Click to expand → renders the details panel inline. + * Status and step_results come live from the parent's Zero query; the + * panel itself only fetches the heavy REST fields on first expand. */ export function RunRow({ run, automationId }: RunRowProps) { const [open, setOpen] = useState(false); @@ -47,7 +47,14 @@ export function RunRow({ run, automationId }: RunRowProps) {
- {open && } + {open && ( + + )}
); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx index 681877523..de156a09c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx @@ -1,15 +1,36 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertCircle, CalendarClock, Clock, Pencil, Save, Trash2 } from "lucide-react"; +import { AlertCircle, MoreHorizontal, Pencil, Trash2 } from "lucide-react"; import { useState } from "react"; import { updateTriggerMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; -import { JsonView } from "@/components/json-view"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { type Trigger, triggerUpdateRequest } from "@/contracts/types/automation.types"; import { describeCron } from "@/lib/automations/describe-cron"; -import { formatRelativeDate, formatRelativeFutureDate } from "@/lib/format-date"; +import { + DEFAULT_SCHEDULE, + fromCron, + type ScheduleFrequency, + toCron, +} from "@/lib/automations/schedule-builder"; +import { formatRelativeFutureDate } from "@/lib/format-date"; +import { TimezoneCombobox } from "../../components/builder/timezone-combobox"; import { DeleteTriggerDialog } from "./delete-trigger-dialog"; interface TriggerCardProps { @@ -19,27 +40,58 @@ interface TriggerCardProps { canDelete: boolean; } +type SimpleFrequency = Extract | "custom"; + interface TriggerDraft { - params: Record; - static_inputs: Record; + frequency: SimpleFrequency; + hour: number; + minute: number; + timezone: string; + cron: string; } +const SIMPLE_FREQUENCIES = new Set(["hourly", "daily", "weekdays"]); + function draftFromTrigger(trigger: Trigger): TriggerDraft { + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : ""; + const timezone = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; + const model = fromCron(cron); + if (model && SIMPLE_FREQUENCIES.has(model.frequency)) { + return { + frequency: model.frequency as SimpleFrequency, + hour: model.hour, + minute: model.minute, + timezone, + cron, + }; + } return { - params: trigger.params, - static_inputs: trigger.static_inputs ?? {}, + frequency: "custom", + hour: DEFAULT_SCHEDULE.hour, + minute: DEFAULT_SCHEDULE.minute, + timezone, + cron, }; } +function pad(value: number): string { + return value.toString().padStart(2, "0"); +} + +function clampInt(raw: string, min: number, max: number): number { + const value = Number.parseInt(raw, 10); + if (Number.isNaN(value)) return min; + return Math.min(max, Math.max(min, value)); +} + /** * One trigger row in the Triggers section of the detail page. Renders: - * - type icon + human-readable schedule + timezone - * - last_fired_at / next_fire_at hints - * - static_inputs as formatted JSON (when present) - * - enable toggle + remove button + inline edit (each gated independently) + * - human-readable schedule + * - compact enable toggle + * - dropdown actions for edit/remove * - * Inline edit covers ``params`` and ``static_inputs`` — the two fields the - * backend ``PATCH /triggers/[id]`` endpoint accepts beyond ``enabled``. + * Inline edit keeps schedule editing intentionally small: common frequencies, + * time, timezone, and raw cron only for schedules outside the simple model. * ``enabled`` stays on the Switch so the two surfaces don't fight. */ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: TriggerCardProps) { @@ -51,10 +103,9 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri const [issues, setIssues] = useState([]); const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : undefined; - const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; const human = cron ? describeCron(cron) : trigger.type; - const triggerLabel = cron ? `${human} · ${tz}` : trigger.type; - const hasStaticInputs = Object.keys(trigger.static_inputs ?? {}).length > 0; + const triggerLabel = human; + const showActions = (canUpdate && !isEditing) || canDelete; async function handleToggle(checked: boolean) { await updateTrigger({ @@ -77,7 +128,22 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri async function saveEdit() { setIssues([]); - const result = triggerUpdateRequest.safeParse(draft); + const params = + draft.frequency === "custom" + ? { cron: draft.cron.trim(), timezone: draft.timezone } + : { + cron: toCron({ + ...DEFAULT_SCHEDULE, + frequency: draft.frequency, + hour: draft.hour, + minute: draft.minute, + }), + timezone: draft.timezone, + }; + const result = triggerUpdateRequest.safeParse({ + params, + static_inputs: trigger.static_inputs ?? {}, + }); if (!result.success) { setIssues( result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) @@ -98,134 +164,212 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri return ( <> -
-
-
- -
-
- {human} - · {tz} -
- {cron && {cron}} -
-
+
+
+
{human}
-
+
{canUpdate && ( -
- - {trigger.enabled ? "Enabled" : "Off"} - - -
+ )} - {canUpdate && !isEditing && ( - - )} - {canDelete && ( - + {showActions && ( + + + + + + {canUpdate && !isEditing && ( + + + Edit + + )} + {canDelete && ( + setDeleteOpen(true)}> + + Delete + + )} + + )}
-
- {isEditing ? ( - <> -
- setDraft(next as TriggerDraft)} - collapsed={false} - /> + {!isEditing && trigger.next_fire_at ? ( +
+
+ Next fire: +
+
+ {formatRelativeFutureDate(trigger.next_fire_at)} +
+
+ ) : null} + + {isEditing ? ( +
+
+
+ +
- {issues.length > 0 && ( -
-
- - {issues.length === 1 ? "1 issue" : `${issues.length} issues`} -
-
    + {draft.frequency === "hourly" ? ( +
    + + + setDraft((prev) => ({ + ...prev, + minute: clampInt(event.target.value, 0, 59), + })) + } + /> +
    + ) : draft.frequency !== "custom" ? ( +
    + + { + const [hour, minute] = event.target.value.split(":"); + setDraft((prev) => ({ + ...prev, + hour: clampInt(hour, 0, 23), + minute: clampInt(minute, 0, 59), + })); + }} + /> +
    + ) : ( +
    + + + setDraft((prev) => ({ ...prev, cron: event.target.value })) + } + /> +
    + )} + +
    +
    Timezone
    + setDraft((prev) => ({ ...prev, timezone }))} + /> +
    +
+ + {issues.length > 0 && ( + + + + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} + + +
    {issues.map((issue) => (
  • {issue}
  • ))}
-
- )} + + + )} -
- - -
- - ) : ( - <> - {(trigger.last_fired_at || trigger.next_fire_at) && ( -
- {trigger.next_fire_at && ( - - )} - {trigger.last_fired_at && ( - - )} -
- )} - - {hasStaticInputs && ( -
-
Static inputs
-
- -
-
- )} - - )} -
+
+ + +
+
+ ) : null}
{canDelete && ( @@ -240,35 +384,3 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri ); } - -function TimeRow({ - label, - iso, - tense, - highlight = false, -}: { - label: string; - iso: string; - tense: "past" | "future"; - highlight?: boolean; -}) { - const formatted = tense === "future" ? formatRelativeFutureDate(iso) : formatRelativeDate(iso); - return ( - <> -
- - {label} -
-
- {formatted} -
- - ); -} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx index 2c9db217d..c05bff7d9 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx @@ -51,9 +51,17 @@ export function AutomationEditContent({ searchSpaceId, automationId }: Automatio } return ( - <> - - - + ( + + )} + /> ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx index 6b2a31822..ca477220e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx @@ -1,15 +1,21 @@ "use client"; import { ArrowLeft } from "lucide-react"; import Link from "next/link"; +import type { ReactNode } from "react"; import { Button } from "@/components/ui/button"; import type { Automation } from "@/contracts/types/automation.types"; interface AutomationEditHeaderProps { automation: Automation; searchSpaceId: number; + modeSwitcher?: ReactNode; } -export function AutomationEditHeader({ automation, searchSpaceId }: AutomationEditHeaderProps) { +export function AutomationEditHeader({ + automation, + searchSpaceId, + modeSwitcher, +}: AutomationEditHeaderProps) { const detailHref = `/dashboard/${searchSpaceId}/automations/${automation.id}`; return ( @@ -20,11 +26,11 @@ export function AutomationEditHeader({ automation, searchSpaceId }: AutomationEd Back to automation -
+

Edit automation

-

{automation.name}

+ {modeSwitcher ?
{modeSwitcher}
: null}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx index 756221d38..d9c949058 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx @@ -1,5 +1,6 @@ "use client"; -import { ShieldAlert } from "lucide-react"; +import { AlertCircle, ShieldAlert } from "lucide-react"; +import { Alert, AlertDescription } from "@/components/ui/alert"; import { useAutomations } from "@/hooks/use-automations"; import { AutomationsEmptyState } from "./components/automations-empty-state"; import { AutomationsHeader } from "./components/automations-header"; @@ -60,9 +61,10 @@ export function AutomationsContent({ searchSpaceId }: AutomationsContentProps) { loading={false} canCreate={perms.canCreate} /> -
-

Couldn't load automations. {error.message}

-
+ + + Couldn't load automations {error.message} + ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx index 229a417dc..95ee23445 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx @@ -8,7 +8,6 @@ import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, - DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import type { AutomationSummary } from "@/contracts/types/automation.types"; @@ -58,25 +57,21 @@ export function AutomationRowActions({ - + {canToggle && ( {pauseLabel} )} - {canToggle && canDelete && } {canDelete && ( - setDeleteOpen(true)} - className="text-destructive focus:text-destructive" - > + setDeleteOpen(true)}> Delete diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx index a59fb4527..74c95cee4 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx @@ -26,35 +26,30 @@ export function AutomationRow({ canDelete, }: AutomationRowProps) { return ( - - -
- - {automation.name} - - {automation.description && ( - - {automation.description} - - )} -
+ + + + {automation.name} + - + - + {formatRelativeDate(automation.updated_at)} - - + +
+ +
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx index ecf171e78..c3cab1dc1 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx @@ -1,5 +1,4 @@ "use client"; -import { Archive, CircleDot, Pause } from "lucide-react"; import type { AutomationStatus } from "@/contracts/types/automation.types"; import { cn } from "@/lib/utils"; @@ -8,41 +7,32 @@ interface AutomationStatusBadgeProps { className?: string; } -// Color + icon per status. Active = green, paused = amber, archived = muted. -const STATUS_STYLES: Record< - AutomationStatus, - { label: string; icon: typeof CircleDot; classes: string } -> = { +// Small borderless status pills, matching model-selector badges. +const STATUS_STYLES: Record = { active: { label: "Active", - icon: CircleDot, - classes: - "bg-emerald-50 text-emerald-700 border border-emerald-200 dark:bg-emerald-950/40 dark:text-emerald-300 dark:border-emerald-900/50", + classes: "bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300", }, paused: { label: "Paused", - icon: Pause, - classes: - "bg-amber-50 text-amber-700 border border-amber-200 dark:bg-amber-950/40 dark:text-amber-300 dark:border-amber-900/50", + classes: "bg-amber-100 text-amber-700 dark:bg-amber-900/50 dark:text-amber-300", }, archived: { label: "Archived", - icon: Archive, - classes: "bg-muted text-muted-foreground border border-border/60", + classes: "bg-muted text-muted-foreground", }, }; export function AutomationStatusBadge({ status, className }: AutomationStatusBadgeProps) { - const { label, icon: Icon, classes } = STATUS_STYLES[status]; + const { label, classes } = STATUS_STYLES[status]; return ( - {label} ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx index cc54c5e94..b2e7b2532 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx @@ -1,5 +1,5 @@ "use client"; -import { MessageSquarePlus, SquarePen, Workflow } from "lucide-react"; +import { Workflow } from "lucide-react"; import Link from "next/link"; import { Button } from "@/components/ui/button"; @@ -28,16 +28,14 @@ export function AutomationsEmptyState({ searchSpaceId, canCreate }: AutomationsE {canCreate ? (
-
) : ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx index 137727f60..5c1fcb507 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx @@ -1,5 +1,4 @@ "use client"; -import { MessageSquarePlus, SquarePen } from "lucide-react"; import Link from "next/link"; import { Button } from "@/components/ui/button"; @@ -41,17 +40,16 @@ export function AutomationsHeader({
{canCreate && showCreateCta && (
-
)} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx index ec3aeeef5..8314a5179 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx @@ -1,5 +1,5 @@ "use client"; -import { Activity, CalendarDays, Workflow } from "lucide-react"; +import { CalendarDays, Info, Workflow } from "lucide-react"; import { Table, TableBody, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import type { AutomationSummary } from "@/contracts/types/automation.types"; import { AutomationRow } from "./automation-row"; @@ -37,7 +37,7 @@ export function AutomationsTable({ - + Status diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx index 740f199af..110de57f6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx @@ -58,7 +58,7 @@ export function AdvancedSection({ return (
- + - + - + {BACKOFF_OPTIONS.map((option) => ( {option.label} @@ -105,7 +105,7 @@ export function AdvancedSection({ - + {CONCURRENCY_OPTIONS.map((option) => ( {option.label} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx index 39904dfa0..59967080f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx @@ -1,8 +1,8 @@ "use client"; import { useAtomValue } from "jotai"; -import { Code2, LayoutList, Save } from "lucide-react"; -import Link from "next/link"; +import { AlertCircle, Code2, LayoutList } from "lucide-react"; import { useRouter } from "next/navigation"; +import type { ReactNode } from "react"; import { useMemo, useState } from "react"; import type { z } from "zod"; import { @@ -12,9 +12,12 @@ import { updateAutomationMutationAtom, updateTriggerMutationAtom, } from "@/atoms/automations/automations-mutation.atoms"; +import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Separator } from "@/components/ui/separator"; import { Spinner } from "@/components/ui/spinner"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { type Automation, @@ -35,7 +38,6 @@ import { hasResolvedModels, hydrateForm, } from "@/lib/automations/builder-schema"; -import { cn } from "@/lib/utils"; import { AdvancedSection } from "./advanced-section"; import { AutomationModelFields } from "./automation-model-fields"; import { BasicsSection } from "./basics-section"; @@ -56,6 +58,7 @@ interface AutomationBuilderFormProps { * eligibility itself is now owned by the in-form pickers. */ submitDisabledReason?: string; + renderModeSwitcher?: (modeSwitcher: ReactNode) => ReactNode; } type Mode = "form" | "json"; @@ -78,6 +81,7 @@ export function AutomationBuilderForm({ searchSpaceId, automation, submitDisabledReason, + renderModeSwitcher, }: AutomationBuilderFormProps) { const router = useRouter(); const { mutateAsync: createAutomation } = useAtomValue(createAutomationMutationAtom); @@ -97,7 +101,7 @@ export function AutomationBuilderForm({ return { mode: "json" as Mode, form: createEmptyForm(), - notice: `This automation ${result.reason}, which the form can't show. Edit it as JSON below.`, + notice: `This automation ${result.reason}, which the form can't show. Edit it as JSON below`, }; } return { mode: "form" as Mode, form: createEmptyForm(), notice: undefined }; @@ -116,11 +120,6 @@ export function AutomationBuilderForm({ const [submitting, setSubmitting] = useState(false); - const cancelHref = - mode === "edit" && automation - ? `/dashboard/${searchSpaceId}/automations/${automation.id}` - : `/dashboard/${searchSpaceId}/automations`; - // Eligible models + the search-space-seeded defaults. Models are chosen per // automation on create; in edit mode the backend preserves the captured // snapshot, so the picker is create-only. @@ -192,7 +191,7 @@ export function AutomationBuilderForm({ // form's own validation enforces completeness on submit. const definition = jsonValue.definition; if (!definition || typeof definition !== "object") { - return { ok: false, issues: [], notice: "Add a definition before switching to the form." }; + return { ok: false, issues: [], notice: "Add a definition before switching to the form" }; } const name = @@ -210,7 +209,7 @@ export function AutomationBuilderForm({ const h = hydrateForm(name, description, definition, triggers); return h.formable ? { ok: true, form: h.form } - : { ok: false, issues: [], notice: `Can't show in the form: it ${h.reason}.` }; + : { ok: false, issues: [], notice: `Can't show in the form: it ${h.reason}` }; } function validateForm(): Record | null { @@ -328,119 +327,133 @@ export function AutomationBuilderForm({ : undefined); // Only gate creation; editing an existing automation isn't blocked here. const submitBlocked = mode === "create" && !!effectiveDisabledReason; + const modeSwitcher = ( + { + if (value === activeMode) return; + if (value === "form") switchToForm(); + else if (value === "json") switchToJson(); + }} + > + + + + Form + + + + Edit as JSON + + + + ); return (
-
-
- (activeMode === "form" ? undefined : switchToForm())} - /> - (activeMode === "json" ? undefined : switchToJson())} - /> -
-
+ {renderModeSwitcher ? ( + renderModeSwitcher(modeSwitcher) + ) : ( +
{modeSwitcher}
+ )} {activeMode === "json" ? ( - - - - - + ) : (
-
- - - Basics - - - - - - - - - Tasks - - - patchForm({ tasks })} - /> - patchForm({ unattended })} - /> - - - - - - Schedule - - - patchForm({ schedule })} - onTimezoneChange={(timezone) => patchForm({ timezone })} - /> - - - - - - Models - - - patchForm({ models: { ...form.models, ...patch } })} - /> - - - - - - Settings - - - - patchForm({ execution: { ...form.execution, ...patch } }) - } - onTagsChange={(tags) => patchForm({ tags })} - /> - +
+ +
+ + Basics + + + + +
+ +
+ + Tasks + + + patchForm({ tasks })} + /> + patchForm({ unattended })} + /> + +
+ +
+ + Schedule + + + patchForm({ schedule })} + onTimezoneChange={(timezone) => patchForm({ timezone })} + /> + +
+ +
+ + Models + + + patchForm({ models: { ...form.models, ...patch } })} + /> + +
+ +
+ + Settings + + + + patchForm({ execution: { ...form.execution, ...patch } }) + } + onTagsChange={(tags) => patchForm({ tags })} + /> + +
- + Summary @@ -452,12 +465,14 @@ export function AutomationBuilderForm({
)} - {rootError &&

{rootError}

} + {rootError && ( + + + {rootError} + + )}
- {submitBlocked ? ( @@ -470,7 +485,6 @@ export function AutomationBuilderForm({ className="cursor-not-allowed opacity-50" onClick={(event) => event.preventDefault()} > - {submitLabel} @@ -481,14 +495,11 @@ export function AutomationBuilderForm({ type="button" size="sm" disabled={submitting} + className="relative" onClick={() => (activeMode === "json" ? submitJson() : submitForm())} > - {submitting ? ( - - ) : ( - - )} - {submitLabel} + {submitLabel} + {submitting && } )}
@@ -496,34 +507,6 @@ export function AutomationBuilderForm({ ); } -function ModeButton({ - active, - icon: Icon, - label, - onClick, -}: { - active: boolean; - icon: typeof Code2; - label: string; - onClick: () => void; -}) { - return ( - - ); -} - function extractTriggers(raw: unknown): HydratableTrigger[] { if (!Array.isArray(raw)) return []; return raw.map((entry) => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx index 8ca8d839c..2c4a0bf60 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx @@ -118,15 +118,14 @@ const ModelSelectField = memo(function ModelSelectField({ if (kind.options.length === 0) { return ( - + No eligible models - - Automations need a premium or your own (BYOK) model. Set one up in{" "} + + Use a premium model or your own (BYOK) model in{" "} role settings - . @@ -155,7 +154,7 @@ const ModelSelectField = memo(function ModelSelectField({ )} - + {premium.length > 0 ? ( Premium diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx index 21a77cb5f..4ba9e4182 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx @@ -1,5 +1,5 @@ "use client"; -import { CalendarClock, CheckCircle2, ListOrdered, type LucideIcon, XCircle } from "lucide-react"; +import { Dot } from "lucide-react"; import { type BuilderForm, scheduleToCron } from "@/lib/automations/builder-schema"; import { describeCron } from "@/lib/automations/describe-cron"; @@ -12,85 +12,64 @@ interface BuilderSummaryProps { * chat ``AutomationDraftPreview`` so the two creation paths feel consistent. */ export function BuilderSummary({ form }: BuilderSummaryProps) { - const scheduleLabel = form.schedule - ? `${describeCron(scheduleToCron(form.schedule))} · ${form.timezone}` - : "No schedule — won't run automatically"; + const automationName = form.name.trim() || "Untitled automation"; + const scheduleDescription = form.schedule ? describeCron(scheduleToCron(form.schedule)) : null; + const taskCountLabel = `${form.tasks.length} task${form.tasks.length === 1 ? "" : "s"}`; + const visibleTasks = form.tasks.slice(0, 2); + const hiddenTaskCount = form.tasks.length - visibleTasks.length; return ( -
-
-

{form.name.trim() || "Untitled automation"}

- {form.description?.trim() && ( -

{form.description.trim()}

- )} +
+
+

+ {automationName} +

-
-

{scheduleLabel}

-
+
-
-
    - {form.tasks.map((task, index) => ( -
  1. - - {index + 1} - - - - {task.query.trim() || ( - No instructions yet - )} +
    + + {scheduleDescription ? ( + + {scheduleDescription} + + {form.timezone} + + ) : ( + No schedule — won't run automatically + )} + + + +
      + {visibleTasks.map((task, index) => ( +
    1. + {index + 1}. + + {task.query.trim() || "No instructions yet"} - {task.mentions.length > 0 && ( - - {task.mentions.map((mention) => ( - - @{mention.title} - - ))} - - )} - -
    2. - ))} -
    -
+ + ))} + {hiddenTaskCount > 0 && ( +
  • +{hiddenTaskCount} more tasks
  • + )} + + -
    - {form.unattended ? ( - - ) : ( - - )} - {form.unattended ? "Runs without approval prompts" : "Will reject approval prompts"} + + {form.unattended ? "Runs without approval prompts" : "Approval prompts are rejected"} +
    ); } -function Section({ - icon: Icon, - label, - children, -}: { - icon: LucideIcon; - label: string; - children: React.ReactNode; -}) { +function SummaryRow({ label, children }: { label: string; children: React.ReactNode }) { return ( -
    -
    - - {label} -
    - {children} +
    +
    {label}
    +
    {children}
    ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx index 1f25f8a61..412533d36 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx @@ -1,6 +1,7 @@ "use client"; -import { AlertCircle } from "lucide-react"; +import { AlertCircle, TriangleAlert } from "lucide-react"; import { JsonView } from "@/components/json-view"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; interface JsonModePanelProps { value: Record; @@ -19,32 +20,32 @@ export function JsonModePanel({ value, issues, notice, onChange }: JsonModePanel return (
    {notice && ( -
    - {notice} -
    + + + {notice} + )} -
    - onChange(next as Record)} - collapsed={false} - /> -
    + onChange(next as Record)} + collapsed={false} + className="max-h-144 overflow-auto rounded-md border border-accent bg-accent/20" + /> {issues.length > 0 && ( -
    -
    - - {issues.length === 1 ? "1 issue" : `${issues.length} issues`} -
    -
      - {issues.map((issue) => ( -
    • {issue}
    • - ))} -
    -
    + + + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} + +
      + {issues.map((issue) => ( +
    • {issue}
    • + ))} +
    +
    +
    )}
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx index 401b4f5cb..810984acd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx @@ -1,5 +1,5 @@ "use client"; -import { CalendarClock, CalendarOff, Plus, X } from "lucide-react"; +import { CalendarClock, CalendarOff, Dot, Plus, X } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { @@ -70,17 +70,18 @@ export function ScheduleSection({ return (
    -
    +
    {label} - · {timezone} + + {timezone}
    diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx index bc3b97542..ed0808bb3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx @@ -35,22 +35,26 @@ export function TimezoneCombobox({ value, onChange }: TimezoneComboboxProps) { variant="outline" role="combobox" aria-expanded={open} - className="w-full justify-between font-normal" + className="w-full justify-between border-popover-border bg-transparent font-normal hover:bg-transparent" > {value || "Select timezone"} - - + + No timezone found. - + {timezones.map((tz) => ( { onChange(tz); setOpen(false); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx index ba665445f..861f22204 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx @@ -1,7 +1,5 @@ "use client"; -import { Info } from "lucide-react"; import { Switch } from "@/components/ui/switch"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; interface UnattendedToggleProps { checked: boolean; @@ -15,26 +13,15 @@ interface UnattendedToggleProps { */ export function UnattendedToggle({ checked, onChange }: UnattendedToggleProps) { return ( -
    +
    Run without asking for approvals - - - - - - Automations run unattended. With this off, any approval the agent asks for is - rejected, which can stall a step. - -

    - Auto-approve actions the agent would normally pause to confirm. + Tasks run automatically without asking for confirmation

    - - - + ( + + )} + /> ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx index ccfbbc9fa..a2f7f85f0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx @@ -1,38 +1,40 @@ "use client"; -import { ArrowLeft, MessageSquarePlus } from "lucide-react"; +import { ArrowLeft } from "lucide-react"; import Link from "next/link"; +import type { ReactNode } from "react"; import { Button } from "@/components/ui/button"; interface AutomationNewHeaderProps { searchSpaceId: number; + modeSwitcher?: ReactNode; } -export function AutomationNewHeader({ searchSpaceId }: AutomationNewHeaderProps) { +export function AutomationNewHeader({ searchSpaceId, modeSwitcher }: AutomationNewHeaderProps) { return (
    - +
    + + {modeSwitcher ?
    {modeSwitcher}
    : null} +

    New automation

    - Set up a task and a schedule. Prefer natural language? Use chat instead. + Configure the task, schedule, and execution settings for this automation.

    - + {modeSwitcher ? ( +
    {modeSwitcher}
    + ) : null}
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx index b77cb20f4..0502d2310 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx @@ -8,7 +8,7 @@ export default async function AutomationsPage({ const { search_space_id } = await params; return ( -
    +
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx index 74bcaff2e..b4ec015b7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx @@ -1,6 +1,5 @@ "use client"; -import { motion } from "motion/react"; import { useState } from "react"; import { BuyPagesContent } from "@/components/settings/buy-pages-content"; import { BuyTokensContent } from "@/components/settings/buy-tokens-content"; @@ -17,12 +16,7 @@ export default function BuyMorePage() { const [activeTab, setActiveTab] = useState("pages"); return ( - +
    { @@ -49,6 +43,6 @@ export default function BuyMorePage() { - +
    ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 759539ce3..3a41b5998 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -21,6 +21,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com import { useFolderSync } from "@/hooks/use-folder-sync"; import { useGlobalLoadingEffect } from "@/hooks/use-global-loading"; import { useElectronAPI } from "@/hooks/use-platform"; +import { isLlmOnboardingComplete } from "@/lib/onboarding"; export function DashboardClientLayout({ children, @@ -47,14 +48,8 @@ export function DashboardClientLayout({ const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); const isOnboardingComplete = useCallback(() => { - // Check that both LLM IDs are set (including 0 for Auto mode) - return ( - preferences.agent_llm_id !== null && - preferences.agent_llm_id !== undefined && - preferences.document_summary_llm_id !== null && - preferences.document_summary_llm_id !== undefined - ); - }, [preferences]); + return isLlmOnboardingComplete(preferences.agent_llm_id, globalConfigs.length > 0); + }, [preferences.agent_llm_id, globalConfigs.length]); const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom); const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false); @@ -100,7 +95,6 @@ export function DashboardClientLayout({ search_space_id: Number(searchSpaceId), data: { agent_llm_id: firstGlobalConfig.id, - document_summary_llm_id: firstGlobalConfig.id, }, }); diff --git a/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx b/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx deleted file mode 100644 index ccb3b35e3..000000000 --- a/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { Skeleton } from "@/components/ui/skeleton"; - -export default function Loading() { - return ( -
    - - -
    - ); -} diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index f8ca9bbc2..75cfa4184 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -18,6 +18,7 @@ import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; import { clearTargetCommentIdAtom, currentThreadAtom, + setCurrentThreadMetadataAtom, setTargetCommentIdAtom, } from "@/atoms/chat/current-thread.atom"; import { @@ -36,7 +37,7 @@ import { closeReportPanelAtom } from "@/atoms/chat/report-panel.atom"; import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; import { closeEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { membersAtom } from "@/atoms/members/members-query.atoms"; -import { removeChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; +import { removeChatTabAtom, syncChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { EditMessageDialog, @@ -50,6 +51,7 @@ import { TokenUsageProvider, } from "@/components/assistant-ui/token-usage-context"; import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; import { type HitlDecision, PendingInterruptProvider, @@ -64,6 +66,7 @@ import { } from "@/hooks/use-agent-actions-query"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; +import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getBearerToken } from "@/lib/auth-utils"; @@ -100,8 +103,6 @@ import { appendMessage, createThread, getRegenerateUrl, - getThreadFull, - getThreadMessages, type ThreadListItem, type ThreadListResponse, type ThreadRecord, @@ -119,7 +120,7 @@ import { trackChatMessageSent, trackChatResponseReceived, } from "@/lib/posthog/events"; -import Loading from "../loading"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; const MobileEditorPanel = dynamic( () => @@ -287,11 +288,78 @@ function computeFallbackTurnCancellingRetryDelay(attempt: number): number { return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); } +function parseUrlChatId(id: string | string[] | undefined): number { + let parsed = 0; + if (Array.isArray(id) && id.length > 0) { + parsed = Number.parseInt(id[0], 10); + } else if (typeof id === "string") { + parsed = Number.parseInt(id, 10); + } + return Number.isNaN(parsed) ? 0 : parsed; +} + +function ThreadMessagesSkeleton() { + return ( +
    +
    +
    +
    +
    + +
    + +
    + + + +
    + +
    + +
    + +
    + + + +
    + +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + ); +} + export default function NewChatPage() { const params = useParams(); const queryClient = useQueryClient(); - const [isInitializing, setIsInitializing] = useState(true); - const [threadId, setThreadId] = useState(null); + const urlChatId = useMemo(() => parseUrlChatId(params.chat_id), [params.chat_id]); + const [threadId, setThreadId] = useState(() => (urlChatId > 0 ? urlChatId : null)); + const activeThreadId = urlChatId > 0 ? urlChatId : threadId; + const handledLoadErrorThreadRef = useRef(null); const [currentThread, setCurrentThread] = useState(null); const [messages, setMessages] = useState([]); const [isRunning, setIsRunning] = useState(false); @@ -375,12 +443,14 @@ export default function NewChatPage() { const mentionedDocuments = useAtomValue(mentionedDocumentsAtom); const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom); - const setCurrentThreadState = useSetAtom(currentThreadAtom); + const currentThreadState = useAtomValue(currentThreadAtom); + const setCurrentThreadMetadata = useSetAtom(setCurrentThreadMetadataAtom); const setPremiumAlertForThread = useSetAtom(setPremiumAlertForThreadAtom); const setTargetCommentId = useSetAtom(setTargetCommentIdAtom); const clearTargetCommentId = useSetAtom(clearTargetCommentIdAtom); const closeReportPanel = useSetAtom(closeReportPanelAtom); const closeEditorPanel = useSetAtom(closeEditorPanelAtom); + const syncChatTab = useSetAtom(syncChatTabAtom); const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom); const removeChatTab = useSetAtom(removeChatTabAtom); const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom); @@ -402,9 +472,11 @@ export default function NewChatPage() { const { data: currentUser } = useAtomValue(currentUserAtom); const { data: agentFlags } = useAtomValue(agentFlagsAtom); const localFilesystemEnabled = agentFlags?.enable_desktop_local_filesystem === true; + const threadDetailQuery = useThreadDetail(activeThreadId); + const threadMessagesQuery = useThreadMessages(activeThreadId); // Live collaboration: sync session state and messages via Zero - useChatSessionStateSync(threadId); + useChatSessionStateSync(activeThreadId); const { data: membersData } = useAtomValue(membersAtom); const handleSyncedMessagesUpdate = useCallback( @@ -465,7 +537,7 @@ export default function NewChatPage() { [isRunning, membersData] ); - useMessagesSync(threadId, handleSyncedMessagesUpdate); + useMessagesSync(activeThreadId, handleSyncedMessagesUpdate); // Extract search_space_id from URL params const searchSpaceId = useMemo(() => { @@ -479,19 +551,7 @@ export default function NewChatPage() { // per-turn Revert button all read). Hydrates from // ``GET /threads/{id}/actions`` and is updated incrementally by the // SSE handlers + revert-batch results below — no atom side-channel. - const { items: agentActionItems } = useAgentActionsQuery(threadId); - - // Extract chat_id from URL params - const urlChatId = useMemo(() => { - const id = params.chat_id; - let parsed = 0; - if (Array.isArray(id) && id.length > 0) { - parsed = Number.parseInt(id[0], 10); - } else if (typeof id === "string") { - parsed = Number.parseInt(id, 10); - } - return Number.isNaN(parsed) ? 0 : parsed; - }, [params.chat_id]); + const { items: agentActionItems } = useAgentActionsQuery(activeThreadId); const handleChatFailure = useCallback( async ({ @@ -630,14 +690,19 @@ export default function NewChatPage() { }); }, []); - // Initialize thread and load messages - // For new chats (no urlChatId), we use lazy creation - thread is created on first message - const initializeThread = useCallback(async () => { - setIsInitializing(true); + const hydratedMessagesRef = useRef<{ + threadId: number | null; + data: typeof threadMessagesQuery.data; + }>({ threadId: null, data: undefined }); - // Reset all state when switching between chats/search spaces to prevent stale data + // Reset thread-local runtime state on route/search-space changes. Data fetching + // is handled by React Query below so the chat shell can render immediately. + useEffect(() => { + const nextThreadId = urlChatId > 0 ? urlChatId : null; + handledLoadErrorThreadRef.current = null; + hydratedMessagesRef.current = { threadId: null, data: undefined }; + setThreadId(nextThreadId); setMessages([]); - setThreadId(null); setCurrentThread(null); setMentionedDocuments([]); tokenUsageStore.clear(); @@ -647,82 +712,105 @@ export default function NewChatPage() { closeEditorPanel(); // Note: agent-action data is keyed by threadId in react-query so // switching threads naturally swaps caches; no explicit reset. - - try { - if (urlChatId > 0) { - // Thread exists - load thread data and messages - setThreadId(urlChatId); - - // Load thread data (for visibility info) and messages in parallel - const [threadData, messagesResponse] = await Promise.all([ - getThreadFull(urlChatId), - getThreadMessages(urlChatId), - ]); - - setCurrentThread(threadData); - - if (messagesResponse.messages && messagesResponse.messages.length > 0) { - const loadedMessages = reconcileInterruptedAssistantMessages( - messagesResponse.messages - ).map(convertToThreadMessage); - setMessages(loadedMessages); - - for (const msg of messagesResponse.messages) { - if (msg.token_usage) { - tokenUsageStore.set(`msg-${msg.id}`, msg.token_usage as TokenUsageData); - } - } - - const restoredDocsMap: Record = {}; - for (const msg of messagesResponse.messages) { - if (msg.role === "user") { - const docs = extractMentionedDocuments(msg.content); - if (docs.length > 0) { - restoredDocsMap[`msg-${msg.id}`] = docs; - } - } - } - if (Object.keys(restoredDocsMap).length > 0) { - setMessageDocumentsMap(restoredDocsMap); - } - } - } - // For new chats (urlChatId === 0), don't create thread yet - // Thread will be created lazily when user sends first message - // This improves UX (instant load) and avoids orphan threads - } catch (error) { - console.error("[NewChatPage] Failed to initialize thread:", error); - if (urlChatId > 0 && error instanceof NotFoundError) { - removeChatTab(urlChatId); - if (typeof window !== "undefined") { - window.history.replaceState(null, "", `/dashboard/${searchSpaceId}/new-chat`); - } - toast.error("This chat was deleted."); - return; - } - // Keep threadId as null - don't use Date.now() as it creates an invalid ID - // that will cause 404 errors on subsequent API calls - setThreadId(null); - setCurrentThread(null); - toast.error("Failed to load chat. Please try again."); - } finally { - setIsInitializing(false); - } }, [ urlChatId, - setMessageDocumentsMap, setMentionedDocuments, + setMessageDocumentsMap, + tokenUsageStore, closeReportPanel, closeEditorPanel, - removeChatTab, - searchSpaceId, + ]); + + useEffect(() => { + if (!activeThreadId) { + setCurrentThread(null); + return; + } + if (threadDetailQuery.data?.id === activeThreadId) { + const thread = threadDetailQuery.data; + setCurrentThread(thread); + syncChatTab({ + chatId: thread.id, + title: thread.title, + chatUrl: `/dashboard/${thread.search_space_id ?? searchSpaceId}/new-chat/${thread.id}`, + searchSpaceId: thread.search_space_id ?? searchSpaceId, + visibility: thread.visibility, + hasComments: thread.has_comments ?? false, + }); + } + }, [activeThreadId, searchSpaceId, syncChatTab, threadDetailQuery.data]); + + useEffect(() => { + const messagesResponse = threadMessagesQuery.data; + if (!activeThreadId || !messagesResponse) return; + + if ( + hydratedMessagesRef.current.threadId === activeThreadId && + hydratedMessagesRef.current.data === messagesResponse + ) { + return; + } + + if (isRunning) { + return; + } + + const loadedMessages = reconcileInterruptedAssistantMessages(messagesResponse.messages).map( + convertToThreadMessage + ); + setMessages(loadedMessages); + + tokenUsageStore.clear(); + const restoredDocsMap: Record = {}; + for (const msg of messagesResponse.messages) { + if (msg.token_usage) { + tokenUsageStore.set(`msg-${msg.id}`, msg.token_usage as TokenUsageData); + } + if (msg.role === "user") { + const docs = extractMentionedDocuments(msg.content); + if (docs.length > 0) { + restoredDocsMap[`msg-${msg.id}`] = docs; + } + } + } + setMessageDocumentsMap(restoredDocsMap); + hydratedMessagesRef.current = { threadId: activeThreadId, data: messagesResponse }; + }, [ + activeThreadId, + isRunning, + setMessageDocumentsMap, + threadMessagesQuery.data, tokenUsageStore, ]); - // Initialize on mount, and re-init when switching search spaces (even if urlChatId is the same) useEffect(() => { - initializeThread(); - }, [initializeThread]); + const loadError = threadDetailQuery.error ?? threadMessagesQuery.error; + if (!activeThreadId || !loadError) return; + if (handledLoadErrorThreadRef.current === activeThreadId) return; + + handledLoadErrorThreadRef.current = activeThreadId; + console.error("[NewChatPage] Failed to load thread:", loadError); + + if (loadError instanceof NotFoundError) { + removeChatTab(activeThreadId); + if (typeof window !== "undefined") { + window.history.replaceState(null, "", `/dashboard/${searchSpaceId}/new-chat`); + } + setThreadId(null); + setCurrentThread(null); + setMessages([]); + toast.error("This chat was deleted."); + return; + } + + toast.error("Failed to load chat. Please try again."); + }, [ + activeThreadId, + removeChatTab, + searchSpaceId, + threadDetailQuery.error, + threadMessagesQuery.error, + ]); // Prefetch document titles for @ mention picker // Runs when user lands on page so data is ready when they type @ @@ -750,7 +838,7 @@ export default function NewChatPage() { const readAndApplyCommentId = () => { const params = new URLSearchParams(window.location.search); const raw = params.get("commentId"); - if (raw && !isInitializing) { + if (raw && activeThreadId) { const commentId = Number.parseInt(raw, 10); if (!Number.isNaN(commentId)) { setTargetCommentId(commentId); @@ -768,17 +856,42 @@ export default function NewChatPage() { window.removeEventListener("popstate", readAndApplyCommentId); clearTargetCommentId(); }; - }, [isInitializing, setTargetCommentId, clearTargetCommentId]); + }, [activeThreadId, setTargetCommentId, clearTargetCommentId]); // Sync current thread state to atom useEffect(() => { - setCurrentThreadState((prev) => ({ - ...prev, - id: currentThread?.id ?? null, - visibility: currentThread?.visibility ?? null, - hasComments: currentThread?.has_comments ?? false, - })); - }, [currentThread, setCurrentThreadState]); + if (!currentThread) { + if (activeThreadId) { + return; + } + setCurrentThreadMetadata({ + id: null, + searchSpaceId: null, + visibility: null, + hasComments: false, + }); + return; + } + + const visibility = + currentThreadState.id === currentThread.id && currentThreadState.visibility !== null + ? currentThreadState.visibility + : currentThread.visibility; + + setCurrentThreadMetadata({ + id: currentThread.id, + searchSpaceId: currentThread.search_space_id ?? searchSpaceId, + visibility, + hasComments: currentThread.has_comments ?? false, + }); + }, [ + activeThreadId, + currentThread, + currentThreadState.id, + currentThreadState.visibility, + searchSpaceId, + setCurrentThreadMetadata, + ]); // Cleanup on unmount - abort any in-flight requests useEffect(() => { @@ -862,6 +975,8 @@ export default function NewChatPage() { setThreadId(currentThreadId); // Set currentThread so share button in header appears immediately setCurrentThread(newThread); + queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); + queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); // Track chat creation trackChatCreated(searchSpaceId, currentThreadId); @@ -1369,6 +1484,14 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + if (currentThreadId) { + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(currentThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(currentThreadId), + }); + } } }, [ @@ -1717,6 +1840,12 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(resumeThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(resumeThreadId), + }); } }, [ @@ -2210,6 +2339,12 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(threadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(threadId), + }); } }, [ @@ -2396,22 +2531,25 @@ export default function NewChatPage() { onCancel: cancelRun, }); - // Show loading state only when loading an existing thread - if (isInitializing) { - return ; - } + const threadLoadError = activeThreadId + ? (threadDetailQuery.error ?? threadMessagesQuery.error) + : null; + const shouldShowThreadLoadError = + !!threadLoadError && !!activeThreadId && !currentThread && messages.length === 0; + const isThreadMessagesLoading = + !!activeThreadId && + threadMessagesQuery.isPending && + messages.length === 0 && + !threadMessagesQuery.error; - // Show error state only if we tried to load an existing thread but failed - // For new chats (urlChatId === 0), threadId being null is expected (lazy creation) - if (!threadId && urlChatId > 0) { + if (shouldShowThreadLoadError) { return (
    Failed to load chat
    - -
    - - members +
    +
    +

    Members

    +
    +
    + +
    @@ -319,51 +320,54 @@ export function TeamContent({ searchSpaceId }: TeamContentProps) { return (
    -
    - {canInvite && - (rolesLoading ? ( - - ) : ( - - ))} - {canInvite && - (invitesLoading ? ( - - ) : ( - activeInvites.length > 0 && ( +
    +
    +

    Members

    +

    + {members.length} {members.length === 1 ? "member" : "members"} +

    +
    + {canInvite && ( +
    + {rolesLoading ? ( + + ) : ( + + )} + {invitesLoading ? ( + + ) : ( - ) - ))} -

    - {members.length} {members.length === 1 ? "member" : "members"} -

    + )} +
    + )}
    @@ -859,7 +863,11 @@ function AllInvitesDialog({ return ( - + ) : null} + +
    +
    +
    + ))} +
    + ); + }; + const renderPairingPanel = (platform: PairingPlatform) => { + if (!pairing || pairingPlatform !== platform) return null; + + return ( +
    +

    Pairing code

    +

    {pairing.code}

    + + Open {platform === "whatsapp" ? "WhatsApp" : "Telegram"} pairing link + +

    + Expires at {new Date(pairing.expires_at).toLocaleString()}. SurfSense stores this + channel's messages for agent memory and operational debugging. +

    +
    + ); + }; + const renderGatewaySkeletons = () => ( + <> + {[0, 1].map((index) => ( + + + + + + + + + + + + ))} + + ); + + return ( +
    + {isGatewayConfigLoading ? renderGatewaySkeletons() : null} + + {!isGatewayConfigLoading && !hasEnabledGateway ? ( + + + No messaging gateways enabled + + + ) : null} + + {telegramGatewayEnabled ? ( + + +
    + Telegram +
    +

    + Connect Telegram to chat with SurfSense. +

    +
    + +
    + {hasTelegramConnection ? null : ( + + )} + +
    + + {hasTelegramConnection ? null : renderPairingPanel("telegram")} + + {renderConnectionRows("telegram", "No Telegram chats connected yet.")} +
    +
    + ) : null} + + {slackGatewayEnabled ? ( + + +
    + Slack +
    +

    + Enable the SurfSense Slack bot so teammates can mention it in Slack. +

    +
    + +
    + + +
    + + {renderConnectionRows("slack", "No Slack workspaces connected yet.")} +
    +
    + ) : null} + + {discordGatewayEnabled ? ( + + +
    + Discord +
    +

    + Enable the SurfSense Discord bot so teammates can mention it in Discord. +

    +
    + +
    + + +
    + + {renderConnectionRows("discord", "No Discord servers connected yet.")} +
    +
    + ) : null} + + {whatsappMode !== "disabled" ? ( + + +
    + WhatsApp +
    +

    + {whatsappMode === "baileys" + ? 'Use "Message Yourself". Other chats are ignored.' + : "Connect WhatsApp to chat with Surfsense."} +

    +
    + + {whatsappMode === "cloud" ? ( +
    +
    + {hasWhatsAppConnection ? null : ( + + )} + +
    + {hasWhatsAppConnection ? null : renderPairingPanel("whatsapp")} +
    + ) : null} + {whatsappMode === "baileys" ? ( +
    + + {baileysQr ? ( +
    +

    WhatsApp QR pairing

    +

    + Scan this QR from WhatsApp > Linked Devices > Link a Device. +

    +
    + +
    +
    + ) : null} + {baileysHealth ? ( +

    + Bridge status: {baileysHealth.status} + {typeof baileysHealth.queueDepth === "number" + ? `, queue: ${baileysHealth.queueDepth}` + : ""} +

    + ) : null} +
    + ) : null} + + {renderConnectionRows("whatsapp", "No WhatsApp chats connected yet.")} +
    +
    + ) : null} +
    + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx index 037568db3..4aac4d2f6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/layout-shell.tsx @@ -5,6 +5,7 @@ import { Keyboard, KeyRound, Library, + MessageCircle, Monitor, ReceiptText, ShieldCheck, @@ -29,7 +30,8 @@ export type UserSettingsTab = | "agent-status" | "purchases" | "desktop" - | "hotkeys"; + | "hotkeys" + | "messaging-channels"; const DEFAULT_TAB: UserSettingsTab = "profile"; @@ -83,6 +85,11 @@ export function UserSettingsLayoutShell({ searchSpaceId, children }: UserSetting label: "Agent Status", icon: , }, + { + value: "messaging-channels" as const, + label: "Messaging Channels", + icon: , + }, { value: "purchases" as const, label: "Purchase History", diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx new file mode 100644 index 000000000..335c7f084 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/messaging-channels/page.tsx @@ -0,0 +1,5 @@ +import { MessagingChannelsContent } from "../components/MessagingChannelsContent"; + +export default function Page() { + return ; +} diff --git a/surfsense_web/atoms/chat/current-thread.atom.ts b/surfsense_web/atoms/chat/current-thread.atom.ts index 131c98309..9e931a9bf 100644 --- a/surfsense_web/atoms/chat/current-thread.atom.ts +++ b/surfsense_web/atoms/chat/current-thread.atom.ts @@ -4,12 +4,27 @@ import { reportPanelAtom } from "./report-panel.atom"; interface CurrentThreadState { id: number | null; + searchSpaceId: number | null; visibility: ChatVisibility | null; hasComments: boolean; } +interface CurrentThreadMetadataPatch { + id: number | null; + searchSpaceId?: number | null; + visibility?: ChatVisibility | null; + hasComments?: boolean; +} + +interface CurrentThreadMetadataUpdate { + id: number; + visibility?: ChatVisibility | null; + hasComments?: boolean; +} + const initialState: CurrentThreadState = { id: null, + searchSpaceId: null, visibility: null, hasComments: false, }; @@ -20,9 +35,52 @@ export const commentsEnabledAtom = atom( (get) => get(currentThreadAtom).visibility === "SEARCH_SPACE" ); -export const setThreadVisibilityAtom = atom(null, (get, set, newVisibility: ChatVisibility) => { - set(currentThreadAtom, { ...get(currentThreadAtom), visibility: newVisibility }); -}); +export const setCurrentThreadMetadataAtom = atom( + null, + (get, set, metadata: CurrentThreadMetadataPatch) => { + const current = get(currentThreadAtom); + const isSameThread = current.id === metadata.id; + + set(currentThreadAtom, { + ...current, + id: metadata.id, + searchSpaceId: + "searchSpaceId" in metadata + ? (metadata.searchSpaceId ?? null) + : isSameThread + ? current.searchSpaceId + : null, + visibility: + "visibility" in metadata + ? (metadata.visibility ?? null) + : isSameThread + ? current.visibility + : null, + hasComments: + "hasComments" in metadata + ? (metadata.hasComments ?? false) + : isSameThread + ? current.hasComments + : false, + }); + } +); + +export const patchCurrentThreadMetadataAtom = atom( + null, + (get, set, patch: CurrentThreadMetadataUpdate) => { + const current = get(currentThreadAtom); + if (current.id !== patch.id) { + return; + } + + set(currentThreadAtom, { + ...current, + visibility: "visibility" in patch ? (patch.visibility ?? null) : current.visibility, + hasComments: "hasComments" in patch ? (patch.hasComments ?? false) : current.hasComments, + }); + } +); export const resetCurrentThreadAtom = atom(null, (_, set) => { set(currentThreadAtom, initialState); diff --git a/surfsense_web/atoms/tabs/tabs.atom.ts b/surfsense_web/atoms/tabs/tabs.atom.ts index dba459cc9..0abbf2e74 100644 --- a/surfsense_web/atoms/tabs/tabs.atom.ts +++ b/surfsense_web/atoms/tabs/tabs.atom.ts @@ -1,5 +1,6 @@ import { atom } from "jotai"; import { atomWithStorage, createJSONStorage } from "jotai/utils"; +import type { ChatVisibility } from "@/lib/chat/thread-persistence"; export type TabType = "chat" | "document"; @@ -10,6 +11,8 @@ export interface Tab { /** For chat tabs */ chatId?: number | null; chatUrl?: string; + visibility?: ChatVisibility; + hasComments?: boolean; /** For document tabs */ documentId?: number; searchSpaceId?: number; @@ -79,11 +82,15 @@ export const syncChatTabAtom = atom( title, chatUrl, searchSpaceId, + visibility, + hasComments, }: { chatId: number | null; title?: string; chatUrl?: string; searchSpaceId: number; + visibility?: ChatVisibility; + hasComments?: boolean; } ) => { if (chatId && get(deletedChatIdsAtom).has(chatId)) { @@ -105,6 +112,8 @@ export const syncChatTabAtom = atom( title: title || t.title, chatUrl: chatUrl || t.chatUrl, searchSpaceId: searchSpaceId ?? t.searchSpaceId, + ...(visibility !== undefined ? { visibility } : {}), + ...(hasComments !== undefined ? { hasComments } : {}), } : t ), @@ -140,6 +149,8 @@ export const syncChatTabAtom = atom( chatId, chatUrl, searchSpaceId, + ...(visibility !== undefined ? { visibility } : {}), + ...(hasComments !== undefined ? { hasComments } : {}), }; let updatedTabs: Tab[]; diff --git a/surfsense_web/changelog/content/2025-12-24.mdx b/surfsense_web/changelog/content/2025-12-24.mdx index dbd241d59..096b5116b 100644 --- a/surfsense_web/changelog/content/2025-12-24.mdx +++ b/surfsense_web/changelog/content/2025-12-24.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.9 - Introducing the Agentic Architecture" -description: "SurfSense v0.0.9 introduces a new agentic architecture with intelligent source selection, temporal query understanding, and MCP compatibility." date: "2025-12-24" -tags: ["Agentic", "Agent", "MCP"] version: "0.0.9" --- @@ -19,28 +16,17 @@ This release introduces the **SurfSense Agent**, a shift from pre-determined wor - **Faster Indexing**: Improved speed when syncing connected sources - **MCP Compatibility**: Extensible architecture supporting Model Context Protocol servers - - - For Developers - -
      -
    • More extensible architecture for adding connectors and tools
    • -
    • Supports long-running agents for complex tasks
    • -
    • Cleaner codebase enables faster contributions
    • -
    -
    -
    - - Notes - -
      -
    • May increase API costs due to dynamic LLM calls
    • -
    • Users with small local models may see varied performance
    • -
    -
    -
    -
    +### Developers + +- More extensible architecture for adding connectors and tools +- Supports long-running agents for complex tasks +- Cleaner codebase enables faster contributions + +### Notes + +- May increase API costs due to dynamic LLM calls +- Users with small local models may see varied performance SurfSense is an AI-powered federated search solution that connects all your knowledge sources in one place. -🎄 Merry Christmas from the SurfSense team! \ No newline at end of file +🎄 Merry Christmas from the SurfSense team! diff --git a/surfsense_web/changelog/content/2026-01-08.mdx b/surfsense_web/changelog/content/2026-01-08.mdx index 0827e2182..25ca7ad85 100644 --- a/surfsense_web/changelog/content/2026-01-08.mdx +++ b/surfsense_web/changelog/content/2026-01-08.mdx @@ -1,12 +1,9 @@ --- -title: "SurfSense v0.0.11 - Connectors And More Connectors" -description: "SurfSense v0.0.11 delivers powerful new integrations for our AI enterprise search platform, including Google Drive and Circleback connectors, multi-account support, and a fully responsive mobile interface." date: "2026-01-08" -tags: ["Mobile", "UX", "Integrations", "Connectors"] version: "0.0.11" --- -SurfSense v0.0.11 - Connectors And More Connectors +SurfSense v0.0.11: Connectors And More Connectors ## What's New in v0.0.11 @@ -21,26 +18,15 @@ This release focuses on **connectivity and ease of use** for your enterprise sea - **Simplified OAuth Authentication**: All supported connectors have been migrated to OAuth, making setup faster and more secure across your enterprise search software stack. - **Multi-Account Support**: Connect multiple accounts for the same service (e.g., Personal and Work Google Drives) to unify all your data sources in one AI-powered search hub. - - - Bug Fixes - -
      -
    • Fixed a login issue affecting specific Google accounts on surfsense.com
    • -
    • Resolved most Docker self-hosting configuration issues for easier deployment
    • -
    -
    -
    - - For Self-Hosters - -
      -
    • Docker configuration has been streamlined for smoother self-hosted deployments
    • -
    • OAuth setup is now consistent across all connectors
    • -
    -
    -
    -
    +### No longer broken + +- Fixed a login issue affecting specific Google accounts on surfsense.com +- Resolved most Docker self-hosting configuration issues for easier deployment + +### For self-hosters + +- Docker configuration has been streamlined for smoother self-hosted deployments +- OAuth setup is now consistent across all connectors SurfSense is an open-source AI enterprise search solution that connects all your knowledge sources, from Google Drive to Slack to meeting notes, in one intelligent, federated search platform. Whether you're looking for enterprise search software for your team or a personal knowledge assistant, SurfSense delivers powerful enterprise search solutions with the flexibility of self-hosting. diff --git a/surfsense_web/changelog/content/2026-01-26.mdx b/surfsense_web/changelog/content/2026-01-26.mdx index e2092bc83..1e5debaf8 100644 --- a/surfsense_web/changelog/content/2026-01-26.mdx +++ b/surfsense_web/changelog/content/2026-01-26.mdx @@ -1,12 +1,9 @@ --- -title: "SurfSense v0.0.12 - New Main UI, Real-time Collaboration and Comments" -description: "SurfSense v0.0.12 transforms the platform with a redesigned chat-first interface, real-time collaboration features, comment threads with @mentions, and instant notifications powered by ElectricSQL + PGlite." date: "2026-01-26" -tags: ["UX", "UI", "Real-time chat", "Collaboration", "Comments"] version: "0.0.12" --- -SurfSense v0.0.12 - New Main UI, Real-time Collaboration and Comments +SurfSense v0.0.12: New Main UI, Real-time Collaboration and Comments ## What's New in v0.0.12 @@ -54,34 +51,23 @@ This release brings major improvements to **collaboration and user experience**. - **5-Minute Periodic Syncing**: Near-real-time updates option - **Real-Time UI Updates**: See connector indexing progress without page refreshes - - - Bug Fixes - -
      -
    • Syncs with no new content now show "Already up to date!" instead of falsely reporting failures
    • -
    • Restored missing indexing options page for Google Drive connector
    • -
    • File mention picker now handles large document counts with server-side search and pagination
    • -
    • Reasoning steps no longer overlap with chat input field
    • -
    • File upload modal is now scrollable when adding many files
    • -
    • OAuth connectors now display properly on mobile devices
    • -
    -
    -
    - - Technical Improvements - -
      -
    • Made Alembic migrations idempotent for safer deployments
    • -
    • Migrations now work on fresh databases from scratch
    • -
    • Major refactoring of chat components for better maintainability
    • -
    • Streamlined sidebar and connector page code
    • -
    • Fixed legacy route handling
    • -
    • Fixed hardcoded Docker values for complex deployments
    • -
    -
    -
    -
    +### No longer broken + +- Syncs with no new content now show "Already up to date!" instead of falsely reporting failures +- Restored missing indexing options page for Google Drive connector +- File mention picker now handles large document counts with server-side search and pagination +- Reasoning steps no longer overlap with chat input field +- File upload modal is now scrollable when adding many files +- OAuth connectors now display properly on mobile devices + +### Developers + +- Made Alembic migrations idempotent for safer deployments +- Migrations now work on fresh databases from scratch +- Major refactoring of chat components for better maintainability +- Streamlined sidebar and connector page code +- Fixed legacy route handling +- Fixed hardcoded Docker values for complex deployments This release transforms SurfSense into a truly collaborative, real-time platform with a redesigned interface that puts chat front and center. The addition of comments, @mentions, and live collaboration features makes it easier than ever for teams to work together without leaving the app. diff --git a/surfsense_web/changelog/content/2026-02-09.mdx b/surfsense_web/changelog/content/2026-02-09.mdx index 8e222a16b..3bbc6f45e 100644 --- a/surfsense_web/changelog/content/2026-02-09.mdx +++ b/surfsense_web/changelog/content/2026-02-09.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.13 - Public Sharing, Image Generation & Redesigned Documents" -description: "SurfSense v0.0.13 introduces public chat sharing with permissions, image generation support, an auto load-balanced model mode, a redesigned Documents page, and numerous bug fixes across connectors and UI." date: "2026-02-09" -tags: ["Public Sharing", "Image Generation", "Documents", "UI", "Bug Fixes"] version: "0.0.13" --- @@ -42,31 +39,20 @@ This update brings **public sharing, image generation**, a redesigned Documents - **New Connector Docs**: Added docs for Luma, Circleback, Elasticsearch, Bookstack, and Obsidian connectors. - - - Bug Fixes - -
      -
    • Fixed cloud scaling issues where document queue congestion occurred under high load
    • -
    • Documents now correctly attribute to the uploading user and de-index on disconnect or deletion
    • -
    • Fixed common backend errors in indexing and large file handling
    • -
    • Fixed Notion indexing failures caused by transcription blocks
    • -
    • Chat refresh button now correctly regenerates AI responses
    • -
    • Restored the previously disabled Role Editor
    • -
    • Fixed Mentions tab appearing empty when document notifications pushed mentions out of the pagination window
    • -
    • Bundled git in the Docker image to fix GitHub connector failures with gitingest
    • -
    • Fixed Google Calendar default date range errors and aligned backend defaults with the frontend
    • -
    -
    -
    - - Technical Improvements - -
      -
    • Rebuilt the GitHub connector on gitingest for more efficient, lower-cost repository fetching
    • -
    -
    -
    -
    +### No longer broken + +- Fixed cloud scaling issues where document queue congestion occurred under high load +- Documents now correctly attribute to the uploading user and de-index on disconnect or deletion +- Fixed common backend errors in indexing and large file handling +- Fixed Notion indexing failures caused by transcription blocks +- Chat refresh button now correctly regenerates AI responses +- Restored the previously disabled Role Editor +- Fixed Mentions tab appearing empty when document notifications pushed mentions out of the pagination window +- Bundled git in the Docker image to fix GitHub connector failures with gitingest +- Fixed Google Calendar default date range errors and aligned backend defaults with the frontend + +### Developers + +- Rebuilt the GitHub connector on gitingest for more efficient, lower-cost repository fetching SurfSense is your AI-powered federated search solution, connecting all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-03-31.mdx b/surfsense_web/changelog/content/2026-03-31.mdx index de2395253..271851fb8 100644 --- a/surfsense_web/changelog/content/2026-03-31.mdx +++ b/surfsense_web/changelog/content/2026-03-31.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.14 - Dropbox, OneDrive, Agent Approvals, Reports & Self-Hosted Docker" -description: "SurfSense v0.0.14 is a massive release: new Dropbox and Microsoft OneDrive connectors, human-in-the-loop approvals across Gmail, Google Calendar, Jira, Confluence, Linear, and Notion, a full report generation system with multi-format export, a faster real-time sync engine, a new desktop app, one-command self-hosted Docker install, and built-in private web search." date: "2026-03-31" -tags: ["Connectors", "Approvals", "Reports", "Desktop App", "Docker", "Dropbox", "OneDrive"] version: "0.0.14" --- @@ -93,39 +90,28 @@ SurfSense now treats every sensitive AI action as an explicit, reviewable step. - **MiniMax**: Complete MiniMax LLM integration joins the growing list of supported models. - - - Bug Fixes - -
      -
    • Fixed citations for live connectors
    • -
    • Fixed Google Drive handling of failed documents and unsupported file types
    • -
    • Fixed Dropbox path retrieval and error handling during indexing
    • -
    • Fixed a build failure in the roles manager page
    • -
    • Re-enabled legacy AI image handling so older chats still render correctly
    • -
    • Fixed sign-in state syncing between the desktop app and web app
    • -
    • Fixed nested button accessibility issues across the roles manager, link toolbar, and hero section
    • -
    • Fixed several memory leaks (onboarding tour, Spotlight, register page)
    • -
    • Fixed Google Calendar event update behavior
    • -
    • Smoother client-side navigation in several places
    • -
    -
    -
    - - Performance & Reliability - -
      -
    • Large document migrations now run in batches to reduce memory usage
    • -
    • Smoother streaming in chat thanks to batched UI updates
    • -
    • Rate limiting now falls back gracefully when the cache is unavailable
    • -
    • Better error tracking and referral attribution
    • -
    • Smaller page bundles for the web app, with heavy editors loaded only when needed
    • -
    • End-to-end tests for the document upload pipeline
    • -
    • Health check endpoint and safer default rate limits
    • -
    -
    -
    -
    +### No longer broken + +- Fixed citations for live connectors +- Fixed Google Drive handling of failed documents and unsupported file types +- Fixed Dropbox path retrieval and error handling during indexing +- Fixed a build failure in the roles manager page +- Re-enabled legacy AI image handling so older chats still render correctly +- Fixed sign-in state syncing between the desktop app and web app +- Fixed nested button accessibility issues across the roles manager, link toolbar, and hero section +- Fixed several memory leaks (onboarding tour, Spotlight, register page) +- Fixed Google Calendar event update behavior +- Smoother client-side navigation in several places + +### Improvements + +- Large document migrations now run in batches to reduce memory usage +- Smoother streaming in chat thanks to batched UI updates +- Rate limiting now falls back gracefully when the cache is unavailable +- Better error tracking and referral attribution +- Smaller page bundles for the web app, with heavy editors loaded only when needed +- End-to-end tests for the document upload pipeline +- Health check endpoint and safer default rate limits v0.0.14 is one of the largest SurfSense releases to date. If you're looking for a **NotebookLM alternative** or comparing **free alternatives to ChatGPT** that you can actually self-host, this release is built for you. diff --git a/surfsense_web/changelog/content/2026-04-08.mdx b/surfsense_web/changelog/content/2026-04-08.mdx index 080a03e3c..7be430933 100644 --- a/surfsense_web/changelog/content/2026-04-08.mdx +++ b/surfsense_web/changelog/content/2026-04-08.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.15 - Watch Local Folders, Vision Autocomplete & New Document Pipeline" -description: "SurfSense v0.0.15 turns your laptop into a living knowledge base: watch any local folder and keep it auto-synced, a vision-powered quick-assist autocomplete in the desktop app, a brand new document processing pipeline with Docling, LlamaCloud, Unstructured, and Azure Document Intelligence, full document version history, and per-user page limits across every connector." date: "2026-04-08" -tags: ["Desktop App", "Local Folder", "Document Pipeline", "Vision AI", "Version History", "Azure Document Intelligence"] version: "0.0.15" --- @@ -74,37 +71,26 @@ The SurfSense desktop app becomes a serious always-on **AI like ChatGPT** that a - **Leaner Pages**: Shared libraries now load smaller, more focused bundles. - **Smoother Self-Hosted Setup**: New installs of the real-time sync service now set themselves up automatically. - - - Bug Fixes - -
      -
    • Fixed several memory leaks when leaving pages during active work
    • -
    • Upload progress bar now clears correctly when leaving the upload screen
    • -
    • Smoother, less jittery thread list and chat refreshes
    • -
    • Smoother client-side navigation in several places
    • -
    • Improved responsiveness of the documents sidebar
    • -
    • Debounced search on the logs page to stop UI jank
    • -
    • Correct browser autocomplete hints on sign-in and register forms
    • -
    • Vision model is no longer triggered unnecessarily by the desktop folder watcher
    • -
    • Silenced noisy auto-updater errors in the desktop app
    • -
    -
    -
    - - Performance & Reliability - -
      -
    • Faster local folder indexing thanks to smarter content hashing
    • -
    • Smoother sidebar open/close animations
    • -
    • Long lists now scroll more smoothly on the documents and logs pages
    • -
    • Search and filter inputs feel snappier while typing
    • -
    • Unified version numbering across the web app, backend, and desktop app
    • -
    • Event tracking now includes the platform (web/desktop) for better analytics
    • -
    -
    -
    -
    +### No longer broken + +- Fixed several memory leaks when leaving pages during active work +- Upload progress bar now clears correctly when leaving the upload screen +- Smoother, less jittery thread list and chat refreshes +- Smoother client-side navigation in several places +- Improved responsiveness of the documents sidebar +- Debounced search on the logs page to stop UI jank +- Correct browser autocomplete hints on sign-in and register forms +- Vision model is no longer triggered unnecessarily by the desktop folder watcher +- Silenced noisy auto-updater errors in the desktop app + +### Improvements + +- Faster local folder indexing thanks to smarter content hashing +- Smoother sidebar open/close animations +- Long lists now scroll more smoothly on the documents and logs pages +- Search and filter inputs feel snappier while typing +- Unified version numbering across the web app, backend, and desktop app +- Event tracking now includes the platform (web/desktop) for better analytics If you've been looking for a tool that works across your cloud connectors **and** your local machine, v0.0.15 is exactly that. It's also one of the most compelling **NotebookLM alternatives** and **free ChatGPT alternatives** for private team use, with full control over your data, your parsers, and your models. diff --git a/surfsense_web/changelog/content/2026-04-16.mdx b/surfsense_web/changelog/content/2026-04-16.mdx index 7b4e712ca..f9c8d74bf 100644 --- a/surfsense_web/changelog/content/2026-04-16.mdx +++ b/surfsense_web/changelog/content/2026-04-16.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.16 - Try SurfSense With No Login, Long-Term Memory, Token Tracking & AI File Sorting" -description: "SurfSense v0.0.16 introduces a no-login online experience so anyone can try this free ChatGPT alternative in seconds, long-term user and team memory, a reworked model selector with full token usage tracking, AI-powered file sorting, one-click Knowledge Base export as ZIP, trusted MCP tools, and OpenRouter integration." date: "2026-04-16" -tags: ["No Login", "Memory", "Token Usage", "AI File Sorting", "KB Export", "OpenRouter", "MCP"] version: "0.0.16" --- @@ -97,39 +94,28 @@ Token visibility is now first-class. - **Composer & Layout**: Loading placeholders in the composer, a better mention picker experience, and refined settings layout. - **Better Editing**: Trailing empty lines in the rich-text editor make long-form writing smoother. - - - Bug Fixes - -
      -
    • Fixed knowledge base search being skipped in certain chat sequences
    • -
    • Fixed a button nesting issue in the mobile upload area
    • -
    • Clearer logs and behavior when a vision model isn't available vs. not configured
    • -
    • Added a safety check for very large images before vision processing
    • -
    • Better fallback when an image can't be processed by the vision model
    • -
    • Fixed duplicate filename collisions during folder upload
    • -
    • Cleaner filenames on folder uploads
    • -
    • More accurate "documents still processing" warning during folder export
    • -
    • Handled display names that are just whitespace without crashing
    • -
    • Added user feedback when snapshot deletion fails
    • -
    • Updated billing text to reflect 500 pages included in demo plans
    • -
    -
    -
    - - Under the Hood - -
      -
    • Old memory storage fully migrated and retired
    • -
    • Smaller bundles on public marketing pages
    • -
    • Consistent approval handling across all connectors
    • -
    • Cleaner file type detection using explicit, well-defined rules
    • -
    • Rich-text editor now powers memory input, with inline alerts and a better layout
    • -
    • Localized folder-upload messages and download button text
    • -
    -
    -
    -
    +### No longer broken + +- Fixed knowledge base search being skipped in certain chat sequences +- Fixed a button nesting issue in the mobile upload area +- Clearer logs and behavior when a vision model isn't available vs. not configured +- Added a safety check for very large images before vision processing +- Better fallback when an image can't be processed by the vision model +- Fixed duplicate filename collisions during folder upload +- Cleaner filenames on folder uploads +- More accurate "documents still processing" warning during folder export +- Handled display names that are just whitespace without crashing +- Added user feedback when snapshot deletion fails +- Updated billing text to reflect 500 pages included in demo plans + +### Developers + +- Old memory storage fully migrated and retired +- Smaller bundles on public marketing pages +- Consistent approval handling across all connectors +- Cleaner file type detection using explicit, well-defined rules +- Rich-text editor now powers memory input, with inline alerts and a better layout +- Localized folder-upload messages and download button text v0.0.16 makes SurfSense the easiest open source AI assistant to actually try: zero friction, no login, online. Whether you're searching for a **ChatGPT alternative free online**, a **free Claude AI alternative**, or a **NotebookLM alternative** you can self-host, SurfSense v0.0.16 is built for you. diff --git a/surfsense_web/changelog/content/2026-04-21.mdx b/surfsense_web/changelog/content/2026-04-21.mdx new file mode 100644 index 000000000..2c96e4543 --- /dev/null +++ b/surfsense_web/changelog/content/2026-04-21.mdx @@ -0,0 +1,60 @@ +--- +date: "2026-04-21" +version: "0.0.19" +--- + +## What's New in v0.0.19 + +v0.0.19 brings a new **AI resume builder** into SurfSense, giving users a practical way to turn structured information into polished, previewable PDF outputs. This release also improves the anonymous chat experience across cross-site deployments, adds desktop startup options, strengthens accessibility, and tightens several dashboard interactions for a faster, cleaner product experience. + +### Resume Builder + +Turn your knowledge and profile details into export-ready resume documents. + +- **AI Resume Generation**: A new resume generation tool can produce structured resume content from user information and report prompts. +- **Typst-Based PDF Output**: Resume reports now support Typst-powered PDF generation, giving users a cleaner path from generated content to shareable documents. +- **PDF Preview Experience**: A built-in PDF viewer lets users preview generated resumes directly in SurfSense before exporting. +- **Resume Tool UI**: A dedicated resume generation interface tracks loading, success, and error states so users understand exactly what is happening during generation. +- **Public Report Preview Endpoint**: Generated resume previews can be accessed through a public report endpoint when needed for sharing and preview workflows. + +### Anonymous Chat & Web Experience + +- **Cross-Site Anonymous Chat Cookies**: Anonymous chat cookies now adapt their SameSite and Secure settings based on deployment context, making hosted and cross-domain setups more reliable. +- **Better Anonymous Chat History**: Message history handling in anonymous chat is more dependable, especially when users move between public chat states. +- **Safer Form Inputs**: Login, registration, profile, search space, and role forms now enforce sensible max-length limits directly in the UI. +- **Cleaner Page Landmarks**: Home and free-chat pages no longer nest main landmarks, improving HTML semantics and screen-reader navigation. +- **SEO Metadata Refresh**: Titles and descriptions across key pages now better communicate SurfSense's open-source, privacy-focused positioning. + +### Desktop App + +- **Launch at Startup**: The desktop app now supports auto-launch settings, including options for starting minimized to the tray. +- **Desktop Analytics Events**: Desktop activation, quitting, folder watching, deep links, connector setup, and update-related actions now emit richer analytics events. +- **Windows Signing Workflow**: The desktop release pipeline gained Windows signing support for production builds. + +### UI & Performance + +- **Faster Dashboard Startup**: Document tab content now lazy-loads to reduce the initial dashboard bundle. +- **More Stable Theme Toggle**: The theme toggle uses a functional state update, keeping callbacks stable and safer under concurrent rendering. +- **Cleaner Dialog Reset Behavior**: Dialog reset logic moved into open-change handlers, reducing effect-driven UI churn. +- **Model Selector Polish**: Model selector reset behavior now runs from event handlers for a cleaner settings experience. + +### No longer broken + +- Fixed scroll-to-citation timers continuing after the source detail panel closes or unmounts +- Removed unused source-detail-panel state that caused unnecessary re-renders +- Fixed anonymous chat message history behavior +- Improved resume generation error handling, loading states, and template validation +- Removed an unnecessary Windows signing publisherName argument +- Improved HTTP exception handling so unexpected 500 responses are sanitized while useful server errors remain visible + +### Developers + +- Reports gained content type support for resume-specific output handling +- PDF handling moved from react-pdf to pdfjs-dist for better control over loading and rendering +- Desktop analytics bridge and IPC channels now support identification and event capture +- Rate limiting now extracts real client IPs more accurately behind Cloudflare-style proxies +- New contributors landed fixes across accessibility, dialog state, and dashboard performance + +v0.0.19 expands SurfSense beyond search and chat into deliverable creation. With the new **AI resume builder**, PDF previews, stronger anonymous chat behavior, and desktop startup controls, SurfSense becomes more useful both in the browser and as a daily desktop companion. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-03.mdx b/surfsense_web/changelog/content/2026-05-03.mdx new file mode 100644 index 000000000..4a45071e5 --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-03.mdx @@ -0,0 +1,63 @@ +--- +date: "2026-05-03" +version: "0.0.20" +--- + +## What's New in v0.0.20 + +v0.0.20 is a platform-wide upgrade for connectors, desktop workflows, and chat. SurfSense now supports **live connector tools** through MCP OAuth and native APIs, adds an **Obsidian vault sync plugin**, introduces local file editing in the desktop app, and makes chat smarter with thread-level model pinning, structured errors, and live usage visibility. + +### Live Connector Tools + +Connectors are no longer just places to index data. They can now expose live actions to SurfSense agents. + +- **Live Tools via MCP OAuth**: SurfSense can discover and use connector tools through MCP OAuth, bringing external services directly into agent workflows. +- **Native Connector APIs**: Connectors can expose native API-backed actions where MCP is not the right fit. +- **OAuth Trust Improvements**: MCP OAuth flows now handle trust, 401 recovery, and connector UX more cleanly. +- **Parallel Tool Discovery**: Connector tool discovery can run in parallel, reducing setup latency for tool-rich integrations. +- **Long Indexing Stability**: Connector indexing refreshes Redis heartbeats during long Phase 1 runs so large imports do not look stale. + +### Desktop Knowledge Work + +- **Obsidian Vault Sync**: A new Obsidian sync plugin helps bring local vault knowledge into SurfSense. +- **Swappable Desktop Filesystem**: The desktop app gained a swappable filesystem architecture for safer local-file workflows. +- **Local File Editing**: Monaco-powered editing lets desktop users open and edit local files directly from SurfSense. +- **Screenshot Assist**: Desktop screenshot capture gives agents more context from the user's screen when needed. +- **Linux RPM Builds**: The desktop build pipeline now includes an RPM target for Linux users. + +### Smarter Chat & Models + +- **Thread-Level Model Pinning**: Chat threads can automatically pin models for more consistent long-running conversations. +- **Structured Chat Errors**: Chat failures now surface cleaner, more actionable error states. +- **Unified Streaming Flow**: Streaming state was reworked for a smoother viewport and mention experience. +- **Auto-Pin Quality Scoring**: Model auto-pinning now uses quality scoring to make better routing decisions. +- **Live Usage Sidebar**: OpenRouter tier refactors and usage visibility make model cost and capacity easier to understand. + +### Docs, Marketing & Performance + +- **Internal Backend URL Config**: Self-hosters can configure internal backend URLs and host gateway behavior more clearly. +- **Ollama Docs**: Documentation now covers Ollama-oriented local model setups. +- **Baidu Search Guide**: Added setup guidance for the Baidu Search connector. +- **Marketing Route Skeletons**: Async marketing routes now show loading skeletons instead of blank waits. +- **Leaner Docs Bundles**: Docs pages now avoid a full lucide barrel import for better bundle performance. + +### No longer broken + +- Fixed Docker zero-cache issues +- Fixed multiple agent runtime bugs and temporarily blocked an unstable GitHub Copilot custom provider path +- Fixed resume page limit functionality +- Deduplicated anonymous chat uploads through the anonymous chat API service +- Improved mentioned document handling in chat +- Moved duplicate tag checks in the HITL edit panel into functional state updates + +### Developers + +- Agent harness internals were updated for the next generation of tool-heavy chat turns +- LLM role manager reset behavior now uses keyed state instead of manual effect resets +- Blog search now derives results during render instead of syncing duplicate state through effects +- A diagnostic notary-status workflow was added for desktop release troubleshooting +- Announcements gained SEO metadata through a server layout + +v0.0.20 makes SurfSense feel more connected, more local, and more agentic. Whether you are syncing an **Obsidian** vault, editing desktop files, giving agents live connector tools, or tracking model usage in chat, this release pushes SurfSense closer to a full knowledge-work operating layer. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-04.mdx b/surfsense_web/changelog/content/2026-05-04.mdx new file mode 100644 index 000000000..7a8e10548 --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-04.mdx @@ -0,0 +1,35 @@ +--- +date: "2026-05-04" +version: "0.0.21" +--- + +## What's New in v0.0.21 + +v0.0.21 is a focused reliability release for SurfSense agents and chat infrastructure. It adds agent-side caches, fixes invalid prompt cache configuration paths, moves chat persistence to the server, and closes security leaks in manual authentication endpoints. + +### Agent Reliability + +- **Agent Caches**: SurfSense agents now use dedicated caches to reduce repeated work and make longer conversations more stable. +- **Prompt Cache Fixes**: Invalid prompt cache configurations are now handled correctly instead of creating brittle runtime behavior. +- **More Reliable Runtime State**: The caching improvements give agent turns a cleaner foundation for repeated tool use and prompt execution. + +### Server-Side Chat Persistence + +- **Chat Persistence Moved Server-Side**: Chat state persistence now lives on the server, reducing fragile client-side assumptions and improving consistency across sessions. +- **Cleaner Chat Continuity**: Server-owned persistence gives SurfSense a stronger base for future multi-agent and long-running chat work. + +### No longer broken + +- Fixed Docker deployment issues found after the previous platform upgrade +- Fixed manual authentication endpoint leaks +- Fixed invalid prompt cache configuration handling + +### Developers + +- Chat persistence responsibility moved from the frontend into backend-managed flows +- Agent cache plumbing was introduced as groundwork for faster, more resilient agent turns +- Security handling around manual auth routes was tightened + +v0.0.21 keeps the momentum from the connector and desktop releases while strengthening the foundation: safer authentication, more reliable Docker behavior, and better server-owned chat state. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-05.mdx b/surfsense_web/changelog/content/2026-05-05.mdx new file mode 100644 index 000000000..dc6976adc --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-05.mdx @@ -0,0 +1,27 @@ +--- +date: "2026-05-05" +version: "0.0.22" +--- + +## What's New in v0.0.22 + +v0.0.22 is a compact backend performance release. The headline change is **24-hour CORS preflight response caching**, which helps browsers avoid repeated preflight work when calling SurfSense APIs from approved origins. + +### API Performance + +- **24-Hour CORS Preflight Cache**: SurfSense now caches CORS preflight responses for 24 hours, reducing repeated OPTIONS traffic in browser-based deployments. +- **Smoother Hosted Usage**: Cloud and cross-origin setups benefit from less repetitive browser negotiation before API calls. +- **Better Self-Hosted Behavior**: Self-hosted deployments with separate frontend and backend origins get a cleaner default API experience. + +### No longer broken + +- No user-facing bug fixes in this release; this was a focused API performance and release maintenance update + +### Developers + +- CORS preflight responses now advertise a 24-hour cache window +- Release metadata and versioning were advanced for v0.0.22 + +v0.0.22 is small by design, but it improves the everyday feel of SurfSense API calls in browser and self-hosted environments by cutting down redundant CORS preflight traffic. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-06.mdx b/surfsense_web/changelog/content/2026-05-06.mdx new file mode 100644 index 000000000..b85c13f6f --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-06.mdx @@ -0,0 +1,46 @@ +--- +date: "2026-05-06" +version: "0.0.23" +--- + +## What's New in v0.0.23 + +v0.0.23 introduces the next major step for SurfSense chat: **opt-in multi-agent orchestration** with bundled human approval. Agents can now coordinate work through a more modular runtime, while improvements to memory extraction, prompt caching, subagent resilience, and document chunking make the system more dependable for complex knowledge tasks. + +### Multi-Agent Chat + +- **Opt-In Multi-Agent Mode**: Users can enable a multi-agent chat path when a task benefits from delegation. +- **Bundled Human Approval**: Human-in-the-loop approval is integrated into the multi-agent experience, so sensitive steps can be reviewed before execution. +- **Agent Orchestration**: SurfSense can coordinate multiple agent roles during a chat turn, opening the door to more specialized reasoning and tool use. +- **Subagent Resilience**: Runtime improvements make delegated work more robust when individual subagent paths fail or need recovery. + +### Agent Runtime Improvements + +- **Modular Middleware Stack**: Agent execution now flows through a more modular middleware architecture. +- **Agent and Prompt Caching**: Repeated agent setup and prompt work can be cached, reducing waste across complex runs. +- **Unit Test Coverage**: New tests lock in behavior around the agent middleware and caching improvements. +- **Memory Extraction Cleanup**: Memory extraction is more streamlined, keeping durable memory useful without collecting unnecessary noise. + +### Better Document Understanding + +- **Table-Aware Chunking**: The hybrid text chunker now understands tables better and avoids splitting documents in the middle of rows. +- **Cleaner Retrieval Context**: Better chunk boundaries improve the quality of document snippets agents receive during search and reasoning. + +### No longer broken + +- Fixed Stripe webhook handling +- Fixed Stripe routes used by billing flows +- Fixed metadata extraction from Stripe checkout sessions +- Improved memory extraction behavior for cleaner long-term memory updates + +### Developers + +- Multi-agent orchestration primitives were added to the chat runtime +- Agent middleware was reorganized into a more composable stack +- Prompt and agent caches were introduced to reduce repeated setup work +- The hybrid chunker gained table-aware logic to preserve row integrity +- New unit tests cover middleware, caching, and subagent resilience paths + +v0.0.23 lays the groundwork for SurfSense's multi-agent future. With orchestration, approval, caching, and better document chunking, SurfSense is better prepared for complex research, retrieval, and action-heavy workflows. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-20.mdx b/surfsense_web/changelog/content/2026-05-20.mdx new file mode 100644 index 000000000..1f42fb2f7 --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-20.mdx @@ -0,0 +1,61 @@ +--- +date: "2026-05-20" +version: "0.0.24" +--- + +## What's New in v0.0.24 + +v0.0.24 is a major product and runtime release for SurfSense. The interface gets a broad **UI revamp**, multi-agent chat gains a hierarchical timeline with **live subagent streaming**, and human approvals become more natural with inline and parallel approval flows. Behind the scenes, this release also introduces a full E2E test harness and hardens Docker build migrations. + +### UI Revamp + +- **Refreshed Product Interface**: SurfSense gets a cleaner, more polished UI across key chat and dashboard surfaces. +- **Improved Mentioned Document Chips**: Mentioned document chips behave more reliably and feel better in active chat workflows. +- **Consistent Mobile Behavior**: Slide-out panels now share the same mobile breakpoint logic used elsewhere in the app. +- **Better Error Toast Ownership**: Mutations that own their own error UX no longer also trigger noisy global error toasts. +- **Clickable Media Accessibility**: Media cards now include accessible labels for better keyboard and screen-reader use. + +### Multi-Agent Chat, Reworked + +- **Hierarchical Timeline**: Multi-agent chat now presents work in a clearer timeline, making it easier to understand what the main agent and subagents are doing. +- **Live Subagent Streaming**: Subagent work can stream into the interface live instead of appearing only after completion. +- **Inline HITL Approvals**: Human-in-the-loop approval cards now fit directly into the chat flow. +- **Parallel Task Delegation**: Multi-agent tasks can be delegated in parallel, letting independent work proceed at the same time. +- **Parallel Human Approvals**: Approval requests can be handled in parallel when multiple subagent actions need review. +- **Knowledge Base Specialist**: Delegation now uses an orchestrator-only main agent with a dedicated knowledge base specialist for retrieval-heavy work. + +### Testing & Release Confidence + +- **E2E Test Suite**: SurfSense now includes an end-to-end test suite for critical product flows. +- **E2E CI**: The E2E suite is wired into CI so regressions can be caught earlier. +- **Docker Migration Hardening**: Docker build migrations were hardened to make self-hosted upgrades safer. +- **Unit and Integration Test Fixes**: Test reliability improved across backend and frontend paths. + +### Citations & Logs + +- **Citation Panel Reset Fixes**: Citation panels now reset expanded state correctly when the active chunk changes. +- **Log Cache Invalidation**: Log mutations now invalidate all relevant log cache keys. +- **Canonical Log Types**: Log hooks now use shared canonical log types from the contracts layer. +- **Stable Cache Key Ordering**: Cache key ordering was made stable to reduce subtle invalidation bugs. + +### No longer broken + +- Fixed redundant auth token storage writes +- Centralized redirect path storage in authentication flows +- Fixed stale zero-cache replica behavior +- Fixed citation panel expanded state when switching chunks +- Fixed log cache invalidation after log mutations +- Suppressed global error toasts when a mutation provides its own error handling + +### Developers + +- Multi-agent chat architecture and streaming runtime were rewritten for timeline-first execution +- Parallel delegation and parallel HITL approval paths were added to the agent runtime +- Shared helpers were extracted for chat sidebar timestamp formatting and domain parsing +- Editor documentation now matches the Mod+Shift+S save shortcut +- The blog was hidden from the navbar until publication readiness +- Full E2E harness coverage was added around key chat and product workflows + +v0.0.24 makes multi-agent SurfSense easier to trust and easier to follow. With a clearer UI, live subagent streaming, parallel approvals, and a real E2E safety net, this release turns the multi-agent runtime into a more visible and dependable part of everyday knowledge work. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-21.mdx b/surfsense_web/changelog/content/2026-05-21.mdx new file mode 100644 index 000000000..1a8388b39 --- /dev/null +++ b/surfsense_web/changelog/content/2026-05-21.mdx @@ -0,0 +1,46 @@ +--- +date: "2026-05-21" +version: "0.0.25" +--- + +## What's New in v0.0.25 + +v0.0.25 focuses on making SurfSense agents faster, cheaper to run, and easier to trust. Agent turns now do less unnecessary LLM work, citations are improved, memory extraction gets more useful, and users can edit memory directly from the document panel. The release also cleans up backend URL handling across the web app and tightens CI triggers. + +### Faster, Lower-Cost Agents + +- **Faster Agent Turns**: Agent execution was optimized so common turns finish with less waiting. +- **Lower LLM Cost**: The agent runtime now avoids more unnecessary model calls, reducing token spend for complex workflows. +- **Improved Agent Speed**: Follow-up runtime improvements make chat feel more responsive, especially in agent-heavy paths. +- **Better Citations**: Citation behavior was fixed and improved so answers remain easier to verify against source documents. + +### Memory Editing + +- **Improved Memory Extraction**: SurfSense extracts durable memory more cleanly, keeping useful context while avoiding unnecessary noise. +- **Document-Panel Memory Editing**: Users can now edit memory from the document panel, bringing memory management closer to the context that created it. +- **More Practical Long-Term Context**: Cleaner memory updates make future chats more personalized without making memory harder to control. + +### Backend URL Cleanup + +- **Centralized BACKEND_URL Usage**: Inline environment reads were replaced with a shared BACKEND_URL path across key frontend areas. +- **Connector Form Consistency**: Connector forms and hooks now use the same backend URL contract. +- **Editor, Chat, Dashboard & Settings Cleanup**: Core app surfaces now rely on the shared backend URL helper instead of scattered environment access. +- **Tool UI Generator Cleanup**: Tool UI generators also moved to the centralized backend URL flow. + +### No longer broken + +- Fixed citation behavior in agent responses +- Updated CI workflow versions +- Scoped test triggers so CI runs are better targeted +- Removed scattered inline process.env reads from frontend runtime paths + +### Developers + +- Agent execution was tuned to reduce LLM calls and improve response latency +- BACKEND_URL was adopted consistently across lib code, connector forms, hooks, editor, chat, dashboard, settings, and tool UI generators +- CI workflows were refreshed with updated versions and narrower trigger scopes +- Memory extraction logic was refined to support cleaner editing and retrieval flows + +v0.0.25 makes SurfSense agents more efficient and easier to inspect. Faster turns, lower LLM cost, better citations, and more direct memory editing all move SurfSense toward a smoother day-to-day AI knowledge workspace. + +SurfSense connects all your knowledge sources in one place. diff --git a/surfsense_web/changelog/content/2026-05-31.mdx b/surfsense_web/changelog/content/2026-05-31.mdx index 2e79effc6..e48f2fbb9 100644 --- a/surfsense_web/changelog/content/2026-05-31.mdx +++ b/surfsense_web/changelog/content/2026-05-31.mdx @@ -1,8 +1,5 @@ --- -title: "SurfSense v0.0.26 - AI Automations: Build, Schedule & Event-Trigger AI Agents From Chat" -description: "SurfSense v0.0.26 introduces open source AI automations across your connectors: describe a workflow in plain English and SurfSense builds it, run AI agents on a schedule, or trigger them when a document lands in a folder, working across Notion, Slack, Google Drive, Gmail, GitHub, Linear, Jira and Confluence. Plus connector @-mentions in chat and a faster anonymous chat experience." date: "2026-05-31" -tags: ["Automations", "AI Agents", "Workflow Automation", "Scheduled Workflows", "Event Triggers", "Connectors", "Notion", "Slack", "Open Source"] version: "0.0.26" --- @@ -41,35 +38,24 @@ Turn one-off prompts into repeatable, hands-off **AI agent workflows**. - **Redesigned Use-Case Showcase**: The homepage now groups demos into clear categories (Desktop App, Deliverable Studio, Search & Chat, Connectors & Integrations, and Automations) so visitors immediately see what SurfSense can do. - **Desktop App, Front and Center**: The desktop experience is highlighted as a set of native extras on top of everything SurfSense already does, not a separate product. - - - Bug Fixes - -
      -
    • Bulk-moving documents now uses ORM objects so folder events fire correctly and trigger automations
    • -
    • Automation enum columns now persist Postgres enum values instead of names
    • -
    • Automation agent tasks use an in-memory checkpointer to avoid Celery pool timeouts
    • -
    • The API client now handles 204 No Content responses without errors
    • -
    • The model role manager now stays in sync when preferences are updated
    • -
    • The JSON editor now coerces numeric strings to numbers on edit
    • -
    -
    -
    - - Under the Hood - -
      -
    • New in-process domain event bus with an event catalog and a document.entered_folder event
    • -
    • SQLAlchemy session hooks publish folder events automatically, registered at app startup
    • -
    • Cron schedule triggers backed by croniter and a Celery beat tick task
    • -
    • Sandboxed template engine with an allowlisted filter and test set for safe workflow templating
    • -
    • Automations reorganized into a vertical-slice architecture (actions and triggers grouped by domain)
    • -
    • Extensive new test coverage locking automation schemas, dispatch, runtime, triggers, and templating
    • -
    • Model eligibility checks when creating automations, so only valid models are selectable
    • -
    -
    -
    -
    +### No longer broken + +- Bulk-moving documents now uses ORM objects so folder events fire correctly and trigger automations +- Automation enum columns now persist Postgres enum values instead of names +- Automation agent tasks use an in-memory checkpointer to avoid Celery pool timeouts +- The API client now handles 204 No Content responses without errors +- The model role manager now stays in sync when preferences are updated +- The JSON editor now coerces numeric strings to numbers on edit + +### Developers + +- New in-process domain event bus with an event catalog and a `document.entered_folder` event +- SQLAlchemy session hooks publish folder events automatically, registered at app startup +- Cron schedule triggers backed by croniter and a Celery beat tick task +- Sandboxed template engine with an allowlisted filter and test set for safe workflow templating +- Automations reorganized into a vertical-slice architecture (actions and triggers grouped by domain) +- Extensive new test coverage locking automation schemas, dispatch, runtime, triggers, and templating +- Model eligibility checks when creating automations, so only valid models are selectable v0.0.26 turns SurfSense from a place you ask questions into a place that does the work for you. Whether you want **scheduled AI workflows**, **event-driven document automation**, or a **self-hosted, open source AI agent** you fully control, this release lets you build it from a single sentence. diff --git a/surfsense_web/components/assistant-ui/chat-viewport.tsx b/surfsense_web/components/assistant-ui/chat-viewport.tsx index cb0fd2005..83308b642 100644 --- a/surfsense_web/components/assistant-ui/chat-viewport.tsx +++ b/surfsense_web/components/assistant-ui/chat-viewport.tsx @@ -1,6 +1,6 @@ "use client"; -import { ThreadPrimitive } from "@assistant-ui/react"; +import { AuiIf, ThreadPrimitive } from "@assistant-ui/react"; import { ArrowDownIcon } from "lucide-react"; import type { FC, ReactNode } from "react"; import { Button } from "@/components/ui/button"; @@ -40,15 +40,17 @@ export const ChatViewport: FC = ({ children, footer }) => ( /> {children} {footer ? ( - -
    - - {footer} -
    -
    + !thread.isEmpty}> + +
    + + {footer} +
    +
    +
    ) : null} ); diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 9b76ea4cb..6ea55f4a5 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -1,18 +1,10 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertTriangle } from "lucide-react"; -import { useRouter } from "next/navigation"; import { forwardRef, useEffect, useImperativeHandle, useMemo, useState } from "react"; import { createPortal } from "react-dom"; import { statusInboxItemsAtom } from "@/atoms/inbox/status-inbox.atom"; -import { - globalNewLLMConfigsAtom, - llmPreferencesAtom, -} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; import { Tabs, TabsContent } from "@/components/ui/tabs"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; @@ -44,28 +36,7 @@ interface ConnectorIndicatorProps { export const ConnectorIndicator = forwardRef( (_props, ref) => { - const router = useRouter(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { data: preferences = {}, isFetching: preferencesLoading } = - useAtomValue(llmPreferencesAtom); - const { data: globalConfigs = [], isFetching: globalConfigsLoading } = - useAtomValue(globalNewLLMConfigsAtom); - - // Check if document summary LLM is properly configured - // - If ID is 0 (Auto mode), we need global configs to be available - // - If ID is positive (user config) or negative (specific global config), it's configured - // - If ID is null/undefined, it's not configured - const docSummaryLlmId = preferences.document_summary_llm_id; - const isAutoMode = docSummaryLlmId === 0; - const hasGlobalConfigs = globalConfigs.length > 0; - - const hasDocumentSummaryLLM = - docSummaryLlmId !== null && - docSummaryLlmId !== undefined && - // If it's Auto mode, we need global configs to actually be available - (!isAutoMode || hasGlobalConfigs); - - const llmConfigLoading = preferencesLoading || globalConfigsLoading; // Real-time document type counts via Zero (updates instantly as docs are indexed) const documentTypeCounts = useZeroDocumentTypeCounts(searchSpaceId); @@ -97,7 +68,6 @@ export const ConnectorIndicator = forwardRef { startIndexing(editingConnector.id); @@ -339,7 +306,6 @@ export const ConnectorIndicator = forwardRef { @@ -378,35 +343,6 @@ export const ConnectorIndicator = forwardRef
    - {/* LLM Configuration Warning */} - {!llmConfigLoading && !hasDocumentSummaryLLM && ( -
    - - - LLM Configuration Required - -

    - {isAutoMode && !hasGlobalConfigs - ? "Auto mode requires a global LLM configuration. Please add one in Settings" - : "A Document Summary LLM is required to process uploads, configure one in Settings"} -

    - -
    -
    -
    - )} - {}} - onConnectNonOAuth={hasDocumentSummaryLLM ? handleConnectNonOAuth : () => {}} - onCreateWebcrawler={ - hasDocumentSummaryLLM ? handleCreateWebcrawler : () => {} - } - onCreateYouTubeCrawler={ - hasDocumentSummaryLLM ? handleCreateYouTubeCrawler : () => {} - } + onConnectOAuth={handleConnectOAuth} + onConnectNonOAuth={handleConnectNonOAuth} + onCreateWebcrawler={handleCreateWebcrawler} + onCreateYouTubeCrawler={handleCreateYouTubeCrawler} onManage={handleStartEdit} onViewAccountsList={handleViewAccountsList} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx deleted file mode 100644 index b9ff69f5f..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx +++ /dev/null @@ -1,25 +0,0 @@ -"use client"; - -import type { FC } from "react"; -import { Switch } from "@/components/ui/switch"; - -interface SummaryConfigProps { - enabled: boolean; - onEnabledChange: (enabled: boolean) => void; -} - -export const SummaryConfig: FC = ({ enabled, onEnabledChange }) => { - return ( -
    -
    -
    -

    Enable AI Summary

    -

    - Improves search quality but adds latency during indexing -

    -
    - -
    -
    - ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx index ed01511ca..78d48070a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-trusted-tools.tsx @@ -1,6 +1,6 @@ "use client"; -import { ShieldCheck, Trash2 } from "lucide-react"; +import { Trash2 } from "lucide-react"; import type { FC } from "react"; import { useState } from "react"; import { toast } from "sonner"; @@ -35,10 +35,7 @@ export const MCPTrustedTools: FC = ({ connector }) => { return (
    -

    - - Trusted Tools -

    +

    Trusted Tools

    diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index 2b86daf65..011eeec96 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -17,7 +17,6 @@ import { BACKEND_URL } from "@/lib/env-config"; import { cn } from "@/lib/utils"; import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; -import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; import { LIVE_CONNECTOR_TYPES } from "../../constants/connector-constants"; import { getConnectorDisplayName } from "../../tabs/all-connectors-tab"; @@ -38,7 +37,6 @@ interface ConnectorEditViewProps { endDate: Date | undefined; periodicEnabled: boolean; frequencyMinutes: string; - enableSummary: boolean; enableVisionLlm: boolean; isSaving: boolean; isDisconnecting: boolean; @@ -48,7 +46,6 @@ interface ConnectorEditViewProps { onEndDateChange: (date: Date | undefined) => void; onPeriodicEnabledChange: (enabled: boolean) => void; onFrequencyChange: (frequency: string) => void; - onEnableSummaryChange: (enabled: boolean) => void; onEnableVisionLlmChange: (enabled: boolean) => void; onSave: () => void; onDisconnect: () => void; @@ -64,7 +61,6 @@ export const ConnectorEditView: FC = ({ endDate, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, isSaving, isDisconnecting, @@ -74,7 +70,6 @@ export const ConnectorEditView: FC = ({ onEndDateChange, onPeriodicEnabledChange, onFrequencyChange, - onEnableSummaryChange, onEnableVisionLlmChange, onSave, onDisconnect, @@ -87,9 +82,13 @@ export const ConnectorEditView: FC = ({ const isAuthExpired = connector.config?.auth_expired === true; const reauthEndpoint = getReauthEndpoint(connector); const [reauthing, setReauthing] = useState(false); + const isMCPBacked = Boolean(connector.config?.server_config); + const isLive = isMCPBacked || LIVE_CONNECTOR_TYPES.has(connector.connector_type); const supportsVisionLlm = VISION_LLM_CONNECTOR_TYPES.has(connector.connector_type); - const showsAiToggles = - connector.is_indexable || connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR; + const showsVisionToggle = + !isLive && + supportsVisionLlm && + (connector.is_indexable || connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR); const handleReauth = useCallback(async () => { const spaceId = searchSpaceId ?? searchSpaceIdAtom; @@ -121,9 +120,6 @@ export const ConnectorEditView: FC = ({ } }, [searchSpaceId, searchSpaceIdAtom, reauthEndpoint, connector.id]); - const isMCPBacked = Boolean(connector.config?.server_config); - const isLive = isMCPBacked || LIVE_CONNECTOR_TYPES.has(connector.connector_type); - // Get connector-specific config component (MCP-backed connectors use a generic view) const ConnectorConfigComponent = useMemo(() => { if (isMCPBacked) return MCPServiceConfig; @@ -280,77 +276,69 @@ export const ConnectorEditView: FC = ({ /> )} - {/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */} - {showsAiToggles && !isLive && ( - <> - {/* AI Summary toggle */} - - - {/* Vision LLM toggle for file/attachment connectors */} - {supportsVisionLlm && ( - - )} - - {/* Date-range and periodic sync stay indexable-only */} - {connector.is_indexable && - connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && - connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - connector.connector_type !== "DROPBOX_CONNECTOR" && - connector.connector_type !== "ONEDRIVE_CONNECTOR" && - connector.connector_type !== "WEBCRAWLER_CONNECTOR" && - connector.connector_type !== "GITHUB_CONNECTOR" && ( - - )} - - {connector.is_indexable && - (() => { - const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; - const isComposioGoogleDrive = - connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; - const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; - const selectedFolders = - (connector.config?.selected_folders as - | Array<{ id: string; name: string }> - | undefined) || []; - const selectedFiles = - (connector.config?.selected_files as - | Array<{ id: string; name: string }> - | undefined) || []; - const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; - const isDisabled = requiresFolderSelection && !hasItemsSelected; - - return ( - - ); - })()} - + {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} + {showsVisionToggle && ( + )} + {/* Date-range and periodic sync stay indexable-only */} + {connector.is_indexable && + connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && + connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + connector.connector_type !== "DROPBOX_CONNECTOR" && + connector.connector_type !== "ONEDRIVE_CONNECTOR" && + connector.connector_type !== "WEBCRAWLER_CONNECTOR" && + connector.connector_type !== "GITHUB_CONNECTOR" && ( + + )} + + {connector.is_indexable && + (() => { + const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; + const isComposioGoogleDrive = + connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; + const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; + const selectedFolders = + (connector.config?.selected_folders as + | Array<{ id: string; name: string }> + | undefined) || []; + const selectedFiles = + (connector.config?.selected_files as + | Array<{ id: string; name: string }> + | undefined) || []; + const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; + const isDisabled = requiresFolderSelection && !hasItemsSelected; + + return ( + + ); + })()} + {/* Info box - hidden for live connectors */} {connector.is_indexable && !isLive && ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx index 74b4fad9f..097360e14 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx @@ -11,7 +11,6 @@ import { getConnectorTypeDisplay } from "@/lib/connectors/utils"; import { cn } from "@/lib/utils"; import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; -import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; import { type IndexingConfigState, @@ -35,7 +34,6 @@ interface IndexingConfigurationViewProps { endDate: Date | undefined; periodicEnabled: boolean; frequencyMinutes: string; - enableSummary: boolean; enableVisionLlm: boolean; isStartingIndexing: boolean; isFromOAuth?: boolean; @@ -43,7 +41,6 @@ interface IndexingConfigurationViewProps { onEndDateChange: (date: Date | undefined) => void; onPeriodicEnabledChange: (enabled: boolean) => void; onFrequencyChange: (frequency: string) => void; - onEnableSummaryChange: (enabled: boolean) => void; onEnableVisionLlmChange: (enabled: boolean) => void; onConfigChange?: (config: Record) => void; onStartIndexing: () => void; @@ -57,7 +54,6 @@ export const IndexingConfigurationView: FC = ({ endDate, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, isStartingIndexing, isFromOAuth = false, @@ -65,7 +61,6 @@ export const IndexingConfigurationView: FC = ({ onEndDateChange, onPeriodicEnabledChange, onFrequencyChange, - onEnableSummaryChange, onEnableVisionLlmChange, onConfigChange, onStartIndexing, @@ -78,9 +73,11 @@ export const IndexingConfigurationView: FC = ({ () => (connector ? getConnectorConfigComponent(connector.connector_type) : null), [connector] ); - const showsAiToggles = - (connector?.is_indexable ?? false) || - connector?.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR; + const showsVisionToggle = + !isLive && + ((connector?.is_indexable ?? false) || + connector?.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR) && + VISION_LLM_CONNECTOR_TYPES.has(config.connectorType); const [isScrolled, setIsScrolled] = useState(false); const [hasMoreContent, setHasMoreContent] = useState(false); const scrollContainerRef = useRef(null); @@ -178,57 +175,49 @@ export const IndexingConfigurationView: FC = ({ )} - {/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */} - {showsAiToggles && !isLive && ( - <> - {/* AI Summary toggle */} - - - {/* Vision LLM toggle for file/attachment connectors */} - {VISION_LLM_CONNECTOR_TYPES.has(config.connectorType) && ( - - )} - - {/* Date-range and periodic sync stay indexable-only */} - {connector?.is_indexable && - config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "DROPBOX_CONNECTOR" && - config.connectorType !== "ONEDRIVE_CONNECTOR" && - config.connectorType !== "WEBCRAWLER_CONNECTOR" && - config.connectorType !== "GITHUB_CONNECTOR" && ( - - )} - - {connector?.is_indexable && - config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "DROPBOX_CONNECTOR" && - config.connectorType !== "ONEDRIVE_CONNECTOR" && ( - - )} - + {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} + {showsVisionToggle && ( + )} + {/* Date-range and periodic sync stay indexable-only */} + {connector?.is_indexable && + config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "DROPBOX_CONNECTOR" && + config.connectorType !== "ONEDRIVE_CONNECTOR" && + config.connectorType !== "WEBCRAWLER_CONNECTOR" && + config.connectorType !== "GITHUB_CONNECTOR" && ( + + )} + + {connector?.is_indexable && + config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "DROPBOX_CONNECTOR" && + config.connectorType !== "ONEDRIVE_CONNECTOR" && ( + + )} + {/* Info box - hidden for live connectors */} {connector?.is_indexable && !isLive && ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 5de623c22..315574aa9 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -243,6 +243,38 @@ export function getConnectorTitle(connectorType: string): string { ); } +/** + * Primary way a user interacts with a connector. + * Drives the two top-level groupings in the connector catalog UI. + */ +export type ConnectorCategory = "knowledge_base" | "tools_live"; + +export const CONNECTOR_CATEGORY_LABELS: Record = { + knowledge_base: "Knowledge Base", + tools_live: "Tools & Live Sources", +}; + +const KNOWLEDGE_BASE_CONNECTOR_TYPES = new Set([ + EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, + EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + EnumConnectorName.ONEDRIVE_CONNECTOR, + EnumConnectorName.DROPBOX_CONNECTOR, + EnumConnectorName.NOTION_CONNECTOR, + EnumConnectorName.CONFLUENCE_CONNECTOR, + EnumConnectorName.YOUTUBE_CONNECTOR, + EnumConnectorName.WEBCRAWLER_CONNECTOR, + EnumConnectorName.BOOKSTACK_CONNECTOR, + EnumConnectorName.GITHUB_CONNECTOR, + EnumConnectorName.ELASTICSEARCH_CONNECTOR, + EnumConnectorName.CIRCLEBACK_CONNECTOR, + EnumConnectorName.OBSIDIAN_CONNECTOR, +]); + +/** Unmapped connectors surface under Tools & Live Sources. */ +export function getConnectorCategory(connectorType: string): ConnectorCategory { + return KNOWLEDGE_BASE_CONNECTOR_TYPES.has(connectorType) ? "knowledge_base" : "tools_live"; +} + // Composio Toolkits (available integrations via Composio) export const COMPOSIO_TOOLKITS = [ { diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index 25ab82e2e..45c174d74 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -82,7 +82,6 @@ export const useConnectorDialog = () => { const [isStartingIndexing, setIsStartingIndexing] = useState(false); const [periodicEnabled, setPeriodicEnabled] = useState(false); const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); - const [enableSummary, setEnableSummary] = useState(false); const [enableVisionLlm, setEnableVisionLlm] = useState(false); // Edit mode state @@ -418,7 +417,6 @@ export const useConnectorDialog = () => { periodic_indexing_enabled: false, indexing_frequency_minutes: null, next_scheduled_at: null, - enable_summary: false, enable_vision_llm: false, }, queryParams: { @@ -520,7 +518,6 @@ export const useConnectorDialog = () => { connector_type: connectorData.connector_type as EnumConnectorName, is_active: true, next_scheduled_at: connectorData.next_scheduled_at as string | null, - enable_summary: false, enable_vision_llm: false, }, queryParams: { @@ -657,7 +654,6 @@ export const useConnectorDialog = () => { setConnectorConfig(connector.config || {}); setPeriodicEnabled(false); setFrequencyMinutes("1440"); - setEnableSummary(connector.enable_summary ?? false); setEnableVisionLlm(connector.enable_vision_llm ?? false); setStartDate(undefined); setEndDate(undefined); @@ -806,13 +802,12 @@ export const useConnectorDialog = () => { const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; - // Update connector with summary, periodic sync settings, and config changes - if (enableSummary || enableVisionLlm || periodicEnabled || indexingConnectorConfig) { + // Update connector with vision, periodic sync settings, and config changes + if (enableVisionLlm || periodicEnabled || indexingConnectorConfig) { const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined; await updateConnector({ id: indexingConfig.connectorId, data: { - enable_summary: enableSummary, enable_vision_llm: enableVisionLlm, ...(periodicEnabled && { periodic_indexing_enabled: true, @@ -940,7 +935,6 @@ export const useConnectorDialog = () => { updateConnector, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, indexingConnectorConfig, setIsOpen, @@ -1005,7 +999,6 @@ export const useConnectorDialog = () => { setConnectorName(connector.name); setPeriodicEnabled(!connector.is_indexable ? false : connector.periodic_indexing_enabled); setFrequencyMinutes(connector.indexing_frequency_minutes?.toString() || "1440"); - setEnableSummary(connector.enable_summary ?? false); setEnableVisionLlm(connector.enable_vision_llm ?? false); setStartDate(undefined); setEndDate(undefined); @@ -1084,7 +1077,6 @@ export const useConnectorDialog = () => { id: editingConnector.id, data: { name: connectorName || editingConnector.name, - enable_summary: enableSummary, enable_vision_llm: enableVisionLlm, periodic_indexing_enabled: !editingConnector.is_indexable ? false : periodicEnabled, indexing_frequency_minutes: !editingConnector.is_indexable ? null : frequency, @@ -1219,7 +1211,6 @@ export const useConnectorDialog = () => { updateConnector, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, getFrequencyLabel, connectorConfig, @@ -1380,7 +1371,6 @@ export const useConnectorDialog = () => { setEndDate(undefined); setPeriodicEnabled(false); setFrequencyMinutes("1440"); - setEnableSummary(false); setEnableVisionLlm(false); } } @@ -1417,7 +1407,6 @@ export const useConnectorDialog = () => { isDisconnecting, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, searchSpaceId, allConnectors, @@ -1432,7 +1421,6 @@ export const useConnectorDialog = () => { setEndDate, setPeriodicEnabled, setFrequencyMinutes, - setEnableSummary, setEnableVisionLlm, setConnectorName, diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 814959ec4..4977219f7 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -9,7 +9,10 @@ import { isSelfHosted } from "@/lib/env-config"; import { ConnectorCard } from "../components/connector-card"; import { COMPOSIO_CONNECTORS, + CONNECTOR_CATEGORY_LABELS, + type ConnectorCategory, CRAWLERS, + getConnectorCategory, OAUTH_CONNECTORS, OTHER_CONNECTORS, } from "../constants/connector-constants"; @@ -20,19 +23,6 @@ type ComposioConnector = (typeof COMPOSIO_CONNECTORS)[number]; type OtherConnector = (typeof OTHER_CONNECTORS)[number]; type CrawlerConnector = (typeof CRAWLERS)[number]; -const DOCUMENT_FILE_CONNECTOR_TYPES = new Set([ - EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, - EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, - EnumConnectorName.ONEDRIVE_CONNECTOR, - EnumConnectorName.DROPBOX_CONNECTOR, -]); - -const OTHER_DOCUMENT_CONNECTOR_TYPES = new Set([ - EnumConnectorName.YOUTUBE_CONNECTOR, - EnumConnectorName.NOTION_CONNECTOR, - EnumConnectorName.AIRTABLE_CONNECTOR, -]); - /** * Extract the display name from a full connector name. * Full names are in format "Base Name - identifier" (e.g., "Gmail - john@example.com"). @@ -106,45 +96,23 @@ export const AllConnectorsTab: FC = ({ c.description.toLowerCase().includes(searchQuery.toLowerCase()) ); - const nativeGoogleDriveConnectors = filteredOAuth.filter( - (c) => c.connectorType === EnumConnectorName.GOOGLE_DRIVE_CONNECTOR - ); - const composioGoogleDriveConnectors = filteredComposio.filter( - (c) => c.connectorType === EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR - ); - const fileStorageConnectors = filteredOAuth.filter( - (c) => - c.connectorType === EnumConnectorName.ONEDRIVE_CONNECTOR || - c.connectorType === EnumConnectorName.DROPBOX_CONNECTOR - ); + const inCategory = + (category: ConnectorCategory) => + (connector: T): boolean => + !!connector.connectorType && getConnectorCategory(connector.connectorType) === category; - const otherDocumentYouTubeConnectors = filteredCrawlers.filter( - (c) => c.connectorType === EnumConnectorName.YOUTUBE_CONNECTOR - ); - const otherDocumentNotionConnectors = filteredOAuth.filter( - (c) => c.connectorType === EnumConnectorName.NOTION_CONNECTOR - ); - const otherDocumentAirtableConnectors = filteredOAuth.filter( - (c) => c.connectorType === EnumConnectorName.AIRTABLE_CONNECTOR - ); - - const moreIntegrationsComposio = filteredComposio.filter( - (c) => - !DOCUMENT_FILE_CONNECTOR_TYPES.has(c.connectorType) && - !OTHER_DOCUMENT_CONNECTOR_TYPES.has(c.connectorType) - ); - const moreIntegrationsOAuth = filteredOAuth.filter( - (c) => - !DOCUMENT_FILE_CONNECTOR_TYPES.has(c.connectorType) && - !OTHER_DOCUMENT_CONNECTOR_TYPES.has(c.connectorType) - ); - const moreIntegrationsOther = filteredOther; - const moreIntegrationsCrawlers = filteredCrawlers.filter( - (c) => - !c.connectorType || - (!DOCUMENT_FILE_CONNECTOR_TYPES.has(c.connectorType) && - !OTHER_DOCUMENT_CONNECTOR_TYPES.has(c.connectorType)) - ); + const knowledgeBase = { + oauth: filteredOAuth.filter(inCategory("knowledge_base")), + composio: filteredComposio.filter(inCategory("knowledge_base")), + other: filteredOther.filter(inCategory("knowledge_base")), + crawlers: filteredCrawlers.filter(inCategory("knowledge_base")), + }; + const toolsLive = { + oauth: filteredOAuth.filter(inCategory("tools_live")), + composio: filteredComposio.filter(inCategory("tools_live")), + other: filteredOther.filter(inCategory("tools_live")), + crawlers: filteredCrawlers.filter(inCategory("tools_live")), + }; const renderOAuthCard = (connector: OAuthConnector | ComposioConnector) => { const isConnected = connectedTypes.has(connector.connectorType); @@ -275,20 +243,18 @@ export const AllConnectorsTab: FC = ({ ); }; - const hasDocumentFileConnectors = - nativeGoogleDriveConnectors.length > 0 || - composioGoogleDriveConnectors.length > 0 || - fileStorageConnectors.length > 0; - const hasMoreIntegrations = - otherDocumentYouTubeConnectors.length > 0 || - otherDocumentNotionConnectors.length > 0 || - otherDocumentAirtableConnectors.length > 0 || - moreIntegrationsComposio.length > 0 || - moreIntegrationsOAuth.length > 0 || - moreIntegrationsOther.length > 0 || - moreIntegrationsCrawlers.length > 0; + const hasKnowledgeBase = + knowledgeBase.oauth.length > 0 || + knowledgeBase.composio.length > 0 || + knowledgeBase.other.length > 0 || + knowledgeBase.crawlers.length > 0; + const hasToolsLive = + toolsLive.oauth.length > 0 || + toolsLive.composio.length > 0 || + toolsLive.other.length > 0 || + toolsLive.crawlers.length > 0; - const hasAnyResults = hasDocumentFileConnectors || hasMoreIntegrations; + const hasAnyResults = hasKnowledgeBase || hasToolsLive; if (!hasAnyResults && searchQuery) { return ( @@ -302,36 +268,34 @@ export const AllConnectorsTab: FC = ({ return (

    - {/* File Storage Integrations */} - {hasDocumentFileConnectors && ( + {hasKnowledgeBase && (

    - File Storage Integrations + {CONNECTOR_CATEGORY_LABELS.knowledge_base}

    - {nativeGoogleDriveConnectors.map(renderOAuthCard)} - {composioGoogleDriveConnectors.map(renderOAuthCard)} - {fileStorageConnectors.map(renderOAuthCard)} + {knowledgeBase.oauth.map(renderOAuthCard)} + {knowledgeBase.composio.map(renderOAuthCard)} + {knowledgeBase.crawlers.map(renderCrawlerCard)} + {knowledgeBase.other.map(renderOtherCard)}
    )} - {/* More Integrations */} - {hasMoreIntegrations && ( + {hasToolsLive && (
    -

    More Integrations

    +

    + {CONNECTOR_CATEGORY_LABELS.tools_live} +

    - {otherDocumentYouTubeConnectors.map(renderCrawlerCard)} - {otherDocumentNotionConnectors.map(renderOAuthCard)} - {otherDocumentAirtableConnectors.map(renderOAuthCard)} - {moreIntegrationsComposio.map(renderOAuthCard)} - {moreIntegrationsOAuth.map(renderOAuthCard)} - {moreIntegrationsOther.map(renderOtherCard)} - {moreIntegrationsCrawlers.map(renderCrawlerCard)} + {toolsLive.oauth.map(renderOAuthCard)} + {toolsLive.composio.map(renderOAuthCard)} + {toolsLive.crawlers.map(renderCrawlerCard)} + {toolsLive.other.map(renderOtherCard)}
    )} diff --git a/surfsense_web/components/assistant-ui/document-upload-popup.tsx b/surfsense_web/components/assistant-ui/document-upload-popup.tsx index 477d7ee77..504f1d8d4 100644 --- a/surfsense_web/components/assistant-ui/document-upload-popup.tsx +++ b/surfsense_web/components/assistant-ui/document-upload-popup.tsx @@ -1,8 +1,6 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertTriangle } from "lucide-react"; -import { useRouter } from "next/navigation"; import { createContext, type FC, @@ -12,14 +10,8 @@ import { useRef, useState, } from "react"; -import { - globalNewLLMConfigsAtom, - llmPreferencesAtom, -} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, @@ -98,12 +90,7 @@ const DocumentUploadPopupContent: FC<{ isOpen: boolean; onOpenChange: (open: boolean) => void; }> = ({ isOpen, onOpenChange }) => { - const router = useRouter(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { data: preferences = {}, isFetching: preferencesLoading } = - useAtomValue(llmPreferencesAtom); - const { data: globalConfigs = [], isFetching: globalConfigsLoading } = - useAtomValue(globalNewLLMConfigsAtom); if (!searchSpaceId) return null; @@ -111,22 +98,6 @@ const DocumentUploadPopupContent: FC<{ onOpenChange(false); }; - // Check if document summary LLM is properly configured - // - If ID is 0 (Auto mode), we need global configs to be available - // - If ID is positive (user config) or negative (specific global config), it's configured - // - If ID is null/undefined, it's not configured - const docSummaryLlmId = preferences.document_summary_llm_id; - const isAutoMode = docSummaryLlmId === 0; - const hasGlobalConfigs = globalConfigs.length > 0; - - const hasDocumentSummaryLLM = - docSummaryLlmId !== null && - docSummaryLlmId !== undefined && - // If it's Auto mode, we need global configs to actually be available - (!isAutoMode || hasGlobalConfigs); - - const isLoading = preferencesLoading || globalConfigsLoading; - return (
    - {!isLoading && !hasDocumentSummaryLLM ? ( -
    - - - LLM Configuration Required - -

    - {isAutoMode && !hasGlobalConfigs - ? "Auto mode requires a global LLM configuration. Please add one in Settings" - : "A Document Summary LLM is required to process uploads, configure one in Settings"} -

    - -
    -
    -
    - ) : ( - - )} +
    diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index 6a8f2e035..59a10739c 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -3,7 +3,7 @@ import { useSetAtom } from "jotai"; import { FileText } from "lucide-react"; import type { FC } from "react"; -import { useState } from "react"; +import { useId, useState } from "react"; import { openCitationPanelAtom } from "@/atoms/citation/citation-panel.atom"; import { useCitationMetadata } from "@/components/assistant-ui/citation-metadata-context"; import { CitationPanelContent } from "@/components/citation-panel/citation-panel"; @@ -120,12 +120,14 @@ interface UrlCitationProps { * page title and snippet (extracted deterministically from web_search tool results). */ export const UrlCitation: FC = ({ url }) => { + const reactId = useId(); + const citationInstanceId = `url-cite-${reactId.replace(/:/g, "")}`; const domain = tryGetHostname(url) ?? url; const meta = useCitationMetadata(url); return ( { const [actionQuery, setActionQuery] = useState(""); const [suggestionAnchorPoint, setSuggestionAnchorPoint] = useState(null); + const [isComposerInputEmpty, setIsComposerInputEmpty] = useState(true); const editorRef = useRef(null); const prevMentionedDocsRef = useRef>(new Map()); const documentPickerRef = useRef(null); @@ -538,6 +539,7 @@ const Composer: FC = () => { // short-circuit keeps pure-text keystrokes from churning the atom. const handleEditorChange = useCallback( (text: string, docs: MentionedDocument[]) => { + setIsComposerInputEmpty(text.trim().length === 0 && docs.length === 0); aui.composer().setText(text); setMentionedDocuments((prev) => { if (prev.length === docs.length) { @@ -653,6 +655,7 @@ const Composer: FC = () => { : action.prompt; editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); + setIsComposerInputEmpty(false); setShowPromptPicker(false); setActionQuery(""); setSuggestionAnchorPoint(null); @@ -664,6 +667,7 @@ const Composer: FC = () => { (prompt: string) => { editorRef.current?.setText(prompt); aui.composer().setText(prompt); + setIsComposerInputEmpty(false); editorRef.current?.focus(); }, [aui] @@ -678,6 +682,7 @@ const Composer: FC = () => { : `${action.prompt}\n\n${clipboardInitialText}`; editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); + setIsComposerInputEmpty(false); setShowPromptPicker(false); setActionQuery(""); setSuggestionAnchorPoint(null); @@ -757,6 +762,7 @@ const Composer: FC = () => { aui.composer().send(); editorRef.current?.clear(); + setIsComposerInputEmpty(true); setMentionedDocuments([]); }, [ showDocumentPopover, @@ -892,10 +898,10 @@ const Composer: FC = () => { ) : null} -
    +
    @@ -906,7 +912,7 @@ const Composer: FC = () => { onDismiss={() => setClipboardInitialText(undefined)} /> )} -
    +
    { onDocumentRemove={handleDocumentRemove} onSubmit={handleSubmit} onKeyDown={handleKeyDown} - className="min-h-[24px] **:data-slate-placeholder:font-normal" + className="min-h-[48px] sm:min-h-[24px] **:data-slate-placeholder:font-normal" />
    @@ -928,7 +934,11 @@ const Composer: FC = () => { isThreadEmpty={isThreadEmpty} onVisibleChange={setConnectToolsTrayVisible} /> - {isThreadEmpty && } + {isThreadEmpty && isComposerInputEmpty ? ( +
    + +
    + ) : null}
    ); diff --git a/surfsense_web/components/documents/download-original-button.tsx b/surfsense_web/components/documents/download-original-button.tsx new file mode 100644 index 000000000..b79b289b4 --- /dev/null +++ b/surfsense_web/components/documents/download-original-button.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { Download } from "lucide-react"; +import { useEffect, useState } from "react"; +import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { authenticatedFetch } from "@/lib/auth-utils"; +import { BACKEND_URL } from "@/lib/env-config"; + +interface DownloadOriginalButtonProps { + documentId: number; +} + +/** Renders only when the document has a stored ORIGINAL file; downloads it on click. */ +export function DownloadOriginalButton({ documentId }: DownloadOriginalButtonProps) { + const [originalFilename, setOriginalFilename] = useState(null); + const [downloading, setDownloading] = useState(false); + + useEffect(() => { + let active = true; + documentsApiService + .getDocumentFiles(documentId) + .then((files) => { + if (!active) return; + const original = files.find((file) => file.kind === "ORIGINAL"); + setOriginalFilename(original?.original_filename ?? null); + }) + .catch(() => { + if (active) setOriginalFilename(null); + }); + return () => { + active = false; + }; + }, [documentId]); + + if (!originalFilename) return null; + + const handleDownload = async () => { + setDownloading(true); + try { + const response = await authenticatedFetch( + `${BACKEND_URL}/api/v1/documents/${documentId}/download-original`, + { method: "GET" } + ); + if (!response.ok) throw new Error("Download failed"); + + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = originalFilename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); + toast.success("Download started"); + } catch { + toast.error("Failed to download original file"); + } finally { + setDownloading(false); + } + }; + + return ( + + ); +} diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 534ff9daa..01983cbe1 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -12,9 +12,10 @@ import { XIcon, } from "lucide-react"; import dynamic from "next/dynamic"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom"; +import { DownloadOriginalButton } from "@/components/documents/download-original-button"; import { VersionHistoryButton } from "@/components/documents/version-history"; import { SourceCodeEditor } from "@/components/editor/source-code-editor"; import { @@ -49,11 +50,13 @@ interface EditorContent { source_markdown: string; content_size_bytes?: number; chunk_count?: number; - truncated?: boolean; + viewer_mode?: ViewerMode; + editor_plate_max_bytes?: number; } const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]); type EditorRenderMode = "rich_markdown" | "source_code"; +type ViewerMode = "plate" | "monaco"; type AgentFilesystemMount = { mount: string; @@ -111,6 +114,20 @@ function EditorPanelSkeleton() { ); } +function getUtf8ByteSize(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function formatBytes(bytes: number): string { + if (bytes >= 1024 * 1024) { + return `${(bytes / 1024 / 1024).toFixed(1)}MB`; + } + if (bytes >= 1024) { + return `${Math.round(bytes / 1024)}KB`; + } + return `${bytes}B`; +} + export function EditorPanelContent({ kind = "document", documentId, @@ -166,7 +183,11 @@ export function EditorPanelContent({ [electronAPI, searchSpaceId] ); - const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; + const plateMaxBytes = editorDoc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD; + const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > plateMaxBytes; + const viewerMode: ViewerMode = isMemoryMode + ? "plate" + : (editorDoc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate")); useEffect(() => { const controller = new AbortController(); @@ -242,8 +263,6 @@ export function EditorPanelContent({ const url = new URL( `${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); - const response = await authenticatedFetch(url.toString(), { method: "GET" }); if (controller.signal.aborted) return; @@ -401,7 +420,12 @@ export function EditorPanelContent({ setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); setEditedMarkdown(null); if (!options?.silent) { - toast.success("Document saved! Reindexing in background..."); + const savedSizeBytes = getUtf8ByteSize(markdownRef.current); + if (savedSizeBytes > plateMaxBytes) { + toast.success("Document saved. It will reopen in raw markdown mode."); + } else { + toast.success("Document saved! Reindexing in background..."); + } } return true; } catch (err) { @@ -422,6 +446,7 @@ export function EditorPanelContent({ localFilePath, memoryLimits, memoryScope, + plateMaxBytes, resolveLocalVirtualPath, searchSpaceId, ] @@ -431,18 +456,21 @@ export function EditorPanelContent({ ? (isMemoryMode || editorRenderMode === "source_code" || EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")) && - !isLargeDocument + viewerMode === "plate" : false; - // Render through PlateEditor for editable doc types (FILE/NOTE). - // Everything else (large docs, non-editable types) falls back to the - // lightweight `MarkdownViewer` — Plate is heavy on multi-MB docs and - // non-editable types don't benefit from its editing UX. + // Render through PlateEditor only when the backend says the rich editor is safe. + // Monaco mode is a raw markdown safety path for large documents. const renderInPlateEditor = isEditableType; const hasUnsavedChanges = editedMarkdown !== null; const showDesktopHeader = !!onClose; const showEditingActions = isEditableType && isEditing; const localFileLanguage = inferMonacoLanguageFromPath(localFilePath); const activeMarkdown = editedMarkdown ?? editorDoc?.source_markdown ?? ""; + const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]); + const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9; + const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes; + const showPlateSizeWarning = + showEditingActions && !isMemoryMode && !isLocalFileMode && isNearPlateLimit; const memoryLimitState = isMemoryMode ? getMemoryLimitState(activeMarkdown.length, memoryLimits) : null; @@ -491,14 +519,14 @@ export function EditorPanelContent({ } }, [documentId, editorDoc?.title, searchSpaceId]); - const largeDocAlert = isLargeDocument && !isLocalFileMode && editorDoc && ( - + const largeDocAlert = viewerMode === "monaco" && !isLocalFileMode && editorDoc && ( + This document is too large for the editor ( {Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "} - {editorDoc.chunk_count ?? 0} chunks). Showing a preview below. + {editorDoc.chunk_count ?? 0} chunks). Showing raw markdown below.
    - ) : isLargeDocument && !isLocalFileMode ? ( - // Large doc — fast Streamdown preview + download CTA. - // Plate is heavy on multi-MB docs. -
    + ) : viewerMode === "monaco" && !isLocalFileMode ? ( + // Large doc — raw markdown in Monaco. Rich renderers are intentionally skipped. +
    {largeDocAlert} - +
    + {}} + /> +
    ) : renderInPlateEditor ? ( // Editable doc (FILE/NOTE) — Plate editing UX.
    + {showPlateSizeWarning && ( + + + + {isOverPlateLimit + ? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.` + : `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`} + + + )}
    + + {mobileDisabledLabel} + + ); + } if (!primary) { return ( diff --git a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx index 1c5d7af6e..9e2a99f2a 100644 --- a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx @@ -3,7 +3,7 @@ import { Inbox, LibraryBig } from "lucide-react"; import { useRouter } from "next/navigation"; import type { ReactNode } from "react"; -import { Fragment, useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useAnonymousMode } from "@/contexts/anonymous-mode"; import { useLoginGate } from "@/contexts/login-gate"; import { useAnnouncements } from "@/hooks/use-announcements"; @@ -110,15 +110,13 @@ export function FreeLayoutDataProvider({ children }: FreeLayoutDataProviderProps navItems={navItems} onNavItemClick={handleNavItemClick} chats={[]} - sharedChats={[]} activeChatId={null} onNewChat={resetChat} onChatSelect={handleChatSelect} onChatRename={gatedAction("rename chats")} onChatDelete={gatedAction("delete chats")} onChatArchive={gatedAction("archive chats")} - onViewAllSharedChats={gatedAction("view shared chats")} - onViewAllPrivateChats={gatedAction("view chat history")} + onViewAllChats={gatedAction("view chat history")} user={{ email: "Guest", name: "Guest", @@ -137,7 +135,7 @@ export function FreeLayoutDataProvider({ children }: FreeLayoutDataProviderProps onOpenChange: setIsDocsSidebarOpen, }} > - {children} + {children} ); } diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 34fd15e3b..46f6ec8ae 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -1,6 +1,6 @@ "use client"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useQuery } from "@tanstack/react-query"; import { useAtom, useAtomValue, useSetAtom } from "jotai"; import { AlertTriangle, Inbox, LibraryBig, Workflow } from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; @@ -41,13 +41,15 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Spinner } from "@/components/ui/spinner"; +import { useActivateChatThread } from "@/hooks/use-activate-chat-thread"; import { useAnnouncements } from "@/hooks/use-announcements"; import { useInbox } from "@/hooks/use-inbox"; import { useIsMobile } from "@/hooks/use-mobile"; +import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations"; import { notificationsApiService } from "@/lib/apis/notifications-api.service"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { getLoginPath, logout } from "@/lib/auth-utils"; -import { deleteThread, fetchThreads, updateThread } from "@/lib/chat/thread-persistence"; +import { fetchThreads } from "@/lib/chat/thread-persistence"; import { resetUser, trackLogout } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import type { ChatItem, NavItem, SearchSpace } from "../types/layout.types"; @@ -77,7 +79,6 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const router = useRouter(); const params = useParams(); const pathname = usePathname(); - const queryClient = useQueryClient(); const { theme, setTheme } = useTheme(); const isMobile = useIsMobile(); @@ -96,6 +97,10 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const resetCurrentThread = useSetAtom(resetCurrentThreadAtom); const syncChatTab = useSetAtom(syncChatTabAtom); const removeChatTab = useSetAtom(removeChatTabAtom); + const { activateChatThread, prefetchChatThread } = useActivateChatThread(); + const { mutateAsync: archiveThread } = useArchiveThread(searchSpaceId); + const { mutateAsync: deleteThread } = useDeleteThread(searchSpaceId); + const { mutateAsync: renameThread } = useRenameThread(searchSpaceId); // Key used to force-remount the page component (e.g. after deleting the active chat // when the router is out of sync due to replaceState) @@ -121,7 +126,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid }); // Unified slide-out panel state (only one can be open at a time) - type SlideoutPanel = "inbox" | "shared" | "private" | null; + type SlideoutPanel = "inbox" | "chats" | null; const [activeSlideoutPanel, setActiveSlideoutPanel] = useState(null); const isInboxSidebarOpen = activeSlideoutPanel === "inbox"; @@ -301,37 +306,21 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid title: chatId ? (thread?.title ?? undefined) : "New Chat", chatUrl, searchSpaceId: Number(searchSpaceId), + ...(thread?.visibility !== undefined ? { visibility: thread.visibility } : {}), }); }, [currentChatId, searchSpaceId, threadsData?.threads, syncChatTab]); - // Transform and split chats into private and shared based on visibility - const { myChats, sharedChats } = useMemo(() => { - if (!threadsData?.threads) return { myChats: [], sharedChats: [] }; + const chats = useMemo(() => { + if (!threadsData?.threads) return []; - const privateChats: ChatItem[] = []; - const sharedChatsList: ChatItem[] = []; - - for (const thread of threadsData.threads) { - const chatItem: ChatItem = { - id: thread.id, - name: thread.title || `Chat ${thread.id}`, - url: `/dashboard/${searchSpaceId}/new-chat/${thread.id}`, - visibility: thread.visibility, - isOwnThread: thread.is_own_thread, - archived: thread.archived, - }; - - // Split based on visibility, not ownership: - // - PRIVATE chats go to "Private Chats" section - // - SEARCH_SPACE chats go to "Shared Chats" section - if (thread.visibility === "SEARCH_SPACE") { - sharedChatsList.push(chatItem); - } else { - privateChats.push(chatItem); - } - } - - return { myChats: privateChats, sharedChats: sharedChatsList }; + return threadsData.threads.map((thread) => ({ + id: thread.id, + name: thread.title || `Chat ${thread.id}`, + url: `/dashboard/${searchSpaceId}/new-chat/${thread.id}`, + visibility: thread.visibility, + isOwnThread: thread.is_own_thread, + archived: thread.archived, + })); }, [threadsData, searchSpaceId]); // Navigation items @@ -478,12 +467,34 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const handleTabSwitch = useCallback( (tab: Tab) => { if (tab.type === "chat") { - const url = tab.chatUrl || `/dashboard/${searchSpaceId}/new-chat`; - router.push(url); + activateChatThread({ + id: tab.chatId ?? null, + title: tab.title, + url: tab.chatUrl, + searchSpaceId: tab.searchSpaceId ?? searchSpaceId, + ...(tab.visibility !== undefined ? { visibility: tab.visibility } : {}), + ...(tab.hasComments !== undefined ? { hasComments: tab.hasComments } : {}), + }); } // Document tabs are handled in-place by LayoutShell — no navigation needed }, - [router, searchSpaceId] + [activateChatThread, searchSpaceId] + ); + + const handleTabPrefetch = useCallback( + (tab: Tab) => { + if (tab.type === "chat") { + prefetchChatThread(tab.chatId); + } + }, + [prefetchChatThread] + ); + + const handleChatPrefetch = useCallback( + (chat: ChatItem) => { + prefetchChatThread(chat.id); + }, + [prefetchChatThread] ); const handleNavItemClick = useCallback( @@ -535,9 +546,15 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const handleChatSelect = useCallback( (chat: ChatItem) => { - router.push(chat.url); + activateChatThread({ + id: chat.id, + title: chat.name, + url: chat.url, + searchSpaceId, + ...(chat.visibility !== undefined ? { visibility: chat.visibility } : {}), + }); }, - [router] + [activateChatThread, searchSpaceId] ); const handleChatDelete = useCallback((chat: ChatItem) => { @@ -559,18 +576,14 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid : tSidebar("chat_unarchived") || "Chat restored"; try { - await updateThread(chat.id, { archived: newArchivedState }); + await archiveThread({ threadId: chat.id, archived: newArchivedState }); toast.success(successMessage); - // Invalidate queries to refresh UI (React Query will only refetch active queries) - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); } catch (error) { console.error("Error archiving thread:", error); toast.error(tSidebar("error_archiving_chat") || "Failed to archive chat"); } }, - [queryClient, searchSpaceId, tSidebar] + [archiveThread, tSidebar] ); const handleSettings = useCallback(() => { @@ -599,12 +612,8 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } }, [router]); - const handleViewAllSharedChats = useCallback(() => { - setActiveSlideoutPanel((prev) => (prev === "shared" ? null : "shared")); - }, []); - - const handleViewAllPrivateChats = useCallback(() => { - setActiveSlideoutPanel((prev) => (prev === "private" ? null : "private")); + const handleViewAllChats = useCallback(() => { + setActiveSlideoutPanel((prev) => (prev === "chats" ? null : "chats")); }, []); // Delete handlers @@ -612,13 +621,21 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid if (!chatToDelete) return; setIsDeletingChat(true); try { - await deleteThread(chatToDelete.id); + await deleteThread({ threadId: chatToDelete.id }); const fallbackTab = removeChatTab(chatToDelete.id); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); if (currentChatId === chatToDelete.id) { resetCurrentThread(); if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); + activateChatThread({ + id: fallbackTab.chatId ?? null, + title: fallbackTab.title, + url: fallbackTab.chatUrl, + searchSpaceId: fallbackTab.searchSpaceId ?? searchSpaceId, + ...(fallbackTab.visibility !== undefined ? { visibility: fallbackTab.visibility } : {}), + ...(fallbackTab.hasComments !== undefined + ? { hasComments: fallbackTab.hasComments } + : {}), + }); } else { const isOutOfSync = currentThreadState.id !== null && !params?.chat_id; if (isOutOfSync) { @@ -638,7 +655,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } }, [ chatToDelete, - queryClient, + deleteThread, searchSpaceId, resetCurrentThread, currentChatId, @@ -646,6 +663,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid params?.chat_id, router, removeChatTab, + activateChatThread, ]); // Rename handler @@ -653,11 +671,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid if (!chatToRename || !newChatTitle.trim()) return; setIsRenamingChat(true); try { - await updateThread(chatToRename.id, { title: newChatTitle.trim() }); + await renameThread({ + threadId: chatToRename.id, + title: newChatTitle.trim(), + previousTitle: chatToRename.name, + }); toast.success(tSidebar("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); } catch (error) { console.error("Error renaming thread:", error); toast.error(tSidebar("error_renaming_chat") || "Failed to rename chat"); @@ -667,7 +686,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid setChatToRename(null); setNewChatTitle(""); } - }, [chatToRename, newChatTitle, queryClient, searchSpaceId, tSidebar]); + }, [chatToRename, newChatTitle, renameThread, tSidebar]); // Detect if we're on the chat page (needs overflow-hidden for chat's own scroll) const isChatPage = pathname?.includes("/new-chat") ?? false; @@ -695,16 +714,15 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid searchSpace={activeSearchSpace} navItems={navItems} onNavItemClick={handleNavItemClick} - chats={myChats} - sharedChats={sharedChats} + chats={chats} activeChatId={currentChatId} onNewChat={handleNewChat} onChatSelect={handleChatSelect} + onChatPrefetch={handleChatPrefetch} onChatRename={handleChatRename} onChatDelete={handleChatDelete} onChatArchive={handleChatArchive} - onViewAllSharedChats={handleViewAllSharedChats} - onViewAllPrivateChats={handleViewAllPrivateChats} + onViewAllChats={handleViewAllChats} user={{ email: user?.email || "", name: user?.display_name || user?.email?.split("@")[0], @@ -727,7 +745,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } workspacePanelContentClassName={ isAutomationsPage - ? "max-w-none" + ? "max-w-none select-none" : isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage ? "max-w-5xl" : undefined @@ -759,10 +777,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid markAllAsRead: statusInbox.markAllAsRead, }, }} - allSharedChatsPanel={{ - searchSpaceId, - }} - allPrivateChatsPanel={{ + allChatsPanel={{ searchSpaceId, }} documentsPanel={{ @@ -770,6 +785,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid onOpenChange: setIsDocumentsSidebarOpen, }} onTabSwitch={handleTabSwitch} + onTabPrefetch={handleTabPrefetch} > {children} @@ -841,7 +857,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid {tSidebar("rename") || "Rename"} - {isRenamingChat && } + {isRenamingChat && ( + + )} diff --git a/surfsense_web/components/layout/types/layout.types.ts b/surfsense_web/components/layout/types/layout.types.ts index 720aaecf1..1bb0a089e 100644 --- a/surfsense_web/components/layout/types/layout.types.ts +++ b/surfsense_web/components/layout/types/layout.types.ts @@ -70,8 +70,7 @@ export interface ChatsSectionProps { activeChatId?: number | null; onChatSelect: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; searchSpaceId?: string; } @@ -96,13 +95,11 @@ export interface SidebarProps { searchSpaceId?: string; navItems: NavItem[]; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; user: User; theme?: string; onSettings?: () => void; diff --git a/surfsense_web/components/layout/ui/header/Header.tsx b/surfsense_web/components/layout/ui/header/Header.tsx index c6ccfddc6..79839622d 100644 --- a/surfsense_web/components/layout/ui/header/Header.tsx +++ b/surfsense_web/components/layout/ui/header/Header.tsx @@ -8,7 +8,7 @@ import { activeTabAtom } from "@/atoms/tabs/tabs.atom"; import { ActionLogButton } from "@/components/agent-action-log/action-log-button"; import { ChatHeader } from "@/components/new-chat/chat-header"; import { ChatShareButton } from "@/components/new-chat/chat-share-button"; -import type { ChatVisibility, ThreadRecord } from "@/lib/chat/thread-persistence"; +import type { ThreadRecord } from "@/lib/chat/thread-persistence"; interface HeaderProps { mobileMenuTrigger?: React.ReactNode; @@ -26,6 +26,14 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { const currentThreadState = useAtomValue(currentThreadAtom); const hasThread = isChatPage && !isDocumentTab && currentThreadState.id !== null; + const activeSearchSpaceId = searchSpaceId ? Number(searchSpaceId) : null; + const canRenderShareButton = + hasThread && + currentThreadState.id !== null && + currentThreadState.visibility !== null && + currentThreadState.searchSpaceId !== null && + activeSearchSpaceId !== null && + currentThreadState.searchSpaceId === activeSearchSpaceId; // Free chat pages have their own header with model selector; only render mobile trigger if (isFreePage) { @@ -37,21 +45,24 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { ); } - const threadForButton: ThreadRecord | null = - hasThread && currentThreadState.id !== null - ? { - id: currentThreadState.id, - visibility: currentThreadState.visibility ?? "PRIVATE", - created_by_id: null, - search_space_id: 0, - title: "", - archived: false, - created_at: "", - updated_at: "", - } - : null; - - const handleVisibilityChange = (_visibility: ChatVisibility) => {}; + let threadForButton: ThreadRecord | null = null; + if ( + canRenderShareButton && + currentThreadState.id !== null && + currentThreadState.visibility !== null && + currentThreadState.searchSpaceId !== null + ) { + threadForButton = { + id: currentThreadState.id, + visibility: currentThreadState.visibility, + created_by_id: null, + search_space_id: currentThreadState.searchSpaceId, + title: "", + archived: false, + created_at: "", + updated_at: "", + }; + } return (
    @@ -66,9 +77,7 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { {/* Right side - Actions */}
    {hasThread && } - {hasThread && ( - - )} + {threadForButton && }
    ); diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 91d85cc1e..1c076a254 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -27,8 +27,7 @@ import { RightPanelToggleButton, } from "../right-panel/RightPanel"; import { - AllPrivateChatsSidebarContent, - AllSharedChatsSidebarContent, + AllChatsSidebarContent, DocumentsSidebar, InboxSidebarContent, MobileSidebar, @@ -94,7 +93,7 @@ interface TabDataSource { markAllAsRead: () => Promise; } -export type ActiveSlideoutPanel = "inbox" | "shared" | "private" | null; +export type ActiveSlideoutPanel = "inbox" | "chats" | null; // Inbox-related props — per-tab data sources with independent loading/pagination interface InboxProps { @@ -115,15 +114,14 @@ interface LayoutShellProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -148,10 +146,7 @@ interface LayoutShellProps { inbox?: InboxProps; isLoadingChats?: boolean; // All chats panel props - allSharedChatsPanel?: { - searchSpaceId: string; - }; - allPrivateChatsPanel?: { + allChatsPanel?: { searchSpaceId: string; }; documentsPanel?: { @@ -159,11 +154,13 @@ interface LayoutShellProps { onOpenChange: (open: boolean) => void; }; onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; } function MainContentPanel({ isChatPage, onTabSwitch, + onTabPrefetch, onNewChat, showRightPanelExpandButton = true, showTopBorder = false, @@ -171,6 +168,7 @@ function MainContentPanel({ }: { isChatPage: boolean; onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; onNewChat?: () => void; showRightPanelExpandButton?: boolean; showTopBorder?: boolean; @@ -185,6 +183,7 @@ function MainContentPanel({ > : null} className="min-w-0" @@ -226,15 +225,14 @@ export function LayoutShell({ navItems, onNavItemClick, chats, - sharedChats, activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, + onViewAllChats, user, onSettings, onManageMembers, @@ -256,10 +254,10 @@ export function LayoutShell({ onSlideoutPanelChange, inbox, isLoadingChats = false, - allSharedChatsPanel, - allPrivateChatsPanel, + allChatsPanel, documentsPanel, onTabSwitch, + onTabPrefetch, }: LayoutShellProps) { const isMobile = useIsMobile(); const electronAPI = useElectronAPI(); @@ -288,13 +286,7 @@ export function LayoutShell({ const anySlideOutOpen = activeSlideoutPanel !== null; const panelAriaLabel = - activeSlideoutPanel === "inbox" - ? "Inbox" - : activeSlideoutPanel === "shared" - ? "Shared Chats" - : activeSlideoutPanel === "private" - ? "Private Chats" - : "Panel"; + activeSlideoutPanel === "inbox" ? "Inbox" : activeSlideoutPanel === "chats" ? "Chats" : "Panel"; // Mobile layout if (isMobile) { @@ -317,17 +309,15 @@ export function LayoutShell({ navItems={navItems} onNavItemClick={onNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={onNewChat} onChatSelect={onChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} - isSharedChatsPanelOpen={activeSlideoutPanel === "shared"} - isPrivateChatsPanelOpen={activeSlideoutPanel === "private"} + onViewAllChats={onViewAllChats} + isChatsPanelOpen={activeSlideoutPanel === "chats"} user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -379,34 +369,18 @@ export function LayoutShell({ /> )} - {activeSlideoutPanel === "shared" && allSharedChatsPanel && ( + {activeSlideoutPanel === "chats" && allChatsPanel && ( - closeSlideout(open)} - searchSpaceId={allSharedChatsPanel.searchSpaceId} - onCloseMobileSidebar={() => setMobileMenuOpen(false)} - /> - - )} - {activeSlideoutPanel === "private" && allPrivateChatsPanel && ( - - closeSlideout(open)} - searchSpaceId={allPrivateChatsPanel.searchSpaceId} + searchSpaceId={allChatsPanel.searchSpaceId} onCloseMobileSidebar={() => setMobileMenuOpen(false)} /> @@ -478,17 +452,15 @@ export function LayoutShell({ navItems={navItems} onNavItemClick={onNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={onNewChat} onChatSelect={onChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} - isSharedChatsPanelOpen={activeSlideoutPanel === "shared"} - isPrivateChatsPanelOpen={activeSlideoutPanel === "private"} + onViewAllChats={onViewAllChats} + isChatsPanelOpen={activeSlideoutPanel === "chats"} user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -554,33 +526,18 @@ export function LayoutShell({ /> )} - {activeSlideoutPanel === "shared" && allSharedChatsPanel && ( + {activeSlideoutPanel === "chats" && allChatsPanel && ( - closeSlideout(open)} - searchSpaceId={allSharedChatsPanel.searchSpaceId} - /> - - )} - {activeSlideoutPanel === "private" && allPrivateChatsPanel && ( - - closeSlideout(open)} - searchSpaceId={allPrivateChatsPanel.searchSpaceId} + searchSpaceId={allChatsPanel.searchSpaceId} /> )} @@ -603,6 +560,7 @@ export function LayoutShell({ void; searchSpaceId: string; onCloseMobileSidebar?: () => void; } -interface AllPrivateChatsSidebarProps extends AllPrivateChatsSidebarContentProps { +interface AllChatsSidebarProps extends AllChatsSidebarContentProps { open: boolean; } -export function AllPrivateChatsSidebarContent({ +export function AllChatsSidebarContent({ onOpenChange, searchSpaceId, onCloseMobileSidebar, -}: AllPrivateChatsSidebarContentProps) { +}: AllChatsSidebarContentProps) { const t = useTranslations("sidebar"); const router = useRouter(); const params = useParams(); const queryClient = useQueryClient(); const isMobile = useIsMobile(); const removeChatTab = useSetAtom(removeChatTabAtom); + const { activateChatThread, prefetchChatThread } = useActivateChatThread(); + const { mutateAsync: deleteThread } = useDeleteThread(searchSpaceId); + const { mutateAsync: archiveThread } = useArchiveThread(searchSpaceId); + const { mutateAsync: renameThread } = useRenameThread(searchSpaceId); const currentChatId = Array.isArray(params.chat_id) ? Number(params.chat_id[0]) @@ -122,57 +124,66 @@ export function AllPrivateChatsSidebarContent({ enabled: !!searchSpaceId && isSearchMode, }); - // Filter to only private chats (PRIVATE visibility or no visibility set) const { activeChats, archivedChats } = useMemo(() => { if (isSearchMode) { - const privateSearchResults = (searchData ?? []).filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); return { - activeChats: privateSearchResults.filter((t) => !t.archived), - archivedChats: privateSearchResults.filter((t) => t.archived), + activeChats: (searchData ?? []).filter((t) => !t.archived), + archivedChats: (searchData ?? []).filter((t) => t.archived), }; } if (!threadsData) return { activeChats: [], archivedChats: [] }; - const activePrivate = threadsData.threads.filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); - const archivedPrivate = threadsData.archived_threads.filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); - - return { activeChats: activePrivate, archivedChats: archivedPrivate }; + return { + activeChats: threadsData.threads, + archivedChats: threadsData.archived_threads, + }; }, [threadsData, searchData, isSearchMode]); const threads = showArchived ? archivedChats : activeChats; const handleThreadClick = useCallback( - (threadId: number) => { - router.push(`/dashboard/${searchSpaceId}/new-chat/${threadId}`); + (thread: ThreadListItem) => { + activateChatThread({ + id: thread.id, + title: thread.title || "New Chat", + searchSpaceId, + visibility: thread.visibility, + }); onOpenChange(false); onCloseMobileSidebar?.(); }, - [router, onOpenChange, searchSpaceId, onCloseMobileSidebar] + [activateChatThread, onOpenChange, searchSpaceId, onCloseMobileSidebar] ); const handleDeleteThread = useCallback( async (threadId: number) => { setDeletingThreadId(threadId); try { - await deleteThread(threadId); + await deleteThread({ threadId }); const fallbackTab = removeChatTab(threadId); toast.success(t("chat_deleted") || "Chat deleted successfully"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); if (currentChatId === threadId) { onOpenChange(false); setTimeout(() => { - if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); + if ( + fallbackTab?.type === "chat" && + fallbackTab.chatUrl && + fallbackTab.chatId !== undefined + ) { + activateChatThread({ + id: fallbackTab.chatId ?? null, + title: fallbackTab.title, + url: fallbackTab.chatUrl, + searchSpaceId: fallbackTab.searchSpaceId ?? searchSpaceId, + ...(fallbackTab.visibility !== undefined + ? { visibility: fallbackTab.visibility } + : {}), + ...(fallbackTab.hasComments !== undefined + ? { hasComments: fallbackTab.hasComments } + : {}), + }); return; } router.push(`/dashboard/${searchSpaceId}/new-chat`); @@ -185,22 +196,28 @@ export function AllPrivateChatsSidebarContent({ setDeletingThreadId(null); } }, - [queryClient, searchSpaceId, t, currentChatId, router, onOpenChange, removeChatTab] + [ + activateChatThread, + deleteThread, + t, + currentChatId, + router, + onOpenChange, + removeChatTab, + searchSpaceId, + ] ); const handleToggleArchive = useCallback( async (threadId: number, currentlyArchived: boolean) => { setArchivingThreadId(threadId); try { - await updateThread(threadId, { archived: !currentlyArchived }); + await archiveThread({ threadId, archived: !currentlyArchived }); toast.success( currentlyArchived ? t("chat_unarchived") || "Chat restored" : t("chat_archived") || "Chat archived" ); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); } catch (error) { console.error("Error archiving thread:", error); toast.error(t("error_archiving_chat") || "Failed to archive chat"); @@ -208,7 +225,7 @@ export function AllPrivateChatsSidebarContent({ setArchivingThreadId(null); } }, - [queryClient, searchSpaceId, t] + [archiveThread, t] ); const handleStartRename = useCallback((threadId: number, title: string) => { @@ -221,14 +238,12 @@ export function AllPrivateChatsSidebarContent({ if (!renamingThread || !newTitle.trim()) return; setIsRenaming(true); try { - await updateThread(renamingThread.id, { title: newTitle.trim() }); - toast.success(t("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ - queryKey: ["threads", searchSpaceId, "detail", String(renamingThread.id)], + await renameThread({ + threadId: renamingThread.id, + title: newTitle.trim(), + previousTitle: renamingThread.title, }); + toast.success(t("chat_renamed") || "Chat renamed"); } catch (error) { console.error("Error renaming thread:", error); toast.error(t("error_renaming_chat") || "Failed to rename chat"); @@ -238,7 +253,7 @@ export function AllPrivateChatsSidebarContent({ setRenamingThread(null); setNewTitle(""); } - }, [renamingThread, newTitle, queryClient, searchSpaceId, t]); + }, [renamingThread, newTitle, renameThread, t]); const handleClearSearch = useCallback(() => { setSearchQuery(""); @@ -265,7 +280,7 @@ export function AllPrivateChatsSidebarContent({ {t("close") || "Close"} )} -

    {t("chats") || "Private Chats"}

    +

    {t("chats") || "Chats"}

    @@ -353,8 +368,10 @@ export function AllPrivateChatsSidebarContent({ variant="ghost" onClick={() => { if (wasLongPress()) return; - handleThreadClick(thread.id); + handleThreadClick(thread); }} + onMouseEnter={() => prefetchChatThread(thread.id)} + onFocus={() => prefetchChatThread(thread.id)} onTouchStart={() => { pendingThreadIdRef.current = thread.id; longPressHandlers.onTouchStart(); @@ -366,11 +383,12 @@ export function AllPrivateChatsSidebarContent({ "h-auto w-full justify-start gap-2 overflow-hidden px-2 py-1.5 text-left font-normal", "group-hover/item:bg-accent group-hover/item:text-accent-foreground", "focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring", + thread.visibility === "SEARCH_SPACE" && "pr-9", isActive && "bg-accent text-accent-foreground", isBusy && "opacity-50 pointer-events-none" )} > - {thread.title || "New Chat"} + {thread.title || "New Chat"} ) : ( @@ -378,17 +396,22 @@ export function AllPrivateChatsSidebarContent({ @@ -407,64 +430,83 @@ export function AllPrivateChatsSidebarContent({ : "bg-gradient-to-l from-sidebar from-60% to-transparent group-hover/item:from-accent", isMobile ? "opacity-0" - : openDropdownId === thread.id + : thread.visibility === "SEARCH_SPACE" || openDropdownId === thread.id ? "opacity-100" : "opacity-0 group-hover/item:opacity-100" )} > - setOpenDropdownId(isOpen ? thread.id : null)} - > - - - - - {!thread.archived && ( - handleStartRename(thread.id, thread.title || "New Chat")} + /> + ) : null} + setOpenDropdownId(isOpen ? thread.id : null)} + > + + + + + {!thread.archived && ( + + handleStartRename(thread.id, thread.title || "New Chat") + } + > + + {t("rename") || "Rename"} + )} - - handleDeleteThread(thread.id)}> - - {t("delete") || "Delete"} - - - + handleToggleArchive(thread.id, thread.archived)} + disabled={isArchiving} + > + {thread.archived ? ( + <> + + {t("unarchive") || "Restore"} + + ) : ( + <> + + {t("archive") || "Archive"} + + )} + + handleDeleteThread(thread.id)}> + + {t("delete") || "Delete"} + + + +
    ); @@ -486,7 +528,7 @@ export function AllPrivateChatsSidebarContent({

    {showArchived ? t("no_archived_chats") || "No archived chats" - : t("no_chats") || "No private chats"} + : t("no_chats") || "No chats"}

    {!showArchived && (

    @@ -527,16 +569,17 @@ export function AllPrivateChatsSidebarContent({ @@ -545,21 +588,17 @@ export function AllPrivateChatsSidebarContent({ ); } -export function AllPrivateChatsSidebar({ +export function AllChatsSidebar({ open, onOpenChange, searchSpaceId, onCloseMobileSidebar, -}: AllPrivateChatsSidebarProps) { +}: AllChatsSidebarProps) { const t = useTranslations("sidebar"); return ( - - + void; - searchSpaceId: string; - onCloseMobileSidebar?: () => void; -} - -interface AllSharedChatsSidebarProps extends AllSharedChatsSidebarContentProps { - open: boolean; -} - -export function AllSharedChatsSidebarContent({ - onOpenChange, - searchSpaceId, - onCloseMobileSidebar, -}: AllSharedChatsSidebarContentProps) { - const t = useTranslations("sidebar"); - const router = useRouter(); - const params = useParams(); - const queryClient = useQueryClient(); - const isMobile = useIsMobile(); - const removeChatTab = useSetAtom(removeChatTabAtom); - - const currentChatId = Array.isArray(params.chat_id) - ? Number(params.chat_id[0]) - : params.chat_id - ? Number(params.chat_id) - : null; - const [deletingThreadId, setDeletingThreadId] = useState(null); - const [archivingThreadId, setArchivingThreadId] = useState(null); - const [searchQuery, setSearchQuery] = useState(""); - const [showArchived, setShowArchived] = useState(false); - const [openDropdownId, setOpenDropdownId] = useState(null); - const [showRenameDialog, setShowRenameDialog] = useState(false); - const [renamingThread, setRenamingThread] = useState<{ id: number; title: string } | null>(null); - const [newTitle, setNewTitle] = useState(""); - const [isRenaming, setIsRenaming] = useState(false); - const debouncedSearchQuery = useDebouncedValue(searchQuery, 300); - - const pendingThreadIdRef = useRef(null); - const { handlers: longPressHandlers, wasLongPress } = useLongPress( - useCallback(() => { - if (pendingThreadIdRef.current !== null) { - setOpenDropdownId(pendingThreadIdRef.current); - } - }, []) - ); - - const isSearchMode = !!debouncedSearchQuery.trim(); - - const { - data: threadsData, - error: threadsError, - isLoading: isLoadingThreads, - } = useQuery({ - queryKey: ["all-threads", searchSpaceId], - queryFn: () => fetchThreads(Number(searchSpaceId)), - enabled: !!searchSpaceId && !isSearchMode, - }); - - const { - data: searchData, - error: searchError, - isLoading: isLoadingSearch, - } = useQuery({ - queryKey: ["search-threads", searchSpaceId, debouncedSearchQuery], - queryFn: () => searchThreads(Number(searchSpaceId), debouncedSearchQuery.trim()), - enabled: !!searchSpaceId && isSearchMode, - }); - - // Filter to only shared chats (SEARCH_SPACE visibility) - const { activeChats, archivedChats } = useMemo(() => { - if (isSearchMode) { - const sharedSearchResults = (searchData ?? []).filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - return { - activeChats: sharedSearchResults.filter((t) => !t.archived), - archivedChats: sharedSearchResults.filter((t) => t.archived), - }; - } - - if (!threadsData) return { activeChats: [], archivedChats: [] }; - - const activeShared = threadsData.threads.filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - const archivedShared = threadsData.archived_threads.filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - - return { activeChats: activeShared, archivedChats: archivedShared }; - }, [threadsData, searchData, isSearchMode]); - - const threads = showArchived ? archivedChats : activeChats; - - const handleThreadClick = useCallback( - (threadId: number) => { - router.push(`/dashboard/${searchSpaceId}/new-chat/${threadId}`); - onOpenChange(false); - onCloseMobileSidebar?.(); - }, - [router, onOpenChange, searchSpaceId, onCloseMobileSidebar] - ); - - const handleDeleteThread = useCallback( - async (threadId: number) => { - setDeletingThreadId(threadId); - try { - await deleteThread(threadId); - const fallbackTab = removeChatTab(threadId); - toast.success(t("chat_deleted") || "Chat deleted successfully"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - - if (currentChatId === threadId) { - onOpenChange(false); - setTimeout(() => { - if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); - return; - } - router.push(`/dashboard/${searchSpaceId}/new-chat`); - }, 250); - } - } catch (error) { - console.error("Error deleting thread:", error); - toast.error(t("error_deleting_chat") || "Failed to delete chat"); - } finally { - setDeletingThreadId(null); - } - }, - [queryClient, searchSpaceId, t, currentChatId, router, onOpenChange, removeChatTab] - ); - - const handleToggleArchive = useCallback( - async (threadId: number, currentlyArchived: boolean) => { - setArchivingThreadId(threadId); - try { - await updateThread(threadId, { archived: !currentlyArchived }); - toast.success( - currentlyArchived - ? t("chat_unarchived") || "Chat restored" - : t("chat_archived") || "Chat archived" - ); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - } catch (error) { - console.error("Error archiving thread:", error); - toast.error(t("error_archiving_chat") || "Failed to archive chat"); - } finally { - setArchivingThreadId(null); - } - }, - [queryClient, searchSpaceId, t] - ); - - const handleStartRename = useCallback((threadId: number, title: string) => { - setRenamingThread({ id: threadId, title }); - setNewTitle(title); - setShowRenameDialog(true); - }, []); - - const handleConfirmRename = useCallback(async () => { - if (!renamingThread || !newTitle.trim()) return; - setIsRenaming(true); - try { - await updateThread(renamingThread.id, { title: newTitle.trim() }); - toast.success(t("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ - queryKey: ["threads", searchSpaceId, "detail", String(renamingThread.id)], - }); - } catch (error) { - console.error("Error renaming thread:", error); - toast.error(t("error_renaming_chat") || "Failed to rename chat"); - } finally { - setIsRenaming(false); - setShowRenameDialog(false); - setRenamingThread(null); - setNewTitle(""); - } - }, [renamingThread, newTitle, queryClient, searchSpaceId, t]); - - const handleClearSearch = useCallback(() => { - setSearchQuery(""); - }, []); - - const isLoading = isSearchMode ? isLoadingSearch : isLoadingThreads; - const error = isSearchMode ? searchError : threadsError; - - const activeCount = activeChats.length; - const archivedCount = archivedChats.length; - - return ( - <> -

    -
    - {isMobile && ( - - )} -

    {t("shared_chats") || "Shared Chats"}

    -
    - -
    - - setSearchQuery(e.target.value)} - className="h-8 border-0 bg-muted pl-8 pr-7 text-sm shadow-none" - /> - {searchQuery && ( - - )} -
    -
    - - {!isSearchMode && ( - setShowArchived(value === "archived")} - className="shrink-0 mx-3 mt-1.5" - > - - - - - Active - - {activeCount} - - - - - - - Archived - - {archivedCount} - - - - - - )} - -
    - {isLoading ? ( -
    - {[75, 90, 55, 80, 65, 85].map((titleWidth) => ( -
    - - -
    - ))} -
    - ) : error ? ( -
    - {t("error_loading_chats") || "Error loading chats"} -
    - ) : threads.length > 0 ? ( -
    - {threads.map((thread) => { - const isDeleting = deletingThreadId === thread.id; - const isArchiving = archivingThreadId === thread.id; - const isBusy = isDeleting || isArchiving; - const isActive = currentChatId === thread.id; - - return ( -
    - {isMobile ? ( - - ) : ( - - - - - -

    - {t("updated") || "Updated"}: {formatThreadTimestamp(thread.updatedAt)} -

    -
    -
    - )} - -
    - setOpenDropdownId(isOpen ? thread.id : null)} - > - - - - - {!thread.archived && ( - handleStartRename(thread.id, thread.title || "New Chat")} - > - - {t("rename") || "Rename"} - - )} - handleToggleArchive(thread.id, thread.archived)} - disabled={isArchiving} - > - {thread.archived ? ( - <> - - {t("unarchive") || "Restore"} - - ) : ( - <> - - {t("archive") || "Archive"} - - )} - - handleDeleteThread(thread.id)}> - - {t("delete") || "Delete"} - - - -
    -
    - ); - })} -
    - ) : isSearchMode ? ( -
    - -

    - {t("no_chats_found") || "No chats found"} -

    -

    - {t("try_different_search") || "Try a different search term"} -

    -
    - ) : ( -
    - -

    - {showArchived - ? t("no_archived_chats") || "No archived chats" - : t("no_shared_chats") || "No shared chats"} -

    - {!showArchived && ( -

    - Share a chat to collaborate with your team -

    - )} -
    - )} -
    - - - - - {t("rename_chat") || "Rename Chat"} - - - {t("rename_chat_description") || "Enter a new name for this conversation."} - - - setNewTitle(e.target.value)} - placeholder={t("chat_title_placeholder") || "Chat title"} - onKeyDown={(e) => { - if (e.key === "Enter" && !isRenaming && newTitle.trim()) { - handleConfirmRename(); - } - }} - /> - - - - - - - - ); -} - -export function AllSharedChatsSidebar({ - open, - onOpenChange, - searchSpaceId, - onCloseMobileSidebar, -}: AllSharedChatsSidebarProps) { - const t = useTranslations("sidebar"); - - return ( - - - - ); -} diff --git a/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx b/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx index ea4f946c2..c854225a2 100644 --- a/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx +++ b/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx @@ -18,10 +18,12 @@ import { cn } from "@/lib/utils"; interface ChatListItemProps { name: string; isActive?: boolean; + isShared?: boolean; archived?: boolean; dropdownOpen?: boolean; onDropdownOpenChange?: (open: boolean) => void; onClick?: () => void; + onPrefetch?: () => void; onRename?: () => void; onArchive?: () => void; onDelete?: () => void; @@ -34,6 +36,7 @@ export function ChatListItem({ dropdownOpen: controlledOpen, onDropdownOpenChange, onClick, + onPrefetch, onRename, onArchive, onDelete, @@ -60,6 +63,8 @@ export function ChatListItem({ type="button" variant="ghost" onClick={handleClick} + onMouseEnter={onPrefetch} + onFocus={onPrefetch} {...(isMobile ? longPressHandlers : {})} className={cn( "h-auto w-full justify-start gap-2 overflow-hidden px-2 py-1.5 text-left font-normal", @@ -68,7 +73,7 @@ export function ChatListItem({ isActive && "bg-accent text-accent-foreground" )} > - {animatedName} + {animatedName} {/* Actions dropdown - trigger hidden on mobile, long-press opens it instead */} diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index a90d6b32e..6c6668319 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -619,7 +619,6 @@ function AuthenticatedDocumentsSidebarBase({ searchSpaceId, excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS, fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()), - enableSummary: false, rootFolderId: folder.id, }); toast.success(`Re-scan complete: ${matched.name}`); diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index 83d423ace..89a01d0c7 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -19,17 +19,15 @@ interface MobileSidebarProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; - isSharedChatsPanelOpen?: boolean; - isPrivateChatsPanelOpen?: boolean; + onViewAllChats?: () => void; + isChatsPanelOpen?: boolean; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -69,17 +67,15 @@ export function MobileSidebar({ navItems, onNavItemClick, chats, - sharedChats, activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, - isSharedChatsPanelOpen = false, - isPrivateChatsPanelOpen = false, + onViewAllChats, + isChatsPanelOpen = false, user, onSettings, onManageMembers, @@ -152,34 +148,25 @@ export function MobileSidebar({ navItems={navItems} onNavItemClick={handleNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={() => { onNewChat(); onOpenChange(false); }} onChatSelect={handleChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={ - onViewAllSharedChats + onViewAllChats={ + onViewAllChats ? () => { onOpenChange(false); - onViewAllSharedChats(); + onViewAllChats(); } : undefined } - onViewAllPrivateChats={ - onViewAllPrivateChats - ? () => { - onOpenChange(false); - onViewAllPrivateChats(); - } - : undefined - } - isSharedChatsPanelOpen={isSharedChatsPanelOpen} - isPrivateChatsPanelOpen={isPrivateChatsPanelOpen} + isChatsPanelOpen={isChatsPanelOpen} user={user} onSettings={ onSettings diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 805f8bfd3..6a4785d98 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -67,17 +67,15 @@ interface SidebarProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; - isSharedChatsPanelOpen?: boolean; - isPrivateChatsPanelOpen?: boolean; + onViewAllChats?: () => void; + isChatsPanelOpen?: boolean; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -106,17 +104,15 @@ export function Sidebar({ navItems, onNavItemClick, chats, - sharedChats = [], activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, - isSharedChatsPanelOpen = false, - isPrivateChatsPanelOpen = false, + onViewAllChats, + isChatsPanelOpen = false, user, onSettings, onManageMembers, @@ -264,73 +260,20 @@ export function Sidebar({
    ) : (
    - {/* Shared Chats Section - takes only space needed, max 50% */} - {!disableTooltips && isSharedChatsPanelOpen ? t("hide") : t("show_all")} - - ) : undefined - } - > - {isLoadingChats ? ( - - ) : sharedChats.length > 0 ? ( -
    -
    4 ? "pb-2" : ""}`} - > - {sharedChats.slice(0, 20).map((chat) => ( - setOpenDropdownChatId(open ? chat.id : null)} - onClick={() => onChatSelect(chat)} - onRename={() => onChatRename?.(chat)} - onArchive={() => onChatArchive?.(chat)} - onDelete={() => onChatDelete?.(chat)} - /> - ))} -
    - {/* Gradient fade indicator when more than 4 items */} - {sharedChats.length > 4 && ( -
    - )} -
    - ) : ( -

    {t("no_shared_chats")}

    - )} - - - {/* Private Chats Section - fills remaining space */} - - {!disableTooltips && isPrivateChatsPanelOpen ? t("hide") : t("show_all")} + {!disableTooltips && isChatsPanelOpen ? t("hide") : t("show_all")} ) : undefined } @@ -347,10 +290,12 @@ export function Sidebar({ key={chat.id} name={chat.name} isActive={chat.id === activeChatId} + isShared={chat.visibility === "SEARCH_SPACE"} archived={chat.archived} dropdownOpen={openDropdownChatId === chat.id} onDropdownOpenChange={(open) => setOpenDropdownChatId(open ? chat.id : null)} onClick={() => onChatSelect(chat)} + onPrefetch={() => onChatPrefetch?.(chat)} onRename={() => onChatRename?.(chat)} onArchive={() => onChatArchive?.(chat)} onDelete={() => onChatDelete?.(chat)} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx index 99162dddf..c041dec86 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx @@ -39,12 +39,12 @@ export function SidebarSection({ className )} > -
    +
    {title} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx index bc3b36efd..ea93ce4d0 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx @@ -139,14 +139,14 @@ export function SidebarUserProfile({ const { locale, setLocale } = useLocaleContext(); const { isDesktop } = usePlatform(); const isDesktopViewport = useMediaQuery("(min-width: 768px)"); - const { os, primary } = usePrimaryDownload(); + const { os, primary, isMobileOS } = usePrimaryDownload(); const [isLoggingOut, setIsLoggingOut] = useState(false); const bgColor = getUserAvatarColor(user.email); const initials = getUserInitials(user.email); const displayName = user.name || user.email.split("@")[0]; const downloadUrl = primary?.url ?? GITHUB_RELEASES_URL; const downloadLabel = t("download_for_os", { os }); - const showDownloadCta = !isDesktop && isDesktopViewport; + const showDownloadCta = !isDesktop && !isMobileOS && isDesktopViewport; const handleLanguageChange = (newLocale: "en" | "es" | "pt" | "hi" | "zh") => { setLocale(newLocale); @@ -221,18 +221,15 @@ export function SidebarUserProfile({ - -
    - -
    -

    {displayName}

    -

    {user.email}

    -
    + +
    + {/*

    {displayName}

    */} +

    + {user.email} +

    - - {t("user_settings")} @@ -327,14 +324,14 @@ export function SidebarUserProfile({ ))} -

    +

    v{APP_VERSION}

    - {!isDesktop && ( + {!isDesktop && !isMobileOS && ( @@ -406,18 +403,15 @@ export function SidebarUserProfile({ - -
    - -
    -

    {displayName}

    -

    {user.email}

    -
    + +
    +

    {displayName}

    +

    + {user.email} +

    - - {t("user_settings")} @@ -512,7 +506,7 @@ export function SidebarUserProfile({ ))} -

    +

    v{APP_VERSION}

    diff --git a/surfsense_web/components/layout/ui/sidebar/index.ts b/surfsense_web/components/layout/ui/sidebar/index.ts index d72f86c8a..e25149b06 100644 --- a/surfsense_web/components/layout/ui/sidebar/index.ts +++ b/surfsense_web/components/layout/ui/sidebar/index.ts @@ -1,5 +1,4 @@ -export { AllPrivateChatsSidebar, AllPrivateChatsSidebarContent } from "./AllPrivateChatsSidebar"; -export { AllSharedChatsSidebar, AllSharedChatsSidebarContent } from "./AllSharedChatsSidebar"; +export { AllChatsSidebar, AllChatsSidebarContent } from "./AllChatsSidebar"; export { ChatListItem } from "./ChatListItem"; export { DocumentsSidebar } from "./DocumentsSidebar"; export { InboxSidebar, InboxSidebarContent } from "./InboxSidebar"; diff --git a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx index 34cf707b0..61b8c3e25 100644 --- a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx +++ b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx @@ -2,9 +2,10 @@ import { Download, FileQuestionMark, FileText, Pencil, RefreshCw } from "lucide-react"; import { useRouter } from "next/navigation"; -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { PlateEditor } from "@/components/editor/plate-editor"; +import { SourceCodeEditor } from "@/components/editor/source-code-editor"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; @@ -21,7 +22,8 @@ interface DocumentContent { source_markdown: string; content_size_bytes?: number; chunk_count?: number; - truncated?: boolean; + viewer_mode?: ViewerMode; + editor_plate_max_bytes?: number; } function DocumentSkeleton() { @@ -51,6 +53,21 @@ interface DocumentTabContentProps { } const EDITABLE_DOCUMENT_TYPES = new Set(["FILE", "NOTE"]); +type ViewerMode = "plate" | "monaco"; + +function getUtf8ByteSize(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function formatBytes(bytes: number): string { + if (bytes >= 1024 * 1024) { + return `${(bytes / 1024 / 1024).toFixed(1)}MB`; + } + if (bytes >= 1024) { + return `${Math.round(bytes / 1024)}KB`; + } + return `${bytes}B`; +} export function DocumentTabContent({ documentId, searchSpaceId, title }: DocumentTabContentProps) { const [doc, setDoc] = useState(null); @@ -65,7 +82,13 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen const changeCountRef = useRef(0); const router = useRouter(); - const isLargeDocument = (doc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; + const plateMaxBytes = doc?.editor_plate_max_bytes ?? LARGE_DOCUMENT_THRESHOLD; + const isLargeDocument = (doc?.content_size_bytes ?? 0) > plateMaxBytes; + const viewerMode: ViewerMode = doc?.viewer_mode ?? (isLargeDocument ? "monaco" : "plate"); + const activeMarkdown = editedMarkdown ?? doc?.source_markdown ?? ""; + const activeMarkdownSizeBytes = useMemo(() => getUtf8ByteSize(activeMarkdown), [activeMarkdown]); + const isNearPlateLimit = activeMarkdownSizeBytes >= plateMaxBytes * 0.9; + const isOverPlateLimit = activeMarkdownSizeBytes > plateMaxBytes; useEffect(() => { const controller = new AbortController(); @@ -88,8 +111,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen const url = new URL( `${BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); - const response = await authenticatedFetch(url.toString(), { method: "GET" }); if (controller.signal.aborted) return; @@ -161,14 +182,19 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen setDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); setEditedMarkdown(null); - toast.success("Document saved! Reindexing in background..."); + const savedSizeBytes = getUtf8ByteSize(markdownRef.current); + if (savedSizeBytes > plateMaxBytes) { + toast.success("Document saved. It will reopen in raw markdown mode."); + } else { + toast.success("Document saved! Reindexing in background..."); + } } catch (err) { console.error("Error saving document:", err); toast.error(err instanceof Error ? err.message : "Failed to save document"); } finally { setSaving(false); } - }, [documentId, searchSpaceId]); + }, [documentId, plateMaxBytes, searchSpaceId]); if (isLoading) return ; @@ -204,9 +230,9 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen ); } - const isEditable = EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? "") && !isLargeDocument; + const isEditable = viewerMode === "plate" && EDITABLE_DOCUMENT_TYPES.has(doc.document_type ?? ""); - if (isEditing && !isLargeDocument) { + if (isEditing && viewerMode === "plate") { return (
    @@ -228,20 +254,32 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen Done editing
    -
    - +
    + {isNearPlateLimit && ( + + + + {isOverPlateLimit + ? `This document is ${formatBytes(activeMarkdownSizeBytes)}, above the rich editor limit of ${formatBytes(plateMaxBytes)}. You can save, but it will reopen in raw markdown mode.` + : `This document is approaching the rich editor limit (${formatBytes(activeMarkdownSizeBytes)} of ${formatBytes(plateMaxBytes)}).`} + + + )} +
    + +
    ); @@ -265,64 +303,74 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen )}
    -
    -
    - {isLargeDocument ? ( - <> - - - - - This document is too large for the editor ( - {Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "} - {doc.chunk_count ?? 0} chunks). Showing a preview below. +
    + {viewerMode === "monaco" ? ( +
    + + + + + This document is too large for the editor ( + {Math.round((doc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "} + {doc.chunk_count ?? 0} chunks). Showing raw markdown below. + + - - + {downloading && } + + + +
    + {}} + /> +
    +
    + ) : ( +
    +
    - - ) : ( - - )} -
    +
    +
    + )}
    ); diff --git a/surfsense_web/components/layout/ui/tabs/TabBar.tsx b/surfsense_web/components/layout/ui/tabs/TabBar.tsx index c5fb91f4d..869c9cee2 100644 --- a/surfsense_web/components/layout/ui/tabs/TabBar.tsx +++ b/surfsense_web/components/layout/ui/tabs/TabBar.tsx @@ -15,6 +15,7 @@ import { cn } from "@/lib/utils"; interface TabBarProps { onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; onNewChat?: () => void; leftActions?: React.ReactNode; rightActions?: React.ReactNode; @@ -36,6 +37,7 @@ function nextTabListScrollLeft(input: { export function TabBar({ onTabSwitch, + onTabPrefetch, onNewChat, leftActions, rightActions, @@ -71,6 +73,15 @@ export function TabBar({ [activeTabId, switchTab, onTabSwitch] ); + const handleTabPrefetch = useCallback( + (tab: Tab) => { + if (tab.type === "chat") { + onTabPrefetch?.(tab); + } + }, + [onTabPrefetch] + ); + const handleTabClose = useCallback( (e: React.MouseEvent, tabId: string) => { e.stopPropagation(); @@ -195,7 +206,11 @@ export function TabBar({ type="button" variant="ghost" onClick={() => handleTabClick(tab)} - onMouseEnter={() => setHoveredTabIndex(index)} + onMouseEnter={() => { + setHoveredTabIndex(index); + handleTabPrefetch(tab); + }} + onFocus={() => handleTabPrefetch(tab)} onMouseLeave={() => setHoveredTabIndex(null)} className={cn( "h-full w-full justify-start overflow-hidden px-3 text-left text-[13px] font-medium transition-colors duration-150", diff --git a/surfsense_web/components/new-chat/chat-example-prompts.tsx b/surfsense_web/components/new-chat/chat-example-prompts.tsx index 4a204386a..98d95b98b 100644 --- a/surfsense_web/components/new-chat/chat-example-prompts.tsx +++ b/surfsense_web/components/new-chat/chat-example-prompts.tsx @@ -1,10 +1,17 @@ "use client"; -import { CornerDownLeft, Lightbulb } from "lucide-react"; -import { memo, useCallback } from "react"; +import { + FilePlus2, + Search, + Settings2, + type LucideIcon, + WandSparkles, + Workflow, + X, +} from "lucide-react"; +import { memo, useCallback, useState } from "react"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { CHAT_EXAMPLE_CATEGORIES } from "@/lib/chat/example-prompts"; interface ChatExamplePromptsProps { @@ -12,6 +19,13 @@ interface ChatExamplePromptsProps { onSelect: (prompt: string) => void; } +const CATEGORY_ICONS: Record = { + search: Search, + create: FilePlus2, + automate: Workflow, + tools: Settings2, +}; + const ExamplePromptButton = memo(function ExamplePromptButton({ prompt, onSelect, @@ -26,54 +40,72 @@ const ExamplePromptButton = memo(function ExamplePromptButton({ type="button" variant="ghost" onClick={handleClick} - className="h-auto w-full items-start justify-start gap-2.5 whitespace-normal rounded-md border bg-background px-3 py-2 text-left font-normal text-muted-foreground hover:bg-accent hover:text-accent-foreground" + className="h-auto w-full items-start justify-start whitespace-normal rounded-lg bg-transparent px-2.5 py-1.5 text-left font-normal text-muted-foreground shadow-none hover:bg-foreground/10 hover:text-foreground sm:rounded-xl sm:px-3 sm:py-2" > -
    -
    -
    - -
    - - {CHAT_EXAMPLE_CATEGORIES.map((category) => ( - - {category.label} - - ))} - +
    + {activeCategory ? null : ( +
    +
    + {CHAT_EXAMPLE_CATEGORIES.map((category) => { + const Icon = CATEGORY_ICONS[category.id] ?? WandSparkles; + + return ( + + ); + })} +
    - {CHAT_EXAMPLE_CATEGORIES.map((category) => ( - - -
      - {category.prompts.map((prompt) => ( -
    • - -
    • - ))} -
    -
    -
    - ))} - + )} + + {activeCategory ? ( +
    +
    +
    + {activeCategory.label} +
    + +
    + +
      + {activeCategory.prompts.map((prompt) => ( +
    • + +
    • + ))} +
    +
    +
    + ) : null}
    ); } diff --git a/surfsense_web/components/new-chat/chat-share-button.tsx b/surfsense_web/components/new-chat/chat-share-button.tsx index 101f73ade..5e5e3fe19 100644 --- a/surfsense_web/components/new-chat/chat-share-button.tsx +++ b/surfsense_web/components/new-chat/chat-share-button.tsx @@ -1,24 +1,21 @@ "use client"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { useAtomValue, useSetAtom } from "jotai"; +import { useAtomValue } from "jotai"; import { Earth, User, Users } from "lucide-react"; import { useRouter } from "next/navigation"; import { useCallback, useState } from "react"; import { toast } from "sonner"; -import { currentThreadAtom, setThreadVisibilityAtom } from "@/atoms/chat/current-thread.atom"; +import { currentThreadAtom } from "@/atoms/chat/current-thread.atom"; import { myAccessAtom } from "@/atoms/members/members-query.atoms"; import { createPublicChatSnapshotMutationAtom } from "@/atoms/public-chat-snapshots/public-chat-snapshots-mutation.atoms"; import { Button } from "@/components/ui/button"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { useUpdateThreadVisibility } from "@/hooks/use-thread-mutations"; import { chatThreadsApiService } from "@/lib/apis/chat-threads-api.service"; -import { - type ChatVisibility, - type ThreadRecord, - updateThreadVisibility, -} from "@/lib/chat/thread-persistence"; +import type { ChatVisibility, ThreadRecord } from "@/lib/chat/thread-persistence"; import { cn } from "@/lib/utils"; interface ChatShareButtonProps { @@ -54,7 +51,7 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS // Use Jotai atom for visibility (single source of truth) const currentThreadState = useAtomValue(currentThreadAtom); - const setThreadVisibility = useSetAtom(setThreadVisibilityAtom); + const { mutateAsync: updateVisibility } = useUpdateThreadVisibility(thread?.search_space_id ?? 0); // Snapshot creation mutation const { mutateAsync: createSnapshot, isPending: isCreatingSnapshot } = useAtomValue( @@ -80,8 +77,9 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS }); const hasPublicSnapshots = (snapshotsData?.snapshots?.length ?? 0) > 0; - // Use Jotai visibility if available (synced from chat page), otherwise fall back to thread prop - const currentVisibility = currentThreadState.visibility ?? thread?.visibility ?? "PRIVATE"; + // Use Jotai visibility if available (synced from chat page), otherwise fall back to thread prop. + // Unknown visibility should not be presented as private while thread detail is still resolving. + const currentVisibility = currentThreadState.visibility ?? thread?.visibility; const handleVisibilityChange = useCallback( async (newVisibility: ChatVisibility) => { @@ -90,30 +88,23 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS return; } - // Update Jotai atom immediately for instant UI feedback - setThreadVisibility(newVisibility); - try { - await updateThreadVisibility(thread.id, newVisibility); - - // Refetch threads list to update sidebar - await queryClient.refetchQueries({ - predicate: (query) => Array.isArray(query.queryKey) && query.queryKey[0] === "threads", + const updatedThread = await updateVisibility({ + thread, + visibility: newVisibility, }); - onVisibilityChange?.(newVisibility); + onVisibilityChange?.(updatedThread.visibility); toast.success( newVisibility === "SEARCH_SPACE" ? "Chat shared with search space" : "Chat is now private" ); setOpen(false); } catch (error) { console.error("Failed to update visibility:", error); - // Revert Jotai state on error - setThreadVisibility(thread.visibility ?? "PRIVATE"); toast.error("Failed to update sharing settings"); } }, - [thread, currentVisibility, onVisibilityChange, queryClient, setThreadVisibility] + [thread, currentVisibility, onVisibilityChange, updateVisibility] ); const handleCreatePublicLink = useCallback(async () => { @@ -130,7 +121,7 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS }, [thread, createSnapshot, queryClient]); // Don't show if no thread (new chat that hasn't been created yet) - if (!thread) { + if (!thread || currentVisibility === undefined) { return null; } diff --git a/surfsense_web/components/settings/agent-model-manager.tsx b/surfsense_web/components/settings/agent-model-manager.tsx index b0e13d3d7..507a263e0 100644 --- a/surfsense_web/components/settings/agent-model-manager.tsx +++ b/surfsense_web/components/settings/agent-model-manager.tsx @@ -228,7 +228,7 @@ export function AgentModelManager({ searchSpaceId }: AgentModelManagerProps) {

    No Models Yet

    {canCreate - ? "Add your first model to power document summarization, chat, and other agent capabilities" + ? "Add your first model to power chat, reports, and other agent capabilities" : "No models have been added to this space yet. Contact a space owner to add one"}

    diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 9af9eec27..c32e79a8e 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -55,15 +55,6 @@ const ROLE_DESCRIPTIONS = { prefKey: "agent_llm_id" as const, configType: "llm" as const, }, - document_summary: { - icon: FileText, - title: "Document Summary LLM", - description: "Handles document summarization and research synthesis", - color: "text-muted-foreground", - bgColor: "bg-muted", - prefKey: "document_summary_llm_id" as const, - configType: "llm" as const, - }, image_generation: { icon: ImageIcon, title: "Image Generation Model", @@ -137,7 +128,6 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { const [assignments, setAssignments] = useState>(() => ({ agent_llm_id: preferences.agent_llm_id ?? null, - document_summary_llm_id: preferences.document_summary_llm_id ?? null, image_generation_config_id: preferences.image_generation_config_id ?? null, vision_llm_config_id: preferences.vision_llm_config_id ?? null, })); @@ -148,13 +138,11 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { useEffect(() => { setAssignments({ agent_llm_id: preferences.agent_llm_id ?? null, - document_summary_llm_id: preferences.document_summary_llm_id ?? null, image_generation_config_id: preferences.image_generation_config_id ?? null, vision_llm_config_id: preferences.vision_llm_config_id ?? null, }); }, [ preferences.agent_llm_id, - preferences.document_summary_llm_id, preferences.image_generation_config_id, preferences.vision_llm_config_id, ]); diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index ce9a5bc74..3f68f6d64 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -139,7 +139,6 @@ export function DocumentUploadTab({ const [files, setFiles] = useState([]); const [uploadProgress, setUploadProgress] = useState(0); const [accordionValue, setAccordionValue] = useState(""); - const [shouldSummarize, setShouldSummarize] = useState(false); const [useVisionLlm, setUseVisionLlm] = useState(false); const [processingMode, setProcessingMode] = useState("basic"); const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom); @@ -366,7 +365,6 @@ export function DocumentUploadTab({ search_space_id: Number(searchSpaceId), relative_paths: batch.map((e) => e.relativePath), root_folder_id: rootFolderId, - enable_summary: shouldSummarize, use_vision_llm: useVisionLlm, processing_mode: processingMode, } @@ -414,7 +412,6 @@ export function DocumentUploadTab({ { files: rawFiles, search_space_id: Number(searchSpaceId), - should_summarize: shouldSummarize, use_vision_llm: useVisionLlm, processing_mode: processingMode, }, @@ -696,16 +693,6 @@ export function DocumentUploadTab({
    )} -
    -
    -

    Enable AI Summary

    -

    - Improves search quality but adds latency -

    -
    - -
    -

    Enable Vision LLM

    diff --git a/surfsense_web/components/sources/FolderWatchDialog.tsx b/surfsense_web/components/sources/FolderWatchDialog.tsx index 4cc566ef6..8c5629276 100644 --- a/surfsense_web/components/sources/FolderWatchDialog.tsx +++ b/surfsense_web/components/sources/FolderWatchDialog.tsx @@ -12,7 +12,6 @@ import { DialogTitle, } from "@/components/ui/dialog"; import { Spinner } from "@/components/ui/spinner"; -import { Switch } from "@/components/ui/switch"; import { type FolderSyncProgress, uploadFolderScan } from "@/lib/folder-sync-upload"; import { getSupportedExtensionsSet } from "@/lib/supported-extensions"; @@ -46,7 +45,6 @@ export function FolderWatchDialog({ initialFolder, }: FolderWatchDialogProps) { const [selectedFolder, setSelectedFolder] = useState(null); - const [shouldSummarize, setShouldSummarize] = useState(false); const [submitting, setSubmitting] = useState(false); const [progress, setProgress] = useState(null); const abortRef = useRef(null); @@ -91,7 +89,6 @@ export function FolderWatchDialog({ searchSpaceId, excludePatterns: DEFAULT_EXCLUDE_PATTERNS, fileExtensions: supportedExtensions, - enableSummary: shouldSummarize, onProgress: setProgress, signal: controller.signal, }); @@ -108,7 +105,6 @@ export function FolderWatchDialog({ toast.success(`Watching folder: ${selectedFolder.name}`); setSelectedFolder(null); - setShouldSummarize(false); setProgress(null); onOpenChange(false); onSuccess?.(); @@ -123,20 +119,12 @@ export function FolderWatchDialog({ setSubmitting(false); setProgress(null); } - }, [ - selectedFolder, - searchSpaceId, - shouldSummarize, - supportedExtensions, - onOpenChange, - onSuccess, - ]); + }, [selectedFolder, searchSpaceId, supportedExtensions, onOpenChange, onSuccess]); const handleOpenChange = useCallback( (nextOpen: boolean) => { if (!nextOpen && !submitting) { setSelectedFolder(null); - setShouldSummarize(false); setProgress(null); } onOpenChange(nextOpen); @@ -206,16 +194,6 @@ export function FolderWatchDialog({ {selectedFolder && ( <> -
    -
    -

    Enable AI Summary

    -

    - Improves search quality but adds latency -

    -
    - -
    - {progressLabel && (

    {progressLabel}

    diff --git a/surfsense_web/components/tool-ui/automation/create-automation.tsx b/surfsense_web/components/tool-ui/automation/create-automation.tsx index d4cc0ec4d..24e9d66bd 100644 --- a/surfsense_web/components/tool-ui/automation/create-automation.tsx +++ b/surfsense_web/components/tool-ui/automation/create-automation.tsx @@ -12,6 +12,7 @@ import { import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { JsonView } from "@/components/json-view"; import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { automationCreateRequest } from "@/contracts/types/automation.types"; import type { HitlDecision, InterruptResult } from "@/features/chat-messages/hitl"; @@ -351,19 +352,21 @@ function JsonEditor({ initialValue, onSave, onCancel }: JsonEditorProps) { />
    {issues.length > 0 && ( -
    -
    - + + + {issues.length} issue{issues.length === 1 ? "" : "s"} -
    -
      - {issues.map((issue) => ( -
    • - {issue} -
    • - ))} -
    -
    + + +
      + {issues.map((issue) => ( +
    • + {issue} +
    • + ))} +
    +
    + )}
    + ) : null} +
    +
    + ))} +
    + ) : ( +

    + No changelog entries yet. +

    + )} +
    + + ); +}; diff --git a/surfsense_web/components/ui/select.tsx b/surfsense_web/components/ui/select.tsx index cf22bf6a3..a57415c99 100644 --- a/surfsense_web/components/ui/select.tsx +++ b/surfsense_web/components/ui/select.tsx @@ -43,9 +43,12 @@ function SelectTrigger({ function SelectContent({ className, children, + matchTriggerWidth = true, position = "popper", ...props -}: React.ComponentProps) { +}: React.ComponentProps & { + matchTriggerWidth?: boolean; +}) { return ( diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx index 0155969cd..60b5e67b6 100644 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ b/surfsense_web/content/docs/docker-installation/docker-compose.mdx @@ -35,6 +35,7 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y | Variable | Description | Default | |----------|-------------|---------| | `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` | +| `SURFSENSE_VARIANT` | Backend image variant. Leave empty for CPU, set `cuda` for CUDA 12.8, or `cuda126` for CUDA 12.6. | *(empty)* | | `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | | `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | | `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | @@ -42,6 +43,62 @@ All configuration lives in a single `docker/.env` file (or `surfsense/.env` if y | `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | | `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | +### Image Variants + +SurfSense publishes CPU and CUDA backend image variants. The frontend image is not variant-specific. + +| Backend tag | Use case | `SURFSENSE_VARIANT` | +|-------------|----------|---------------------| +| `:latest` | CPU-only default | *(empty)* | +| `:latest-cuda` | NVIDIA CUDA 12.8 backend image | `cuda` | +| `:latest-cuda126` | NVIDIA CUDA 12.6 backend image for older driver stacks | `cuda126` | + +All backend variants are published for `linux/amd64` and `linux/arm64`. CUDA on `linux/arm64` is best-effort. + + +GPU acceleration needs two settings: `SURFSENSE_VARIANT` selects the CUDA image, and `COMPOSE_FILE` enables the GPU device overlay. The host must have the NVIDIA Container Toolkit installed. + + +### NVIDIA GPU Acceleration + +For most NVIDIA systems, add these values to `.env` to use the CUDA 12.8 image: + +```dotenv +SURFSENSE_VARIANT=cuda +COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml +SURFSENSE_GPU_COUNT=1 +``` + +Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks or older GPUs that need the CUDA 12.6 fallback image. + +On Windows, use `;` instead of `:` in `COMPOSE_FILE` inside `.env`: + +```dotenv +COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml +``` + +To switch variants later, edit `SURFSENSE_VARIANT` and `COMPOSE_FILE` in `.env`, then run: + +```bash +docker compose pull +docker compose up -d --wait +``` + +### Automatic Updates + +Manual Docker Compose installs do not start Watchtower automatically. To enable external automatic updates, run Watchtower separately: + +```bash +docker run -d --name watchtower \ + --restart unless-stopped \ + -v /var/run/docker.sock:/var/run/docker.sock \ + nickfedor/watchtower \ + --label-enable \ + --interval 86400 +``` + +SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services. + ### Ports | Variable | Description | Default | @@ -103,7 +160,6 @@ Create credentials at the [Google Cloud Console](https://console.cloud.google.co | Variable | Description | |----------|-------------| -| `FIRECRAWL_API_KEY` | [Firecrawl](https://www.firecrawl.dev/) API key for web crawling | | `UNSTRUCTURED_API_KEY` | [Unstructured.io](https://unstructured.io/) API key (required if `ETL_SERVICE=UNSTRUCTURED`) | | `LLAMA_CLOUD_API_KEY` | [LlamaCloud](https://cloud.llamaindex.ai/) API key (required if `ETL_SERVICE=LLAMACLOUD`) | @@ -124,6 +180,19 @@ Uncomment the connectors you want to use. Redirect URIs follow the pattern `http | Microsoft (Teams & OneDrive) | `MICROSOFT_CLIENT_ID`, `MICROSOFT_CLIENT_SECRET`, `TEAMS_REDIRECT_URI`, `ONEDRIVE_REDIRECT_URI` | | Dropbox | `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, `DROPBOX_REDIRECT_URI` | +### Messaging Channels + +Configure these in the same `docker/.env` file when you want users to chat with +SurfSense from external apps. See [Messaging Channels](/docs/messaging-channels) +for full setup. + +| Channel | Variables | +|---------|-----------| +| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` | +| WhatsApp | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` | +| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` | +| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` | + ### Observability (optional) | Variable | Description | @@ -187,9 +256,9 @@ Postgres. Before this design, a silent migration failure would leave The backend exposes two endpoints: -- `GET /health` — lightweight liveness probe (always returns 200 if the +- `GET /health`: lightweight liveness probe (always returns 200 if the process is up). -- `GET /ready` — readiness probe that confirms `zero_publication` exists. +- `GET /ready`: readiness probe that confirms `zero_publication` exists. Returns 503 if not. The compose `backend.healthcheck` uses `/ready` so the container only reports `healthy` once the schema is actually usable by zero-cache. @@ -247,7 +316,7 @@ docker compose exec db psql -U surfsense -d surfsense \ ``` The default migration timeout is 900 seconds. Slow disks (Windows / WSL2) -may need more — set `MIGRATION_TIMEOUT` in `.env` to increase it. +may need more. Set `MIGRATION_TIMEOUT` in `.env` to increase it. ### Zero-cache stuck on `Unknown or invalid publications` @@ -257,11 +326,13 @@ Symptom (in `docker compose logs zero-cache`): Error: Unknown or invalid publications. Specified: [zero_publication]. Found: [] ``` -This means `zero-cache` started before `zero_publication` was created. With -the current compose files this should be impossible — the `migrations` -service blocks `zero-cache` from starting. If you see it, your stack -predates the fix or you brought up `zero-cache` manually with `docker -compose up zero-cache` before the migrations service ran. +This means `zero-cache` started before `zero_publication` was created or the +publication does not match SurfSense's canonical Zero shape. With the current +compose files this should be impossible: the `migrations` service blocks +`zero-cache` from starting and verifies the publication before exiting +successfully. If you see it, your stack predates the fix or you brought up +`zero-cache` manually with `docker compose up zero-cache` before the migrations +service ran. Recovery: @@ -271,18 +342,13 @@ docker volume rm surfsense-zero-cache # wipe half-built SQLite replica docker compose up -d # migrations runs first, then zero-cache ``` -The install script (`install.ps1` / `install.sh`) detects this case -automatically: if it finds a `surfsense-zero-cache` volume from a previous -install with no matching `surfsense-zero-init` volume, it removes the stale -volume before bringing the stack up. - ### Zero-cache crashes with `_zero.tableMetadata` errors This indicates a half-initialized SQLite replica left behind by a previous -crash. The `migrations` service writes a marker file on a shared volume -(`surfsense-zero-init`) when the publication oid changes; zero-cache wipes -its replica and re-syncs on next start. If the marker mechanism somehow did -not trigger, run the recovery one-liner above. +crash. Zero's own event triggers and `ZERO_AUTO_RESET` handle schema and +replication halts automatically. If the local SQLite replica is wedged, run the +recovery one-liner above to wipe `surfsense-zero-cache`; zero-cache will +re-sync from Postgres on the next start. ### Ensuring `wal_level = logical` diff --git a/surfsense_web/content/docs/docker-installation/install-script.mdx b/surfsense_web/content/docs/docker-installation/install-script.mdx index 50ccc7288..9f8acf9e5 100644 --- a/surfsense_web/content/docs/docker-installation/install-script.mdx +++ b/surfsense_web/content/docs/docker-installation/install-script.mdx @@ -3,7 +3,7 @@ title: One-Line Install Script description: One-command installation of SurfSense using Docker --- -Downloads the compose files, generates a `SECRET_KEY`, starts all services, and sets up [Watchtower](https://github.com/nicholas-fedor/watchtower) for automatic daily updates. +Downloads the compose files, generates a `SECRET_KEY`, starts all services with `docker compose up -d --wait`, and starts [Watchtower](https://github.com/nicholas-fedor/watchtower) as an external updater for automatic daily updates. **Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running. @@ -19,9 +19,38 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex ``` -This creates a `./surfsense/` directory with `docker-compose.yml` and `.env`, then runs `docker compose up -d`. +This creates a `./surfsense/` directory with `docker-compose.yml`, `docker-compose.gpu.yml`, and `.env`, then runs `docker compose up -d --wait`. -To skip Watchtower (e.g. in production where you manage updates yourself): +If an NVIDIA GPU and NVIDIA Container Toolkit are detected, the installer asks whether to use GPU acceleration and chooses the compatible backend image automatically. Non-interactive installs default to CPU unless you pass an explicit flag. + +Interactive installs also ask whether to enable automatic daily updates with Watchtower, noting that updates may download several GB in the background. + +### GPU options + +Linux/macOS: + +```bash +# CUDA 12.8 +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda + +# CUDA 12.6 fallback for older driver stacks +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda126 + +# Reserve all available GPUs +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --gpu --gpu-count=all +``` + +PowerShell: + +```powershell +# Save the script locally first when passing PowerShell parameters. +.\install.ps1 -Variant cuda +.\install.ps1 -Variant cuda126 -GpuCount all +``` + +The installer writes the same `.env` settings you would configure manually: `SURFSENSE_VARIANT` selects the backend image and `COMPOSE_FILE` enables the GPU overlay. + +To skip Watchtower (e.g. in production where you manage updates yourself, or to avoid large background image downloads): ```bash curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower @@ -29,6 +58,16 @@ curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scr To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`. +Manual updates use the same compose state stored in `.env`, so GPU overlays and variants are preserved: + +```bash +cd surfsense +docker compose pull +docker compose up -d --wait +``` + +If Watchtower is enabled, it preserves the running image variant tag automatically. Because SurfSense images are large, use `--no-watchtower` when you prefer to manage update timing yourself. + --- ## Access SurfSense diff --git a/surfsense_web/content/docs/how-to/zero-sync.mdx b/surfsense_web/content/docs/how-to/zero-sync.mdx index 1a7762f23..7007e6637 100644 --- a/surfsense_web/content/docs/how-to/zero-sync.mdx +++ b/surfsense_web/content/docs/how-to/zero-sync.mdx @@ -104,6 +104,7 @@ Zero syncs the following tables for real-time features: | `new_chat_messages` | Live chat message sync for shared chats | | `chat_comments` | Real-time comment threads on AI responses | | `chat_session_state` | Collaboration indicators (who is typing) | +| `automation_runs` | Live run status and per-step progress (thin column set; heavy fields stay on REST) | ## Troubleshooting diff --git a/surfsense_web/content/docs/index.mdx b/surfsense_web/content/docs/index.mdx index 2204e4e34..4a321b376 100644 --- a/surfsense_web/content/docs/index.mdx +++ b/surfsense_web/content/docs/index.mdx @@ -5,7 +5,7 @@ icon: BookOpen --- import { Card, Cards } from 'fumadocs-ui/components/card'; -import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart } from 'lucide-react'; +import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle } from 'lucide-react'; Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense. Dive in to explore how SurfSense can be your AI-powered research companion. @@ -40,6 +40,12 @@ Welcome to **SurfSense's Documentation!** Here, you'll find everything you need description="Integrate with third-party services" href="/docs/connectors" /> + } + title="Messaging Channels" + description="Chat with SurfSense from Telegram, WhatsApp, Slack, and Discord" + href="/docs/messaging-channels" + /> } title="How-To Guides" diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 599cb6238..22a8ff5a1 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -39,6 +39,15 @@ Complete all the [setup steps](/docs), including: The backend is the core of SurfSense. Follow these steps to set it up: +### Optional: Messaging Channels + +SurfSense can expose the same backend agent through Telegram, WhatsApp, Slack, +and Discord. For manual installs, configure the relevant channel variables in +`surfsense_backend/.env`. + +See [Messaging Channels](/docs/messaging-channels) for the channel-specific +setup guides. + ### 1. Environment Configuration First, create and configure your environment variables by copying the example file: @@ -85,12 +94,10 @@ Edit the `.env` file and set the following variables: | STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | | STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | | STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | -| FIRECRAWL_API_KEY | (Optional) API key for Firecrawl service for web crawling | | ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | | UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) | | LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) | -| CELERY_BROKER_URL | Redis connection URL for Celery broker (e.g., `redis://localhost:6379/0`) | -| CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | +| REDIS_URL | Single Redis connection URL for the Celery broker, result backend, and app cache (e.g., `redis://localhost:6379/0`). Optionally override per use with `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, or `REDIS_APP_URL`. | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | @@ -350,7 +357,7 @@ redis-cli ping ### 6. Start Celery Worker -In a new terminal window, start the Celery worker to handle background tasks: +In a new terminal window, start the Celery worker to handle background tasks. For external chat surfaces, Celery only runs maintenance tasks; agent turns run inside the FastAPI process. **If using uv:** @@ -358,9 +365,9 @@ In a new terminal window, start the Celery worker to handle background tasks: # Make sure you're in the surfsense_backend directory cd surfsense_backend -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **If using pip/venv:** @@ -374,9 +381,9 @@ source .venv/bin/activate # Linux/macOS # OR .venv\Scripts\activate # Windows -# Start Celery worker (consume both default and connectors queues) +# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors" +celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` **Optional: Start Flower for monitoring Celery tasks:** @@ -457,7 +464,7 @@ If everything is set up correctly, you should see output indicating the server i ## Zero-Cache Setup -**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup — without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data. +**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup. Without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data. For an overview of how Zero works and the list of synced tables, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide. @@ -539,7 +546,7 @@ cd ../docker docker compose -f docker-compose.deps-only.yml up -d ``` -The deps-only stack exposes zero-cache on port `4848` (default) — keep `NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` in your `surfsense_web/.env`. +The deps-only stack exposes zero-cache on port `4848` by default. Keep `NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` in your `surfsense_web/.env`. ## Frontend Setup @@ -675,13 +682,13 @@ To verify your installation: 1. Open your browser and navigate to `http://localhost:3000` 2. Sign in with your Google account (or local credentials if `AUTH_TYPE=LOCAL`) 3. Create a search space and try uploading a document -4. Watch the upload status update live without refreshing — this confirms zero-cache is wired up correctly +4. Watch the upload status update live without refreshing. This confirms zero-cache is wired up correctly 5. Test the chat functionality with your uploaded content ## Troubleshooting - **Database Connection Issues**: Verify your PostgreSQL server is running and pgvector is properly installed -- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `CELERY_BROKER_URL` and `CELERY_RESULT_BACKEND` are correctly set in your `.env` file +- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `REDIS_URL` is correctly set in your `.env` file - **Celery Worker Issues**: Make sure the Celery worker is running in a separate terminal. Check worker logs for any errors - **Authentication Problems**: Check your Google OAuth configuration and ensure redirect URIs are set correctly - **LLM Errors**: Confirm your LLM API keys are valid and the selected models are accessible diff --git a/surfsense_web/content/docs/messaging-channels/discord.mdx b/surfsense_web/content/docs/messaging-channels/discord.mdx new file mode 100644 index 000000000..c0874dfe3 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/discord.mdx @@ -0,0 +1,76 @@ +--- +title: Discord +description: Enable the SurfSense bot for in-Discord agent chat +--- + +# Discord Messaging Channel + +The Discord messaging channel lets users mention the SurfSense bot in Discord +and chat with the SurfSense backend agent from a Discord channel. + +This is separate from the Discord connector. The messaging channel handles bot +mentions and replies; the connector gives the agent Discord channel/message read +tools. + +## Discord Application Settings + +Create or reuse a Discord application in the +[Discord Developer Portal](https://discord.com/developers/applications). + +In **OAuth2 > Redirects**, add both callback URLs if the same application powers +the connector and messaging channel: + +```bash +https://your-backend.example.com/api/v1/auth/discord/connector/callback +https://your-backend.example.com/api/v1/gateway/discord/callback +``` + +For local OAuth testing, replace the host with your local or public tunnel URL, +and make sure `DISCORD_REDIRECT_URI` and `GATEWAY_DISCORD_REDIRECT_URI` match +the Discord dashboard exactly. + +## Bot Permissions And Intents + +In **Bot > Privileged Gateway Intents**, enable: + +- **Message Content Intent** so SurfSense can read text after a bot mention. + +When installing the bot, grant: + +- View Channels +- Send Messages +- Send Messages in Threads +- Read Message History + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +DISCORD_CLIENT_ID=your_discord_client_id +DISCORD_CLIENT_SECRET=your_discord_client_secret +DISCORD_BOT_TOKEN=your_discord_bot_token +GATEWAY_DISCORD_ENABLED=TRUE +GATEWAY_DISCORD_REDIRECT_URI=https://your-backend.example.com/api/v1/gateway/discord/callback +``` + +The messaging channel uses the same Discord app credentials as the Discord +connector. `DISCORD_REDIRECT_URI` remains the connector callback; +`GATEWAY_DISCORD_REDIRECT_URI` is the separate messaging channel install +callback. + +## Runtime Behavior + +1. Discord sends a `MESSAGE_CREATE` event over its WebSocket API. +2. SurfSense stores the event in the durable gateway inbox. +3. SurfSense resolves the Discord user binding to a SurfSense user and search space. +4. SurfSense runs the backend agent with that user's permissions. +5. The agent reply is posted back to the same Discord channel. + +## Deployment Note + +Only one running backend process should connect to Discord with a +given bot token. For multi-replica deployments, enable +`GATEWAY_DISCORD_ENABLED=TRUE` in a single backend process and leave it disabled +in other API replicas. diff --git a/surfsense_web/content/docs/messaging-channels/docker.mdx b/surfsense_web/content/docs/messaging-channels/docker.mdx new file mode 100644 index 000000000..3a4d4177f --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/docker.mdx @@ -0,0 +1,60 @@ +--- +title: Docker Setup +description: Configure messaging channels for Docker and one-line installs +--- + +# Docker Setup + +For Docker and one-line installs, configure messaging channels in the generated +`docker/.env` file. You do not need to edit `surfsense_backend/.env.example`. + +The Compose stack passes `docker/.env` into the backend, worker, and beat +containers. Database, Redis, SearXNG, and internal Docker networking are already +wired by Compose. + +## Public URLs + +For localhost-only testing, the defaults are enough for the SurfSense UI, but +public webhooks from Telegram, WhatsApp, and Slack require a public HTTPS backend +URL. Use your deployed backend URL or a tunnel such as Cloudflare Tunnel or +ngrok. + +When using a custom domain or tunnel, set: + +```bash +BACKEND_URL=https://api.example.com +GATEWAY_BASE_URL=https://api.example.com +NEXT_FRONTEND_URL=https://app.example.com +NEXT_PUBLIC_FASTAPI_BACKEND_URL=https://api.example.com +``` + +## Environment Variables + +Uncomment only the channel you are enabling in `docker/.env`. + +| Channel | Main variables | +| --- | --- | +| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` | +| WhatsApp Cloud API | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` | +| WhatsApp Baileys | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_BRIDGE_URL`, `WHATSAPP_MODE` | +| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` | +| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` | + +After editing `docker/.env`, restart the stack: + +```bash +docker compose up -d +``` + +For WhatsApp Baileys, start the Compose profile: + +```bash +docker compose --profile whatsapp up -d +``` + +Then follow the per-channel setup pages: + +- [Telegram](/docs/messaging-channels/telegram) +- [WhatsApp](/docs/messaging-channels/whatsapp) +- [Slack](/docs/messaging-channels/slack) +- [Discord](/docs/messaging-channels/discord) diff --git a/surfsense_web/content/docs/messaging-channels/index.mdx b/surfsense_web/content/docs/messaging-channels/index.mdx new file mode 100644 index 000000000..d15dc0e6e --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/index.mdx @@ -0,0 +1,42 @@ +--- +title: Messaging Channels +description: Chat with SurfSense from Telegram, WhatsApp, Slack, and Discord +--- + +import { Card, Cards } from 'fumadocs-ui/components/card'; + +Choose the external chat app you want to connect to SurfSense. Each guide shows +the required app setup, environment variables, and pairing flow. + + + + + + + + + diff --git a/surfsense_web/content/docs/messaging-channels/meta.json b/surfsense_web/content/docs/messaging-channels/meta.json new file mode 100644 index 000000000..594fd95b9 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Messaging Channels", + "icon": "MessageCircle", + "pages": ["telegram", "whatsapp", "slack", "discord", "docker", "troubleshooting"], + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/messaging-channels/slack.mdx b/surfsense_web/content/docs/messaging-channels/slack.mdx new file mode 100644 index 000000000..4e001d13a --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/slack.mdx @@ -0,0 +1,84 @@ +--- +title: Slack +description: Enable the SurfSense bot for in-Slack agent chat +--- + +# Slack Messaging Channel + +The Slack messaging channel lets users mention the SurfSense bot in Slack and +chat with the SurfSense backend agent from a Slack thread. + +This is separate from the Slack connector. The messaging channel handles bot +mentions and replies; the connector gives the agent Slack search/read tools. + +## Required Slack App Scopes + +Add these **Bot Token Scopes** in Slack OAuth & Permissions: + +| Scope | Purpose | +| --- | --- | +| `app_mentions:read` | Receive bot mention events | +| `chat:write` | Reply in Slack threads | +| `channels:read` | Read public channel metadata | +| `groups:read` | Read private channel metadata where the bot is present | +| `im:write` | Send onboarding or direct replies | +| `users:read` | Resolve Slack users | +| `team:read` | Resolve workspace metadata | + +Optional scopes: + +- `im:history` if you support direct message chat with the bot. +- `commands` if you add slash commands. + +Avoid `channels:history` and `groups:history` for the messaging channel unless +you specifically need gateway-side context reads. Slack workspace search should +stay with the Slack connector. + +## Event Subscriptions + +Enable Slack Events API and subscribe to: + +- `app_mention` + +Set the request URL to: + +```bash +https://your-backend.example.com/api/v1/gateway/webhooks/slack +``` + +Slack must be able to reach this URL. Do not use `localhost` for event +subscriptions. + +## OAuth Redirect URLs + +If the same Slack app powers both the connector and messaging channel, add both +redirect URLs in **OAuth & Permissions**: + +```bash +https://your-backend.example.com/api/v1/auth/slack/connector/callback +https://your-backend.example.com/api/v1/gateway/slack/callback +``` + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +SLACK_CLIENT_ID=your_slack_client_id +SLACK_CLIENT_SECRET=your_slack_client_secret +GATEWAY_SLACK_ENABLED=TRUE +GATEWAY_SLACK_SIGNING_SECRET=your_slack_signing_secret +GATEWAY_SLACK_REDIRECT_URI=https://your-backend.example.com/api/v1/gateway/slack/callback +``` + +After changing Slack scopes, redirect URLs, or event subscriptions, reinstall +the Slack app to your workspace so Slack grants the updated permissions. + +## Runtime Behavior + +1. Slack sends an `app_mention` event to SurfSense. +2. SurfSense verifies the Slack signature and stores the event in the gateway inbox. +3. SurfSense resolves the Slack user binding to a SurfSense user and search space. +4. SurfSense runs the backend agent with that user's permissions. +5. The agent reply is posted back in the same Slack thread. diff --git a/surfsense_web/content/docs/messaging-channels/telegram.mdx b/surfsense_web/content/docs/messaging-channels/telegram.mdx new file mode 100644 index 000000000..3487da864 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/telegram.mdx @@ -0,0 +1,62 @@ +--- +title: Telegram +description: Enable SurfSense chat from Telegram +--- + +# Telegram Messaging Channel + +Telegram lets users chat with the SurfSense agent from a Telegram bot. Users pair +their Telegram chat from **User Settings > Messaging Channels**. + +## Environment Variables + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +TELEGRAM_SHARED_BOT_TOKEN=123456:bot-token-from-botfather +TELEGRAM_SHARED_BOT_USERNAME=your_bot_username +TELEGRAM_WEBHOOK_SECRET=generate-a-long-random-secret +GATEWAY_BASE_URL=https://api.example.com +GATEWAY_TELEGRAM_INTAKE_MODE=webhook +``` + +`TELEGRAM_WEBHOOK_SECRET` must be 1-256 characters and contain only `A-Z`, `a-z`, +`0-9`, `_`, or `-`. + +## Intake Modes + +| Mode | Use when | +| --- | --- | +| `webhook` | Production or any deployment with a public HTTPS backend URL | +| `longpoll` | Single-replica self-host installs that cannot expose a public HTTPS webhook | +| `disabled` | You do not want Telegram intake enabled | + +For SaaS-style or multi-replica deployments, use `webhook`. Long polling should +only run in a single backend process. + +## Webhook URL + +Telegram webhooks use this shape: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/telegram/{account_id} +``` + +After deploying the backend, register the webhook: + +```bash +cd surfsense_backend +uv run python scripts/register_webhook.py +``` + +If switching a bot from long polling to webhooks, delete any existing Telegram +webhook or pending `getUpdates` session before relying on the new mode. + +## Pairing Flow + +1. The user opens **User Settings > Messaging Channels**. +2. The user starts Telegram pairing. +3. SurfSense provides a pairing code or bot link. +4. The user sends the pairing command to the Telegram bot. +5. SurfSense binds that Telegram chat to the selected search space. diff --git a/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx b/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx new file mode 100644 index 000000000..bdd385e28 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/troubleshooting.mdx @@ -0,0 +1,66 @@ +--- +title: Troubleshooting +description: Common messaging channel pairing, webhook, and bot reply issues +--- + +# Messaging Channels Troubleshooting + +## The Bot Does Not Reply + +Check that: + +- The channel is enabled in the backend environment. +- The backend restarted after the environment change. +- The external platform can reach your public HTTPS backend URL. +- The user paired the channel from **User Settings > Messaging Channels**. +- Redis is running, because gateway inbox processing uses backend coordination + and rate-limit state. + +## Telegram + +Check that: + +- `TELEGRAM_SHARED_BOT_TOKEN` and `TELEGRAM_SHARED_BOT_USERNAME` are correct. +- `GATEWAY_TELEGRAM_INTAKE_MODE` is one of `webhook`, `longpoll`, or `disabled`. +- `TELEGRAM_WEBHOOK_SECRET` contains only `A-Z`, `a-z`, `0-9`, `_`, or `-`. +- Webhook mode uses a public HTTPS `GATEWAY_BASE_URL`. +- Long polling runs in only one backend process. + +## WhatsApp + +For Meta Cloud API, check that: + +- `GATEWAY_WHATSAPP_INTAKE_MODE=cloud`. +- The Meta webhook URL is `${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/whatsapp`. +- The Meta verify token matches `WHATSAPP_WEBHOOK_VERIFY_TOKEN`. +- `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER` contains the public WhatsApp number + users should message. + +For Baileys, check that: + +- `GATEWAY_WHATSAPP_INTAKE_MODE=baileys`. +- The `whatsapp` Compose profile is running. +- The bridge is paired and healthy. +- You are messaging the account's Message Yourself chat. + +## Slack + +Check that: + +- `GATEWAY_SLACK_ENABLED=TRUE`. +- The Slack signing secret matches `GATEWAY_SLACK_SIGNING_SECRET`. +- Slack Events API is enabled and subscribed to `app_mention`. +- The Slack event request URL is public HTTPS and points to + `/api/v1/gateway/webhooks/slack`. +- The Slack app was reinstalled after scope or redirect URL changes. + +## Discord + +Check that: + +- `GATEWAY_DISCORD_ENABLED=TRUE`. +- The bot token is valid. +- Message Content Intent is enabled. +- The bot can view and send messages in the channel. +- Exactly one backend process is running the Discord listener. +- The Discord user is paired to a SurfSense user and search space. diff --git a/surfsense_web/content/docs/messaging-channels/whatsapp.mdx b/surfsense_web/content/docs/messaging-channels/whatsapp.mdx new file mode 100644 index 000000000..56015fd20 --- /dev/null +++ b/surfsense_web/content/docs/messaging-channels/whatsapp.mdx @@ -0,0 +1,75 @@ +--- +title: WhatsApp +description: Enable SurfSense chat from WhatsApp +--- + +# WhatsApp Messaging Channel + +WhatsApp supports two intake modes: + +- `cloud` uses the official Meta WhatsApp Cloud API with a SurfSense-owned system + WhatsApp Business Account. +- `baileys` uses the unofficial Baileys WebSocket bridge for self-hosted, + one-tenant Message Yourself installs. + +Use `cloud` for production and shared deployments. + +## Meta Cloud API + +Create a Meta app, provision a WhatsApp Business Account and phone number, and +create a long-lived system user token with WhatsApp permissions. + +Point the Meta app webhook to: + +```text +${GATEWAY_BASE_URL}/api/v1/gateway/webhooks/whatsapp +``` + +Set the webhook verify token in Meta to the same value as +`WHATSAPP_WEBHOOK_VERIFY_TOKEN`. + +For Docker installs, add these to `docker/.env`. For manual installs, add them to +`surfsense_backend/.env`. + +```bash +GATEWAY_WHATSAPP_INTAKE_MODE=cloud +WHATSAPP_SHARED_BUSINESS_TOKEN=your-system-user-token +WHATSAPP_SHARED_PHONE_NUMBER_ID=your-meta-phone-number-id +WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER=15551234567 +WHATSAPP_SHARED_WABA_ID=your-waba-id +WHATSAPP_GRAPH_API_VERSION=v25.0 +WHATSAPP_WEBHOOK_VERIFY_TOKEN=generate-a-long-random-secret +WHATSAPP_WEBHOOK_APP_SECRET=your-meta-app-secret +``` + +Users open **User Settings > Messaging Channels**, click **Pair WhatsApp**, and +open the returned `wa.me` link. WhatsApp pre-fills `/start CODE`; the user must +press send to bind the chat. + +## Baileys Self-Hosted Mode + +Baileys is unofficial. Use it only for single-tenant self-hosted installs where +the operator accepts the risk of a personal WhatsApp session bridge. + +```bash +GATEWAY_WHATSAPP_INTAKE_MODE=baileys +WHATSAPP_BRIDGE_URL=http://whatsapp-bridge:9929 +WHATSAPP_MODE=self-chat +docker compose --profile whatsapp up -d +``` + +After pairing, use WhatsApp's Message Yourself chat. The bridge only forwards +messages from your own self-chat and ignores groups, other chats, and other +people. + +The `whatsapp-bridge` container stores Baileys auth state in the +`surfsense-whatsapp-sessions` Docker volume. That volume contains account +takeover material. Treat it like a secret. + +To intentionally reset pairing: + +```bash +docker compose --profile whatsapp down +docker volume rm surfsense-whatsapp-sessions +docker compose --profile whatsapp up -d +``` diff --git a/surfsense_web/content/docs/meta.json b/surfsense_web/content/docs/meta.json index 13b599118..74be10600 100644 --- a/surfsense_web/content/docs/meta.json +++ b/surfsense_web/content/docs/meta.json @@ -9,6 +9,7 @@ "installation", "manual-installation", "docker-installation", + "messaging-channels", "connectors", "how-to", "---Developers---", diff --git a/surfsense_web/contracts/enums/llm-models.ts b/surfsense_web/contracts/enums/llm-models.ts index 74cc056f3..9647c9d31 100644 --- a/surfsense_web/contracts/enums/llm-models.ts +++ b/surfsense_web/contracts/enums/llm-models.ts @@ -1528,14 +1528,20 @@ export const LLM_MODELS: LLMModel[] = [ // MiniMax { - value: "MiniMax-M2.5", - label: "MiniMax M2.5", + value: "MiniMax-M3", + label: "MiniMax M3", + provider: "MINIMAX", + contextWindow: "512K", + }, + { + value: "MiniMax-M2.7", + label: "MiniMax M2.7", provider: "MINIMAX", contextWindow: "204K", }, { - value: "MiniMax-M2.5-highspeed", - label: "MiniMax M2.5 Highspeed", + value: "MiniMax-M2.7-highspeed", + label: "MiniMax M2.7 Highspeed", provider: "MINIMAX", contextWindow: "204K", }, diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts index ce2b6afe9..c04a44923 100644 --- a/surfsense_web/contracts/enums/llm-providers.ts +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -184,8 +184,8 @@ export const LLM_PROVIDERS: LLMProvider[] = [ { value: "MINIMAX", label: "MiniMax", - example: "MiniMax-M2.5, MiniMax-M2.5-highspeed", - description: "High-performance models with 204K context", + example: "MiniMax-M3, MiniMax-M2.7", + description: "High-performance models with up to 512K context", apiBase: "https://api.minimax.io/v1", }, { diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts index 61d5ffc94..7c3dbb043 100644 --- a/surfsense_web/contracts/types/connector.types.ts +++ b/surfsense_web/contracts/types/connector.types.ts @@ -43,7 +43,6 @@ export const searchSourceConnector = z.object({ is_active: z.boolean().default(true), last_indexed_at: z.string().nullable(), config: z.record(z.string(), z.any()), - enable_summary: z.boolean().default(false), enable_vision_llm: z.boolean().default(false), periodic_indexing_enabled: z.boolean(), indexing_frequency_minutes: z.number().nullable(), @@ -98,7 +97,6 @@ export const createConnectorRequest = z.object({ is_active: true, last_indexed_at: true, config: true, - enable_summary: true, enable_vision_llm: true, periodic_indexing_enabled: true, indexing_frequency_minutes: true, @@ -124,7 +122,6 @@ export const updateConnectorRequest = z.object({ is_active: true, last_indexed_at: true, config: true, - enable_summary: true, enable_vision_llm: true, periodic_indexing_enabled: true, indexing_frequency_minutes: true, diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts index 82c6cbdaf..da1dac537 100644 --- a/surfsense_web/contracts/types/document.types.ts +++ b/surfsense_web/contracts/types/document.types.ts @@ -130,7 +130,6 @@ export const processingModeEnum = z.enum(["basic", "premium"]); export const uploadDocumentRequest = z.object({ files: z.array(z.instanceof(File)), search_space_id: z.number(), - should_summarize: z.boolean().default(false), use_vision_llm: z.boolean().default(false), processing_mode: processingModeEnum.default("basic"), }); @@ -281,6 +280,23 @@ export const deleteDocumentResponse = z.object({ message: z.literal("Document deleted successfully"), }); +/** + * Document files (stored originals / derived artifacts) + */ +export const documentFileKindEnum = z.enum(["ORIGINAL", "REDACTED", "FILLED_FORM"]); + +export const documentFileRead = z.object({ + id: z.number(), + document_id: z.number(), + kind: documentFileKindEnum, + original_filename: z.string(), + mime_type: z.string().nullable().optional(), + size_bytes: z.number(), + created_at: z.string(), +}); + +export const getDocumentFilesResponse = z.array(documentFileRead); + export type Document = z.infer; export type DocumentTitleRead = z.infer; export type GetDocumentsRequest = z.infer; @@ -314,3 +330,6 @@ export type GetDocumentChunksRequest = z.infer; export type GetDocumentChunksResponse = z.infer; export type ChunkRead = z.infer; export type ProcessingMode = z.infer; +export type DocumentFileKind = z.infer; +export type DocumentFileRead = z.infer; +export type GetDocumentFilesResponse = z.infer; diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index b52b98ae4..2fa7a37be 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -384,11 +384,9 @@ export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig); export const llmPreferences = z.object({ agent_llm_id: z.union([z.number(), z.null()]).optional(), - document_summary_llm_id: z.union([z.number(), z.null()]).optional(), image_generation_config_id: z.union([z.number(), z.null()]).optional(), vision_llm_config_id: z.union([z.number(), z.null()]).optional(), agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), - document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), }); @@ -409,7 +407,6 @@ export const updateLLMPreferencesRequest = z.object({ search_space_id: z.number(), data: llmPreferences.pick({ agent_llm_id: true, - document_summary_llm_id: true, image_generation_config_id: true, vision_llm_config_id: true, }), diff --git a/surfsense_web/hooks/use-activate-chat-thread.ts b/surfsense_web/hooks/use-activate-chat-thread.ts new file mode 100644 index 000000000..2367093b2 --- /dev/null +++ b/surfsense_web/hooks/use-activate-chat-thread.ts @@ -0,0 +1,79 @@ +"use client"; + +import { useQueryClient } from "@tanstack/react-query"; +import { useSetAtom } from "jotai"; +import { useRouter } from "next/navigation"; +import { useCallback } from "react"; +import { setCurrentThreadMetadataAtom } from "@/atoms/chat/current-thread.atom"; +import { syncChatTabAtom } from "@/atoms/tabs/tabs.atom"; +import type { ChatVisibility } from "@/lib/chat/thread-persistence"; +import { prefetchThreadData } from "./use-thread-queries"; + +interface ActivateChatThreadInput { + id: number | null; + title?: string; + url?: string; + searchSpaceId: number | string; + visibility?: ChatVisibility; + hasComments?: boolean; +} + +function getSearchSpaceId(searchSpaceId: number | string): number { + const parsed = + typeof searchSpaceId === "number" ? searchSpaceId : Number.parseInt(searchSpaceId, 10); + return Number.isNaN(parsed) ? 0 : parsed; +} + +function getChatUrl(searchSpaceId: number | string, threadId: number | null): string { + return threadId + ? `/dashboard/${searchSpaceId}/new-chat/${threadId}` + : `/dashboard/${searchSpaceId}/new-chat`; +} + +export function useActivateChatThread() { + const router = useRouter(); + const queryClient = useQueryClient(); + const syncChatTab = useSetAtom(syncChatTabAtom); + const setCurrentThreadMetadata = useSetAtom(setCurrentThreadMetadataAtom); + + const prefetchChatThread = useCallback( + (threadId: number | null | undefined) => { + if (typeof threadId === "number" && threadId > 0) { + prefetchThreadData(queryClient, threadId); + } + }, + [queryClient] + ); + + const activateChatThread = useCallback( + ({ id, title, url, searchSpaceId, visibility, hasComments }: ActivateChatThreadInput) => { + const numericSearchSpaceId = getSearchSpaceId(searchSpaceId); + const chatUrl = url ?? getChatUrl(searchSpaceId, id); + + syncChatTab({ + chatId: id, + title: id ? title : (title ?? "New Chat"), + chatUrl, + searchSpaceId: numericSearchSpaceId, + ...(visibility !== undefined ? { visibility } : {}), + ...(hasComments !== undefined ? { hasComments } : {}), + }); + + setCurrentThreadMetadata({ + id, + searchSpaceId: numericSearchSpaceId, + ...(visibility !== undefined ? { visibility } : {}), + ...(hasComments !== undefined ? { hasComments } : {}), + }); + + if (id) { + prefetchThreadData(queryClient, id); + } + + router.push(chatUrl); + }, + [queryClient, router, setCurrentThreadMetadata, syncChatTab] + ); + + return { activateChatThread, prefetchChatThread }; +} diff --git a/surfsense_web/hooks/use-automation-runs.ts b/surfsense_web/hooks/use-automation-runs.ts index c91c7bd6e..69e51ddc6 100644 --- a/surfsense_web/hooks/use-automation-runs.ts +++ b/surfsense_web/hooks/use-automation-runs.ts @@ -1,42 +1,109 @@ "use client"; -import { useQuery } from "@tanstack/react-query"; -import type { Run, RunListResponse } from "@/contracts/types/automation.types"; +import { useQuery as useZeroQuery } from "@rocicorp/zero/react"; +import { useQuery as useReactQuery } from "@tanstack/react-query"; +import { useMemo } from "react"; +import type { Run, RunStepResult, RunSummary } from "@/contracts/types/automation.types"; import { automationsApiService } from "@/lib/apis/automations-api.service"; import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queries } from "@/zero/queries"; const DEFAULT_LIMIT = 50; -const DEFAULT_OFFSET = 0; + +/** + * Thin live row sourced from Zero. Strict superset of {@link RunSummary} — + * existing consumers that only look at the summary fields keep working, + * while the run detail panel can read ``step_results`` directly for the + * live step ticker without a second REST round-trip. + */ +export interface LiveRunSummary extends RunSummary { + step_results: RunStepResult[]; +} export interface UseAutomationRunsOptions { limit?: number; - offset?: number; - enabled?: boolean; } -/** Paginated run history for one automation. Newest-first per backend. */ +interface UseAutomationRunsResult { + data: { items: LiveRunSummary[]; total: number } | undefined; + isLoading: boolean; + error: Error | null; +} + +/** + * Live run history for one automation, newest-first. Sourced from Zero's + * thin ``automation_runs`` publication so status and per-step progress + * tick in real time without polling. Heavy fields (output, artifacts, + * inputs, error, definition_snapshot) are still fetched lazily via + * {@link useAutomationRun}. + */ export function useAutomationRuns( automationId: number | undefined, - { limit = DEFAULT_LIMIT, offset = DEFAULT_OFFSET, enabled = true }: UseAutomationRunsOptions = {} -) { - return useQuery({ - queryKey: cacheKeys.automations.runs(automationId ?? 0, limit, offset), - queryFn: () => automationsApiService.listRuns(automationId as number, { limit, offset }), - enabled: enabled && !!automationId, - staleTime: 30_000, - }); + { limit = DEFAULT_LIMIT }: UseAutomationRunsOptions = {} +): UseAutomationRunsResult { + const [rows, result] = useZeroQuery( + queries.automationRuns.byAutomation({ automationId: automationId ?? -1 }) + ); + + const items = useMemo(() => { + if (!automationId) return []; + return rows.slice(0, limit).map(toLiveRunSummary); + }, [automationId, rows, limit]); + + const total = automationId ? rows.length : 0; + + // Pre-hydration window: nothing visible AND Zero hasn't confirmed + // completeness yet. After the first sync (even an empty set) we stop + // showing the skeleton so the empty-state copy can take over. + const isLoading = !!automationId && result.type !== "complete" && rows.length === 0; + + return { + data: automationId ? { items, total } : undefined, + isLoading, + error: null, + }; } -/** Single run with the full snapshot, step results, output and artifacts. */ +/** + * Full run record (definition snapshot, inputs, output, artifacts, error). + * Stays on REST: these fields are large and largely static after the run + * finishes, so they're not worth replicating to every connected client. + */ export function useAutomationRun( automationId: number | undefined, runId: number | undefined, options: { enabled?: boolean } = {} ) { const { enabled = true } = options; - return useQuery({ + return useReactQuery({ queryKey: cacheKeys.automations.run(automationId ?? 0, runId ?? 0), queryFn: () => automationsApiService.getRun(automationId as number, runId as number), enabled: enabled && !!automationId && !!runId, staleTime: 30_000, }); } + +interface ZeroAutomationRunRow { + id: number; + automationId: number; + triggerId?: number | null; + status: string; + stepResults: unknown; + startedAt?: number | null; + finishedAt?: number | null; + createdAt: number; +} + +/** Adapt a Zero camelCase row (epoch ms timestamps) to the snake_case + * ISO-string ``RunSummary`` shape the existing UI already consumes. */ +function toLiveRunSummary(row: ZeroAutomationRunRow): LiveRunSummary { + return { + id: row.id, + automation_id: row.automationId, + trigger_id: row.triggerId ?? null, + status: row.status as RunSummary["status"], + started_at: row.startedAt ? new Date(row.startedAt).toISOString() : null, + finished_at: row.finishedAt ? new Date(row.finishedAt).toISOString() : null, + created_at: new Date(row.createdAt).toISOString(), + step_results: Array.isArray(row.stepResults) ? (row.stepResults as RunStepResult[]) : [], + }; +} diff --git a/surfsense_web/hooks/use-connectors-sync.ts b/surfsense_web/hooks/use-connectors-sync.ts index d36728118..00e7350f8 100644 --- a/surfsense_web/hooks/use-connectors-sync.ts +++ b/surfsense_web/hooks/use-connectors-sync.ts @@ -24,7 +24,6 @@ export function useConnectorsSync(searchSpaceId: number | string | null) { is_active: true, last_indexed_at: c.lastIndexedAt ? new Date(c.lastIndexedAt).toISOString() : null, config: (c.config as Record) ?? {}, - enable_summary: c.enableSummary, periodic_indexing_enabled: c.periodicIndexingEnabled, indexing_frequency_minutes: c.indexingFrequencyMinutes ?? null, next_scheduled_at: c.nextScheduledAt ? new Date(c.nextScheduledAt).toISOString() : null, diff --git a/surfsense_web/hooks/use-thread-mutations.ts b/surfsense_web/hooks/use-thread-mutations.ts new file mode 100644 index 000000000..e3ae35e6b --- /dev/null +++ b/surfsense_web/hooks/use-thread-mutations.ts @@ -0,0 +1,158 @@ +"use client"; + +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useAtomValue, useSetAtom } from "jotai"; +import { + currentThreadAtom, + patchCurrentThreadMetadataAtom, + resetCurrentThreadAtom, +} from "@/atoms/chat/current-thread.atom"; +import { + moveThreadArchiveState, + patchThreadEverywhere, + removeThreadEverywhere, + replaceThreadEverywhere, +} from "@/lib/chat/thread-cache"; +import { + type ChatVisibility, + deleteThread, + type ThreadRecord, + updateThread, + updateThreadVisibility, +} from "@/lib/chat/thread-persistence"; + +type SearchSpaceKey = number | string; + +interface VisibilityVariables { + thread: ThreadRecord; + visibility: ChatVisibility; +} + +interface RenameVariables { + threadId: number; + title: string; + previousTitle?: string; +} + +interface ArchiveVariables { + threadId: number; + archived: boolean; +} + +interface DeleteVariables { + threadId: number; +} + +interface VisibilityRollback { + threadId: number; + visibility: ChatVisibility; +} + +interface RenameRollback { + threadId: number; + title?: string; +} + +interface ArchiveRollback { + threadId: number; + archived: boolean; +} + +export function useUpdateThreadVisibility(searchSpaceId: SearchSpaceKey) { + const queryClient = useQueryClient(); + const currentThread = useAtomValue(currentThreadAtom); + const patchCurrentThreadMetadata = useSetAtom(patchCurrentThreadMetadataAtom); + + return useMutation({ + mutationFn: ({ thread, visibility }) => updateThreadVisibility(thread.id, visibility), + onMutate: ({ thread, visibility }) => { + const previousVisibility = thread.visibility ?? "PRIVATE"; + + patchThreadEverywhere(queryClient, searchSpaceId, thread.id, { visibility }); + if (currentThread.id === thread.id) { + patchCurrentThreadMetadata({ id: thread.id, visibility }); + } + + return { threadId: thread.id, visibility: previousVisibility }; + }, + onError: (_error, _variables, rollback) => { + if (!rollback) return; + patchThreadEverywhere(queryClient, searchSpaceId, rollback.threadId, { + visibility: rollback.visibility, + }); + if (currentThread.id === rollback.threadId) { + patchCurrentThreadMetadata({ + id: rollback.threadId, + visibility: rollback.visibility, + }); + } + }, + onSuccess: (thread) => { + replaceThreadEverywhere(queryClient, searchSpaceId, thread); + if (currentThread.id === thread.id) { + patchCurrentThreadMetadata({ + id: thread.id, + visibility: thread.visibility, + ...(thread.has_comments !== undefined ? { hasComments: thread.has_comments } : {}), + }); + } + }, + }); +} + +export function useRenameThread(searchSpaceId: SearchSpaceKey) { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: ({ threadId, title }) => updateThread(threadId, { title }), + onMutate: ({ threadId, title, previousTitle }) => { + patchThreadEverywhere(queryClient, searchSpaceId, threadId, { title }); + return { threadId, title: previousTitle }; + }, + onError: (_error, _variables, rollback) => { + if (!rollback || rollback.title === undefined) return; + patchThreadEverywhere(queryClient, searchSpaceId, rollback.threadId, { + title: rollback.title, + }); + }, + onSuccess: (thread) => { + replaceThreadEverywhere(queryClient, searchSpaceId, thread); + }, + }); +} + +export function useArchiveThread(searchSpaceId: SearchSpaceKey) { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: ({ threadId, archived }) => updateThread(threadId, { archived }), + onMutate: ({ threadId, archived }) => { + moveThreadArchiveState(queryClient, searchSpaceId, threadId, archived); + return { threadId, archived: !archived }; + }, + onError: (_error, _variables, rollback) => { + if (!rollback) return; + moveThreadArchiveState(queryClient, searchSpaceId, rollback.threadId, rollback.archived); + }, + onSuccess: (thread) => { + replaceThreadEverywhere(queryClient, searchSpaceId, thread); + moveThreadArchiveState(queryClient, searchSpaceId, thread.id, thread.archived); + }, + }); +} + +export function useDeleteThread(searchSpaceId: SearchSpaceKey) { + const queryClient = useQueryClient(); + const currentThread = useAtomValue(currentThreadAtom); + const resetCurrentThread = useSetAtom(resetCurrentThreadAtom); + + return useMutation({ + mutationFn: ({ threadId }) => deleteThread(threadId), + onSuccess: (_data, { threadId }) => { + removeThreadEverywhere(queryClient, searchSpaceId, threadId); + if (currentThread.id === threadId) { + resetCurrentThread(); + } + }, + }); +} diff --git a/surfsense_web/hooks/use-thread-queries.ts b/surfsense_web/hooks/use-thread-queries.ts new file mode 100644 index 000000000..89a62f5b1 --- /dev/null +++ b/surfsense_web/hooks/use-thread-queries.ts @@ -0,0 +1,52 @@ +"use client"; + +import { type QueryClient, useQuery } from "@tanstack/react-query"; +import { + getThreadFull, + getThreadMessages, + type ThreadHistoryLoadResponse, + type ThreadRecord, +} from "@/lib/chat/thread-persistence"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +const THREAD_DETAIL_STALE_TIME_MS = 60 * 1000; +const THREAD_MESSAGES_STALE_TIME_MS = 30 * 1000; + +function isValidThreadId(threadId: number | null | undefined): threadId is number { + return typeof threadId === "number" && threadId > 0; +} + +export function useThreadDetail(threadId: number | null | undefined) { + return useQuery({ + queryKey: cacheKeys.threads.detail(threadId ?? 0), + queryFn: () => getThreadFull(threadId as number), + enabled: isValidThreadId(threadId), + staleTime: THREAD_DETAIL_STALE_TIME_MS, + }); +} + +export function useThreadMessages(threadId: number | null | undefined) { + return useQuery({ + queryKey: cacheKeys.threads.messages(threadId ?? 0), + queryFn: () => getThreadMessages(threadId as number), + enabled: isValidThreadId(threadId), + staleTime: THREAD_MESSAGES_STALE_TIME_MS, + }); +} + +export function prefetchThreadData(queryClient: QueryClient, threadId: number): void { + if (!isValidThreadId(threadId)) return; + + void Promise.all([ + queryClient.prefetchQuery({ + queryKey: cacheKeys.threads.detail(threadId), + queryFn: () => getThreadFull(threadId), + staleTime: THREAD_DETAIL_STALE_TIME_MS, + }), + queryClient.prefetchQuery({ + queryKey: cacheKeys.threads.messages(threadId), + queryFn: () => getThreadMessages(threadId), + staleTime: THREAD_MESSAGES_STALE_TIME_MS, + }), + ]); +} diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts index f9785c8a8..5b50db0c1 100644 --- a/surfsense_web/lib/apis/documents-api.service.ts +++ b/surfsense_web/lib/apis/documents-api.service.ts @@ -3,6 +3,7 @@ import { createDocumentRequest, createDocumentResponse, type DeleteDocumentRequest, + type DocumentFileRead, deleteDocumentRequest, deleteDocumentResponse, documentTitleRead, @@ -16,6 +17,7 @@ import { getDocumentByChunkResponse, getDocumentChunksRequest, getDocumentChunksResponse, + getDocumentFilesResponse, getDocumentRequest, getDocumentResponse, getDocumentsRequest, @@ -124,8 +126,7 @@ class DocumentsApiService { throw new ValidationError(`Invalid request: ${errorMessage}`); } - const { files, search_space_id, should_summarize, use_vision_llm, processing_mode } = - parsedRequest.data; + const { files, search_space_id, use_vision_llm, processing_mode } = parsedRequest.data; const UPLOAD_BATCH_SIZE = 5; const batches: File[][] = []; @@ -143,7 +144,6 @@ class DocumentsApiService { const formData = new FormData(); for (const file of batch) formData.append("files", file); formData.append("search_space_id", String(search_space_id)); - formData.append("should_summarize", String(should_summarize)); formData.append("use_vision_llm", String(use_vision_llm)); formData.append("processing_mode", processing_mode); @@ -381,6 +381,14 @@ class DocumentsApiService { }); }; + /** + * List the stored files for a document (e.g. its original upload). + * Used to gate the "Download original" affordance. + */ + getDocumentFiles = async (documentId: number): Promise => { + return baseApiService.get(`/api/v1/documents/${documentId}/files`, getDocumentFilesResponse); + }; + listDocumentVersions = async (documentId: number) => { return baseApiService.get(`/api/v1/documents/${documentId}/versions`); }; @@ -410,7 +418,6 @@ class DocumentsApiService { search_space_id: number; relative_paths: string[]; root_folder_id?: number | null; - enable_summary?: boolean; use_vision_llm?: boolean; processing_mode?: "basic" | "premium"; }, @@ -426,7 +433,6 @@ class DocumentsApiService { if (metadata.root_folder_id != null) { formData.append("root_folder_id", String(metadata.root_folder_id)); } - formData.append("enable_summary", String(metadata.enable_summary ?? false)); formData.append("use_vision_llm", String(metadata.use_vision_llm ?? false)); formData.append("processing_mode", metadata.processing_mode ?? "basic"); diff --git a/surfsense_web/lib/chat/example-prompts.ts b/surfsense_web/lib/chat/example-prompts.ts index 76b64b2ba..4dd7a00c0 100644 --- a/surfsense_web/lib/chat/example-prompts.ts +++ b/surfsense_web/lib/chat/example-prompts.ts @@ -23,44 +23,40 @@ export const CHAT_EXAMPLE_CATEGORIES: ChatExampleCategory[] = [ id: "search", label: "Search & Summarize", prompts: [ - "Summarize the key points across all the documents in this space.", - "What do my files say about [topic]? Answer with citations.", - "Find every mention of [keyword] and list the sources.", - "Give me a cited briefing on the documents I added this week.", - "Compare these two documents and highlight the differences.", + "Summarize the key points across all the documents in this space", + "What do my files say about [topic]? Answer with citations", + "Find every mention of [keyword] and list the sources", + "Compare these two documents and highlight the differences", ], }, { id: "create", label: "Create", prompts: [ - "Write a cited research report on [topic] from my documents.", - "Turn this folder into a two-host podcast I can listen to.", - "Create a slide deck and a narrated video overview from these sources.", - "Generate an image to illustrate [concept] for my report.", - "Tailor my resume to this job description so it gets past ATS and lands an interview.", + "Write a cited research report on [topic] from my documents", + "Turn this folder into a two-host podcast I can listen to", + "Create a slide deck and a narrated video overview from these sources", + "Tailor my resume to this job description so it gets past ATS and lands an interview", ], }, { id: "automate", label: "Automate", prompts: [ - "Email me a daily brief of new documents in my knowledge base every morning.", - "When a PDF lands in my Research folder, generate a cited AI summary.", - "Generate a weekly status report from my Slack and Gmail every Friday.", - "Build an automation that turns new meeting notes into minutes with action items.", - "Run a monthly competitor analysis report and save it to my workspace.", + "Email me a daily brief of new documents in my knowledge base every morning", + "When a PDF lands in my Research folder, generate a cited AI summary", + "Generate a weekly status report from my Slack and Gmail every Friday", + "Build an automation that turns new meeting notes into minutes with action items", ], }, { id: "tools", label: "Across your tools", prompts: [ - "Search across my Notion, Slack, Google Drive and Gmail for [topic].", - "Post this research summary to my Notion workspace.", - "Send these meeting action items to our team Slack channel.", - "Create a Jira ticket from this bug report.", - "Open a Linear issue from this feature request.", + "Search across my Notion, Slack, Google Drive and Gmail for [topic]", + "Post this research summary to my Notion workspace", + "Send these meeting action items to our team Slack channel", + "Create a Jira ticket from this bug report", ], }, ]; diff --git a/surfsense_web/lib/chat/thread-cache.ts b/surfsense_web/lib/chat/thread-cache.ts new file mode 100644 index 000000000..823979833 --- /dev/null +++ b/surfsense_web/lib/chat/thread-cache.ts @@ -0,0 +1,250 @@ +import type { QueryClient, QueryKey } from "@tanstack/react-query"; +import type { + ThreadListItem, + ThreadListResponse, + ThreadRecord, +} from "@/lib/chat/thread-persistence"; + +type SearchSpaceKey = number | string; + +type ThreadMetadataPatch = Partial & + Partial & { + has_comments?: boolean; + }; + +function isSameSearchSpace(keyValue: unknown, searchSpaceId: SearchSpaceKey): boolean { + return String(keyValue) === String(searchSpaceId); +} + +function isThreadListResponse(value: unknown): value is ThreadListResponse { + return ( + typeof value === "object" && + value !== null && + Array.isArray((value as ThreadListResponse).threads) && + Array.isArray((value as ThreadListResponse).archived_threads) + ); +} + +function isThreadListItemArray(value: unknown): value is ThreadListItem[] { + return Array.isArray(value); +} + +function listItemPatchFromMetadata(patch: ThreadMetadataPatch): Partial { + const listPatch: Partial = {}; + + if (patch.title !== undefined) listPatch.title = patch.title; + if (patch.archived !== undefined) listPatch.archived = patch.archived; + if (patch.visibility !== undefined) listPatch.visibility = patch.visibility; + if (patch.created_by_id !== undefined) listPatch.created_by_id = patch.created_by_id; + if (patch.created_at !== undefined) listPatch.createdAt = patch.created_at; + if (patch.updated_at !== undefined) listPatch.updatedAt = patch.updated_at; + if (patch.createdAt !== undefined) listPatch.createdAt = patch.createdAt; + if (patch.updatedAt !== undefined) listPatch.updatedAt = patch.updatedAt; + + return listPatch; +} + +function patchListItem( + item: ThreadListItem, + threadId: number, + patch: ThreadMetadataPatch +): ThreadListItem { + if (item.id !== threadId) return item; + return { + ...item, + ...listItemPatchFromMetadata(patch), + }; +} + +function patchThreadListResponse( + response: ThreadListResponse, + threadId: number, + patch: ThreadMetadataPatch +): ThreadListResponse { + return { + ...response, + threads: response.threads.map((item) => patchListItem(item, threadId, patch)), + archived_threads: response.archived_threads.map((item) => patchListItem(item, threadId, patch)), + }; +} + +function patchThreadListItems( + items: ThreadListItem[], + threadId: number, + patch: ThreadMetadataPatch +): ThreadListItem[] { + return items.map((item) => patchListItem(item, threadId, patch)); +} + +function patchThreadRecord( + record: ThreadRecord, + threadId: number, + patch: ThreadMetadataPatch +): ThreadRecord { + if (record.id !== threadId) return record; + return { + ...record, + ...patch, + }; +} + +function threadListQueryFilter(searchSpaceId: SearchSpaceKey) { + return { + predicate: ({ queryKey }: { queryKey: QueryKey }) => + Array.isArray(queryKey) && + queryKey[0] === "threads" && + isSameSearchSpace(queryKey[1], searchSpaceId), + }; +} + +function allThreadsQueryFilter(searchSpaceId: SearchSpaceKey) { + return { + predicate: ({ queryKey }: { queryKey: QueryKey }) => + Array.isArray(queryKey) && + queryKey[0] === "all-threads" && + isSameSearchSpace(queryKey[1], searchSpaceId), + }; +} + +function searchThreadsQueryFilter(searchSpaceId: SearchSpaceKey) { + return { + predicate: ({ queryKey }: { queryKey: QueryKey }) => + Array.isArray(queryKey) && + queryKey[0] === "search-threads" && + isSameSearchSpace(queryKey[1], searchSpaceId), + }; +} + +function threadDetailQueryFilter(threadId: number) { + return { + predicate: ({ queryKey }: { queryKey: QueryKey }) => + Array.isArray(queryKey) && + queryKey[0] === "threads" && + queryKey[1] === "detail" && + Number(queryKey[2]) === threadId, + }; +} + +function threadMessagesQueryFilter(threadId: number) { + return { + predicate: ({ queryKey }: { queryKey: QueryKey }) => + Array.isArray(queryKey) && + queryKey[0] === "threads" && + queryKey[1] === "messages" && + Number(queryKey[2]) === threadId, + }; +} + +function updateThreadListResponse( + queryClient: QueryClient, + filter: ReturnType, + threadId: number, + patch: ThreadMetadataPatch +): void { + queryClient.setQueriesData(filter, (old) => { + if (!isThreadListResponse(old)) return old; + return patchThreadListResponse(old, threadId, patch); + }); +} + +export function patchThreadEverywhere( + queryClient: QueryClient, + searchSpaceId: SearchSpaceKey, + threadId: number, + patch: ThreadMetadataPatch +): void { + updateThreadListResponse(queryClient, threadListQueryFilter(searchSpaceId), threadId, patch); + updateThreadListResponse(queryClient, allThreadsQueryFilter(searchSpaceId), threadId, patch); + + queryClient.setQueriesData(searchThreadsQueryFilter(searchSpaceId), (old) => { + if (!isThreadListItemArray(old)) return old; + return patchThreadListItems(old, threadId, patch); + }); + + queryClient.setQueriesData(threadDetailQueryFilter(threadId), (old) => { + if (!old) return old; + return patchThreadRecord(old, threadId, patch); + }); +} + +export function replaceThreadEverywhere( + queryClient: QueryClient, + searchSpaceId: SearchSpaceKey, + thread: ThreadRecord +): void { + patchThreadEverywhere(queryClient, searchSpaceId, thread.id, thread); +} + +export function removeThreadEverywhere( + queryClient: QueryClient, + searchSpaceId: SearchSpaceKey, + threadId: number +): void { + const removeFromListResponse = (old: ThreadListResponse | undefined) => { + if (!isThreadListResponse(old)) return old; + return { + ...old, + threads: old.threads.filter((thread) => thread.id !== threadId), + archived_threads: old.archived_threads.filter((thread) => thread.id !== threadId), + }; + }; + + queryClient.setQueriesData( + threadListQueryFilter(searchSpaceId), + removeFromListResponse + ); + queryClient.setQueriesData( + allThreadsQueryFilter(searchSpaceId), + removeFromListResponse + ); + queryClient.setQueriesData(searchThreadsQueryFilter(searchSpaceId), (old) => { + if (!isThreadListItemArray(old)) return old; + return old.filter((thread) => thread.id !== threadId); + }); + queryClient.removeQueries(threadDetailQueryFilter(threadId)); + queryClient.removeQueries(threadMessagesQueryFilter(threadId)); +} + +export function moveThreadArchiveState( + queryClient: QueryClient, + searchSpaceId: SearchSpaceKey, + threadId: number, + archived: boolean +): void { + const moveInListResponse = (old: ThreadListResponse | undefined) => { + if (!isThreadListResponse(old)) return old; + + const activeWithoutThread = old.threads.filter((thread) => thread.id !== threadId); + const archivedWithoutThread = old.archived_threads.filter((thread) => thread.id !== threadId); + const existing = + old.threads.find((thread) => thread.id === threadId) ?? + old.archived_threads.find((thread) => thread.id === threadId); + + if (!existing) return old; + + const updated = { ...existing, archived }; + + return { + ...old, + threads: archived ? activeWithoutThread : [updated, ...activeWithoutThread], + archived_threads: archived ? [updated, ...archivedWithoutThread] : archivedWithoutThread, + }; + }; + + queryClient.setQueriesData( + threadListQueryFilter(searchSpaceId), + moveInListResponse + ); + queryClient.setQueriesData( + allThreadsQueryFilter(searchSpaceId), + moveInListResponse + ); + queryClient.setQueriesData(searchThreadsQueryFilter(searchSpaceId), (old) => { + if (!isThreadListItemArray(old)) return old; + return old.map((thread) => (thread.id === threadId ? { ...thread, archived } : thread)); + }); + queryClient.setQueriesData(threadDetailQueryFilter(threadId), (old) => { + if (!old || old.id !== threadId) return old; + return { ...old, archived }; + }); +} diff --git a/surfsense_web/lib/desktop-download-utils.ts b/surfsense_web/lib/desktop-download-utils.ts index d4e2d4e68..5b861d7d9 100644 --- a/surfsense_web/lib/desktop-download-utils.ts +++ b/surfsense_web/lib/desktop-download-utils.ts @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from "react"; export type OSInfo = { - os: "macOS" | "Windows" | "Linux"; + os: "macOS" | "Windows" | "Linux" | "Android" | "iOS"; arch: "arm64" | "x64"; }; @@ -12,7 +12,13 @@ export function useUserOS(): OSInfo { let os: OSInfo["os"] = "macOS"; let arch: OSInfo["arch"] = "x64"; - if (/Windows/i.test(ua)) { + if (/Android/i.test(ua)) { + os = "Android"; + arch = "arm64"; + } else if (/iPhone|iPad|iPod/i.test(ua)) { + os = "iOS"; + arch = "arm64"; + } else if (/Windows/i.test(ua)) { os = "Windows"; arch = "x64"; } else if (/Linux/i.test(ua)) { @@ -88,9 +94,11 @@ export const GITHUB_RELEASES_URL = "https://github.com/MODSetter/SurfSense/relea export function usePrimaryDownload() { const { os, arch } = useUserOS(); const assets = useLatestRelease(); + const isMobileOS = os === "Android" || os === "iOS"; const { primary, alternatives } = useMemo(() => { if (assets.length === 0) return { primary: null, alternatives: [] }; + if (isMobileOS) return { primary: null, alternatives: assets }; const matchers: Record boolean> = { Windows: (n) => n.endsWith(".exe"), @@ -102,7 +110,7 @@ export function usePrimaryDownload() { const primary = assets.find((a) => match(a.name)) ?? null; const alternatives = assets.filter((a) => a !== primary); return { primary, alternatives }; - }, [assets, os, arch]); + }, [assets, os, arch, isMobileOS]); - return { os, arch, assets, primary, alternatives }; + return { os, arch, assets, primary, alternatives, isMobileOS }; } diff --git a/surfsense_web/lib/folder-sync-upload.ts b/surfsense_web/lib/folder-sync-upload.ts index 46ee7230a..334d9550d 100644 --- a/surfsense_web/lib/folder-sync-upload.ts +++ b/surfsense_web/lib/folder-sync-upload.ts @@ -16,7 +16,6 @@ export interface FolderSyncParams { searchSpaceId: number; excludePatterns: string[]; fileExtensions: string[]; - enableSummary: boolean; processingMode?: "basic" | "premium"; rootFolderId?: number | null; onProgress?: (progress: FolderSyncProgress) => void; @@ -62,7 +61,6 @@ async function uploadBatchesWithConcurrency( folderName: string; searchSpaceId: number; rootFolderId: number | null; - enableSummary: boolean; processingMode?: "basic" | "premium"; signal?: AbortSignal; onBatchComplete?: (filesInBatch: number) => void; @@ -100,7 +98,6 @@ async function uploadBatchesWithConcurrency( search_space_id: params.searchSpaceId, relative_paths: batch.map((e) => e.relativePath), root_folder_id: resolvedRootFolderId, - enable_summary: params.enableSummary, processing_mode: params.processingMode, }, params.signal @@ -147,7 +144,6 @@ export async function uploadFolderScan(params: FolderSyncParams): Promise { diff --git a/surfsense_web/lib/onboarding.ts b/surfsense_web/lib/onboarding.ts new file mode 100644 index 000000000..b87f822a0 --- /dev/null +++ b/surfsense_web/lib/onboarding.ts @@ -0,0 +1,8 @@ +export function isLlmOnboardingComplete( + agentLlmId: number | null | undefined, + hasGlobalConfigs: boolean +): boolean { + if (agentLlmId === null || agentLlmId === undefined) return false; + if (agentLlmId === 0) return hasGlobalConfigs; + return true; +} diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 49bcd8d0e..6f8885d7e 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -19,10 +19,8 @@ function stableEntries(obj: Record | null | undefined): unknown export const cacheKeys = { // New chat threads (assistant-ui) threads: { - list: (searchSpaceId: number) => ["threads", searchSpaceId] as const, detail: (threadId: number) => ["threads", "detail", threadId] as const, - search: (searchSpaceId: number, query: string) => - ["threads", "search", searchSpaceId, query] as const, + messages: (threadId: number) => ["threads", "messages", threadId] as const, }, documents: { globalQueryParams: (queries: GetDocumentsRequest["queryParams"]) => diff --git a/surfsense_web/lib/source.ts b/surfsense_web/lib/source.ts index 62fbb362b..f71e8b688 100644 --- a/surfsense_web/lib/source.ts +++ b/surfsense_web/lib/source.ts @@ -7,6 +7,7 @@ import { Download, FlaskConical, Heart, + MessageCircle, Radar, Unplug, Wrench, @@ -27,6 +28,7 @@ const DOCS_ICONS: Record = { Download, FlaskConical, Heart, + MessageCircle, Radar, Unplug, Wrench, diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index 97ae05101..a13942e64 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -650,13 +650,15 @@ "created": "Created" }, "sidebar": { - "chats": "Private Chats", + "recents": "Recents", + "chats": "Chats", "shared_chats": "Shared Chats", "search_chats": "Search chats", "no_chats_found": "No chats found", "no_shared_chats": "No shared chats", + "shared_chat": "Shared chat", "view_all_shared_chats": "View all shared chats", - "view_all_private_chats": "View all private chats", + "view_all_chats": "View all chats", "show_all": "Show all", "hide": "Hide", "no_chats": "No chats", diff --git a/surfsense_web/messages/es.json b/surfsense_web/messages/es.json index 2ac168d0a..33ae79c52 100644 --- a/surfsense_web/messages/es.json +++ b/surfsense_web/messages/es.json @@ -650,13 +650,15 @@ "created": "Creado" }, "sidebar": { - "chats": "Chats privados", + "recents": "Recientes", + "chats": "Chats", "shared_chats": "Chats compartidos", "search_chats": "Buscar chats", "no_chats_found": "No se encontraron chats", "no_shared_chats": "No hay chats compartidos", + "shared_chat": "Chat compartido", "view_all_shared_chats": "Ver todos los chats compartidos", - "view_all_private_chats": "Ver todos los chats privados", + "view_all_chats": "Ver todos los chats", "show_all": "Ver todo", "hide": "Ocultar", "no_chats": "Sin chats", diff --git a/surfsense_web/messages/hi.json b/surfsense_web/messages/hi.json index 2bfa9a59f..7a26d0c1d 100644 --- a/surfsense_web/messages/hi.json +++ b/surfsense_web/messages/hi.json @@ -650,13 +650,15 @@ "created": "बनाया गया" }, "sidebar": { - "chats": "निजी चैट", + "recents": "हालिया", + "chats": "चैट", "shared_chats": "साझा चैट", "search_chats": "चैट खोजें", "no_chats_found": "कोई चैट नहीं मिला", "no_shared_chats": "कोई साझा चैट नहीं", + "shared_chat": "साझा चैट", "view_all_shared_chats": "सभी साझा चैट देखें", - "view_all_private_chats": "सभी निजी चैट देखें", + "view_all_chats": "सभी चैट देखें", "show_all": "सभी देखें", "hide": "छिपाएँ", "no_chats": "कोई चैट नहीं", diff --git a/surfsense_web/messages/pt.json b/surfsense_web/messages/pt.json index 029959689..61c22e086 100644 --- a/surfsense_web/messages/pt.json +++ b/surfsense_web/messages/pt.json @@ -650,13 +650,15 @@ "created": "Criado" }, "sidebar": { - "chats": "Chats privados", + "recents": "Recentes", + "chats": "Chats", "shared_chats": "Chats compartilhados", "search_chats": "Pesquisar chats", "no_chats_found": "Nenhum chat encontrado", "no_shared_chats": "Nenhum chat compartilhado", + "shared_chat": "Chat compartilhado", "view_all_shared_chats": "Ver todos os chats compartilhados", - "view_all_private_chats": "Ver todos os chats privados", + "view_all_chats": "Ver todos os chats", "show_all": "Ver tudo", "hide": "Ocultar", "no_chats": "Nenhum chat", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index 6f5b2cafb..7d0419cbd 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -634,13 +634,15 @@ "created": "创建于" }, "sidebar": { - "chats": "私人对话", + "recents": "最近", + "chats": "对话", "shared_chats": "共享对话", "search_chats": "搜索对话...", "no_chats_found": "未找到对话", "no_shared_chats": "暂无共享对话", + "shared_chat": "共享对话", "view_all_shared_chats": "查看所有共享对话", - "view_all_private_chats": "查看所有私人对话", + "view_all_chats": "查看所有对话", "show_all": "查看全部", "hide": "隐藏", "no_chats": "无对话", diff --git a/surfsense_web/package.json b/surfsense_web/package.json index cc55d0d5d..2e999b42c 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.26", + "version": "0.0.27", "private": true, "packageManager": "pnpm@10.26.0", "description": "SurfSense Frontend", @@ -127,6 +127,7 @@ "postgres": "^3.4.7", "posthog-js": "^1.336.1", "posthog-node": "^5.24.4", + "qrcode.react": "^4.2.0", "radix-ui": "^1.4.3", "react": "^19.2.3", "react-day-picker": "^9.13.2", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index 7cbff6923..652eff8f5 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -302,6 +302,9 @@ importers: posthog-node: specifier: ^5.24.4 version: 5.24.17 + qrcode.react: + specifier: ^4.2.0 + version: 4.2.0(react@19.2.4) radix-ui: specifier: ^1.4.3 version: 1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -7740,6 +7743,11 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + qrcode.react@4.2.0: + resolution: {integrity: sha512-QpgqWi8rD9DsS9EP3z7BT+5lY5SFhsqGjpgW5DY/i3mK4M9DTBNz3ErMi8BWYEfI3L0d8GIbGmcdFAS1uIRGjA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + query-selector-shadow-dom@1.0.1: resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} @@ -16887,6 +16895,10 @@ snapshots: punycode@2.3.1: {} + qrcode.react@4.2.0(react@19.2.4): + dependencies: + react: 19.2.4 + query-selector-shadow-dom@1.0.1: {} queue-microtask@1.2.3: {} diff --git a/surfsense_web/source.config.ts b/surfsense_web/source.config.ts index 7ba4b89b5..8dbb15880 100644 --- a/surfsense_web/source.config.ts +++ b/surfsense_web/source.config.ts @@ -1,4 +1,5 @@ import { defineConfig, defineDocs, frontmatterSchema } from "fumadocs-mdx/config"; +import lastModified from "fumadocs-mdx/plugins/last-modified"; import { z } from "zod"; export const docs = defineDocs({ @@ -8,9 +9,8 @@ export const docs = defineDocs({ export const changelog = defineDocs({ dir: "changelog/content", docs: { - schema: frontmatterSchema.extend({ + schema: frontmatterSchema.omit({ title: true, description: true }).extend({ date: z.string(), - tags: z.array(z.string()).optional(), version: z.string().optional(), }), }, @@ -36,7 +36,7 @@ export const blog = defineDocs({ }); export default defineConfig({ - lastModifiedTime: "git", + plugins: [lastModified()], mdxOptions: { providerImportSource: "@/mdx-components", }, diff --git a/surfsense_web/tests/auth.setup.ts b/surfsense_web/tests/auth.setup.ts index a33a81b3c..7c1e37a39 100644 --- a/surfsense_web/tests/auth.setup.ts +++ b/surfsense_web/tests/auth.setup.ts @@ -1,5 +1,6 @@ import path from "node:path"; import { expect, test as setup } from "@playwright/test"; +import { announcements } from "../lib/announcements/announcements-data"; import { acquireTestToken } from "./helpers/api/auth"; /** @@ -7,21 +8,58 @@ import { acquireTestToken } from "./helpers/api/auth"; * e2e user (rate-limit-free /__e2e__/auth/token first, /auth/jwt/login * fallback) and persists it via localStorage so every test in the * chromium project starts already authenticated. + * + * Also pre-seeds the localStorage flags that gate the two new-user UI + * overlays so they never intercept clicks in journeys: + * - `surfsense_announcements_state` — the blocking AnnouncementSpotlight + * dialog (e.g. "Introducing AI Automations") plus its toasts. + * - `surfsense-tour-` — the OnboardingTour spotlight for new users. */ const authFile = path.join(__dirname, "..", "playwright", ".auth", "user.json"); const STORAGE_KEY = "surfsense_bearer_token"; +const ANNOUNCEMENTS_KEY = "surfsense_announcements_state"; + +/** Decode the user id (`sub`) from a JWT without verifying the signature. */ +function decodeUserId(token: string): string | null { + try { + const payload = token.split(".")[1]; + if (!payload) return null; + const json = Buffer.from(payload, "base64").toString("utf8"); + const obj = JSON.parse(json) as { sub?: string }; + return obj.sub ?? null; + } catch { + return null; + } +} setup("authenticate", async ({ page, request }) => { const access_token = await acquireTestToken(request); expect(access_token, "Failed to acquire e2e bearer token").toBeTruthy(); + const userId = decodeUserId(access_token); + // Mark every known announcement read + toasted so spotlight/toast + // announcements never overlay the dashboard during journeys. Sourced + // from the real data file so future announcements are covered too. + const announcementIds = announcements.map((a) => a.id); + const announcementState = { readIds: announcementIds, toastedIds: announcementIds }; + await page.addInitScript( - ({ key, token }) => { + ({ key, token, announcementsKey, state, uid }) => { localStorage.setItem(key, token); + localStorage.setItem(announcementsKey, JSON.stringify(state)); + if (uid) { + localStorage.setItem(`surfsense-tour-${uid}`, "true"); + } }, - { key: STORAGE_KEY, token: access_token } + { + key: STORAGE_KEY, + token: access_token, + announcementsKey: ANNOUNCEMENTS_KEY, + state: announcementState, + uid: userId, + } ); // Use a public page so the init script can write localStorage without diff --git a/surfsense_web/tests/helpers/api/documents.ts b/surfsense_web/tests/helpers/api/documents.ts index 92055602b..9658feead 100644 --- a/surfsense_web/tests/helpers/api/documents.ts +++ b/surfsense_web/tests/helpers/api/documents.ts @@ -43,7 +43,8 @@ export type EditorContent = { source_markdown: string; content_size_bytes: number; chunk_count: number; - truncated: boolean; + viewer_mode?: "plate" | "monaco"; + editor_plate_max_bytes?: number; }; // Same endpoint the UI hits when a user opens a document in the dashboard. diff --git a/surfsense_web/zero/queries/automations.ts b/surfsense_web/zero/queries/automations.ts new file mode 100644 index 000000000..79772eb1f --- /dev/null +++ b/surfsense_web/zero/queries/automations.ts @@ -0,0 +1,12 @@ +import { defineQuery } from "@rocicorp/zero"; +import { z } from "zod"; +import { zql } from "../schema/index"; + +// Mirrors chat byThread: client passes the parent id, the REST route still +// authorizes via `automation_id -> search_space`. No search_space_id on the +// table by design. +export const automationRunQueries = { + byAutomation: defineQuery(z.object({ automationId: z.number() }), ({ args: { automationId } }) => + zql.automation_runs.where("automationId", automationId).orderBy("createdAt", "desc") + ), +}; diff --git a/surfsense_web/zero/queries/index.ts b/surfsense_web/zero/queries/index.ts index fbf1bd76e..fe711f5d3 100644 --- a/surfsense_web/zero/queries/index.ts +++ b/surfsense_web/zero/queries/index.ts @@ -1,4 +1,5 @@ import { defineQueries } from "@rocicorp/zero"; +import { automationRunQueries } from "./automations"; import { chatSessionQueries, commentQueries, messageQueries } from "./chat"; import { connectorQueries, documentQueries } from "./documents"; import { folderQueries } from "./folders"; @@ -14,4 +15,5 @@ export const queries = defineQueries({ comments: commentQueries, chatSession: chatSessionQueries, user: userQueries, + automationRuns: automationRunQueries, }); diff --git a/surfsense_web/zero/schema/automations.ts b/surfsense_web/zero/schema/automations.ts new file mode 100644 index 000000000..4d6ebfac7 --- /dev/null +++ b/surfsense_web/zero/schema/automations.ts @@ -0,0 +1,18 @@ +import { json, number, string, table } from "@rocicorp/zero"; + +// Thin live row: status + per-step progress only. Heavy fields +// (definition_snapshot, inputs, output, artifacts, error) stay on REST +// (`GET /automations/{id}/runs/{run_id}`) and load on detail expand. +// Mirrors the publication shape in migration 148. +export const automationRunTable = table("automation_runs") + .columns({ + id: number(), + automationId: number().from("automation_id"), + triggerId: number().optional().from("trigger_id"), + status: string(), + stepResults: json().from("step_results"), + startedAt: number().optional().from("started_at"), + finishedAt: number().optional().from("finished_at"), + createdAt: number().from("created_at"), + }) + .primaryKey("id"); diff --git a/surfsense_web/zero/schema/chat.ts b/surfsense_web/zero/schema/chat.ts index fb3d7651e..8da41ee45 100644 --- a/surfsense_web/zero/schema/chat.ts +++ b/surfsense_web/zero/schema/chat.ts @@ -8,6 +8,8 @@ export const newChatMessageTable = table("new_chat_messages") threadId: number().from("thread_id"), authorId: string().optional().from("author_id"), createdAt: number().from("created_at"), + source: string(), + platformMetadata: json().optional().from("platform_metadata"), // Per-turn correlation id sourced from ``configurable.turn_id`` // at streaming time. Required by the inline Revert button's // (chat_turn_id, tool_name, position) fallback in tool-fallback.tsx diff --git a/surfsense_web/zero/schema/documents.ts b/surfsense_web/zero/schema/documents.ts index d1ada4bc4..988056297 100644 --- a/surfsense_web/zero/schema/documents.ts +++ b/surfsense_web/zero/schema/documents.ts @@ -21,7 +21,6 @@ export const searchSourceConnectorTable = table("search_source_connectors") isIndexable: boolean().from("is_indexable"), lastIndexedAt: number().optional().from("last_indexed_at"), config: json(), - enableSummary: boolean().from("enable_summary"), periodicIndexingEnabled: boolean().from("periodic_indexing_enabled"), indexingFrequencyMinutes: number().optional().from("indexing_frequency_minutes"), nextScheduledAt: number().optional().from("next_scheduled_at"), diff --git a/surfsense_web/zero/schema/index.ts b/surfsense_web/zero/schema/index.ts index 3cca0f24a..d6731e371 100644 --- a/surfsense_web/zero/schema/index.ts +++ b/surfsense_web/zero/schema/index.ts @@ -1,4 +1,5 @@ import { createBuilder, createSchema, relationships } from "@rocicorp/zero"; +import { automationRunTable } from "./automations"; import { chatCommentTable, chatSessionStateTable, newChatMessageTable } from "./chat"; import { documentTable, searchSourceConnectorTable } from "./documents"; import { folderTable } from "./folders"; @@ -36,6 +37,7 @@ export const schema = createSchema({ chatCommentTable, chatSessionStateTable, userTable, + automationRunTable, ], relationships: [chatCommentRelationships, newChatMessageRelationships], });