Merge pull request #863 from AnishSarkar22/feat/test-ci

feat: add GA CI pipeline for backend tests & few fixes

.github/workflows/backend-tests.yml

@@ -0,0 +1,161 @@
+name: Backend Tests
+
+on:
+  pull_request:
+    branches: [main, dev]
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    env:
+      EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Check if backend files changed
+        id: backend-changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            backend:
+              - 'surfsense_backend/**'
+
+      - name: Set up Python
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install UV
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: astral-sh/setup-uv@v7
+
+      - name: Cache dependencies
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/uv
+            surfsense_backend/.venv
+          key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }}
+          restore-keys: |
+            python-deps-
+
+      - name: Cache HuggingFace models
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/huggingface
+          key: hf-models-${{ env.EMBEDDING_MODEL }}
+
+      - name: Install dependencies
+        if: steps.backend-changes.outputs.backend == 'true'
+        working-directory: surfsense_backend
+        run: uv sync
+
+      - name: Run unit tests
+        if: steps.backend-changes.outputs.backend == 'true'
+        working-directory: surfsense_backend
+        run: uv run pytest -m unit
+
+  integration-tests:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    env:
+      EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: surfsense_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d surfsense_test"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Check if backend files changed
+        id: backend-changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            backend:
+              - 'surfsense_backend/**'
+
+      - name: Set up Python
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install UV
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: astral-sh/setup-uv@v7
+
+      - name: Cache dependencies
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/uv
+            surfsense_backend/.venv
+          key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }}
+          restore-keys: |
+            python-deps-
+
+      - name: Cache HuggingFace models
+        if: steps.backend-changes.outputs.backend == 'true'
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/huggingface
+          key: hf-models-${{ env.EMBEDDING_MODEL }}
+
+      - name: Install dependencies
+        if: steps.backend-changes.outputs.backend == 'true'
+        working-directory: surfsense_backend
+        run: uv sync
+
+      - name: Run integration tests
+        if: steps.backend-changes.outputs.backend == 'true'
+        working-directory: surfsense_backend
+        env:
+          TEST_DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense_test
+          SECRET_KEY: ci-test-secret-key-not-for-production
+          ETL_SERVICE: DOCLING
+        run: uv run pytest -m integration
+
+  test-gate:
+    name: Test Gate
+    runs-on: ubuntu-latest
+    needs: [unit-tests, integration-tests]
+    if: always()
+
+    steps:
+      - name: Check all test jobs
+        run: |
+          if [[ "${{ needs.unit-tests.result }}" == "failure" ||
+                "${{ needs.integration-tests.result }}" == "failure" ]]; then
+            echo "Backend tests failed"
+            exit 1
+          else
+            echo "All backend tests passed"
+          fi

surfsense_backend/tests/integration/document_upload/conftest.py

@@ -22,7 +22,7 @@ from sqlalchemy import text
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.pool import NullPool
 
-from app.app import app
+from app.app import app, limiter
 from app.config import config as app_config
 from app.db import Base
 from app.services.task_dispatcher import get_task_dispatcher
@@ -35,6 +35,8 @@ from tests.utils.helpers import (
     get_search_space_id,
 )
 
+limiter.enabled = False
+
 _EMBEDDING_DIM = app_config.embedding_model_instance.dimension
 _ASYNCPG_URL = TEST_DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://")
 

surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py

@@ -4,6 +4,8 @@ from pydantic import ValidationError
 from app.db import DocumentType
 from app.indexing_pipeline.connector_document import ConnectorDocument
 
+pytestmark = pytest.mark.unit
+
 
 def test_valid_document_created_with_required_fields():
     """All optional fields default correctly when only required fields are supplied."""