diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 000000000..6e0a6db6d --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,161 @@ +name: Backend Tests + +on: + pull_request: + branches: [main, dev] + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + env: + EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Check if backend files changed + id: backend-changes + uses: dorny/paths-filter@v3 + with: + filters: | + backend: + - 'surfsense_backend/**' + + - name: Set up Python + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install UV + if: steps.backend-changes.outputs.backend == 'true' + uses: astral-sh/setup-uv@v7 + + - name: Cache dependencies + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v5 + with: + path: | + ~/.cache/uv + surfsense_backend/.venv + key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }} + restore-keys: | + python-deps- + + - name: Cache HuggingFace models + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v5 + with: + path: ~/.cache/huggingface + key: hf-models-${{ env.EMBEDDING_MODEL }} + + - name: Install dependencies + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: uv sync + + - name: Run unit tests + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: uv run pytest -m unit + + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + env: + EMBEDDING_MODEL: sentence-transformers/all-MiniLM-L6-v2 + + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: surfsense_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d surfsense_test" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Check if backend files changed + id: backend-changes + uses: dorny/paths-filter@v3 + with: + filters: | + backend: + - 'surfsense_backend/**' + + - name: Set up Python + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install UV + if: steps.backend-changes.outputs.backend == 'true' + uses: astral-sh/setup-uv@v7 + + - name: Cache dependencies + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v5 + with: + path: | + ~/.cache/uv + surfsense_backend/.venv + key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }} + restore-keys: | + python-deps- + + - name: Cache HuggingFace models + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v5 + with: + path: ~/.cache/huggingface + key: hf-models-${{ env.EMBEDDING_MODEL }} + + - name: Install dependencies + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: uv sync + + - name: Run integration tests + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + env: + TEST_DATABASE_URL: postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense_test + SECRET_KEY: ci-test-secret-key-not-for-production + ETL_SERVICE: DOCLING + run: uv run pytest -m integration + + test-gate: + name: Test Gate + runs-on: ubuntu-latest + needs: [unit-tests, integration-tests] + if: always() + + steps: + - name: Check all test jobs + run: | + if [[ "${{ needs.unit-tests.result }}" == "failure" || + "${{ needs.integration-tests.result }}" == "failure" ]]; then + echo "Backend tests failed" + exit 1 + else + echo "All backend tests passed" + fi diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index 6daa171d6..41639fc2f 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -22,7 +22,7 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.pool import NullPool -from app.app import app +from app.app import app, limiter from app.config import config as app_config from app.db import Base from app.services.task_dispatcher import get_task_dispatcher @@ -35,6 +35,8 @@ from tests.utils.helpers import ( get_search_space_id, ) +limiter.enabled = False + _EMBEDDING_DIM = app_config.embedding_model_instance.dimension _ASYNCPG_URL = TEST_DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://") diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py index 228777626..2136f2152 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py @@ -4,6 +4,8 @@ from pydantic import ValidationError from app.db import DocumentType from app.indexing_pipeline.connector_document import ConnectorDocument +pytestmark = pytest.mark.unit + def test_valid_document_created_with_required_fields(): """All optional fields default correctly when only required fields are supplied."""