diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 000000000..7695369a0 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,216 @@ +name: Code Quality Checks + +on: + pull_request: + branches: [main, dev] + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + file-quality: + name: File Quality Checks + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch base branch + run: | + # Ensure we have the base branch reference for comparison + git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} 2>/dev/null || git fetch origin ${{ github.base_ref }} 2>/dev/null || true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pre-commit + run: pip install pre-commit + + - name: Cache pre-commit hooks + uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + pre-commit- + + - name: Install hook environments (cache) + run: pre-commit install-hooks + + - name: Run file quality checks on changed files + run: | + # Use pre-commit's native diff detection to run only on changed files + if git show-ref --verify --quiet refs/heads/${{ github.base_ref }}; then + echo "Running pre-commit with native diff detection against ${{ github.base_ref }} branch" + pre-commit run --from-ref ${{ github.base_ref }} --to-ref HEAD \ + check-yaml check-json check-toml check-merge-conflict \ + check-added-large-files debug-statements check-case-conflict + elif git show-ref --verify --quiet refs/remotes/origin/${{ github.base_ref }}; then + echo "Running pre-commit with native diff detection against origin/${{ github.base_ref }}" + pre-commit run --from-ref origin/${{ github.base_ref }} --to-ref HEAD \ + check-yaml check-json check-toml check-merge-conflict \ + check-added-large-files debug-statements check-case-conflict + else + echo "Base branch reference not found, running pre-commit on all files" + echo "⚠️ This may take longer and show more issues than normal" + pre-commit run --all-files \ + check-yaml check-json check-toml check-merge-conflict \ + check-added-large-files debug-statements check-case-conflict + fi + + security-scan: + name: Security Scan + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch base branch + run: | + git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} 2>/dev/null || git fetch origin ${{ github.base_ref }} 2>/dev/null || true + + - name: Get changed files + id: changed-files + run: | + if git show-ref --verify --quiet refs/heads/${{ github.base_ref }}; then + BASE_REF="${{ github.base_ref }}" + elif git show-ref --verify --quiet refs/remotes/origin/${{ github.base_ref }}; then + BASE_REF="origin/${{ github.base_ref }}" + else + echo "changed_files=all" >> $GITHUB_OUTPUT + exit 0 + fi + + # Get list of changed files, excluding the patterns we don't want to scan + CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT $BASE_REF...HEAD | \ + grep -v -E '\.(env\.example|env\.template)$|/tests/|test.*\.py$|^test_.*\.py$|\.github/workflows/.*\.ya?ml$|pnpm-lock\.yaml$|alembic\.ini$|alembic/versions/.*\.py$|\.mdx$' || true) + + if [ -z "$CHANGED_FILES" ]; then + echo "No relevant files changed for security scan" + echo "changed_files=none" >> $GITHUB_OUTPUT + else + echo "Changed files for security scan:" + echo "$CHANGED_FILES" + # Convert to space-separated string for detect-secrets + CHANGED_FILES_STR=$(echo "$CHANGED_FILES" | tr '\n' ' ') + echo "changed_files=$CHANGED_FILES_STR" >> $GITHUB_OUTPUT + fi + + - name: Set up Python + if: steps.changed-files.outputs.changed_files != 'none' + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install detect-secrets + if: steps.changed-files.outputs.changed_files != 'none' + run: pip install detect-secrets + + - name: Run detect-secrets scan on changed files + if: steps.changed-files.outputs.changed_files != 'none' && steps.changed-files.outputs.changed_files != 'all' + run: | + CHANGED_FILES="${{ steps.changed-files.outputs.changed_files }}" + if [ -n "$CHANGED_FILES" ]; then + detect-secrets scan --baseline .secrets.baseline $CHANGED_FILES + fi + + - name: Run detect-secrets scan on all files + if: steps.changed-files.outputs.changed_files == 'all' + run: | + detect-secrets scan --baseline .secrets.baseline --exclude-files '.*\.env\.example|.*\.env\.template|.*/tests/.*|.*test.*\.py|test_.*\.py|.github/workflows/.*\.yml|.github/workflows/.*\.yaml|.*pnpm-lock\.yaml|.*alembic\.ini|.*alembic/versions/.*\.py|.*\.mdx$' + + python-backend: + name: Python Backend Quality + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install UV + uses: astral-sh/setup-uv@v3 + + - name: Check if backend files changed + id: backend-changes + uses: dorny/paths-filter@v3 + with: + filters: | + backend: + - 'surfsense_backend/**' + + - name: Cache dependencies + if: steps.backend-changes.outputs.backend == 'true' + uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + surfsense_backend/.venv + key: python-deps-${{ hashFiles('surfsense_backend/uv.lock') }} + + - name: Install dependencies + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: uv sync + + - name: Run Ruff linting + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: | + # Ruff will automatically use pyproject.toml configuration + uv run ruff check . --output-format=github + + - name: Run Ruff formatting check + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: | + # Ruff will automatically use pyproject.toml configuration + uv run ruff format --check . --diff + + - name: Install Bandit + if: steps.backend-changes.outputs.backend == 'true' + run: pip install bandit + + - name: Run Bandit security scan + if: steps.backend-changes.outputs.backend == 'true' + working-directory: surfsense_backend + run: | + bandit -r . -f json --severity-level high --confidence-level high --exclude ./tests/,./test_*.py,./*test*.py,./alembic/ + + quality-gate: + name: Quality Gate + runs-on: ubuntu-latest + needs: [file-quality, security-scan, python-backend, frontend-quality] + if: always() + + steps: + - name: Check all jobs status + run: | + if [[ "${{ needs.file-quality.result }}" == "failure" || + "${{ needs.security-scan.result }}" == "failure" || + "${{ needs.python-backend.result }}" == "failure" || + "${{ needs.frontend-quality.result }}" == "failure" ]]; then + echo "❌ Code quality checks failed" + exit 1 + else + echo "✅ All code quality checks passed" + fi \ No newline at end of file diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 348ed8516..4a57aab05 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,7 +3,7 @@ name: pre-commit on: push: pull_request: - branches: [main] + branches: [main, dev] jobs: pre-commit: