feat: added celery and removed background_tasks for MQ's

- removed pre commit hooks
- updated docker setup
- updated github docker actions
- updated docs

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,16 +1,13 @@
-<!--- Provide a general summary of your changes in the Title above -->
+<!--- Summarize your pull request in a few sentences -->
 
 ## Description
-<!--- Describe your changes in detail -->
+<!--- Clearly describe what has changed in this pull request -->
 
 ## Motivation and Context
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If this PR relates to an open issue, please link to the issue here: FIX #123 -->
 FIX #
 
-## Changes Overview
-<!-- List the primary changes/improvements made in this PR -->
-- 
 
 ## Screenshots
 <!-- If applicable, add screenshots or images to demonstrate the changes visually -->
@@ -19,27 +16,26 @@ FIX #
 <!-- Document any API changes if applicable -->
 - [ ] This PR includes API changes
 
-## Types of changes
-<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
-- [ ] Bug fix (non-breaking change which fixes an issue)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Performance improvement (non-breaking change which enhances performance)
-- [ ] Documentation update
-- [ ] Breaking change (fix or feature that would cause existing functionality to change)
+## Change Type
+<!--- Indicate what kind(s) of changes this PR includes: -->
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Performance improvement
+- [ ] Refactoring
+- [ ] Documentation
+- [ ] Dependency/Build system
+- [ ] Breaking change
+- [ ] Other (specify):
 
-## Testing
-<!-- Describe the tests that have been run to verify your changes -->
-- [ ] I have tested these changes locally
-- [ ] I have added/updated unit tests
-- [ ] I have added/updated integration tests
+## Testing Performed
+<!--- Briefly describe how you have tested these changes and what verification was performed -->
+- [ ] Tested locally
+- [ ] Manual/QA verification
 
-## Checklist:
-<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
-<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the code style of this project
-- [ ] My change requires documentation updates
-- [ ] I have updated the documentation accordingly
-- [ ] My change requires dependency updates
-- [ ] I have updated the dependencies accordingly
-- [ ] My code builds clean without any errors or warnings
-- [ ] All new and existing tests passed 
+## Checklist
+<!--- Please confirm the following by marking with an 'x' as appropriate -->
+- [ ] Follows project coding standards and conventions
+- [ ] Documentation updated as needed
+- [ ] Dependencies updated as needed
+- [ ] No lint/build errors or new warnings
+- [ ] All relevant tests are passing

.github/workflows/docker-publish.yml

@@ -4,40 +4,40 @@ on:
   workflow_dispatch:
 
 jobs:
-  build_and_push_backend:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+  # build_and_push_backend:
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     contents: read
+  #     packages: write
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v3
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
 
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+  #     - name: Log in to GitHub Container Registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.actor }}
+  #         password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push backend image
-        uses: docker/build-push-action@v5
-        with:
-          context: ./surfsense_backend
-          file: ./surfsense_backend/Dockerfile
-          push: true
-          tags: ghcr.io/${{ github.repository_owner }}/surfsense_backend:${{ github.sha }}
-          platforms: linux/amd64,linux/arm64
-          labels: |
-            org.opencontainers.image.source=${{ github.repositoryUrl }}
-            org.opencontainers.image.created=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
-            org.opencontainers.image.revision=${{ github.sha }}
+  #     - name: Build and push backend image
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: ./surfsense_backend
+  #         file: ./surfsense_backend/Dockerfile
+  #         push: true
+  #         tags: ghcr.io/${{ github.repository_owner }}/surfsense_backend:${{ github.sha }}
+  #         platforms: linux/amd64,linux/arm64
+  #         labels: |
+  #           org.opencontainers.image.source=${{ github.repositoryUrl }}
+  #           org.opencontainers.image.created=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }}
+  #           org.opencontainers.image.revision=${{ github.sha }}
 
   build_and_push_frontend:
     runs-on: ubuntu-latest

.github/workflows/docker_build.yaml

@@ -124,52 +124,52 @@ jobs:
           git ls-remote --tags origin | grep "refs/tags/${{ steps.tag_version.outputs.next_version }}" || (echo "Tag push verification failed!" && exit 1)
           echo "Tag successfully pushed."
   
-  build_and_push_backend_image: 
-    runs-on: ubuntu-latest
-    needs: tag_release # Depends on the tag being created successfully
-    permissions:
-      packages: write # Need permission to write to GHCR
-      contents: read # Need permission to read repo contents (checkout)
+  # build_and_push_backend_image: 
+  #   runs-on: ubuntu-latest
+  #   needs: tag_release # Depends on the tag being created successfully
+  #   permissions:
+  #     packages: write # Need permission to write to GHCR
+  #     contents: read # Need permission to read repo contents (checkout)
 
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+  #   steps:
+  #     - name: Checkout code
+  #       uses: actions/checkout@v4
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+  #     - name: Login to GitHub Container Registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: ghcr.io
+  #         username: ${{ github.repository_owner }}
+  #         password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v3
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
 
-      - name: Extract metadata (tags, labels) for Docker build
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ghcr.io/${{ github.repository_owner }}/surfsense_backend
-          tags: |
-            # Use the tag generated in the previous job
-            type=raw,value=${{ needs.tag_release.outputs.new_tag }}
-            # Optionally add 'latest' tag if building from the default branch
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch }}
+  #     - name: Extract metadata (tags, labels) for Docker build
+  #       id: meta
+  #       uses: docker/metadata-action@v5
+  #       with:
+  #         images: ghcr.io/${{ github.repository_owner }}/surfsense_backend
+  #         tags: |
+  #           # Use the tag generated in the previous job
+  #           type=raw,value=${{ needs.tag_release.outputs.new_tag }}
+  #           # Optionally add 'latest' tag if building from the default branch
+  #           type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || github.event.inputs.branch == github.event.repository.default_branch }}
 
-      - name: Build and push surfsense backend
-        uses: docker/build-push-action@v5
-        with:
-          context: ./surfsense_backend
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64,linux/arm64
-          # Optional: Add build cache for faster builds
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+  #     - name: Build and push surfsense backend
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: ./surfsense_backend
+  #         push: true
+  #         tags: ${{ steps.meta.outputs.tags }}
+  #         labels: ${{ steps.meta.outputs.labels }}
+  #         platforms: linux/amd64,linux/arm64
+  #         # Optional: Add build cache for faster builds
+  #         cache-from: type=gha
+  #         cache-to: type=gha,mode=max
 
   build_and_push_ui_image: 
     runs-on: ubuntu-latest

.github/workflows/pre-commit.yml

@@ -1,59 +0,0 @@
-name: pre-commit
-
-on:
-  push:
-  pull_request:
-    branches: [main, dev]
-
-jobs:
-  pre-commit:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # Required for detecting diffs
-
-      - name: Fetch main branch
-        run: |
-          # Ensure we have the main branch reference for comparison
-          git fetch origin main:main 2>/dev/null || git fetch origin main 2>/dev/null || true
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-
-      - name: Cache pre-commit environments
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pre-commit
-          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
-          restore-keys: |
-            pre-commit-
-
-      - name: Install pre-commit
-        run: |
-          pip install pre-commit
-
-      - name: Install hook environments (cache)
-        run: |
-          pre-commit install-hooks
-
-      - name: Run pre-commit on changed files
-        run: |
-          # Use pre-commit's native diff detection with fallback strategies
-          if git show-ref --verify --quiet refs/heads/main; then
-            # Main branch exists locally, use pre-commit's native diff mode
-            echo "Running pre-commit with native diff detection against main branch"
-            pre-commit run --from-ref main --to-ref HEAD
-          elif git show-ref --verify --quiet refs/remotes/origin/main; then
-            # Origin/main exists, use it as reference
-            echo "Running pre-commit with native diff detection against origin/main"
-            pre-commit run --from-ref origin/main --to-ref HEAD
-          else
-            # Fallback: run on all files (for first commits or when main is unavailable)
-            echo "Main branch reference not found, running pre-commit on all files"
-            echo "⚠️  This may take longer and show more issues than normal"
-            pre-commit run --all-files
-          fi