From 30cfb4d95d4a7f89932488d4996ac56a8c3581f0 Mon Sep 17 00:00:00 2001 From: Rohan Verma <122026167+MODSetter@users.noreply.github.com> Date: Mon, 20 Oct 2025 01:29:28 -0700 Subject: [PATCH 1/3] Update README to remove future work and details Removed future work section and some details about Redis, Celery, and Flower. --- README.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 643a717cb..e8023e22e 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,6 @@ Open source and easy to deploy locally. - Supports all major Rerankers (Pinecode, Cohere, Flashrank etc) - Uses Hierarchical Indices (2 tiered RAG setup). - Utilizes Hybrid Search (Semantic + Full Text Search combined with Reciprocal Rank Fusion). -- RAG as a Service API Backend. ### ℹ️ **External Sources** - Search Engines (Tavily, LinkUp) @@ -148,8 +147,6 @@ SurfSense provides two installation methods: - Supports environment variable customization via `.env` file - Flexible deployment options (full stack or core services only) - No need to manually edit configuration files between environments - - See [Docker Setup Guide](DOCKER_SETUP.md) for detailed instructions - - For deployment scenarios and options, see [Deployment Guide](DEPLOYMENT_GUIDE.md) 2. **[Manual Installation (Recommended)](https://www.surfsense.net/docs/manual-installation)** - For users who prefer more control over their setup or need to customize their deployment. @@ -220,6 +217,12 @@ Before installation, make sure to complete the [prerequisite setup steps](https: - **pgvector**: PostgreSQL extension for efficient vector similarity operations +- **Redis**: In-memory data structure store used as message broker and result backend for Celery + +- **Celery**: Distributed task queue for handling asynchronous background jobs (document processing, podcast generation, etc.) + +- **Flower**: Real-time monitoring and administration tool for Celery task queues + - **Chonkie**: Advanced document chunking and embedding library - Uses `AutoEmbeddings` for flexible embedding model selection - `LateChunker` for optimized document chunking based on embedding model's max sequence length @@ -269,12 +272,6 @@ Before installation, make sure to complete the [prerequisite setup steps](https: ### **Extension** Manifest v3 on Plasmo -## Future Work -- Add More Connectors. -- Patch minor bugs. -- Document Podcasts - - ## Contribute From 52713a6227d8cde2a53dc6b43fa95aa6a3d1f220 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 20 Oct 2025 01:33:21 -0700 Subject: [PATCH 2/3] chore: readded pre-commit-config.yaml --- ‎.pre-commit-config.yaml | 95 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 ‎.pre-commit-config.yaml diff --git a/‎.pre-commit-config.yaml b/‎.pre-commit-config.yaml new file mode 100644 index 000000000..7463c1e2b --- /dev/null +++ b/‎.pre-commit-config.yaml @@ -0,0 +1,95 @@ +# Pre-commit configuration for SurfSense +# See https://pre-commit.com for more information + +repos: + # General file quality hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + args: [--multi, --unsafe] + - id: check-json + exclude: '(tsconfig\.json|\.vscode/.*\.json)$' + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=10240] # 10MB limit + - id: debug-statements + - id: check-case-conflict + + # Security - detect secrets across all file types + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: | + (?x)^( + .*\.env\.example| + .*\.env\.template| + .*/tests/.*| + .*test.*\.py| + test_.*\.py| + .github/workflows/.*\.yml| + .github/workflows/.*\.yaml| + .*pnpm-lock\.yaml| + .*alembic\.ini| + .*alembic/versions/.*\.py| + .*\.mdx$ + )$ + + # Python Backend Hooks (surfsense_backend) - Using Ruff for linting and formatting + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.5 + hooks: + - id: ruff + name: ruff-check + files: ^surfsense_backend/ + exclude: ^surfsense_backend/(test_.*\.py|.*test.*\.py) + args: [--fix] + - id: ruff-format + name: ruff-format + files: ^surfsense_backend/ + exclude: ^surfsense_backend/(test_.*\.py|.*test.*\.py) + + - repo: https://github.com/PyCQA/bandit + rev: 1.8.6 + hooks: + - id: bandit + files: ^surfsense_backend/ + args: ['-f', 'json', '--severity-level', 'high', '--confidence-level', 'high'] + exclude: ^surfsense_backend/(tests/|test_.*\.py|.*test.*\.py|alembic/) + + # Biome hooks for TypeScript/JavaScript projects + - repo: local + hooks: + # Biome check for surfsense_web + - id: biome-check-web + name: biome-check-web + entry: bash -c 'cd surfsense_web && npx @biomejs/biome check --diagnostic-level=error .' + language: system + files: ^surfsense_web/ + pass_filenames: false + always_run: true + stages: [pre-commit] + + # Biome check for surfsense_browser_extension + # - id: biome-check-extension + # name: biome-check-extension + # entry: bash -c 'cd surfsense_browser_extension && npx @biomejs/biome check --diagnostic-level=error .' + # language: system + # files: ^surfsense_browser_extension/ + # pass_filenames: false + # always_run: true + # stages: [pre-commit] + + # Commit message linting + - repo: https://github.com/commitizen-tools/commitizen + rev: v4.8.3 + hooks: + - id: commitizen + stages: [commit-msg] + +# Global configuration +default_stages: [pre-commit] +fail_fast: false \ No newline at end of file From a7251393ab2367bfc744e2e256bcac9926663e40 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 20 Oct 2025 01:36:36 -0700 Subject: [PATCH 3/3] fix hook --- ‎.pre-commit-config.yaml => .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename ‎.pre-commit-config.yaml => .pre-commit-config.yaml (99%) diff --git a/‎.pre-commit-config.yaml b/.pre-commit-config.yaml similarity index 99% rename from ‎.pre-commit-config.yaml rename to .pre-commit-config.yaml index 7463c1e2b..7b7ce6340 100644 --- a/‎.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,4 +92,5 @@ repos: # Global configuration default_stages: [pre-commit] -fail_fast: false \ No newline at end of file +fail_fast: false +