apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-29 19:35:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(fix): add PAGES_LIMIT configuration for ETL services

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2025-12-11 00:29:56 -08:00
parent 0c6a519c37
commit 3207fbf9ea
5 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
surfsense_backend/.env.example
View file

@ -78,6 +78,10 @@ STT_SERVICE=local/base
# STT_SERVICE_API_BASE= # STT_SERVICE_API_BASE=
# (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version)
PAGES_LIMIT=500
FIRECRAWL_API_KEY=fcr-01J0000000000000000000000 FIRECRAWL_API_KEY=fcr-01J0000000000000000000000
# File Parser Service # File Parser Service

3
surfsense_backend/app/config/__init__.py
View file

@ -200,6 +200,9 @@ class Config:
# ETL Service # ETL Service
ETL_SERVICE = os.getenv("ETL_SERVICE") ETL_SERVICE = os.getenv("ETL_SERVICE")
# Pages limit for ETL services (default to very high number for OSS unlimited usage)
PAGES_LIMIT = int(os.getenv("PAGES_LIMIT", "999999999"))
if ETL_SERVICE == "UNSTRUCTURED": if ETL_SERVICE == "UNSTRUCTURED":
# Unstructured API Key # Unstructured API Key
UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY") UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY")

4
surfsense_backend/app/db.py
View file

@ -710,7 +710,7 @@ if config.AUTH_TYPE == "GOOGLE":
) )
# Page usage tracking for ETL services # Page usage tracking for ETL services
pages_limit = Column(Integer, nullable=False, default=500, server_default="500") pages_limit = Column(Integer, nullable=False, default=config.PAGES_LIMIT, server_default=str(config.PAGES_LIMIT))
pages_used = Column(Integer, nullable=False, default=0, server_default="0") pages_used = Column(Integer, nullable=False, default=0, server_default="0")
else: else:
@ -731,7 +731,7 @@ else:
) )
# Page usage tracking for ETL services # Page usage tracking for ETL services
pages_limit = Column(Integer, nullable=False, default=500, server_default="500") pages_limit = Column(Integer, nullable=False, default=config.PAGES_LIMIT, server_default=str(config.PAGES_LIMIT))
pages_used = Column(Integer, nullable=False, default=0, server_default="0") pages_used = Column(Integer, nullable=False, default=0, server_default="0")

1
surfsense_web/content/docs/docker-installation.mdx
View file

@ -229,6 +229,7 @@ Before you begin, ensure you have:
| CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | | CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) |
| SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) |
| REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) |
| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) |
**Optional Backend LangSmith Observability:** **Optional Backend LangSmith Observability:**

1
surfsense_web/content/docs/manual-installation.mdx
View file

@ -89,6 +89,7 @@ Edit the `.env` file and set the following variables:
| CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | | CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) |
| SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `<number><unit>` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) |
| REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) |
| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) |
**(Optional) Backend LangSmith Observability:** **(Optional) Backend LangSmith Observability:**