diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 349cb0307..91dfb35de 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -78,6 +78,10 @@ STT_SERVICE=local/base # STT_SERVICE_API_BASE= +# (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) +PAGES_LIMIT=500 + + FIRECRAWL_API_KEY=fcr-01J0000000000000000000000 # File Parser Service diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index efa51d2db..f279f2708 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -200,6 +200,9 @@ class Config: # ETL Service ETL_SERVICE = os.getenv("ETL_SERVICE") + # Pages limit for ETL services (default to very high number for OSS unlimited usage) + PAGES_LIMIT = int(os.getenv("PAGES_LIMIT", "999999999")) + if ETL_SERVICE == "UNSTRUCTURED": # Unstructured API Key UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 7ad01b01a..c6cdf672d 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -710,7 +710,7 @@ if config.AUTH_TYPE == "GOOGLE": ) # Page usage tracking for ETL services - pages_limit = Column(Integer, nullable=False, default=500, server_default="500") + pages_limit = Column(Integer, nullable=False, default=config.PAGES_LIMIT, server_default=str(config.PAGES_LIMIT)) pages_used = Column(Integer, nullable=False, default=0, server_default="0") else: @@ -731,7 +731,7 @@ else: ) # Page usage tracking for ETL services - pages_limit = Column(Integer, nullable=False, default=500, server_default="500") + pages_limit = Column(Integer, nullable=False, default=config.PAGES_LIMIT, server_default=str(config.PAGES_LIMIT)) pages_used = Column(Integer, nullable=False, default=0, server_default="0") diff --git a/surfsense_web/content/docs/docker-installation.mdx b/surfsense_web/content/docs/docker-installation.mdx index 32532725b..575e6bf5e 100644 --- a/surfsense_web/content/docs/docker-installation.mdx +++ b/surfsense_web/content/docs/docker-installation.mdx @@ -229,6 +229,7 @@ Before you begin, ensure you have: | CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | +| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | **Optional Backend LangSmith Observability:** diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index 228eeb190..f1d81e3a6 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -89,6 +89,7 @@ Edit the `.env` file and set the following variables: | CELERY_RESULT_BACKEND | Redis connection URL for Celery result backend (e.g., `redis://localhost:6379/0`) | | SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | | REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | +| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | **(Optional) Backend LangSmith Observability:**