feat(fix): add PAGES_LIMIT configuration for ETL services

This commit is contained in:
DESKTOP-RTLN3BA\$punk 2025-12-11 00:29:56 -08:00
parent 0c6a519c37
commit 3207fbf9ea
5 changed files with 11 additions and 2 deletions

View file

@ -200,6 +200,9 @@ class Config:
# ETL Service
ETL_SERVICE = os.getenv("ETL_SERVICE")
# Pages limit for ETL services (default to very high number for OSS unlimited usage)
PAGES_LIMIT = int(os.getenv("PAGES_LIMIT", "999999999"))
if ETL_SERVICE == "UNSTRUCTURED":
# Unstructured API Key
UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY")