From 500bc60d02c58eee558629d353911c6e6587f165 Mon Sep 17 00:00:00 2001 From: Differ <1467673018@qq.com> Date: Fri, 5 Dec 2025 09:58:49 +0800 Subject: [PATCH] fix: add input validation, retry limit, code formatting, and exclude i18n from secret detection --- .pre-commit-config.yaml | 3 +- .../app/connectors/bookstack_connector.py | 30 +++++++++++++++---- .../connector_indexers/bookstack_indexer.py | 22 +++++++------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b7ce634..c02d2068 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,8 @@ repos: .*pnpm-lock\.yaml| .*alembic\.ini| .*alembic/versions/.*\.py| - .*\.mdx$ + .*\.mdx$| + .*/messages/.*\.json$ )$ # Python Backend Hooks (surfsense_backend) - Using Ruff for linting and formatting diff --git a/surfsense_backend/app/connectors/bookstack_connector.py b/surfsense_backend/app/connectors/bookstack_connector.py index 547dffa5..138f2a82 100644 --- a/surfsense_backend/app/connectors/bookstack_connector.py +++ b/surfsense_backend/app/connectors/bookstack_connector.py @@ -42,9 +42,7 @@ class BookStackConnector: self.token_secret = token_secret self._last_request_time = 0.0 - def set_credentials( - self, base_url: str, token_id: str, token_secret: str - ) -> None: + def set_credentials(self, base_url: str, token_id: str, token_secret: str) -> None: """ Set the BookStack credentials. @@ -52,7 +50,17 @@ class BookStackConnector: base_url: BookStack instance base URL token_id: BookStack API Token ID token_secret: BookStack API Token Secret + + Raises: + ValueError: If any required credential is missing or invalid """ + if not base_url or not isinstance(base_url, str): + raise ValueError("base_url must be a non-empty string") + if not token_id or not isinstance(token_id, str): + raise ValueError("token_id must be a non-empty string") + if not token_secret or not isinstance(token_secret, str): + raise ValueError("token_secret must be a non-empty string") + self.base_url = base_url.rstrip("/") self.token_id = token_id self.token_secret = token_secret @@ -86,11 +94,15 @@ class BookStackConnector: time.sleep(self.REQUEST_INTERVAL - elapsed) self._last_request_time = time.time() + # Maximum retries for rate limit errors + MAX_RATE_LIMIT_RETRIES = 3 + def make_api_request( self, endpoint: str, params: dict[str, Any] | None = None, raw_response: bool = False, + _retry_count: int = 0, ) -> dict[str, Any] | str: """ Make a request to the BookStack API. @@ -128,9 +140,17 @@ class BookStackConnector: except requests.exceptions.HTTPError as e: if e.response.status_code == 429: - logger.warning("Rate limit exceeded, waiting 60 seconds...") + if _retry_count >= self.MAX_RATE_LIMIT_RETRIES: + raise Exception( + f"BookStack API rate limit exceeded after {self.MAX_RATE_LIMIT_RETRIES} retries" + ) from e + logger.warning( + f"Rate limit exceeded, waiting 60 seconds... (retry {_retry_count + 1}/{self.MAX_RATE_LIMIT_RETRIES})" + ) time.sleep(60) - return self.make_api_request(endpoint, params, raw_response) + return self.make_api_request( + endpoint, params, raw_response, _retry_count + 1 + ) raise Exception(f"BookStack API request failed: {e!s}") from e except requests.exceptions.RequestException as e: raise Exception(f"BookStack API request failed: {e!s}") from e diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index f3994a45..6dc9de46 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -87,7 +87,11 @@ async def index_bookstack_pages( bookstack_token_id = connector.config.get("BOOKSTACK_TOKEN_ID") bookstack_token_secret = connector.config.get("BOOKSTACK_TOKEN_SECRET") - if not bookstack_base_url or not bookstack_token_id or not bookstack_token_secret: + if ( + not bookstack_base_url + or not bookstack_token_id + or not bookstack_token_secret + ): await task_logger.log_task_failure( log_entry, f"BookStack credentials not found in connector config for connector {connector_id}", @@ -196,9 +200,7 @@ async def index_bookstack_pages( page_id, use_markdown=True ) except Exception as e: - logger.warning( - f"Failed to fetch content for page {page_name}: {e}" - ) + logger.warning(f"Failed to fetch content for page {page_name}: {e}") skipped_pages.append(f"{page_name} (content fetch error)") documents_skipped += 1 continue @@ -275,7 +277,9 @@ async def index_bookstack_pages( full_content, user_llm, summary_metadata ) else: - summary_content = f"BookStack Page: {page_name}\n\nBook ID: {book_id}\n\n" + summary_content = ( + f"BookStack Page: {page_name}\n\nBook ID: {book_id}\n\n" + ) if page_content: content_preview = page_content[:1000] if len(page_content) > 1000: @@ -299,9 +303,7 @@ async def index_bookstack_pages( existing_document.chunks = chunks documents_indexed += 1 - logger.info( - f"Successfully updated BookStack page {page_name}" - ) + logger.info(f"Successfully updated BookStack page {page_name}") continue # Document doesn't exist - create new one @@ -388,9 +390,7 @@ async def index_bookstack_pages( f"Final commit: Total {documents_indexed} BookStack pages processed" ) await session.commit() - logger.info( - "Successfully committed all BookStack document changes to database" - ) + logger.info("Successfully committed all BookStack document changes to database") # Log success await task_logger.log_task_success(