From 9975e085aa17b624bc3a41e4e11e2b148c04a01a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 2 May 2026 16:10:30 +0530 Subject: [PATCH 1/7] refactor(memory): streamline memory extraction by utilizing extract_text_content utility --- .../app/agents/new_chat/memory_extraction.py | 13 +-- .../agents/new_chat/tools/update_memory.py | 17 ++-- surfsense_backend/app/routes/memory_routes.py | 7 +- .../app/routes/search_spaces_routes.py | 7 +- .../new_chat/test_memory_response_content.py | 87 +++++++++++++++++++ 5 files changed, 106 insertions(+), 25 deletions(-) create mode 100644 surfsense_backend/tests/unit/agents/new_chat/test_memory_response_content.py diff --git a/surfsense_backend/app/agents/new_chat/memory_extraction.py b/surfsense_backend/app/agents/new_chat/memory_extraction.py index 221c4c75a..e31774a7c 100644 --- a/surfsense_backend/app/agents/new_chat/memory_extraction.py +++ b/surfsense_backend/app/agents/new_chat/memory_extraction.py @@ -16,6 +16,7 @@ from sqlalchemy import select from app.agents.new_chat.tools.update_memory import _save_memory from app.db import SearchSpace, User, shielded_async_session +from app.utils.content_utils import extract_text_content logger = logging.getLogger(__name__) @@ -144,11 +145,7 @@ async def extract_and_save_memory( [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal", "memory-extraction"]}, ) - text = ( - response.content - if isinstance(response.content, str) - else str(response.content) - ).strip() + text = extract_text_content(response.content).strip() if text == "NO_UPDATE" or not text: logger.debug("Memory extraction: no update needed (user %s)", uid) @@ -207,11 +204,7 @@ async def extract_and_save_team_memory( [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal", "team-memory-extraction"]}, ) - text = ( - response.content - if isinstance(response.content, str) - else str(response.content) - ).strip() + text = extract_text_content(response.content).strip() if text == "NO_UPDATE" or not text: logger.debug( diff --git a/surfsense_backend/app/agents/new_chat/tools/update_memory.py b/surfsense_backend/app/agents/new_chat/tools/update_memory.py index 4128ac0dc..ceaddb80f 100644 --- a/surfsense_backend/app/agents/new_chat/tools/update_memory.py +++ b/surfsense_backend/app/agents/new_chat/tools/update_memory.py @@ -27,6 +27,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.db import SearchSpace, User +from app.utils.content_utils import extract_text_content logger = logging.getLogger(__name__) @@ -188,11 +189,7 @@ async def _forced_rewrite(content: str, llm: Any) -> str | None: [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal"]}, ) - text = ( - response.content - if isinstance(response.content, str) - else str(response.content) - ) + text = extract_text_content(response.content) return text.strip() except Exception: logger.exception("Forced rewrite LLM call failed") @@ -235,6 +232,16 @@ async def _save_memory( label : str Human label for log messages (e.g. "user memory", "team memory"). """ + if not isinstance(updated_memory, str): + logger.warning( + "Refusing non-string memory payload (type=%s)", + type(updated_memory).__name__, + ) + return { + "status": "error", + "message": "Internal error: memory payload must be a string.", + } + content = updated_memory # --- forced rewrite if over the hard limit --- diff --git a/surfsense_backend/app/routes/memory_routes.py b/surfsense_backend/app/routes/memory_routes.py index f5df45cf1..e57ca4055 100644 --- a/surfsense_backend/app/routes/memory_routes.py +++ b/surfsense_backend/app/routes/memory_routes.py @@ -16,6 +16,7 @@ from app.agents.new_chat.llm_config import ( from app.agents.new_chat.tools.update_memory import MEMORY_HARD_LIMIT, _save_memory from app.db import User, get_async_session from app.users import current_active_user +from app.utils.content_utils import extract_text_content logger = logging.getLogger(__name__) @@ -123,11 +124,7 @@ async def edit_user_memory( [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal", "memory-edit"]}, ) - updated = ( - response.content - if isinstance(response.content, str) - else str(response.content) - ).strip() + updated = extract_text_content(response.content).strip() except Exception as e: logger.exception("Memory edit LLM call failed: %s", e) raise HTTPException(status_code=500, detail="Memory edit failed.") from e diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index 72715ea5b..e44455ad3 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -35,6 +35,7 @@ from app.schemas import ( SearchSpaceWithStats, ) from app.users import current_active_user +from app.utils.content_utils import extract_text_content from app.utils.rbac import check_permission, check_search_space_access logger = logging.getLogger(__name__) @@ -356,11 +357,7 @@ async def edit_team_memory( [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal", "memory-edit"]}, ) - updated = ( - response.content - if isinstance(response.content, str) - else str(response.content) - ).strip() + updated = extract_text_content(response.content).strip() except Exception as e: logger.exception("Team memory edit LLM call failed: %s", e) raise HTTPException(status_code=500, detail="Team memory edit failed.") from e diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_memory_response_content.py b/surfsense_backend/tests/unit/agents/new_chat/test_memory_response_content.py new file mode 100644 index 000000000..535e4e940 --- /dev/null +++ b/surfsense_backend/tests/unit/agents/new_chat/test_memory_response_content.py @@ -0,0 +1,87 @@ +"""Unit tests for extracting text from LLM memory responses.""" + +import pytest + +from app.agents.new_chat.tools.update_memory import _save_memory +from app.utils.content_utils import extract_text_content + +pytestmark = pytest.mark.unit + + +class _Recorder: + def __init__(self) -> None: + self.applied_content: str | None = None + self.commit_calls = 0 + self.rollback_calls = 0 + + def apply(self, content: str) -> None: + self.applied_content = content + + async def commit(self) -> None: + self.commit_calls += 1 + + async def rollback(self) -> None: + self.rollback_calls += 1 + + +def test_extract_text_content_keeps_no_update_bare_string_from_content_blocks() -> None: + content = [ + {"type": "thinking", "thinking": "No"}, + {"type": "thinking", "thinking": " memorizable info."}, + "NO_UPDATE", + ] + + assert extract_text_content(content).strip() == "NO_UPDATE" + + +def test_extract_text_content_ignores_thinking_blocks_and_keeps_markdown_text() -> None: + markdown = ( + "## Work Context\n" + "- (2026-05-02) [fact] Anish is hardening SurfSense memory extraction.\n" + ) + content = [ + {"type": "thinking", "thinking": "This is durable context."}, + {"type": "text", "text": markdown}, + ] + + assert extract_text_content(content).strip() == markdown.strip() + + +def test_extract_text_content_returns_empty_when_only_thinking_blocks_are_present() -> None: + content = [ + {"type": "thinking", "thinking": "No durable fact."}, + {"type": "thinking", "thinking": "Return no update."}, + ] + + assert extract_text_content(content) == "" + + +def test_extract_text_content_preserves_plain_string_responses() -> None: + markdown = ( + "## Preferences\n" + "- (2026-05-02) [pref] Anish prefers no regex for memory validation.\n" + ) + + assert extract_text_content(markdown) == markdown + + +@pytest.mark.asyncio +async def test_save_memory_rejects_non_string_payload_before_commit() -> None: + recorder = _Recorder() + + result = await _save_memory( + updated_memory=["NO_UPDATE"], # type: ignore[arg-type] + old_memory=None, + llm=None, + apply_fn=recorder.apply, + commit_fn=recorder.commit, + rollback_fn=recorder.rollback, + label="memory", + scope="user", + ) + + assert result["status"] == "error" + assert "must be a string" in result["message"] + assert recorder.applied_content is None + assert recorder.commit_calls == 0 + assert recorder.rollback_calls == 0 From e38e20b48407da9205ca7b7e26a62f8b8959b126 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 4 May 2026 12:18:09 +0530 Subject: [PATCH 2/7] fix: handle empty response in forced rewrite function - Updated the `_forced_rewrite` function to strip whitespace from the extracted text and added a warning log if the response is empty, preventing potential issues with empty rewrites. --- .../app/agents/new_chat/tools/update_memory.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/agents/new_chat/tools/update_memory.py b/surfsense_backend/app/agents/new_chat/tools/update_memory.py index 42148967c..062668aac 100644 --- a/surfsense_backend/app/agents/new_chat/tools/update_memory.py +++ b/surfsense_backend/app/agents/new_chat/tools/update_memory.py @@ -189,8 +189,11 @@ async def _forced_rewrite(content: str, llm: Any) -> str | None: [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal"]}, ) - text = extract_text_content(response.content) - return text.strip() + text = extract_text_content(response.content).strip() + if not text: + logger.warning("Forced rewrite returned empty text; aborting rewrite") + return None + return text except Exception: logger.exception("Forced rewrite LLM call failed") return None From cb6357f57706ed895bc259c47907a765bb93b6a2 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 4 May 2026 19:55:19 -0700 Subject: [PATCH 3/7] feat: add CORS preflight response caching for 24 hours --- surfsense_backend/app/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index 08194e7fb..5057e7d00 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -754,6 +754,12 @@ app.add_middleware( allow_credentials=True, allow_methods=["*"], # Allows all methods allow_headers=["*"], # Allows all headers + # Cache CORS preflight (OPTIONS) responses for 24h. Browsers clamp: + # Chrome/Edge cap at 7200s, Firefox honours up to 86400s. Setting the + # higher value lets each browser cache for as long as it allows. This + # eliminates an OPTIONS round-trip on every non-simple request from + # FRONTEND_URL to BACKEND_URL. + max_age=86400, ) # Password / email-based auth routers are only mounted when not running in From 0b4bd50bce3abb842e827f75d680e9d4bbb39ffe Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 4 May 2026 21:08:08 -0700 Subject: [PATCH 4/7] feat: bumped version to 0.0.22 --- VERSION | 2 +- surfsense_backend/pyproject.toml | 2 +- surfsense_backend/uv.lock | 2 +- surfsense_browser_extension/package.json | 2 +- surfsense_desktop/package.json | 2 +- surfsense_web/package.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/VERSION b/VERSION index 236c7ad08..818944f5b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.21 +0.0.22 diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index da8c4b7d1..4235ac962 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "surf-new-backend" -version = "0.0.21" +version = "0.0.22" description = "SurfSense Backend" requires-python = ">=3.12" dependencies = [ diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 3e371cecc..4dd5156e7 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -7947,7 +7947,7 @@ wheels = [ [[package]] name = "surf-new-backend" -version = "0.0.21" +version = "0.0.22" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/surfsense_browser_extension/package.json b/surfsense_browser_extension/package.json index f127b85c0..b8b5cb2ec 100644 --- a/surfsense_browser_extension/package.json +++ b/surfsense_browser_extension/package.json @@ -1,7 +1,7 @@ { "name": "surfsense_browser_extension", "displayName": "Surfsense Browser Extension", - "version": "0.0.21", + "version": "0.0.22", "description": "Extension to collect Browsing History for SurfSense.", "author": "https://github.com/MODSetter", "engines": { diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 4826b904e..744ab65ab 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -1,6 +1,6 @@ { "name": "surfsense-desktop", - "version": "0.0.21", + "version": "0.0.22", "description": "SurfSense Desktop App", "main": "dist/main.js", "scripts": { diff --git a/surfsense_web/package.json b/surfsense_web/package.json index a34e8a269..2adec8638 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -1,6 +1,6 @@ { "name": "surfsense_web", - "version": "0.0.21", + "version": "0.0.22", "private": true, "description": "SurfSense Frontend", "scripts": { From 5ff6baedb392207a33e724c7531c4aa9005cf053 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 5 May 2026 00:17:44 -0700 Subject: [PATCH 5/7] fix: stripe weebhook --- surfsense_backend/app/routes/stripe_routes.py | 96 ++++++++++++------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/surfsense_backend/app/routes/stripe_routes.py b/surfsense_backend/app/routes/stripe_routes.py index aed74ec8d..2ae4ce79f 100644 --- a/surfsense_backend/app/routes/stripe_routes.py +++ b/surfsense_backend/app/routes/stripe_routes.py @@ -88,10 +88,22 @@ def _normalize_optional_string(value: Any) -> str | None: def _get_metadata(checkout_session: Any) -> dict[str, str]: + """Extract checkout session metadata as a plain ``str -> str`` dict. + + Works for both ``dict`` (e.g. when the metadata round-tripped through + JSON) and Stripe's ``StripeObject`` wrapper. Recent Stripe SDK + versions stopped subclassing ``dict`` for ``StripeObject``, so + ``isinstance(metadata, dict)`` is False and ``dict(metadata)`` falls + into the sequence protocol, looking up integer indices and raising + ``KeyError: 0``. ``.items()`` is exposed by both shapes via the + Mapping protocol, so we always use that. + """ metadata = getattr(checkout_session, "metadata", None) or {} - if isinstance(metadata, dict): - return {str(key): str(value) for key, value in metadata.items()} - return dict(metadata) + try: + items = metadata.items() + except AttributeError: + return {} + return {str(key): str(value) for key, value in items} async def _get_or_create_purchase_from_checkout_session( @@ -439,41 +451,55 @@ async def stripe_webhook( detail="Invalid Stripe webhook signature.", ) from exc - if event.type in { - "checkout.session.completed", - "checkout.session.async_payment_succeeded", - }: - checkout_session = event.data.object - payment_status = getattr(checkout_session, "payment_status", None) - - if event.type == "checkout.session.completed" and payment_status not in { - "paid", - "no_payment_required", + try: + if event.type in { + "checkout.session.completed", + "checkout.session.async_payment_succeeded", }: - logger.info( - "Received checkout.session.completed for unpaid session %s; waiting for async success.", - checkout_session.id, - ) - return StripeWebhookResponse() + checkout_session = event.data.object + payment_status = getattr(checkout_session, "payment_status", None) - metadata = _get_metadata(checkout_session) - purchase_type = metadata.get("purchase_type", "page_packs") - if purchase_type == "premium_tokens": - return await _fulfill_completed_token_purchase(db_session, checkout_session) - return await _fulfill_completed_purchase(db_session, checkout_session) + if event.type == "checkout.session.completed" and payment_status not in { + "paid", + "no_payment_required", + }: + logger.info( + "Received checkout.session.completed for unpaid session %s; waiting for async success.", + checkout_session.id, + ) + return StripeWebhookResponse() - if event.type in { - "checkout.session.async_payment_failed", - "checkout.session.expired", - }: - checkout_session = event.data.object - metadata = _get_metadata(checkout_session) - purchase_type = metadata.get("purchase_type", "page_packs") - if purchase_type == "premium_tokens": - return await _mark_token_purchase_failed( - db_session, str(checkout_session.id) - ) - return await _mark_purchase_failed(db_session, str(checkout_session.id)) + metadata = _get_metadata(checkout_session) + purchase_type = metadata.get("purchase_type", "page_packs") + if purchase_type == "premium_tokens": + return await _fulfill_completed_token_purchase( + db_session, checkout_session + ) + return await _fulfill_completed_purchase(db_session, checkout_session) + + if event.type in { + "checkout.session.async_payment_failed", + "checkout.session.expired", + }: + checkout_session = event.data.object + metadata = _get_metadata(checkout_session) + purchase_type = metadata.get("purchase_type", "page_packs") + if purchase_type == "premium_tokens": + return await _mark_token_purchase_failed( + db_session, str(checkout_session.id) + ) + return await _mark_purchase_failed(db_session, str(checkout_session.id)) + except Exception: + # Re-raise so FastAPI returns 500 and Stripe retries this delivery. + # Logging here gives us a structured trail with event id + type so + # future webhook bugs surface immediately in the logs without + # having to grep by request_id. + logger.exception( + "Stripe webhook handler failed for event id=%s type=%s — Stripe will retry", + getattr(event, "id", "?"), + getattr(event, "type", "?"), + ) + raise return StripeWebhookResponse() From 6e1dd405978510d78924dd3c56c75757e4416c18 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 5 May 2026 01:01:12 -0700 Subject: [PATCH 6/7] feat: implement finalize checkout endpoint and update purchase success handling - Added a new endpoint `/stripe/finalize-checkout` to synchronously fulfill a checkout session, addressing the webhook-vs-redirect race condition. - Updated the `PurchaseSuccessPage` component to handle various states of the checkout process, including loading, completed, pending, and failed states. - Introduced a new response model `FinalizeCheckoutResponse` to provide immediate feedback on the purchase status. - Enhanced the Stripe API service to include the new finalize checkout functionality. --- surfsense_backend/app/routes/stripe_routes.py | 158 +++++++++++++++++- surfsense_backend/app/schemas/stripe.py | 20 +++ .../purchase-success/page.tsx | 140 +++++++++++++++- surfsense_web/contracts/types/stripe.types.ts | 14 ++ surfsense_web/lib/apis/stripe-api.service.ts | 16 ++ 5 files changed, 333 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/routes/stripe_routes.py b/surfsense_backend/app/routes/stripe_routes.py index 2ae4ce79f..43fa8c4b7 100644 --- a/surfsense_backend/app/routes/stripe_routes.py +++ b/surfsense_backend/app/routes/stripe_routes.py @@ -26,6 +26,7 @@ from app.schemas.stripe import ( CreateCheckoutSessionResponse, CreateTokenCheckoutSessionRequest, CreateTokenCheckoutSessionResponse, + FinalizeCheckoutResponse, PagePurchaseHistoryResponse, StripeStatusResponse, StripeWebhookResponse, @@ -65,7 +66,15 @@ def _get_checkout_urls(search_space_id: int) -> tuple[str, str]: ) base_url = config.NEXT_FRONTEND_URL.rstrip("/") - success_url = f"{base_url}/dashboard/{search_space_id}/purchase-success" + # Stripe substitutes ``{CHECKOUT_SESSION_ID}`` with the actual session id + # at redirect time. The frontend uses it to call /stripe/finalize-checkout + # which fulfils synchronously without waiting for the webhook — fixing the + # webhook-vs-redirect race where users land on /purchase-success before + # checkout.session.completed has been delivered. + success_url = ( + f"{base_url}/dashboard/{search_space_id}/purchase-success" + f"?session_id={{CHECKOUT_SESSION_ID}}" + ) cancel_url = f"{base_url}/dashboard/{search_space_id}/purchase-cancel" return success_url, cancel_url @@ -106,6 +115,17 @@ def _get_metadata(checkout_session: Any) -> dict[str, str]: return {str(key): str(value) for key, value in items} +# Canonical purchase_type metadata values. ``premium_credit`` was emitted +# by an earlier release of ``create_token_checkout_session`` so it's still +# accepted on the read side for backward compat with in-flight sessions. +_PURCHASE_TYPE_TOKEN_VALUES = frozenset({"premium_tokens", "premium_credit"}) + + +def _is_token_purchase(metadata: dict[str, str]) -> bool: + """Return True for premium-credit (a.k.a. premium_token) purchases.""" + return metadata.get("purchase_type", "page_packs") in _PURCHASE_TYPE_TOKEN_VALUES + + async def _get_or_create_purchase_from_checkout_session( db_session: AsyncSession, checkout_session: Any, @@ -470,8 +490,7 @@ async def stripe_webhook( return StripeWebhookResponse() metadata = _get_metadata(checkout_session) - purchase_type = metadata.get("purchase_type", "page_packs") - if purchase_type == "premium_tokens": + if _is_token_purchase(metadata): return await _fulfill_completed_token_purchase( db_session, checkout_session ) @@ -483,8 +502,7 @@ async def stripe_webhook( }: checkout_session = event.data.object metadata = _get_metadata(checkout_session) - purchase_type = metadata.get("purchase_type", "page_packs") - if purchase_type == "premium_tokens": + if _is_token_purchase(metadata): return await _mark_token_purchase_failed( db_session, str(checkout_session.id) ) @@ -504,6 +522,124 @@ async def stripe_webhook( return StripeWebhookResponse() +@router.get("/finalize-checkout", response_model=FinalizeCheckoutResponse) +async def finalize_checkout( + session_id: str, + user: User = Depends(current_active_user), + db_session: AsyncSession = Depends(get_async_session), +) -> FinalizeCheckoutResponse: + """Synchronously fulfil a checkout session from the success page. + + Solves the webhook-vs-redirect race: the user lands on + ``/dashboard//purchase-success?session_id=cs_...`` typically a + few hundred ms after paying, but Stripe's + ``checkout.session.completed`` webhook can take 5-30s+ to arrive. + Calling this endpoint on success-page mount fulfils the purchase + immediately by retrieving the session from Stripe's API and + invoking the same idempotent helpers the webhook uses. + + Idempotency: if the webhook has already fulfilled this purchase + (status=COMPLETED), the helpers short-circuit and we just return + the latest balance. Concurrent webhook + finalize calls are safe + because both acquire ``SELECT ... FOR UPDATE`` on the purchase row. + + Authorization: the session's ``client_reference_id`` must match the + authenticated user's id. This prevents a user from finalising + someone else's checkout session if they happen to know the id. + """ + stripe_client = get_stripe_client() + + try: + checkout_session = stripe_client.v1.checkout.sessions.retrieve(session_id) + except StripeError as exc: + logger.warning( + "finalize_checkout: stripe lookup failed for session=%s user=%s: %s", + session_id, + user.id, + exc, + ) + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Checkout session not found.", + ) from exc + + # Authorization check: the user finalising must be the user who + # initiated the checkout. ``client_reference_id`` is set in + # ``create_checkout_session`` / ``create_token_checkout_session``. + client_reference_id = getattr(checkout_session, "client_reference_id", None) + if client_reference_id != str(user.id): + logger.warning( + "finalize_checkout: ownership mismatch session=%s client_ref=%s user=%s", + session_id, + client_reference_id, + user.id, + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="This checkout session does not belong to you.", + ) + + metadata = _get_metadata(checkout_session) + is_token = _is_token_purchase(metadata) + payment_status = getattr(checkout_session, "payment_status", None) + session_status = getattr(checkout_session, "status", None) + + is_paid = payment_status in {"paid", "no_payment_required"} + is_expired = session_status == "expired" + + if is_paid: + if is_token: + await _fulfill_completed_token_purchase(db_session, checkout_session) + else: + await _fulfill_completed_purchase(db_session, checkout_session) + elif is_expired: + if is_token: + await _mark_token_purchase_failed(db_session, str(checkout_session.id)) + else: + await _mark_purchase_failed(db_session, str(checkout_session.id)) + # Otherwise (e.g. payment_status="unpaid", session_status="open"), + # leave the purchase row alone — frontend will keep polling and the + # webhook will eventually win the race. + + # Refresh the user row so the response reflects any update applied + # by the fulfilment helpers in this same session. + await db_session.refresh(user) + + if is_token: + purchase = ( + await db_session.execute( + select(PremiumTokenPurchase).where( + PremiumTokenPurchase.stripe_checkout_session_id + == str(checkout_session.id) + ) + ) + ).scalar_one_or_none() + return FinalizeCheckoutResponse( + purchase_type="premium_tokens", + status=purchase.status.value if purchase else "pending", + premium_credit_micros_limit=user.premium_credit_micros_limit, + premium_credit_micros_used=user.premium_credit_micros_used, + premium_credit_micros_granted=( + purchase.credit_micros_granted if purchase else None + ), + ) + + purchase = ( + await db_session.execute( + select(PagePurchase).where( + PagePurchase.stripe_checkout_session_id == str(checkout_session.id) + ) + ) + ).scalar_one_or_none() + return FinalizeCheckoutResponse( + purchase_type="page_packs", + status=purchase.status.value if purchase else "pending", + pages_limit=user.pages_limit, + pages_used=user.pages_used, + pages_granted=purchase.pages_granted if purchase else None, + ) + + @router.get("/purchases", response_model=PagePurchaseHistoryResponse) async def get_page_purchases( user: User = Depends(current_active_user), @@ -550,7 +686,11 @@ def _get_token_checkout_urls(search_space_id: int) -> tuple[str, str]: detail="NEXT_FRONTEND_URL is not configured.", ) base_url = config.NEXT_FRONTEND_URL.rstrip("/") - success_url = f"{base_url}/dashboard/{search_space_id}/purchase-success" + # See ``_get_checkout_urls`` for why session_id is appended. + success_url = ( + f"{base_url}/dashboard/{search_space_id}/purchase-success" + f"?session_id={{CHECKOUT_SESSION_ID}}" + ) cancel_url = f"{base_url}/dashboard/{search_space_id}/purchase-cancel" return success_url, cancel_url @@ -601,7 +741,11 @@ async def create_token_checkout_session( "user_id": str(user.id), "quantity": str(body.quantity), "credit_micros_per_unit": str(config.STRIPE_CREDIT_MICROS_PER_UNIT), - "purchase_type": "premium_credit", + # Canonical value matched by ``_is_token_purchase``. + # The legacy ``"premium_credit"`` is still accepted on + # the read side for any in-flight sessions started + # before this rename. + "purchase_type": "premium_tokens", }, } ) diff --git a/surfsense_backend/app/schemas/stripe.py b/surfsense_backend/app/schemas/stripe.py index 57265ec8e..ad13ddf04 100644 --- a/surfsense_backend/app/schemas/stripe.py +++ b/surfsense_backend/app/schemas/stripe.py @@ -56,6 +56,26 @@ class StripeWebhookResponse(BaseModel): received: bool = True +class FinalizeCheckoutResponse(BaseModel): + """Response from /stripe/finalize-checkout. + + Returned by the success page so the UI can show the post-purchase + balance immediately, even when the Stripe webhook hasn't been + delivered yet. ``status`` mirrors the underlying purchase row + (``pending`` / ``completed`` / ``failed``); the FE polls this + endpoint until it sees ``completed`` or a final ``failed``. + """ + + purchase_type: str # "page_packs" | "premium_tokens" + status: str # PagePurchaseStatus / PremiumTokenPurchaseStatus value + pages_limit: int | None = None + pages_used: int | None = None + pages_granted: int | None = None + premium_credit_micros_limit: int | None = None + premium_credit_micros_used: int | None = None + premium_credit_micros_granted: int | None = None + + class CreateTokenCheckoutSessionRequest(BaseModel): """Request body for creating a premium token purchase checkout session.""" diff --git a/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx index 85bc4aaa6..b3d504ed5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx @@ -1,8 +1,9 @@ "use client"; -import { CheckCircle2 } from "lucide-react"; +import { AlertCircle, CheckCircle2, Loader2 } from "lucide-react"; import Link from "next/link"; -import { useParams } from "next/navigation"; +import { useParams, useSearchParams } from "next/navigation"; +import { useEffect, useRef, useState } from "react"; import { Button } from "@/components/ui/button"; import { Card, @@ -12,23 +13,133 @@ import { CardHeader, CardTitle, } from "@/components/ui/card"; +import type { FinalizeCheckoutResponse } from "@/contracts/types/stripe.types"; +import { stripeApiService } from "@/lib/apis/stripe-api.service"; + +type FinalizeState = + | { kind: "loading" } + | { kind: "completed"; data: FinalizeCheckoutResponse } + | { kind: "pending"; data: FinalizeCheckoutResponse } + | { kind: "still_pending"; data: FinalizeCheckoutResponse } + | { kind: "failed"; data: FinalizeCheckoutResponse } + | { kind: "error"; message: string } + | { kind: "no_session" }; + +const POLL_INTERVAL_MS = 2000; +const MAX_POLL_ATTEMPTS = 15; // ~30s total before falling back to the still_pending state export default function PurchaseSuccessPage() { const params = useParams(); + const searchParams = useSearchParams(); const searchSpaceId = String(params.search_space_id ?? ""); + const sessionId = searchParams.get("session_id"); + + const [state, setState] = useState( + sessionId ? { kind: "loading" } : { kind: "no_session" } + ); + // Tracks active polling so component unmount cancels it + const cancelledRef = useRef(false); + + useEffect(() => { + if (!sessionId) return; + + cancelledRef.current = false; + + const poll = async (attempt: number): Promise => { + if (cancelledRef.current) return; + try { + const data = await stripeApiService.finalizeCheckout(sessionId); + if (cancelledRef.current) return; + + if (data.status === "completed") { + setState({ kind: "completed", data }); + return; + } + if (data.status === "failed") { + setState({ kind: "failed", data }); + return; + } + + // Status is "pending" - either the user paid via async + // payment method (Klarna, ACH) or webhook + finalize both + // raced and lost. Keep polling up to MAX_POLL_ATTEMPTS, + // then fall back to a friendlier message that explains + // fulfilment may complete asynchronously. + if (attempt < MAX_POLL_ATTEMPTS) { + setState({ kind: "pending", data }); + setTimeout(() => poll(attempt + 1), POLL_INTERVAL_MS); + } else { + setState({ kind: "still_pending", data }); + } + } catch (err) { + if (cancelledRef.current) return; + const message = err instanceof Error ? err.message : "Unable to finalize checkout."; + setState({ kind: "error", message }); + } + }; + + void poll(1); + + return () => { + cancelledRef.current = true; + }; + }, [sessionId]); return (
- - Purchase complete - Your purchase is being applied to your account now. + {state.kind === "loading" || state.kind === "pending" ? ( + + ) : state.kind === "completed" ? ( + + ) : ( + + )} + + {state.kind === "loading" && "Confirming payment…"} + {state.kind === "pending" && "Processing your payment…"} + {state.kind === "still_pending" && "Payment still processing"} + {state.kind === "completed" && "Purchase complete"} + {state.kind === "failed" && "Purchase failed"} + {state.kind === "error" && "Couldn't confirm payment"} + {state.kind === "no_session" && "Purchase complete"} + + + {state.kind === "loading" && "We're verifying your payment with Stripe."} + {state.kind === "pending" && + "Your bank is taking a moment to confirm. This usually takes 5–30 seconds."} + {state.kind === "still_pending" && + "Your payment is still being processed by your bank. We'll apply your purchase as soon as it clears — usually within a few minutes. You can safely close this page."} + {state.kind === "completed" && + (state.data.purchase_type === "page_packs" + ? `Added ${formatNumber(state.data.pages_granted ?? 0)} pages to your account.` + : `Added ${formatCredit(state.data.premium_credit_micros_granted ?? 0)} of premium credit to your account.`)} + {state.kind === "failed" && + "Stripe reported the checkout as failed or expired. Your card was not charged."} + {state.kind === "error" && + "Don't worry — if your card was charged, your purchase will still apply within a minute or two."} + {state.kind === "no_session" && + "Your purchase is being applied to your account."} + -

- Your usage meters should refresh automatically in a moment. -

+ {state.kind === "completed" && state.data.purchase_type === "page_packs" && ( +

+ New balance: {formatNumber(state.data.pages_limit ?? 0)} total pages + {typeof state.data.pages_used === "number" + ? ` (${formatNumber((state.data.pages_limit ?? 0) - state.data.pages_used)} remaining)` + : ""} +

+ )} + {state.kind === "completed" && state.data.purchase_type === "premium_tokens" && ( +

+ New premium credit balance: {formatCredit(state.data.premium_credit_micros_limit ?? 0)} +

+ )} + {state.kind === "error" && ( +

{state.message}

+ )}
); } + +function formatNumber(n: number): string { + return new Intl.NumberFormat("en-US").format(n); +} + +function formatCredit(micros: number): string { + const dollars = micros / 1_000_000; + return new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 2, + }).format(dollars); +} diff --git a/surfsense_web/contracts/types/stripe.types.ts b/surfsense_web/contracts/types/stripe.types.ts index 251f7a176..35ec0cb17 100644 --- a/surfsense_web/contracts/types/stripe.types.ts +++ b/surfsense_web/contracts/types/stripe.types.ts @@ -73,6 +73,19 @@ export const getTokenPurchasesResponse = z.object({ purchases: z.array(tokenPurchase), }); +// Response from /stripe/finalize-checkout. Either page or token fields +// are populated depending on purchase_type. +export const finalizeCheckoutResponse = z.object({ + purchase_type: z.enum(["page_packs", "premium_tokens"]), + status: pagePurchaseStatusEnum, + pages_limit: z.number().nullable().optional(), + pages_used: z.number().nullable().optional(), + pages_granted: z.number().nullable().optional(), + premium_credit_micros_limit: z.number().nullable().optional(), + premium_credit_micros_used: z.number().nullable().optional(), + premium_credit_micros_granted: z.number().nullable().optional(), +}); + export type PagePurchaseStatus = z.infer; export type CreateCheckoutSessionRequest = z.infer; export type CreateCheckoutSessionResponse = z.infer; @@ -85,3 +98,4 @@ export type TokenStripeStatusResponse = z.infer; export type TokenPurchase = z.infer; export type GetTokenPurchasesResponse = z.infer; +export type FinalizeCheckoutResponse = z.infer; diff --git a/surfsense_web/lib/apis/stripe-api.service.ts b/surfsense_web/lib/apis/stripe-api.service.ts index 6e74d7edc..f119fbf6a 100644 --- a/surfsense_web/lib/apis/stripe-api.service.ts +++ b/surfsense_web/lib/apis/stripe-api.service.ts @@ -5,6 +5,8 @@ import { type CreateTokenCheckoutSessionResponse, createCheckoutSessionResponse, createTokenCheckoutSessionResponse, + type FinalizeCheckoutResponse, + finalizeCheckoutResponse, type GetPagePurchasesResponse, type GetTokenPurchasesResponse, getPagePurchasesResponse, @@ -54,6 +56,20 @@ class StripeApiService { getTokenPurchases = async (): Promise => { return baseApiService.get("/api/v1/stripe/token-purchases", getTokenPurchasesResponse); }; + + /** + * Synchronously fulfil a checkout session from the success page. + * + * Solves the race where the user lands on /purchase-success before + * Stripe's checkout.session.completed webhook arrives. Idempotent — + * safe to call concurrently with the webhook. + */ + finalizeCheckout = async (sessionId: string): Promise => { + return baseApiService.get( + `/api/v1/stripe/finalize-checkout?session_id=${encodeURIComponent(sessionId)}`, + finalizeCheckoutResponse + ); + }; } export const stripeApiService = new StripeApiService(); From dd8c503eb05fb4f3d44907c33323bc4b5e05411b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 5 May 2026 01:26:41 -0700 Subject: [PATCH 7/7] fix: metadata extraction in Stripe checkout session - Updated the `_get_metadata` function to handle changes in the Stripe SDK, specifically for `StripeObject` which is no longer a subclass of `dict` in `stripe>=15.0`. - Implemented a fallback mechanism in `finalize_checkout` to recover purchase type from the database if metadata extraction fails, ensuring robust handling of checkout sessions. --- surfsense_backend/app/routes/stripe_routes.py | 95 ++++++++++++++++--- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/surfsense_backend/app/routes/stripe_routes.py b/surfsense_backend/app/routes/stripe_routes.py index 43fa8c4b7..fc5fded84 100644 --- a/surfsense_backend/app/routes/stripe_routes.py +++ b/surfsense_backend/app/routes/stripe_routes.py @@ -99,20 +99,49 @@ def _normalize_optional_string(value: Any) -> str | None: def _get_metadata(checkout_session: Any) -> dict[str, str]: """Extract checkout session metadata as a plain ``str -> str`` dict. - Works for both ``dict`` (e.g. when the metadata round-tripped through - JSON) and Stripe's ``StripeObject`` wrapper. Recent Stripe SDK - versions stopped subclassing ``dict`` for ``StripeObject``, so - ``isinstance(metadata, dict)`` is False and ``dict(metadata)`` falls - into the sequence protocol, looking up integer indices and raising - ``KeyError: 0``. ``.items()`` is exposed by both shapes via the - Mapping protocol, so we always use that. + In ``stripe>=15.0`` ``StripeObject`` is no longer a ``dict`` subclass + and exposes neither ``items()`` nor ``__iter__`` nor ``keys()``. + ``dict(obj)`` falls into the sequence protocol and raises + ``KeyError: 0``; ``obj.items()`` raises ``AttributeError``. The + supported way to materialize a ``StripeObject`` as a plain dict is + its ``to_dict()`` method (added in stripe-python 8.x, present in 15.x). """ - metadata = getattr(checkout_session, "metadata", None) or {} - try: - items = metadata.items() - except AttributeError: + metadata = getattr(checkout_session, "metadata", None) + if metadata is None: return {} - return {str(key): str(value) for key, value in items} + + # 1. Plain dict (older SDKs that subclassed dict, JSON-decoded events + # in tests, etc.). + if isinstance(metadata, dict): + return {str(k): str(v) for k, v in metadata.items()} + + # 2. Modern Stripe SDK: every ``StripeObject`` has ``to_dict()``. + # ``recursive=False`` is correct because Stripe metadata values + # are always primitive strings. + to_dict = getattr(metadata, "to_dict", None) + if callable(to_dict): + try: + d = to_dict(recursive=False) + if isinstance(d, dict): + return {str(k): str(v) for k, v in d.items()} + except Exception: + logger.exception( + "Stripe metadata.to_dict() failed for session %s", + getattr(checkout_session, "id", "?"), + ) + + # 3. Last-resort: read the SDK's private ``_data`` backing dict. + # Stable across stripe-python 6.x -> 15.x. + inner = getattr(metadata, "_data", None) + if isinstance(inner, dict): + return {str(k): str(v) for k, v in inner.items()} + + logger.warning( + "Could not extract metadata from checkout session %s (metadata type=%s)", + getattr(checkout_session, "id", "?"), + type(metadata).__name__, + ) + return {} # Canonical purchase_type metadata values. ``premium_credit`` was emitted @@ -584,6 +613,48 @@ async def finalize_checkout( payment_status = getattr(checkout_session, "payment_status", None) session_status = getattr(checkout_session, "status", None) + # Defensive fallback: if metadata can't be read for any reason + # (extraction failure, manually-created session in Stripe dashboard, + # SDK upgrade breaking ``to_dict``, etc.) we'd otherwise route every + # purchase to the page_packs handler and get stuck. Resolve the + # purchase_type by checking which table actually has the row keyed + # by this Stripe session id. + if not metadata: + existing_token_purchase = ( + await db_session.execute( + select(PremiumTokenPurchase.id).where( + PremiumTokenPurchase.stripe_checkout_session_id + == str(checkout_session.id) + ) + ) + ).scalar_one_or_none() + if existing_token_purchase is not None: + is_token = True + else: + existing_page_purchase = ( + await db_session.execute( + select(PagePurchase.id).where( + PagePurchase.stripe_checkout_session_id + == str(checkout_session.id) + ) + ) + ).scalar_one_or_none() + if existing_page_purchase is None: + logger.error( + "finalize_checkout: no purchase row in either table " + "and metadata is empty for session=%s user=%s", + session_id, + user.id, + ) + # Fall through; downstream path will short-circuit on + # missing-row + empty-metadata. + logger.info( + "finalize_checkout: recovered purchase_type=%s for session=%s " + "via DB fallback (metadata was empty)", + "premium_tokens" if is_token else "page_packs", + session_id, + ) + is_paid = payment_status in {"paid", "no_payment_required"} is_expired = session_status == "expired"