diff --git a/docker/.env.example b/docker/.env.example index cafc74af9..54ca489b2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -166,25 +166,26 @@ EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2 # REDIS_URL=redis://redis:6379/0 # ------------------------------------------------------------------------------ -# Stripe (pay-as-you-go page packs, disabled by default) +# Stripe (unified credit wallet, disabled by default) # ------------------------------------------------------------------------------ -# Set TRUE to allow users to buy additional page packs via Stripe Checkout -STRIPE_PAGE_BUYING_ENABLED=FALSE +# Set TRUE to allow users to buy credit packs via Stripe Checkout. $1 buys +# 1_000_000 micro-USD of credit; both ETL page processing and premium turns +# debit this balance at the actual per-call provider cost from LiteLLM. +STRIPE_CREDIT_BUYING_ENABLED=FALSE # STRIPE_SECRET_KEY=sk_test_... # STRIPE_WEBHOOK_SECRET=whsec_... -# STRIPE_PRICE_ID=price_... -# STRIPE_PAGES_PER_UNIT=1000 +# STRIPE_CREDIT_PRICE_ID=price_... +# STRIPE_CREDIT_MICROS_PER_UNIT=1000000 # STRIPE_RECONCILIATION_INTERVAL=10m # STRIPE_RECONCILIATION_LOOKBACK_MINUTES=10 # STRIPE_RECONCILIATION_BATCH_SIZE=100 -# Premium credit purchases via Stripe ($1 buys 1_000_000 micro-USD of -# credit; premium turns debit the actual per-call provider cost -# reported by LiteLLM, so cheap and expensive models bill proportionally) -# STRIPE_TOKEN_BUYING_ENABLED=FALSE -# STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... -# STRIPE_CREDIT_MICROS_PER_UNIT=1000000 +# Auto-reload: top up via a saved Stripe card when the balance drops below +# the user-chosen threshold. Off by default. +# AUTO_RELOAD_ENABLED=FALSE +# AUTO_RELOAD_MIN_AMOUNT_MICROS=1000000 +# AUTO_RELOAD_COOLDOWN_MINUTES=10 # ------------------------------------------------------------------------------ # TTS & STT (Text-to-Speech / Speech-to-Text) @@ -407,13 +408,16 @@ SURFSENSE_ENABLE_DOOM_LOOP=true # ACCESS_TOKEN_LIFETIME_SECONDS=86400 # REFRESH_TOKEN_LIFETIME_SECONDS=1209600 -# Pages limit per user for ETL (default: unlimited) -# PAGES_LIMIT=500 +# Unified credit wallet starting balance for new users, in micro-USD +# (default: $5). Funds both ETL page processing and premium model calls, +# debited at the actual per-call provider cost reported by LiteLLM. +# DEFAULT_CREDIT_MICROS_BALANCE=5000000 -# Premium credit quota per registered user, in micro-USD (default: $5). -# Premium turns are debited at the actual per-call provider cost reported -# by LiteLLM. Only applies to models with billing_tier=premium. -# PREMIUM_CREDIT_MICROS_LIMIT=5000000 +# Debit the credit wallet for ETL page processing. Default FALSE keeps ETL +# effectively free for self-hosted installs. 1 page == MICROS_PER_PAGE +# micro-USD ($0.001); premium ETL mode is 10x. +# ETL_CREDIT_BILLING_ENABLED=FALSE +# MICROS_PER_PAGE=1000 # Safety ceiling on per-call premium reservation, in micro-USD ($1.00 default). # QUOTA_MAX_RESERVE_MICROS=1000000 diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 6e49a7132..b4f67328c 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -75,23 +75,16 @@ SECRET_KEY=SECRET NEXT_FRONTEND_URL=http://localhost:3000 -# Stripe Checkout for pay-as-you-go page packs -# Configure STRIPE_PRICE_ID to point at your 1,000-page price in Stripe. -# Pages granted per purchase = quantity * STRIPE_PAGES_PER_UNIT. +# Stripe Checkout for the unified credit wallet. +# Each pack grants STRIPE_CREDIT_MICROS_PER_UNIT micro-USD of credit +# (default 1_000_000 = $1.00). Both ETL page processing and premium model +# turns are billed against this single balance at actual provider cost. STRIPE_SECRET_KEY=sk_test_... STRIPE_WEBHOOK_SECRET=whsec_... -STRIPE_PRICE_ID=price_... -STRIPE_PAGES_PER_UNIT=1000 -# Set FALSE to disable new checkout session creation temporarily -STRIPE_PAGE_BUYING_ENABLED=TRUE - -# Premium credit purchases via Stripe (for premium-tier model usage). -# Each pack grants STRIPE_CREDIT_MICROS_PER_UNIT micro-USD of credit -# (default 1_000_000 = $1.00). Premium turns are billed at the actual -# per-call provider cost reported by LiteLLM. -STRIPE_TOKEN_BUYING_ENABLED=FALSE -STRIPE_PREMIUM_TOKEN_PRICE_ID=price_... +STRIPE_CREDIT_PRICE_ID=price_... STRIPE_CREDIT_MICROS_PER_UNIT=1000000 +# Set FALSE to disable new checkout session creation temporarily +STRIPE_CREDIT_BUYING_ENABLED=FALSE # Periodic Stripe safety net for purchases left in PENDING (minutes old) STRIPE_RECONCILIATION_LOOKBACK_MINUTES=10 @@ -221,15 +214,25 @@ VIDEO_PRESENTATION_FPS=30 VIDEO_PRESENTATION_DEFAULT_DURATION_IN_FRAMES=300 -# (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) -PAGES_LIMIT=500 +# Unified credit wallet starting balance for new users, in micro-USD +# (default: 5,000,000 == $5.00). The same balance funds ETL page processing +# and premium model calls, debited at actual provider cost. +DEFAULT_CREDIT_MICROS_BALANCE=5000000 -# Premium credit quota per registered user, in micro-USD -# (default: 5,000,000 == $5.00 of credit). Premium turns are debited at the -# actual per-call provider cost reported by LiteLLM, so cheap and expensive -# models bill proportionally. Applies only to models with -# billing_tier=premium in global_llm_config.yaml. -PREMIUM_CREDIT_MICROS_LIMIT=5000000 +# Debit the credit wallet for ETL page processing. Default FALSE keeps ETL +# effectively free for self-hosted/OSS installs; hosted deployments set TRUE. +# 1 page == MICROS_PER_PAGE micro-USD ($0.001); premium ETL mode is 10x. +ETL_CREDIT_BILLING_ENABLED=FALSE +MICROS_PER_PAGE=1000 + +# Low-balance warning threshold (micro-USD), surfaced to the UI. Default $0.50. +CREDIT_LOW_BALANCE_WARNING_MICROS=500000 + +# Auto-reload: automatically top up via a saved Stripe card when the balance +# drops below the user-chosen threshold. Off by default. +AUTO_RELOAD_ENABLED=FALSE +AUTO_RELOAD_MIN_AMOUNT_MICROS=1000000 +AUTO_RELOAD_COOLDOWN_MINUTES=10 # Safety ceiling on per-call premium reservation, in micro-USD. # stream_new_chat estimates an upper-bound cost from the model's diff --git a/surfsense_backend/alembic/versions/156_unify_credits_wallet.py b/surfsense_backend/alembic/versions/156_unify_credits_wallet.py new file mode 100644 index 000000000..33e6d9087 --- /dev/null +++ b/surfsense_backend/alembic/versions/156_unify_credits_wallet.py @@ -0,0 +1,218 @@ +"""unify page limits and premium credits into a single credit_micros_balance wallet + +Collapses the two separate economies (ETL ``pages_limit``/``pages_used`` and +premium ``premium_credit_micros_limit``/``premium_credit_micros_used``) into one +USD-micro wallet column ``user.credit_micros_balance`` that decreases on use and +increases on purchase / grant. ``premium_credit_micros_reserved`` is kept (renamed +to ``credit_micros_reserved``) for in-flight reservation holds. + +Backfill (per existing user row): + + balance = GREATEST(0, premium_credit_micros_limit - premium_credit_micros_used) + + (CASE WHEN pages_limit < 100000000 + THEN GREATEST(0, pages_limit - pages_used) * 1000 + ELSE 0 END) + +The ``pages_limit < 100000000`` guard skips the OSS "unlimited" default +(``PAGES_LIMIT=999999999``) so self-hosters don't get a ~$1M credit grant. +1 page == 1000 micros == $0.001 (matches the prior $1 / 1000 pages price). + +Table / type renames: + + premium_token_purchases -> credit_purchases + premiumtokenpurchasestatus (enum)-> creditpurchasestatus + user_incentive_tasks.pages_awarded -> credit_micros_awarded (backfilled * 1000) + +The "user" table is in zero_publication's column list, so this migration updates +the publication via ``apply_publication`` (canonical reconcile, per migration 155) +BEFORE dropping the old columns it referenced. + +IMPORTANT - before AND after running this migration (same as migration 140): + 1. Stop zero-cache (it holds replication locks that will deadlock DDL) + 2. Run: alembic upgrade head + 3. Delete / reset the zero-cache data volume + 4. Restart zero-cache (it will do a fresh initial sync) + +Revision ID: 156 +Revises: 155 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op +from app.zero_publication import apply_publication + +revision: str = "156" +down_revision: str | None = "155" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = :col" + ), + {"tbl": table, "col": column}, + ).fetchone() + is not None + ) + + +def _table_exists(conn, table: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.tables " + "WHERE table_name = :tbl AND table_schema = current_schema()" + ), + {"tbl": table}, + ).fetchone() + is not None + ) + + +def _terminate_blocked_pids(conn, table: str) -> None: + """Kill backends whose locks on *table* would block our AccessExclusiveLock.""" + conn.execute( + sa.text( + "SELECT pg_terminate_backend(l.pid) " + "FROM pg_locks l " + "JOIN pg_class c ON c.oid = l.relation " + "WHERE c.relname = :tbl " + " AND l.pid != pg_backend_pid()" + ), + {"tbl": table}, + ) + + +def upgrade() -> None: + conn = op.get_bind() + + # ------------------------------------------------------------------ + # 1. Add credit_micros_balance + backfill from both legacy economies. + # ------------------------------------------------------------------ + if not _column_exists(conn, "user", "credit_micros_balance"): + op.add_column( + "user", + sa.Column( + "credit_micros_balance", + sa.BigInteger(), + nullable=False, + server_default="5000000", + ), + ) + + # Backfill only when the legacy source columns are present (fresh DBs + # created from current models won't have them). + if _column_exists( + conn, "user", "premium_credit_micros_limit" + ) and _column_exists(conn, "user", "pages_limit"): + conn.execute( + sa.text( + 'UPDATE "user" SET credit_micros_balance = ' + "GREATEST(0, premium_credit_micros_limit - premium_credit_micros_used) " + "+ (CASE WHEN pages_limit < 100000000 " + " THEN GREATEST(0, pages_limit - pages_used) * 1000 " + " ELSE 0 END)" + ) + ) + + # ------------------------------------------------------------------ + # 2. Rename premium_credit_micros_reserved -> credit_micros_reserved. + # ------------------------------------------------------------------ + if _column_exists( + conn, "user", "premium_credit_micros_reserved" + ) and not _column_exists(conn, "user", "credit_micros_reserved"): + op.alter_column( + "user", + "premium_credit_micros_reserved", + new_column_name="credit_micros_reserved", + ) + + # ------------------------------------------------------------------ + # 3. Reconcile the Zero publication to the new column list + # (id, credit_micros_balance) BEFORE dropping the columns it used + # to reference, otherwise Postgres rejects the column drops with + # "cannot drop column ... referenced by publication". + # ------------------------------------------------------------------ + conn.execute(sa.text("SET lock_timeout = '10s'")) + _terminate_blocked_pids(conn, "user") + apply_publication(conn) + + # ------------------------------------------------------------------ + # 4. Drop the legacy quota columns now that nothing references them. + # ------------------------------------------------------------------ + for col in ( + "premium_credit_micros_limit", + "premium_credit_micros_used", + "pages_limit", + "pages_used", + ): + if _column_exists(conn, "user", col): + op.drop_column("user", col) + + # ------------------------------------------------------------------ + # 5. Rename premium_token_purchases -> credit_purchases and its enum. + # ------------------------------------------------------------------ + op.execute( + """ + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'premiumtokenpurchasestatus') + AND NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'creditpurchasestatus') + THEN + ALTER TYPE premiumtokenpurchasestatus RENAME TO creditpurchasestatus; + END IF; + END + $$; + """ + ) + + if _table_exists(conn, "premium_token_purchases") and not _table_exists( + conn, "credit_purchases" + ): + op.rename_table("premium_token_purchases", "credit_purchases") + + # ``source`` distinguishes user checkout from auto-reload top-ups. + if _table_exists(conn, "credit_purchases") and not _column_exists( + conn, "credit_purchases", "source" + ): + op.add_column( + "credit_purchases", + sa.Column( + "source", + sa.String(length=20), + nullable=False, + server_default="checkout", + ), + ) + + # ------------------------------------------------------------------ + # 6. Rename user_incentive_tasks.pages_awarded -> credit_micros_awarded + # and convert page counts to micros (1 page == 1000 micros). + # ------------------------------------------------------------------ + if _column_exists( + conn, "user_incentive_tasks", "pages_awarded" + ) and not _column_exists(conn, "user_incentive_tasks", "credit_micros_awarded"): + op.alter_column( + "user_incentive_tasks", + "pages_awarded", + new_column_name="credit_micros_awarded", + type_=sa.BigInteger(), + ) + conn.execute( + sa.text( + "UPDATE user_incentive_tasks " + "SET credit_micros_awarded = credit_micros_awarded * 1000" + ) + ) + + +def downgrade() -> None: + """No-op. This is a one-way data-model unification; the legacy split + columns cannot be faithfully reconstructed from a single balance.""" diff --git a/surfsense_backend/alembic/versions/157_add_auto_reload_columns.py b/surfsense_backend/alembic/versions/157_add_auto_reload_columns.py new file mode 100644 index 000000000..9a0dc2e48 --- /dev/null +++ b/surfsense_backend/alembic/versions/157_add_auto_reload_columns.py @@ -0,0 +1,91 @@ +"""add auto-reload (off-session Stripe top-up) columns to user + +Adds the saved-card + threshold plumbing that powers feature-flagged credit +auto-reload (``AUTO_RELOAD_ENABLED``): + + user.stripe_customer_id (text, nullable) + user.auto_reload_enabled (bool, default false) + user.auto_reload_threshold_micros (bigint, nullable) + user.auto_reload_amount_micros (bigint, nullable) + user.auto_reload_payment_method_id (text, nullable) + user.auto_reload_failed_at (timestamptz, nullable) + +None of these columns are part of the Zero publication (``USER_COLS`` is +``["id", "credit_micros_balance"]``), so this migration does NOT touch the +publication and is safe to run without the zero-cache stop/reset dance that +migration 156 required. + +The ``credit_purchases.source`` column (``checkout`` | ``auto_reload``) was +already added in migration 156, so it is not repeated here. + +Revision ID: 157 +Revises: 156 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "157" +down_revision: str | None = "156" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :tbl AND column_name = :col" + ), + {"tbl": table, "col": column}, + ).fetchone() + is not None + ) + + +_COLUMNS: list[tuple[str, sa.Column]] = [ + ("stripe_customer_id", sa.Column("stripe_customer_id", sa.String(), nullable=True)), + ( + "auto_reload_enabled", + sa.Column( + "auto_reload_enabled", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ), + ( + "auto_reload_threshold_micros", + sa.Column("auto_reload_threshold_micros", sa.BigInteger(), nullable=True), + ), + ( + "auto_reload_amount_micros", + sa.Column("auto_reload_amount_micros", sa.BigInteger(), nullable=True), + ), + ( + "auto_reload_payment_method_id", + sa.Column("auto_reload_payment_method_id", sa.String(), nullable=True), + ), + ( + "auto_reload_failed_at", + sa.Column("auto_reload_failed_at", sa.TIMESTAMP(timezone=True), nullable=True), + ), +] + + +def upgrade() -> None: + conn = op.get_bind() + for name, column in _COLUMNS: + if not _column_exists(conn, "user", name): + op.add_column("user", column) + + +def downgrade() -> None: + conn = op.get_bind() + for name, _ in reversed(_COLUMNS): + if _column_exists(conn, "user", name): + op.drop_column("user", name) diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 0e852b801..6a909a3df 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -189,6 +189,7 @@ celery_app = Celery( "app.tasks.celery_tasks.document_reindex_tasks", "app.tasks.celery_tasks.stale_notification_cleanup_task", "app.tasks.celery_tasks.stripe_reconciliation_task", + "app.tasks.celery_tasks.auto_reload_task", "app.tasks.celery_tasks.gateway_tasks", "app.automations.tasks.execute_run", "app.automations.triggers.builtin.schedule.selector", @@ -281,16 +282,9 @@ celery_app.conf.beat_schedule = { "expires": 60, # Task expires after 60 seconds if not picked up }, }, - # Reconcile Stripe purchases that were paid but remained pending - "reconcile-pending-stripe-page-purchases": { - "task": "reconcile_pending_stripe_page_purchases", - "schedule": crontab(**stripe_reconciliation_schedule_params), - "options": { - "expires": 60, - }, - }, - "reconcile-pending-stripe-token-purchases": { - "task": "reconcile_pending_stripe_token_purchases", + # Reconcile Stripe credit purchases that were paid but remained pending + "reconcile-pending-stripe-credit-purchases": { + "task": "reconcile_pending_stripe_credit_purchases", "schedule": crontab(**stripe_reconciliation_schedule_params), "options": { "expires": 60, diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 75af17d11..bbaf3ac55 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -640,14 +640,9 @@ class Config: ) GATEWAY_DISCORD_REDIRECT_URI = os.getenv("GATEWAY_DISCORD_REDIRECT_URI") - # Stripe checkout for pay-as-you-go page packs + # Stripe checkout (shared secrets for the unified credit wallet) STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY") STRIPE_WEBHOOK_SECRET = os.getenv("STRIPE_WEBHOOK_SECRET") - STRIPE_PRICE_ID = os.getenv("STRIPE_PRICE_ID") - STRIPE_PAGES_PER_UNIT = int(os.getenv("STRIPE_PAGES_PER_UNIT", "1000")) - STRIPE_PAGE_BUYING_ENABLED = ( - os.getenv("STRIPE_PAGE_BUYING_ENABLED", "TRUE").upper() == "TRUE" - ) STRIPE_RECONCILIATION_LOOKBACK_MINUTES = int( os.getenv("STRIPE_RECONCILIATION_LOOKBACK_MINUTES", "10") ) @@ -655,27 +650,56 @@ class Config: os.getenv("STRIPE_RECONCILIATION_BATCH_SIZE", "100") ) - # Premium credit (micro-USD) quota settings. + # Unified credit wallet (micro-USD) settings. # - # Storage unit is integer micro-USD (1_000_000 = $1.00). The legacy - # ``PREMIUM_TOKEN_LIMIT`` and ``STRIPE_TOKENS_PER_UNIT`` env vars are - # still honoured for one release as fall-back values — the prior - # $1-per-1M-tokens Stripe price means every existing value maps 1:1 - # to micros, so operators upgrading without changing their .env still - # get correct behaviour. A startup deprecation warning fires below if - # they're set. - PREMIUM_CREDIT_MICROS_LIMIT = int( - os.getenv("PREMIUM_CREDIT_MICROS_LIMIT") + # Storage unit is integer micro-USD (1_000_000 = $1.00). A single + # ``credit_micros_balance`` funds both ETL page processing and premium + # model calls. New users start with ``DEFAULT_CREDIT_MICROS_BALANCE`` + # ($5 by default). + # + # Legacy env names (``PREMIUM_CREDIT_MICROS_LIMIT`` / ``PREMIUM_TOKEN_LIMIT``, + # ``STRIPE_PREMIUM_TOKEN_PRICE_ID``, ``STRIPE_CREDIT_MICROS_PER_UNIT`` / + # ``STRIPE_TOKENS_PER_UNIT``, ``STRIPE_TOKEN_BUYING_ENABLED``) are still + # honoured as fall-backs for one release; deprecation warnings fire below. + DEFAULT_CREDIT_MICROS_BALANCE = int( + os.getenv("DEFAULT_CREDIT_MICROS_BALANCE") + or os.getenv("PREMIUM_CREDIT_MICROS_LIMIT") or os.getenv("PREMIUM_TOKEN_LIMIT", "5000000") ) - STRIPE_PREMIUM_TOKEN_PRICE_ID = os.getenv("STRIPE_PREMIUM_TOKEN_PRICE_ID") + STRIPE_CREDIT_PRICE_ID = os.getenv("STRIPE_CREDIT_PRICE_ID") or os.getenv( + "STRIPE_PREMIUM_TOKEN_PRICE_ID" + ) STRIPE_CREDIT_MICROS_PER_UNIT = int( os.getenv("STRIPE_CREDIT_MICROS_PER_UNIT") or os.getenv("STRIPE_TOKENS_PER_UNIT", "1000000") ) - STRIPE_TOKEN_BUYING_ENABLED = ( - os.getenv("STRIPE_TOKEN_BUYING_ENABLED", "FALSE").upper() == "TRUE" + STRIPE_CREDIT_BUYING_ENABLED = ( + os.getenv("STRIPE_CREDIT_BUYING_ENABLED") + or os.getenv("STRIPE_TOKEN_BUYING_ENABLED", "FALSE") + ).upper() == "TRUE" + + # ETL page processing debits the credit wallet only when enabled. Defaults + # to FALSE so self-hosted / OSS installs keep effectively-free ETL; hosted + # deployments set this TRUE. 1 page == ``MICROS_PER_PAGE`` micro-USD. + ETL_CREDIT_BILLING_ENABLED = ( + os.getenv("ETL_CREDIT_BILLING_ENABLED", "FALSE").upper() == "TRUE" ) + MICROS_PER_PAGE = int(os.getenv("MICROS_PER_PAGE", "1000")) + + # Low-balance WARNING threshold (micro-USD). Surfaced by the quota service + # so the UI can nudge the user to top up / enable auto-reload. $0.50. + CREDIT_LOW_BALANCE_WARNING_MICROS = int( + os.getenv("CREDIT_LOW_BALANCE_WARNING_MICROS", "500000") + ) + + # Auto-reload (off-session Stripe top-up) feature flag and guards. + AUTO_RELOAD_ENABLED = os.getenv("AUTO_RELOAD_ENABLED", "FALSE").upper() == "TRUE" + # Minimum configurable reload amount (micro-USD). $1.00 to match pack pricing. + AUTO_RELOAD_MIN_AMOUNT_MICROS = int( + os.getenv("AUTO_RELOAD_MIN_AMOUNT_MICROS", "1000000") + ) + # Cooldown so a burst of debits can't fire multiple charges (minutes). + AUTO_RELOAD_COOLDOWN_MINUTES = int(os.getenv("AUTO_RELOAD_COOLDOWN_MINUTES", "10")) # Safety ceiling on the per-call premium reservation. ``stream_new_chat`` # estimates an upper-bound cost from ``litellm.get_model_info`` x the @@ -685,14 +709,13 @@ class Config: # reserve_tokens ≈ $0.36) with headroom. QUOTA_MAX_RESERVE_MICROS = int(os.getenv("QUOTA_MAX_RESERVE_MICROS", "1000000")) - if os.getenv("PREMIUM_TOKEN_LIMIT") and not os.getenv( - "PREMIUM_CREDIT_MICROS_LIMIT" - ): + if ( + os.getenv("PREMIUM_TOKEN_LIMIT") or os.getenv("PREMIUM_CREDIT_MICROS_LIMIT") + ) and not os.getenv("DEFAULT_CREDIT_MICROS_BALANCE"): print( - "Warning: PREMIUM_TOKEN_LIMIT is deprecated; rename to " - "PREMIUM_CREDIT_MICROS_LIMIT (1:1 numerical mapping under the " - "current Stripe price). The old key will be removed in a " - "future release." + "Warning: PREMIUM_TOKEN_LIMIT / PREMIUM_CREDIT_MICROS_LIMIT are " + "deprecated; rename to DEFAULT_CREDIT_MICROS_BALANCE. The old keys " + "will be removed in a future release." ) if os.getenv("STRIPE_TOKENS_PER_UNIT") and not os.getenv( "STRIPE_CREDIT_MICROS_PER_UNIT" @@ -702,6 +725,22 @@ class Config: "STRIPE_CREDIT_MICROS_PER_UNIT (1:1 numerical mapping). " "The old key will be removed in a future release." ) + if os.getenv("STRIPE_PREMIUM_TOKEN_PRICE_ID") and not os.getenv( + "STRIPE_CREDIT_PRICE_ID" + ): + print( + "Warning: STRIPE_PREMIUM_TOKEN_PRICE_ID is deprecated; rename to " + "STRIPE_CREDIT_PRICE_ID. The old key will be removed in a future " + "release." + ) + if os.getenv("STRIPE_TOKEN_BUYING_ENABLED") and not os.getenv( + "STRIPE_CREDIT_BUYING_ENABLED" + ): + print( + "Warning: STRIPE_TOKEN_BUYING_ENABLED is deprecated; rename to " + "STRIPE_CREDIT_BUYING_ENABLED. The old key will be removed in a " + "future release." + ) # Anonymous / no-login mode settings NOLOGIN_MODE_ENABLED = os.getenv("NOLOGIN_MODE_ENABLED", "FALSE").upper() == "TRUE" @@ -903,9 +942,6 @@ class Config: # ETL Service ETL_SERVICE = os.getenv("ETL_SERVICE") - # Pages limit for ETL services (default to very high number for OSS unlimited usage) - PAGES_LIMIT = int(os.getenv("PAGES_LIMIT", "999999999")) - if ETL_SERVICE == "UNSTRUCTURED": # Unstructured API Key UNSTRUCTURED_API_KEY = os.getenv("UNSTRUCTURED_API_KEY") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 6117caecb..ee86ad86d 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -320,7 +320,7 @@ class PagePurchaseStatus(StrEnum): FAILED = "failed" -class PremiumTokenPurchaseStatus(StrEnum): +class CreditPurchaseStatus(StrEnum): PENDING = "pending" COMPLETED = "completed" FAILED = "failed" @@ -332,26 +332,27 @@ INCENTIVE_TASKS_CONFIG = { IncentiveTaskType.GITHUB_STAR: { "title": "Star our GitHub repository", "description": "Show your support by starring SurfSense on GitHub", - "pages_reward": 30, + # Credit reward in USD micro-units (1_000_000 == $1.00). $0.03. + "credit_micros_reward": 30000, "action_url": "https://github.com/MODSetter/SurfSense", }, IncentiveTaskType.REDDIT_FOLLOW: { "title": "Join our Subreddit", "description": "Join the SurfSense community on Reddit", - "pages_reward": 30, + "credit_micros_reward": 30000, "action_url": "https://www.reddit.com/r/SurfSense/", }, IncentiveTaskType.DISCORD_JOIN: { "title": "Join our Discord", "description": "Join the SurfSense community on Discord", - "pages_reward": 40, + "credit_micros_reward": 40000, "action_url": "https://discord.gg/ejRNvftDp9", }, # Future tasks can be configured here: # IncentiveTaskType.GITHUB_ISSUE: { # "title": "Create an issue", # "description": "Help improve SurfSense by reporting bugs or suggesting features", - # "pages_reward": 50, + # "credit_micros_reward": 50000, # "action_url": "https://github.com/MODSetter/SurfSense/issues/new/choose", # }, } @@ -2069,7 +2070,7 @@ class UserIncentiveTask(BaseModel, TimestampMixin): """ Tracks completed incentive tasks for users. Each user can only complete each task type once. - When a task is completed, the user's pages_limit is increased. + When a task is completed, the user's credit_micros_balance is increased. """ __tablename__ = "user_incentive_tasks" @@ -2088,7 +2089,8 @@ class UserIncentiveTask(BaseModel, TimestampMixin): index=True, ) task_type = Column(SQLAlchemyEnum(IncentiveTaskType), nullable=False, index=True) - pages_awarded = Column(Integer, nullable=False) + # Credit reward granted in USD micro-units (1_000_000 == $1.00). + credit_micros_awarded = Column(BigInteger, nullable=False) completed_at = Column( TIMESTAMP(timezone=True), nullable=False, @@ -2131,18 +2133,18 @@ class PagePurchase(Base, TimestampMixin): user = relationship("User", back_populates="page_purchases") -class PremiumTokenPurchase(Base, TimestampMixin): - """Tracks Stripe checkout sessions used to grant additional premium credit (USD micro-units). +class CreditPurchase(Base, TimestampMixin): + """Tracks Stripe checkout sessions used to grant credit (USD micro-units). - Note: the table name is preserved (``premium_token_purchases``) for - operational continuity even though the unit is now USD micro-credits - instead of raw tokens. The ``credit_micros_granted`` column replaced - the legacy ``tokens_granted`` in migration 140; the stored values - were not transformed because the prior $1 = 1M tokens Stripe price - makes the unit conversion 1:1 numerically. + Renamed from ``premium_token_purchases`` in migration 156 as part of the + unified-credits wallet. ``credit_micros_granted`` stores the USD-micro + amount added to ``user.credit_micros_balance`` on fulfillment. + + ``source`` distinguishes a user-initiated checkout from an automatic + off-session top-up (auto-reload), added in the auto-reload migration. """ - __tablename__ = "premium_token_purchases" + __tablename__ = "credit_purchases" __allow_unmapped__ = True id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) @@ -2160,15 +2162,18 @@ class PremiumTokenPurchase(Base, TimestampMixin): credit_micros_granted = Column(BigInteger, nullable=False) amount_total = Column(Integer, nullable=True) currency = Column(String(10), nullable=True) + source = Column( + String(20), nullable=False, default="checkout", server_default="checkout" + ) status = Column( - SQLAlchemyEnum(PremiumTokenPurchaseStatus), + SQLAlchemyEnum(CreditPurchaseStatus), nullable=False, - default=PremiumTokenPurchaseStatus.PENDING, + default=CreditPurchaseStatus.PENDING, index=True, ) completed_at = Column(TIMESTAMP(timezone=True), nullable=True) - user = relationship("User", back_populates="premium_token_purchases") + user = relationship("User", back_populates="credit_purchases") class SearchSpaceRole(BaseModel, TimestampMixin): @@ -2448,33 +2453,40 @@ if config.AUTH_TYPE == "GOOGLE": back_populates="user", cascade="all, delete-orphan", ) - premium_token_purchases = relationship( - "PremiumTokenPurchase", + credit_purchases = relationship( + "CreditPurchase", back_populates="user", cascade="all, delete-orphan", ) - # Page usage tracking for ETL services - pages_limit = Column( - Integer, - nullable=False, - default=config.PAGES_LIMIT, - server_default=str(config.PAGES_LIMIT), - ) - pages_used = Column(Integer, nullable=False, default=0, server_default="0") - - premium_credit_micros_limit = Column( + # Unified credit wallet (USD micro-units, 1_000_000 == $1.00). + # Decreases on use (ETL pages + premium model calls), increases on + # purchase / incentive grant / auto-reload. May dip slightly negative + # when an actual cost exceeds its pre-charge estimate; UI clamps at $0. + credit_micros_balance = Column( BigInteger, nullable=False, - default=config.PREMIUM_CREDIT_MICROS_LIMIT, - server_default=str(config.PREMIUM_CREDIT_MICROS_LIMIT), + default=config.DEFAULT_CREDIT_MICROS_BALANCE, + server_default=str(config.DEFAULT_CREDIT_MICROS_BALANCE), ) - premium_credit_micros_used = Column( + # In-flight reservation holds (released/settled at finalize). + credit_micros_reserved = Column( BigInteger, nullable=False, default=0, server_default="0" ) - premium_credit_micros_reserved = Column( - BigInteger, nullable=False, default=0, server_default="0" + + # Auto-reload (off-session Stripe top-up), behind AUTO_RELOAD_ENABLED. + # ``stripe_customer_id`` + ``auto_reload_payment_method_id`` are the + # saved-card plumbing; thresholds are micro-USD. ``auto_reload_failed_at`` + # is set (and ``auto_reload_enabled`` flipped off) when an off-session + # charge is declined so the UI can prompt the user to fix their card. + stripe_customer_id = Column(String, nullable=True) + auto_reload_enabled = Column( + Boolean, nullable=False, default=False, server_default="false" ) + auto_reload_threshold_micros = Column(BigInteger, nullable=True) + auto_reload_amount_micros = Column(BigInteger, nullable=True) + auto_reload_payment_method_id = Column(String, nullable=True) + auto_reload_failed_at = Column(TIMESTAMP(timezone=True), nullable=True) # User profile from OAuth display_name = Column(String, nullable=True) @@ -2587,33 +2599,40 @@ else: back_populates="user", cascade="all, delete-orphan", ) - premium_token_purchases = relationship( - "PremiumTokenPurchase", + credit_purchases = relationship( + "CreditPurchase", back_populates="user", cascade="all, delete-orphan", ) - # Page usage tracking for ETL services - pages_limit = Column( - Integer, - nullable=False, - default=config.PAGES_LIMIT, - server_default=str(config.PAGES_LIMIT), - ) - pages_used = Column(Integer, nullable=False, default=0, server_default="0") - - premium_credit_micros_limit = Column( + # Unified credit wallet (USD micro-units, 1_000_000 == $1.00). + # Decreases on use (ETL pages + premium model calls), increases on + # purchase / incentive grant / auto-reload. May dip slightly negative + # when an actual cost exceeds its pre-charge estimate; UI clamps at $0. + credit_micros_balance = Column( BigInteger, nullable=False, - default=config.PREMIUM_CREDIT_MICROS_LIMIT, - server_default=str(config.PREMIUM_CREDIT_MICROS_LIMIT), + default=config.DEFAULT_CREDIT_MICROS_BALANCE, + server_default=str(config.DEFAULT_CREDIT_MICROS_BALANCE), ) - premium_credit_micros_used = Column( + # In-flight reservation holds (released/settled at finalize). + credit_micros_reserved = Column( BigInteger, nullable=False, default=0, server_default="0" ) - premium_credit_micros_reserved = Column( - BigInteger, nullable=False, default=0, server_default="0" + + # Auto-reload (off-session Stripe top-up), behind AUTO_RELOAD_ENABLED. + # ``stripe_customer_id`` + ``auto_reload_payment_method_id`` are the + # saved-card plumbing; thresholds are micro-USD. ``auto_reload_failed_at`` + # is set (and ``auto_reload_enabled`` flipped off) when an off-session + # charge is declined so the UI can prompt the user to fix their card. + stripe_customer_id = Column(String, nullable=True) + auto_reload_enabled = Column( + Boolean, nullable=False, default=False, server_default="false" ) + auto_reload_threshold_micros = Column(BigInteger, nullable=True) + auto_reload_amount_micros = Column(BigInteger, nullable=True) + auto_reload_payment_method_id = Column(String, nullable=True) + auto_reload_failed_at = Column(TIMESTAMP(timezone=True), nullable=True) # User profile (can be set manually for non-OAuth users) display_name = Column(String, nullable=True) diff --git a/surfsense_backend/app/notifications/api/api.py b/surfsense_backend/app/notifications/api/api.py index ddca09c66..9a136ca7b 100644 --- a/surfsense_backend/app/notifications/api/api.py +++ b/surfsense_backend/app/notifications/api/api.py @@ -275,7 +275,7 @@ async def list_notifications( query = query.where(unread_filter) count_query = count_query.where(unread_filter) elif filter == "errors": - error_filter = (Notification.type == "page_limit_exceeded") | ( + error_filter = (Notification.type == "insufficient_credits") | ( Notification.notification_metadata["status"].astext == "failed" ) query = query.where(error_filter) diff --git a/surfsense_backend/app/notifications/constants.py b/surfsense_backend/app/notifications/constants.py index e8bd8391d..6fc13e3c7 100644 --- a/surfsense_backend/app/notifications/constants.py +++ b/surfsense_backend/app/notifications/constants.py @@ -12,6 +12,7 @@ CATEGORY_TYPES: dict[str, tuple[str, ...]] = { "connector_indexing", "connector_deletion", "document_processing", - "page_limit_exceeded", + "insufficient_credits", + "auto_reload_failed", ), } diff --git a/surfsense_backend/app/notifications/service/facade.py b/surfsense_backend/app/notifications/service/facade.py index 63154301c..9f4ad50d0 100644 --- a/surfsense_backend/app/notifications/service/facade.py +++ b/surfsense_backend/app/notifications/service/facade.py @@ -10,11 +10,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.notifications.persistence import Notification from app.notifications.service.handlers import ( + AutoReloadFailedNotificationHandler, CommentReplyNotificationHandler, ConnectorIndexingNotificationHandler, DocumentProcessingNotificationHandler, + InsufficientCreditsNotificationHandler, MentionNotificationHandler, - PageLimitNotificationHandler, ) logger = logging.getLogger(__name__) @@ -27,7 +28,8 @@ class NotificationService: document_processing = DocumentProcessingNotificationHandler() mention = MentionNotificationHandler() comment_reply = CommentReplyNotificationHandler() - page_limit = PageLimitNotificationHandler() + insufficient_credits = InsufficientCreditsNotificationHandler() + auto_reload_failed = AutoReloadFailedNotificationHandler() @staticmethod async def create_notification( diff --git a/surfsense_backend/app/notifications/service/handlers/__init__.py b/surfsense_backend/app/notifications/service/handlers/__init__.py index 8c32dea3b..1a6168e37 100644 --- a/surfsense_backend/app/notifications/service/handlers/__init__.py +++ b/surfsense_backend/app/notifications/service/handlers/__init__.py @@ -2,16 +2,18 @@ from __future__ import annotations +from .auto_reload_failed import AutoReloadFailedNotificationHandler from .comment_reply import CommentReplyNotificationHandler from .connector_indexing import ConnectorIndexingNotificationHandler from .document_processing import DocumentProcessingNotificationHandler +from .insufficient_credits import InsufficientCreditsNotificationHandler from .mention import MentionNotificationHandler -from .page_limit import PageLimitNotificationHandler __all__ = [ + "AutoReloadFailedNotificationHandler", "CommentReplyNotificationHandler", "ConnectorIndexingNotificationHandler", "DocumentProcessingNotificationHandler", + "InsufficientCreditsNotificationHandler", "MentionNotificationHandler", - "PageLimitNotificationHandler", ] diff --git a/surfsense_backend/app/notifications/service/handlers/auto_reload_failed.py b/surfsense_backend/app/notifications/service/handlers/auto_reload_failed.py new file mode 100644 index 000000000..0234a436d --- /dev/null +++ b/surfsense_backend/app/notifications/service/handlers/auto_reload_failed.py @@ -0,0 +1,54 @@ +"""Notifications for failed off-session credit auto-reload charges.""" + +from __future__ import annotations + +import logging +from uuid import UUID + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.notifications.persistence import Notification +from app.notifications.service.base import BaseNotificationHandler +from app.notifications.service.messages import auto_reload_failed as msg + +logger = logging.getLogger(__name__) + + +class AutoReloadFailedNotificationHandler(BaseNotificationHandler): + """Notifications for declined auto-reload top-ups.""" + + def __init__(self): + super().__init__("auto_reload_failed") + + async def notify_auto_reload_failed( + self, + session: AsyncSession, + user_id: UUID, + amount_micros: int, + payment_intent_id: str | None = None, + reason: str | None = None, + ) -> Notification: + """Notify that an off-session auto-reload charge was declined. + + Not tied to a search space (``search_space_id`` is None); the action + links to the billing settings so the user can fix their card. + """ + op_id = msg.operation_id(payment_intent_id or "") + title, message = msg.summary(amount_micros, reason) + + return await self.find_or_create_notification( + session=session, + user_id=user_id, + operation_id=op_id, + title=title, + message=message, + search_space_id=None, + initial_metadata={ + "amount_micros": amount_micros, + "payment_intent_id": payment_intent_id, + "status": "failed", + "error_type": "auto_reload_failed", + "action_url": "/dashboard", + "action_label": "Update card", + }, + ) diff --git a/surfsense_backend/app/notifications/service/handlers/page_limit.py b/surfsense_backend/app/notifications/service/handlers/insufficient_credits.py similarity index 55% rename from surfsense_backend/app/notifications/service/handlers/page_limit.py rename to surfsense_backend/app/notifications/service/handlers/insufficient_credits.py index 90722dc62..46124f222 100644 --- a/surfsense_backend/app/notifications/service/handlers/page_limit.py +++ b/surfsense_backend/app/notifications/service/handlers/insufficient_credits.py @@ -1,4 +1,4 @@ -"""Notifications for exceeding the page limit.""" +"""Notifications for running out of credit during document processing.""" from __future__ import annotations @@ -9,46 +9,42 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.notifications.persistence import Notification from app.notifications.service.base import BaseNotificationHandler -from app.notifications.service.messages import page_limit as msg +from app.notifications.service.messages import insufficient_credits as msg logger = logging.getLogger(__name__) -class PageLimitNotificationHandler(BaseNotificationHandler): - """Notifications for exceeding the page limit.""" +class InsufficientCreditsNotificationHandler(BaseNotificationHandler): + """Notifications for running out of credit during document processing.""" def __init__(self): - super().__init__("page_limit_exceeded") + super().__init__("insufficient_credits") - async def notify_page_limit_exceeded( + async def notify_insufficient_credits( self, session: AsyncSession, user_id: UUID, document_name: str, document_type: str, search_space_id: int, - pages_used: int, - pages_limit: int, - pages_to_add: int, + balance_micros: int, + required_micros: int, ) -> Notification: - """Notify that a document was blocked by the page limit.""" + """Notify that a document was blocked by insufficient credit.""" operation_id = msg.operation_id(document_name, search_space_id) - title, message = msg.summary( - document_name, pages_used, pages_limit, pages_to_add - ) + title, message = msg.summary(document_name, balance_micros, required_micros) metadata = { "operation_id": operation_id, "document_name": document_name, "document_type": document_type, - "pages_used": pages_used, - "pages_limit": pages_limit, - "pages_to_add": pages_to_add, + "balance_micros": balance_micros, + "required_micros": required_micros, "status": "failed", - "error_type": "page_limit_exceeded", + "error_type": "insufficient_credits", # Where the inbox item links to. - "action_url": f"/dashboard/{search_space_id}/more-pages", - "action_label": "Upgrade Plan", + "action_url": f"/dashboard/{search_space_id}/buy-more", + "action_label": "Buy credits", } notification = Notification( @@ -63,6 +59,7 @@ class PageLimitNotificationHandler(BaseNotificationHandler): await session.commit() await session.refresh(notification) logger.info( - f"Created page_limit_exceeded notification {notification.id} for user {user_id}" + f"Created insufficient_credits notification {notification.id} " + f"for user {user_id}" ) return notification diff --git a/surfsense_backend/app/notifications/service/messages/auto_reload_failed.py b/surfsense_backend/app/notifications/service/messages/auto_reload_failed.py new file mode 100644 index 000000000..5af19623c --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/auto_reload_failed.py @@ -0,0 +1,31 @@ +"""Pure presentation logic for auto-reload-failure notifications.""" + +from __future__ import annotations + +from datetime import UTC, datetime + + +def operation_id(payment_intent_id: str) -> str: + """Build a unique id for an auto-reload-failure notification. + + Keyed on the failed PaymentIntent so retries of the same charge collapse + into a single inbox item rather than spamming the user. + """ + if payment_intent_id: + return f"auto_reload_failed_{payment_intent_id}" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + return f"auto_reload_failed_{timestamp}" + + +def summary(amount_micros: int, reason: str | None) -> tuple[str, str]: + """Compute the title and message for a failed off-session auto-reload charge.""" + amount_usd = max(0, amount_micros) / 1_000_000 + title = "Auto-reload failed" + base = ( + f"We couldn't automatically add ${amount_usd:.2f} of credit because your " + "saved card was declined. Auto-reload has been turned off — update your " + "card and re-enable it to keep topping up automatically." + ) + if reason: + base = f"{base} (Reason: {reason}.)" + return title, base diff --git a/surfsense_backend/app/notifications/service/messages/insufficient_credits.py b/surfsense_backend/app/notifications/service/messages/insufficient_credits.py new file mode 100644 index 000000000..fad26ad91 --- /dev/null +++ b/surfsense_backend/app/notifications/service/messages/insufficient_credits.py @@ -0,0 +1,30 @@ +"""Pure presentation logic for insufficient-credit notifications.""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime + +from app.notifications.service.messages.text import truncate + + +def operation_id(document_name: str, search_space_id: int) -> str: + """Build a unique id for an insufficient-credits notification.""" + timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") + doc_hash = hashlib.md5(document_name.encode()).hexdigest()[:8] + return f"insufficient_credits_{search_space_id}_{timestamp}_{doc_hash}" + + +def summary( + document_name: str, balance_micros: int, required_micros: int +) -> tuple[str, str]: + """Compute the title and message for a blocked-by-insufficient-credits document.""" + display_name = truncate(document_name, 40) + title = f"Insufficient credits: {display_name}" + balance_usd = max(0, balance_micros) / 1_000_000 + required_usd = max(0, required_micros) / 1_000_000 + message = ( + f"This document costs about ${required_usd:.2f} to process but you have " + f"${balance_usd:.2f} of credit left. Add more credits to continue." + ) + return title, message diff --git a/surfsense_backend/app/notifications/service/messages/page_limit.py b/surfsense_backend/app/notifications/service/messages/page_limit.py deleted file mode 100644 index 54e5cbdec..000000000 --- a/surfsense_backend/app/notifications/service/messages/page_limit.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Pure presentation logic for page-limit notifications.""" - -from __future__ import annotations - -import hashlib -from datetime import UTC, datetime - -from app.notifications.service.messages.text import truncate - - -def operation_id(document_name: str, search_space_id: int) -> str: - """Build a unique id for a page-limit notification.""" - timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f") - doc_hash = hashlib.md5(document_name.encode()).hexdigest()[:8] - return f"page_limit_{search_space_id}_{timestamp}_{doc_hash}" - - -def summary( - document_name: str, pages_used: int, pages_limit: int, pages_to_add: int -) -> tuple[str, str]: - """Compute the title and message for a blocked-by-page-limit document.""" - display_name = truncate(document_name, 40) - title = f"Page limit exceeded: {display_name}" - message = f"This document has ~{pages_to_add} page(s) but you've used {pages_used}/{pages_limit} pages. Upgrade to process more documents." - return title, message diff --git a/surfsense_backend/app/notifications/types.py b/surfsense_backend/app/notifications/types.py index bb8bcfab1..f2974e584 100644 --- a/surfsense_backend/app/notifications/types.py +++ b/surfsense_backend/app/notifications/types.py @@ -10,7 +10,8 @@ NotificationType = Literal[ "document_processing", "new_mention", "comment_reply", - "page_limit_exceeded", + "insufficient_credits", + "auto_reload_failed", ] NotificationCategory = Literal["comments", "status"] diff --git a/surfsense_backend/app/routes/image_generation_routes.py b/surfsense_backend/app/routes/image_generation_routes.py index 018234ad5..33caf8453 100644 --- a/surfsense_backend/app/routes/image_generation_routes.py +++ b/surfsense_backend/app/routes/image_generation_routes.py @@ -622,11 +622,10 @@ async def create_image_generation( detail={ "error_code": "premium_quota_exhausted", "usage_type": exc.usage_type, - "used_micros": exc.used_micros, - "limit_micros": exc.limit_micros, + "balance_micros": exc.balance_micros, "remaining_micros": exc.remaining_micros, "message": ( - "Out of premium credits for image generation. " + "Out of credits for image generation. " "Purchase additional credits or switch to a free model." ), }, diff --git a/surfsense_backend/app/routes/incentive_tasks_routes.py b/surfsense_backend/app/routes/incentive_tasks_routes.py index 496b07d06..1dae09a2d 100644 --- a/surfsense_backend/app/routes/incentive_tasks_routes.py +++ b/surfsense_backend/app/routes/incentive_tasks_routes.py @@ -1,6 +1,6 @@ """ Incentive Tasks API routes. -Allows users to complete tasks (like starring GitHub repo) to earn free pages. +Allows users to complete tasks (like starring GitHub repo) to earn free credits. Each task can only be completed once per user. """ @@ -42,21 +42,21 @@ async def get_incentive_tasks( # Build task list with completion status tasks = [] - total_pages_earned = 0 + total_credit_micros_earned = 0 for task_type, config in INCENTIVE_TASKS_CONFIG.items(): completed_task = completed_tasks.get(task_type) is_completed = completed_task is not None if is_completed: - total_pages_earned += completed_task.pages_awarded + total_credit_micros_earned += completed_task.credit_micros_awarded tasks.append( IncentiveTaskInfo( task_type=task_type, title=config["title"], description=config["description"], - pages_reward=config["pages_reward"], + credit_micros_reward=config["credit_micros_reward"], action_url=config["action_url"], completed=is_completed, completed_at=completed_task.completed_at if completed_task else None, @@ -65,7 +65,7 @@ async def get_incentive_tasks( return IncentiveTasksResponse( tasks=tasks, - total_pages_earned=total_pages_earned, + total_credit_micros_earned=total_credit_micros_earned, ) @@ -79,10 +79,10 @@ async def complete_task( session: AsyncSession = Depends(get_async_session), ) -> CompleteTaskResponse | TaskAlreadyCompletedResponse: """ - Mark an incentive task as completed and award pages to the user. + Mark an incentive task as completed and award credit to the user. Each task can only be completed once. If the task was already completed, - returns the existing completion information without awarding additional pages. + returns the existing completion information without awarding additional credit. """ # Validate task type exists in config task_config = INCENTIVE_TASKS_CONFIG.get(task_type) @@ -109,25 +109,23 @@ async def complete_task( ) # Create the task completion record - pages_reward = task_config["pages_reward"] + credit_micros_reward = task_config["credit_micros_reward"] new_task = UserIncentiveTask( user_id=user.id, task_type=task_type, - pages_awarded=pages_reward, + credit_micros_awarded=credit_micros_reward, ) session.add(new_task) - # pages_used can exceed pages_limit when a document's final page count is - # determined after processing. Base the new limit on the higher of the two - # so the rewarded pages are fully usable above the current high-water mark. - user.pages_limit = max(user.pages_used, user.pages_limit) + pages_reward + # Add the reward directly to the user's spendable wallet balance. + user.credit_micros_balance = user.credit_micros_balance + credit_micros_reward await session.commit() await session.refresh(user) return CompleteTaskResponse( success=True, - message=f"Task completed! You earned {pages_reward} pages.", - pages_awarded=pages_reward, - new_pages_limit=user.pages_limit, + message=f"Task completed! You earned ${credit_micros_reward / 1_000_000:.2f} of credit.", + credit_micros_awarded=credit_micros_reward, + new_balance_micros=user.credit_micros_balance, ) diff --git a/surfsense_backend/app/routes/stripe_routes.py b/surfsense_backend/app/routes/stripe_routes.py index fc5fded84..69dabb311 100644 --- a/surfsense_backend/app/routes/stripe_routes.py +++ b/surfsense_backend/app/routes/stripe_routes.py @@ -1,4 +1,10 @@ -"""Stripe routes for pay-as-you-go page purchases.""" +"""Stripe routes for the unified credit wallet. + +Buying credit packs ($1 == 1_000_000 micro-USD by default) tops up +``user.credit_micros_balance``. The same balance is debited for ETL page +processing and premium model calls. Legacy page-pack buying has been removed; +``page_purchases`` history is still readable via ``GET /stripe/purchases``. +""" from __future__ import annotations @@ -14,24 +20,24 @@ from stripe import SignatureVerificationError, StripeClient, StripeError from app.config import config from app.db import ( + CreditPurchase, + CreditPurchaseStatus, PagePurchase, - PagePurchaseStatus, - PremiumTokenPurchase, - PremiumTokenPurchaseStatus, User, get_async_session, ) from app.schemas.stripe import ( - CreateCheckoutSessionRequest, - CreateCheckoutSessionResponse, - CreateTokenCheckoutSessionRequest, - CreateTokenCheckoutSessionResponse, + AutoReloadSettingsResponse, + CreateAutoReloadSetupSessionRequest, + CreateAutoReloadSetupSessionResponse, + CreateCreditCheckoutSessionRequest, + CreateCreditCheckoutSessionResponse, + CreditPurchaseHistoryResponse, + CreditStripeStatusResponse, FinalizeCheckoutResponse, PagePurchaseHistoryResponse, - StripeStatusResponse, StripeWebhookResponse, - TokenPurchaseHistoryResponse, - TokenStripeStatusResponse, + UpdateAutoReloadSettingsRequest, ) from app.users import current_active_user @@ -50,11 +56,11 @@ def get_stripe_client() -> StripeClient: return StripeClient(config.STRIPE_SECRET_KEY) -def _ensure_page_buying_enabled() -> None: - if not config.STRIPE_PAGE_BUYING_ENABLED: +def _ensure_credit_buying_enabled() -> None: + if not config.STRIPE_CREDIT_BUYING_ENABLED: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Page purchases are temporarily unavailable.", + detail="Credit purchases are temporarily unavailable.", ) @@ -79,13 +85,62 @@ def _get_checkout_urls(search_space_id: int) -> tuple[str, str]: return success_url, cancel_url -def _get_required_stripe_price_id() -> str: - if not config.STRIPE_PRICE_ID: +def _get_required_credit_price_id() -> str: + if not config.STRIPE_CREDIT_PRICE_ID: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="STRIPE_PRICE_ID is not configured.", + detail="STRIPE_CREDIT_PRICE_ID is not configured.", ) - return config.STRIPE_PRICE_ID + return config.STRIPE_CREDIT_PRICE_ID + + +def _ensure_auto_reload_enabled() -> None: + if not config.AUTO_RELOAD_ENABLED: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Auto-reload is not available.", + ) + + +async def _get_or_create_stripe_customer( + stripe_client: StripeClient, db_session: AsyncSession, user: User +) -> str: + """Return the user's Stripe Customer id, creating + persisting one if needed. + + A Customer object is required to save and later reuse a card off-session + (Stripe: save-and-reuse). New checkouts attach to this customer so the same + saved card powers both manual top-ups and auto-reload. + """ + if user.stripe_customer_id: + return user.stripe_customer_id + + customer = stripe_client.v1.customers.create( + params={ + "email": user.email, + "metadata": {"user_id": str(user.id)}, + } + ) + customer_id = str(customer.id) + + # Persist on the live row with a lock to avoid two concurrent checkouts + # creating duplicate customers. + locked = ( + ( + await db_session.execute( + select(User).where(User.id == user.id).with_for_update(of=User) + ) + ) + .unique() + .scalar_one_or_none() + ) + if locked is not None: + if locked.stripe_customer_id: + # Another request won the race; reuse theirs. + customer_id = locked.stripe_customer_id + else: + locked.stripe_customer_id = customer_id + await db_session.commit() + return customer_id def _normalize_optional_string(value: Any) -> str | None: @@ -110,14 +165,9 @@ def _get_metadata(checkout_session: Any) -> dict[str, str]: if metadata is None: return {} - # 1. Plain dict (older SDKs that subclassed dict, JSON-decoded events - # in tests, etc.). if isinstance(metadata, dict): return {str(k): str(v) for k, v in metadata.items()} - # 2. Modern Stripe SDK: every ``StripeObject`` has ``to_dict()``. - # ``recursive=False`` is correct because Stripe metadata values - # are always primitive strings. to_dict = getattr(metadata, "to_dict", None) if callable(to_dict): try: @@ -130,8 +180,6 @@ def _get_metadata(checkout_session: Any) -> dict[str, str]: getattr(checkout_session, "id", "?"), ) - # 3. Last-resort: read the SDK's private ``_data`` backing dict. - # Stable across stripe-python 6.x -> 15.x. inner = getattr(metadata, "_data", None) if isinstance(inner, dict): return {str(k): str(v) for k, v in inner.items()} @@ -144,166 +192,50 @@ def _get_metadata(checkout_session: Any) -> dict[str, str]: return {} -# Canonical purchase_type metadata values. ``premium_credit`` was emitted -# by an earlier release of ``create_token_checkout_session`` so it's still -# accepted on the read side for backward compat with in-flight sessions. -_PURCHASE_TYPE_TOKEN_VALUES = frozenset({"premium_tokens", "premium_credit"}) +# Canonical purchase_type metadata value is ``credits``. ``premium_tokens`` and +# ``premium_credit`` were emitted by earlier releases so they're still accepted +# on the read side for any in-flight checkout sessions. +_PURCHASE_TYPE_CREDIT_VALUES = frozenset( + {"credits", "premium_tokens", "premium_credit"} +) -def _is_token_purchase(metadata: dict[str, str]) -> bool: - """Return True for premium-credit (a.k.a. premium_token) purchases.""" - return metadata.get("purchase_type", "page_packs") in _PURCHASE_TYPE_TOKEN_VALUES +def _is_credit_purchase(metadata: dict[str, str]) -> bool: + """Return True for a credit purchase (default for all live checkouts).""" + return metadata.get("purchase_type", "credits") in _PURCHASE_TYPE_CREDIT_VALUES -async def _get_or_create_purchase_from_checkout_session( - db_session: AsyncSession, - checkout_session: Any, -) -> PagePurchase | None: - """Look up a PagePurchase by checkout session ID (with FOR UPDATE lock). - - If the row doesn't exist yet (e.g. the webhook arrived before the API - response committed), create one from the Stripe session metadata. - """ - checkout_session_id = str(checkout_session.id) - purchase = ( - await db_session.execute( - select(PagePurchase) - .where(PagePurchase.stripe_checkout_session_id == checkout_session_id) - .with_for_update() - ) - ).scalar_one_or_none() - if purchase is not None: - return purchase - - metadata = _get_metadata(checkout_session) - user_id = metadata.get("user_id") - quantity = int(metadata.get("quantity", "0")) - pages_per_unit = int(metadata.get("pages_per_unit", "0")) - - if not user_id or quantity <= 0 or pages_per_unit <= 0: - logger.error( - "Skipping Stripe fulfillment for session %s due to incomplete metadata: %s", - checkout_session_id, - metadata, - ) - return None - - purchase = PagePurchase( - user_id=uuid.UUID(user_id), - stripe_checkout_session_id=checkout_session_id, - stripe_payment_intent_id=_normalize_optional_string( - getattr(checkout_session, "payment_intent", None) - ), - quantity=quantity, - pages_granted=quantity * pages_per_unit, - amount_total=getattr(checkout_session, "amount_total", None), - currency=getattr(checkout_session, "currency", None), - status=PagePurchaseStatus.PENDING, - ) - db_session.add(purchase) - await db_session.flush() - return purchase - - -async def _mark_purchase_failed( +async def _mark_credit_purchase_failed( db_session: AsyncSession, checkout_session_id: str ) -> StripeWebhookResponse: purchase = ( await db_session.execute( - select(PagePurchase) - .where(PagePurchase.stripe_checkout_session_id == checkout_session_id) + select(CreditPurchase) + .where(CreditPurchase.stripe_checkout_session_id == checkout_session_id) .with_for_update() ) ).scalar_one_or_none() - if purchase is not None and purchase.status == PagePurchaseStatus.PENDING: - purchase.status = PagePurchaseStatus.FAILED + if purchase is not None and purchase.status == CreditPurchaseStatus.PENDING: + purchase.status = CreditPurchaseStatus.FAILED await db_session.commit() return StripeWebhookResponse() -async def _mark_token_purchase_failed( - db_session: AsyncSession, checkout_session_id: str -) -> StripeWebhookResponse: - purchase = ( - await db_session.execute( - select(PremiumTokenPurchase) - .where( - PremiumTokenPurchase.stripe_checkout_session_id == checkout_session_id - ) - .with_for_update() - ) - ).scalar_one_or_none() - - if purchase is not None and purchase.status == PremiumTokenPurchaseStatus.PENDING: - purchase.status = PremiumTokenPurchaseStatus.FAILED - await db_session.commit() - - return StripeWebhookResponse() - - -async def _fulfill_completed_purchase( +async def _fulfill_completed_credit_purchase( db_session: AsyncSession, checkout_session: Any ) -> StripeWebhookResponse: - """Grant pages to the user after a confirmed Stripe payment. + """Grant credit to the user after a confirmed Stripe payment. - Uses SELECT ... FOR UPDATE on both the PagePurchase and User rows to + Uses ``SELECT ... FOR UPDATE`` on both the CreditPurchase and User rows to prevent double-granting when Stripe retries the webhook concurrently. """ - purchase = await _get_or_create_purchase_from_checkout_session( - db_session, checkout_session - ) - if purchase is None: - return StripeWebhookResponse() - - if purchase.status == PagePurchaseStatus.COMPLETED: - return StripeWebhookResponse() - - user = ( - ( - await db_session.execute( - select(User).where(User.id == purchase.user_id).with_for_update(of=User) - ) - ) - .unique() - .scalar_one_or_none() - ) - if user is None: - logger.error( - "Skipping Stripe fulfillment for session %s because user %s was not found.", - purchase.stripe_checkout_session_id, - purchase.user_id, - ) - return StripeWebhookResponse() - - purchase.status = PagePurchaseStatus.COMPLETED - purchase.completed_at = datetime.now(UTC) - purchase.amount_total = getattr(checkout_session, "amount_total", None) - purchase.currency = getattr(checkout_session, "currency", None) - purchase.stripe_payment_intent_id = _normalize_optional_string( - getattr(checkout_session, "payment_intent", None) - ) - # pages_used can exceed pages_limit when a document's final page count is - # determined after processing. Base the new limit on the higher of the two - # so the purchased pages are fully usable above the current high-water mark. - user.pages_limit = max(user.pages_used, user.pages_limit) + purchase.pages_granted - - await db_session.commit() - return StripeWebhookResponse() - - -async def _fulfill_completed_token_purchase( - db_session: AsyncSession, checkout_session: Any -) -> StripeWebhookResponse: - """Grant premium tokens to the user after a confirmed Stripe payment.""" checkout_session_id = str(checkout_session.id) purchase = ( await db_session.execute( - select(PremiumTokenPurchase) - .where( - PremiumTokenPurchase.stripe_checkout_session_id == checkout_session_id - ) + select(CreditPurchase) + .where(CreditPurchase.stripe_checkout_session_id == checkout_session_id) .with_for_update() ) ).scalar_one_or_none() @@ -312,10 +244,8 @@ async def _fulfill_completed_token_purchase( metadata = _get_metadata(checkout_session) user_id = metadata.get("user_id") quantity = int(metadata.get("quantity", "0")) - # Read the new metadata key first, fall back to the legacy one so - # in-flight checkout sessions created before the cost-credits - # release still fulfil correctly (the unit is numerically the - # same: $1 buys 1_000_000 micro-USD == 1_000_000 tokens). + # Read the new metadata key first, fall back to legacy ones so + # in-flight checkout sessions created before the rename still fulfil. credit_micros_per_unit = int( metadata.get("credit_micros_per_unit") or metadata.get("tokens_per_unit", "0") @@ -323,13 +253,13 @@ async def _fulfill_completed_token_purchase( if not user_id or quantity <= 0 or credit_micros_per_unit <= 0: logger.error( - "Skipping token fulfillment for session %s: incomplete metadata %s", + "Skipping credit fulfillment for session %s: incomplete metadata %s", checkout_session_id, metadata, ) return StripeWebhookResponse() - purchase = PremiumTokenPurchase( + purchase = CreditPurchase( user_id=uuid.UUID(user_id), stripe_checkout_session_id=checkout_session_id, stripe_payment_intent_id=_normalize_optional_string( @@ -339,12 +269,13 @@ async def _fulfill_completed_token_purchase( credit_micros_granted=quantity * credit_micros_per_unit, amount_total=getattr(checkout_session, "amount_total", None), currency=getattr(checkout_session, "currency", None), - status=PremiumTokenPurchaseStatus.PENDING, + source="checkout", + status=CreditPurchaseStatus.PENDING, ) db_session.add(purchase) await db_session.flush() - if purchase.status == PremiumTokenPurchaseStatus.COMPLETED: + if purchase.status == CreditPurchaseStatus.COMPLETED: return StripeWebhookResponse() user = ( @@ -358,45 +289,188 @@ async def _fulfill_completed_token_purchase( ) if user is None: logger.error( - "Skipping token fulfillment for session %s: user %s not found", + "Skipping credit fulfillment for session %s: user %s not found", purchase.stripe_checkout_session_id, purchase.user_id, ) return StripeWebhookResponse() - purchase.status = PremiumTokenPurchaseStatus.COMPLETED + purchase.status = CreditPurchaseStatus.COMPLETED purchase.completed_at = datetime.now(UTC) purchase.amount_total = getattr(checkout_session, "amount_total", None) purchase.currency = getattr(checkout_session, "currency", None) purchase.stripe_payment_intent_id = _normalize_optional_string( getattr(checkout_session, "payment_intent", None) ) - # Top up the user's credit balance by the granted micro-USD amount. - # ``max(used, limit)`` clamps the case where the legacy code wrote a - # used value above the limit (e.g. underbilling rounding) so adding - # ``credit_micros_granted`` always lifts the limit by the full pack - # size rather than disappearing into past overuse. - user.premium_credit_micros_limit = ( - max(user.premium_credit_micros_used, user.premium_credit_micros_limit) - + purchase.credit_micros_granted + # Add the granted micro-USD directly to the spendable wallet balance. + user.credit_micros_balance = ( + user.credit_micros_balance + purchase.credit_micros_granted ) await db_session.commit() return StripeWebhookResponse() -@router.post("/create-checkout-session", response_model=CreateCheckoutSessionResponse) -async def create_checkout_session( - body: CreateCheckoutSessionRequest, +async def _handle_setup_session_completed( + stripe_client: StripeClient, + db_session: AsyncSession, + checkout_session: Any, +) -> StripeWebhookResponse: + """Persist the saved card from a completed ``mode=setup`` checkout session. + + The setup session saves a card on the customer (Stripe save-and-reuse). We + pull the resulting payment method off the SetupIntent and store it as the + user's ``auto_reload_payment_method_id`` so the off-session charge can use + it. Auto-reload itself is only armed once the user enables it via the + settings endpoint. + """ + metadata = _get_metadata(checkout_session) + user_id = metadata.get("user_id") + if not user_id: + logger.warning( + "Setup session %s completed without user_id metadata", + getattr(checkout_session, "id", "?"), + ) + return StripeWebhookResponse() + + setup_intent_id = _normalize_optional_string( + getattr(checkout_session, "setup_intent", None) + ) + payment_method_id: str | None = None + if setup_intent_id: + try: + setup_intent = stripe_client.v1.setup_intents.retrieve(setup_intent_id) + payment_method_id = _normalize_optional_string( + getattr(setup_intent, "payment_method", None) + ) + except StripeError: + logger.exception( + "Failed to retrieve setup intent %s for session %s", + setup_intent_id, + getattr(checkout_session, "id", "?"), + ) + + if not payment_method_id: + logger.warning( + "Setup session %s completed without a payment method", + getattr(checkout_session, "id", "?"), + ) + return StripeWebhookResponse() + + user = ( + ( + await db_session.execute( + select(User) + .where(User.id == uuid.UUID(user_id)) + .with_for_update(of=User) + ) + ) + .unique() + .scalar_one_or_none() + ) + if user is None: + return StripeWebhookResponse() + + customer_id = _normalize_optional_string( + getattr(checkout_session, "customer", None) + ) + if customer_id and not user.stripe_customer_id: + user.stripe_customer_id = customer_id + user.auto_reload_payment_method_id = payment_method_id + await db_session.commit() + + # Make this the customer's default for future off-session charges. + if user.stripe_customer_id: + try: + stripe_client.v1.customers.update( + user.stripe_customer_id, + params={ + "invoice_settings": {"default_payment_method": payment_method_id} + }, + ) + except StripeError: + logger.warning( + "Failed to set default payment method for customer %s", + user.stripe_customer_id, + exc_info=True, + ) + + return StripeWebhookResponse() + + +async def _reconcile_auto_reload_payment_intent( + db_session: AsyncSession, + payment_intent: Any, + *, + succeeded: bool, +) -> StripeWebhookResponse: + """Backstop for the off-session auto-reload charge via webhook. + + The Celery task confirms the PaymentIntent synchronously and grants credit + inline, but the ``payment_intent.succeeded`` / ``payment_intent.payment_failed`` + webhook acts as a safety net. We locate the matching ``auto_reload`` + CreditPurchase by payment-intent id and only transition PENDING rows so we + never double-grant. + """ + payment_intent_id = str(payment_intent.id) + purchase = ( + await db_session.execute( + select(CreditPurchase) + .where(CreditPurchase.stripe_payment_intent_id == payment_intent_id) + .with_for_update() + ) + ).scalar_one_or_none() + + if purchase is None or purchase.status != CreditPurchaseStatus.PENDING: + return StripeWebhookResponse() + + if succeeded: + user = ( + ( + await db_session.execute( + select(User) + .where(User.id == purchase.user_id) + .with_for_update(of=User) + ) + ) + .unique() + .scalar_one_or_none() + ) + if user is None: + return StripeWebhookResponse() + purchase.status = CreditPurchaseStatus.COMPLETED + purchase.completed_at = datetime.now(UTC) + user.credit_micros_balance = ( + user.credit_micros_balance + purchase.credit_micros_granted + ) + else: + purchase.status = CreditPurchaseStatus.FAILED + + await db_session.commit() + return StripeWebhookResponse() + + +@router.post( + "/create-credit-checkout-session", + response_model=CreateCreditCheckoutSessionResponse, +) +async def create_credit_checkout_session( + body: CreateCreditCheckoutSessionRequest, user: User = Depends(current_active_user), db_session: AsyncSession = Depends(get_async_session), -) -> CreateCheckoutSessionResponse: - """Create a Stripe Checkout Session for buying page packs.""" - _ensure_page_buying_enabled() +) -> CreateCreditCheckoutSessionResponse: + """Create a Stripe Checkout Session for buying credit packs. + + Each pack grants ``STRIPE_CREDIT_MICROS_PER_UNIT`` micro-USD of credit + (default 1_000_000 = $1.00). The balance is debited at the actual provider + cost reported by LiteLLM (premium calls) or ``MICROS_PER_PAGE`` per page + (ETL), so $1 of credit always buys $1 worth of usage at cost. + """ + _ensure_credit_buying_enabled() stripe_client = get_stripe_client() - price_id = _get_required_stripe_price_id() + price_id = _get_required_credit_price_id() success_url, cancel_url = _get_checkout_urls(body.search_space_id) - pages_granted = body.quantity * config.STRIPE_PAGES_PER_UNIT + credit_micros_granted = body.quantity * config.STRIPE_CREDIT_MICROS_PER_UNIT try: checkout_session = stripe_client.v1.checkout.sessions.create( @@ -415,14 +489,14 @@ async def create_checkout_session( "metadata": { "user_id": str(user.id), "quantity": str(body.quantity), - "pages_per_unit": str(config.STRIPE_PAGES_PER_UNIT), - "purchase_type": "page_packs", + "credit_micros_per_unit": str(config.STRIPE_CREDIT_MICROS_PER_UNIT), + "purchase_type": "credits", }, } ) except StripeError as exc: logger.exception( - "Failed to create Stripe checkout session for user %s", user.id + "Failed to create credit checkout session for user %s", user.id ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, @@ -437,28 +511,23 @@ async def create_checkout_session( ) db_session.add( - PagePurchase( + CreditPurchase( user_id=user.id, stripe_checkout_session_id=str(checkout_session.id), stripe_payment_intent_id=_normalize_optional_string( getattr(checkout_session, "payment_intent", None) ), quantity=body.quantity, - pages_granted=pages_granted, + credit_micros_granted=credit_micros_granted, amount_total=getattr(checkout_session, "amount_total", None), currency=getattr(checkout_session, "currency", None), - status=PagePurchaseStatus.PENDING, + source="checkout", + status=CreditPurchaseStatus.PENDING, ) ) await db_session.commit() - return CreateCheckoutSessionResponse(checkout_url=checkout_url) - - -@router.get("/status", response_model=StripeStatusResponse) -async def get_stripe_status() -> StripeStatusResponse: - """Return page-buying availability for frontend feature gating.""" - return StripeStatusResponse(page_buying_enabled=config.STRIPE_PAGE_BUYING_ENABLED) + return CreateCreditCheckoutSessionResponse(checkout_url=checkout_url) @router.post("/webhook", response_model=StripeWebhookResponse) @@ -466,7 +535,7 @@ async def stripe_webhook( request: Request, db_session: AsyncSession = Depends(get_async_session), ) -> StripeWebhookResponse: - """Handle Stripe webhooks and grant purchased pages after payment.""" + """Handle Stripe webhooks and grant purchased credit after payment.""" if not config.STRIPE_WEBHOOK_SECRET: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -518,12 +587,37 @@ async def stripe_webhook( ) return StripeWebhookResponse() + # mode=setup sessions carry no line items / payment; they save a + # card for off-session auto-reload. + if getattr(checkout_session, "mode", None) == "setup": + return await _handle_setup_session_completed( + stripe_client, db_session, checkout_session + ) + metadata = _get_metadata(checkout_session) - if _is_token_purchase(metadata): - return await _fulfill_completed_token_purchase( + if _is_credit_purchase(metadata): + return await _fulfill_completed_credit_purchase( db_session, checkout_session ) - return await _fulfill_completed_purchase(db_session, checkout_session) + # Legacy page-pack purchase: page buying is removed, so log and + # ignore rather than fulfilling. + logger.info( + "Ignoring non-credit checkout session %s (purchase_type=%s); " + "page buying is removed.", + getattr(checkout_session, "id", "?"), + metadata.get("purchase_type"), + ) + return StripeWebhookResponse() + + if event.type == "payment_intent.succeeded": + return await _reconcile_auto_reload_payment_intent( + db_session, event.data.object, succeeded=True + ) + + if event.type == "payment_intent.payment_failed": + return await _reconcile_auto_reload_payment_intent( + db_session, event.data.object, succeeded=False + ) if event.type in { "checkout.session.async_payment_failed", @@ -531,16 +625,12 @@ async def stripe_webhook( }: checkout_session = event.data.object metadata = _get_metadata(checkout_session) - if _is_token_purchase(metadata): - return await _mark_token_purchase_failed( + if _is_credit_purchase(metadata): + return await _mark_credit_purchase_failed( db_session, str(checkout_session.id) ) - return await _mark_purchase_failed(db_session, str(checkout_session.id)) + return StripeWebhookResponse() except Exception: - # Re-raise so FastAPI returns 500 and Stripe retries this delivery. - # Logging here gives us a structured trail with event id + type so - # future webhook bugs surface immediately in the logs without - # having to grep by request_id. logger.exception( "Stripe webhook handler failed for event id=%s type=%s — Stripe will retry", getattr(event, "id", "?"), @@ -557,24 +647,17 @@ async def finalize_checkout( user: User = Depends(current_active_user), db_session: AsyncSession = Depends(get_async_session), ) -> FinalizeCheckoutResponse: - """Synchronously fulfil a checkout session from the success page. + """Synchronously fulfil a credit checkout session from the success page. Solves the webhook-vs-redirect race: the user lands on ``/dashboard//purchase-success?session_id=cs_...`` typically a - few hundred ms after paying, but Stripe's - ``checkout.session.completed`` webhook can take 5-30s+ to arrive. - Calling this endpoint on success-page mount fulfils the purchase - immediately by retrieving the session from Stripe's API and - invoking the same idempotent helpers the webhook uses. - - Idempotency: if the webhook has already fulfilled this purchase - (status=COMPLETED), the helpers short-circuit and we just return - the latest balance. Concurrent webhook + finalize calls are safe - because both acquire ``SELECT ... FOR UPDATE`` on the purchase row. + few hundred ms after paying, but Stripe's ``checkout.session.completed`` + webhook can take 5-30s+ to arrive. Calling this endpoint on success-page + mount fulfils the purchase immediately via the same idempotent helper the + webhook uses. Authorization: the session's ``client_reference_id`` must match the - authenticated user's id. This prevents a user from finalising - someone else's checkout session if they happen to know the id. + authenticated user's id. """ stripe_client = get_stripe_client() @@ -592,9 +675,6 @@ async def finalize_checkout( detail="Checkout session not found.", ) from exc - # Authorization check: the user finalising must be the user who - # initiated the checkout. ``client_reference_id`` is set in - # ``create_checkout_session`` / ``create_token_checkout_session``. client_reference_id = getattr(checkout_session, "client_reference_id", None) if client_reference_id != str(user.id): logger.warning( @@ -608,109 +688,75 @@ async def finalize_checkout( detail="This checkout session does not belong to you.", ) - metadata = _get_metadata(checkout_session) - is_token = _is_token_purchase(metadata) payment_status = getattr(checkout_session, "payment_status", None) session_status = getattr(checkout_session, "status", None) - - # Defensive fallback: if metadata can't be read for any reason - # (extraction failure, manually-created session in Stripe dashboard, - # SDK upgrade breaking ``to_dict``, etc.) we'd otherwise route every - # purchase to the page_packs handler and get stuck. Resolve the - # purchase_type by checking which table actually has the row keyed - # by this Stripe session id. - if not metadata: - existing_token_purchase = ( - await db_session.execute( - select(PremiumTokenPurchase.id).where( - PremiumTokenPurchase.stripe_checkout_session_id - == str(checkout_session.id) - ) - ) - ).scalar_one_or_none() - if existing_token_purchase is not None: - is_token = True - else: - existing_page_purchase = ( - await db_session.execute( - select(PagePurchase.id).where( - PagePurchase.stripe_checkout_session_id - == str(checkout_session.id) - ) - ) - ).scalar_one_or_none() - if existing_page_purchase is None: - logger.error( - "finalize_checkout: no purchase row in either table " - "and metadata is empty for session=%s user=%s", - session_id, - user.id, - ) - # Fall through; downstream path will short-circuit on - # missing-row + empty-metadata. - logger.info( - "finalize_checkout: recovered purchase_type=%s for session=%s " - "via DB fallback (metadata was empty)", - "premium_tokens" if is_token else "page_packs", - session_id, - ) - is_paid = payment_status in {"paid", "no_payment_required"} is_expired = session_status == "expired" if is_paid: - if is_token: - await _fulfill_completed_token_purchase(db_session, checkout_session) - else: - await _fulfill_completed_purchase(db_session, checkout_session) + await _fulfill_completed_credit_purchase(db_session, checkout_session) elif is_expired: - if is_token: - await _mark_token_purchase_failed(db_session, str(checkout_session.id)) - else: - await _mark_purchase_failed(db_session, str(checkout_session.id)) - # Otherwise (e.g. payment_status="unpaid", session_status="open"), - # leave the purchase row alone — frontend will keep polling and the - # webhook will eventually win the race. + await _mark_credit_purchase_failed(db_session, str(checkout_session.id)) + # Otherwise leave the row alone — frontend keeps polling and the webhook + # will eventually win the race. - # Refresh the user row so the response reflects any update applied - # by the fulfilment helpers in this same session. await db_session.refresh(user) - if is_token: - purchase = ( - await db_session.execute( - select(PremiumTokenPurchase).where( - PremiumTokenPurchase.stripe_checkout_session_id - == str(checkout_session.id) - ) - ) - ).scalar_one_or_none() - return FinalizeCheckoutResponse( - purchase_type="premium_tokens", - status=purchase.status.value if purchase else "pending", - premium_credit_micros_limit=user.premium_credit_micros_limit, - premium_credit_micros_used=user.premium_credit_micros_used, - premium_credit_micros_granted=( - purchase.credit_micros_granted if purchase else None - ), - ) - purchase = ( await db_session.execute( - select(PagePurchase).where( - PagePurchase.stripe_checkout_session_id == str(checkout_session.id) + select(CreditPurchase).where( + CreditPurchase.stripe_checkout_session_id == str(checkout_session.id) ) ) ).scalar_one_or_none() return FinalizeCheckoutResponse( - purchase_type="page_packs", status=purchase.status.value if purchase else "pending", - pages_limit=user.pages_limit, - pages_used=user.pages_used, - pages_granted=purchase.pages_granted if purchase else None, + credit_micros_balance=user.credit_micros_balance, + credit_micros_granted=(purchase.credit_micros_granted if purchase else None), ) +@router.get("/credit-status", response_model=CreditStripeStatusResponse) +async def get_credit_status( + user: User = Depends(current_active_user), +) -> CreditStripeStatusResponse: + """Return credit-buying availability and current balance for the frontend. + + ``credit_micros_balance`` is in micro-USD (1_000_000 = $1.00); the FE + divides by 1M when displaying. + """ + return CreditStripeStatusResponse( + credit_buying_enabled=config.STRIPE_CREDIT_BUYING_ENABLED, + credit_micros_balance=user.credit_micros_balance, + ) + + +@router.get("/credit-purchases", response_model=CreditPurchaseHistoryResponse) +async def get_credit_purchases( + user: User = Depends(current_active_user), + db_session: AsyncSession = Depends(get_async_session), + offset: int = 0, + limit: int = 50, +) -> CreditPurchaseHistoryResponse: + """Return the authenticated user's credit purchase history.""" + limit = min(limit, 100) + purchases = ( + ( + await db_session.execute( + select(CreditPurchase) + .where(CreditPurchase.user_id == user.id) + .order_by(CreditPurchase.created_at.desc()) + .offset(offset) + .limit(limit) + ) + ) + .scalars() + .all() + ) + + return CreditPurchaseHistoryResponse(purchases=purchases) + + @router.get("/purchases", response_model=PagePurchaseHistoryResponse) async def get_page_purchases( user: User = Depends(current_active_user), @@ -718,7 +764,10 @@ async def get_page_purchases( offset: int = 0, limit: int = 50, ) -> PagePurchaseHistoryResponse: - """Return the authenticated user's page-purchase history.""" + """Return the authenticated user's legacy page-purchase history (read-only). + + Page buying is removed; this endpoint stays for historical records. + """ limit = min(limit, 100) purchases = ( ( @@ -737,163 +786,152 @@ async def get_page_purchases( return PagePurchaseHistoryResponse(purchases=purchases) -# ============================================================================= -# Premium Token Purchase Routes -# ============================================================================= +def _auto_reload_settings_response(user: User) -> AutoReloadSettingsResponse: + return AutoReloadSettingsResponse( + feature_enabled=config.AUTO_RELOAD_ENABLED, + enabled=bool(user.auto_reload_enabled), + threshold_micros=user.auto_reload_threshold_micros, + amount_micros=user.auto_reload_amount_micros, + min_amount_micros=config.AUTO_RELOAD_MIN_AMOUNT_MICROS, + has_payment_method=bool(user.auto_reload_payment_method_id), + failed_at=user.auto_reload_failed_at, + ) -def _ensure_token_buying_enabled() -> None: - if not config.STRIPE_TOKEN_BUYING_ENABLED: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Premium token purchases are temporarily unavailable.", - ) +@router.post( + "/auto-reload/setup", + response_model=CreateAutoReloadSetupSessionResponse, +) +async def create_auto_reload_setup_session( + body: CreateAutoReloadSetupSessionRequest, + user: User = Depends(current_active_user), + db_session: AsyncSession = Depends(get_async_session), +) -> CreateAutoReloadSetupSessionResponse: + """Start a ``mode=setup`` checkout session to save a card for auto-reload. - -def _get_token_checkout_urls(search_space_id: int) -> tuple[str, str]: + Uses a SetupIntent (no immediate charge) attached to the user's Stripe + Customer so the card can later be charged off-session. On completion the + webhook stores the resulting payment method on the user. + """ + _ensure_auto_reload_enabled() + _ensure_credit_buying_enabled() + stripe_client = get_stripe_client() if not config.NEXT_FRONTEND_URL: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="NEXT_FRONTEND_URL is not configured.", ) + customer_id = await _get_or_create_stripe_customer(stripe_client, db_session, user) + base_url = config.NEXT_FRONTEND_URL.rstrip("/") - # See ``_get_checkout_urls`` for why session_id is appended. success_url = ( - f"{base_url}/dashboard/{search_space_id}/purchase-success" - f"?session_id={{CHECKOUT_SESSION_ID}}" + f"{base_url}/dashboard/{body.search_space_id}/user-settings/purchases" + f"?auto_reload_setup=success" + ) + cancel_url = ( + f"{base_url}/dashboard/{body.search_space_id}/user-settings/purchases" + f"?auto_reload_setup=cancel" ) - cancel_url = f"{base_url}/dashboard/{search_space_id}/purchase-cancel" - return success_url, cancel_url - - -def _get_required_token_price_id() -> str: - if not config.STRIPE_PREMIUM_TOKEN_PRICE_ID: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="STRIPE_PREMIUM_TOKEN_PRICE_ID is not configured.", - ) - return config.STRIPE_PREMIUM_TOKEN_PRICE_ID - - -@router.post("/create-token-checkout-session") -async def create_token_checkout_session( - body: CreateTokenCheckoutSessionRequest, - user: User = Depends(current_active_user), - db_session: AsyncSession = Depends(get_async_session), -): - """Create a Stripe Checkout Session for buying premium credit packs. - - Each pack grants ``STRIPE_CREDIT_MICROS_PER_UNIT`` micro-USD of - credit (default 1_000_000 = $1.00). The user's balance is debited - at the actual provider cost reported by LiteLLM at finalize time, - so $1 of credit always buys $1 worth of provider usage at cost. - """ - _ensure_token_buying_enabled() - stripe_client = get_stripe_client() - price_id = _get_required_token_price_id() - success_url, cancel_url = _get_token_checkout_urls(body.search_space_id) - credit_micros_granted = body.quantity * config.STRIPE_CREDIT_MICROS_PER_UNIT try: checkout_session = stripe_client.v1.checkout.sessions.create( params={ - "mode": "payment", + "mode": "setup", "success_url": success_url, "cancel_url": cancel_url, - "line_items": [ - { - "price": price_id, - "quantity": body.quantity, - } - ], + "customer": customer_id, "client_reference_id": str(user.id), - "customer_email": user.email, "metadata": { "user_id": str(user.id), - "quantity": str(body.quantity), - "credit_micros_per_unit": str(config.STRIPE_CREDIT_MICROS_PER_UNIT), - # Canonical value matched by ``_is_token_purchase``. - # The legacy ``"premium_credit"`` is still accepted on - # the read side for any in-flight sessions started - # before this rename. - "purchase_type": "premium_tokens", + "purchase_type": "auto_reload_setup", }, } ) except StripeError as exc: - logger.exception("Failed to create token checkout session for user %s", user.id) + logger.exception( + "Failed to create auto-reload setup session for user %s", user.id + ) raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail="Unable to create Stripe checkout session.", + detail="Unable to create Stripe setup session.", ) from exc checkout_url = getattr(checkout_session, "url", None) if not checkout_url: raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, - detail="Stripe checkout session did not return a URL.", + detail="Stripe setup session did not return a URL.", ) - db_session.add( - PremiumTokenPurchase( - user_id=user.id, - stripe_checkout_session_id=str(checkout_session.id), - stripe_payment_intent_id=_normalize_optional_string( - getattr(checkout_session, "payment_intent", None) - ), - quantity=body.quantity, - credit_micros_granted=credit_micros_granted, - amount_total=getattr(checkout_session, "amount_total", None), - currency=getattr(checkout_session, "currency", None), - status=PremiumTokenPurchaseStatus.PENDING, - ) - ) - await db_session.commit() - - return CreateTokenCheckoutSessionResponse(checkout_url=checkout_url) + return CreateAutoReloadSetupSessionResponse(checkout_url=checkout_url) -@router.get("/token-status") -async def get_token_status( +@router.get("/auto-reload", response_model=AutoReloadSettingsResponse) +async def get_auto_reload_settings( user: User = Depends(current_active_user), -): - """Return token-buying availability and current premium credit quota for frontend. - - Values are in micro-USD (1_000_000 = $1.00); the FE divides by 1M - when displaying. The route name is preserved for back-compat with - pinned client deployments. - """ - used = user.premium_credit_micros_used - limit = user.premium_credit_micros_limit - return TokenStripeStatusResponse( - token_buying_enabled=config.STRIPE_TOKEN_BUYING_ENABLED, - premium_credit_micros_used=used, - premium_credit_micros_limit=limit, - premium_credit_micros_remaining=max(0, limit - used), - ) +) -> AutoReloadSettingsResponse: + """Return the user's auto-reload configuration and saved-card state.""" + return _auto_reload_settings_response(user) -@router.get("/token-purchases") -async def get_token_purchases( +@router.put("/auto-reload", response_model=AutoReloadSettingsResponse) +async def update_auto_reload_settings( + body: UpdateAutoReloadSettingsRequest, user: User = Depends(current_active_user), db_session: AsyncSession = Depends(get_async_session), - offset: int = 0, - limit: int = 50, -): - """Return the authenticated user's premium token purchase history.""" - limit = min(limit, 100) - purchases = ( +) -> AutoReloadSettingsResponse: + """Update auto-reload preferences. + + Enabling requires a saved card plus a positive threshold and an amount of + at least ``AUTO_RELOAD_MIN_AMOUNT_MICROS``. Disabling always succeeds and + clears any prior failure flag. + """ + _ensure_auto_reload_enabled() + + locked = ( ( await db_session.execute( - select(PremiumTokenPurchase) - .where(PremiumTokenPurchase.user_id == user.id) - .order_by(PremiumTokenPurchase.created_at.desc()) - .offset(offset) - .limit(limit) + select(User).where(User.id == user.id).with_for_update(of=User) ) ) - .scalars() - .all() + .unique() + .scalar_one() ) - return TokenPurchaseHistoryResponse(purchases=purchases) + if body.enabled: + if not locked.auto_reload_payment_method_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Add a payment method before enabling auto-reload.", + ) + if not body.threshold_micros or body.threshold_micros <= 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="A positive low-balance threshold is required.", + ) + if ( + body.amount_micros is None + or body.amount_micros < config.AUTO_RELOAD_MIN_AMOUNT_MICROS + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "Reload amount must be at least " + f"{config.AUTO_RELOAD_MIN_AMOUNT_MICROS} micro-USD." + ), + ) + locked.auto_reload_enabled = True + locked.auto_reload_threshold_micros = body.threshold_micros + locked.auto_reload_amount_micros = body.amount_micros + # Re-enabling clears the prior failure flag so the user can retry. + locked.auto_reload_failed_at = None + else: + locked.auto_reload_enabled = False + if body.threshold_micros is not None: + locked.auto_reload_threshold_micros = body.threshold_micros + if body.amount_micros is not None: + locked.auto_reload_amount_micros = body.amount_micros + + await db_session.commit() + await db_session.refresh(locked) + return _auto_reload_settings_response(locked) diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index fdf34672b..3b9c73cd5 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -111,11 +111,13 @@ from .search_space import ( SearchSpaceWithStats, ) from .stripe import ( - CreateCheckoutSessionRequest, - CreateCheckoutSessionResponse, + CreateCreditCheckoutSessionRequest, + CreateCreditCheckoutSessionResponse, + CreditPurchaseHistoryResponse, + CreditPurchaseRead, + CreditStripeStatusResponse, PagePurchaseHistoryResponse, PagePurchaseRead, - StripeStatusResponse, StripeWebhookResponse, ) from .users import UserCreate, UserRead, UserUpdate @@ -143,8 +145,11 @@ __all__ = [ "ChunkCreate", "ChunkRead", "ChunkUpdate", - "CreateCheckoutSessionRequest", - "CreateCheckoutSessionResponse", + "CreateCreditCheckoutSessionRequest", + "CreateCreditCheckoutSessionResponse", + "CreditPurchaseHistoryResponse", + "CreditPurchaseRead", + "CreditStripeStatusResponse", "DefaultSystemInstructionsResponse", # Document schemas "DocumentBase", @@ -257,7 +262,6 @@ __all__ = [ "SearchSpaceRead", "SearchSpaceUpdate", "SearchSpaceWithStats", - "StripeStatusResponse", "StripeWebhookResponse", "ThreadHistoryLoadResponse", "ThreadListItem", diff --git a/surfsense_backend/app/schemas/incentive_tasks.py b/surfsense_backend/app/schemas/incentive_tasks.py index 52c2a5182..7b9b39cd1 100644 --- a/surfsense_backend/app/schemas/incentive_tasks.py +++ b/surfsense_backend/app/schemas/incentive_tasks.py @@ -15,7 +15,8 @@ class IncentiveTaskInfo(BaseModel): task_type: IncentiveTaskType title: str description: str - pages_reward: int + # Credit reward in USD micro-units (1_000_000 == $1.00). + credit_micros_reward: int action_url: str completed: bool completed_at: datetime | None = None @@ -25,7 +26,7 @@ class IncentiveTasksResponse(BaseModel): """Response containing all available incentive tasks with completion status.""" tasks: list[IncentiveTaskInfo] - total_pages_earned: int + total_credit_micros_earned: int class CompleteTaskRequest(BaseModel): @@ -39,8 +40,8 @@ class CompleteTaskResponse(BaseModel): success: bool message: str - pages_awarded: int - new_pages_limit: int + credit_micros_awarded: int + new_balance_micros: int class TaskAlreadyCompletedResponse(BaseModel): diff --git a/surfsense_backend/app/schemas/stripe.py b/surfsense_backend/app/schemas/stripe.py index ad13ddf04..39c1c653a 100644 --- a/surfsense_backend/app/schemas/stripe.py +++ b/surfsense_backend/app/schemas/stripe.py @@ -1,4 +1,4 @@ -"""Schemas for Stripe-backed page purchases.""" +"""Schemas for Stripe-backed credit purchases.""" import uuid from datetime import datetime @@ -8,27 +8,59 @@ from pydantic import BaseModel, ConfigDict, Field from app.db import PagePurchaseStatus -class CreateCheckoutSessionRequest(BaseModel): - """Request body for creating a page-purchase checkout session.""" +class CreateCreditCheckoutSessionRequest(BaseModel): + """Request body for creating a credit-purchase checkout session.""" quantity: int = Field(ge=1, le=100) search_space_id: int = Field(ge=1) -class CreateCheckoutSessionResponse(BaseModel): +class CreateCreditCheckoutSessionResponse(BaseModel): """Response containing the Stripe-hosted checkout URL.""" checkout_url: str -class StripeStatusResponse(BaseModel): - """Response describing Stripe page-buying availability.""" +class CreditPurchaseRead(BaseModel): + """Serialized credit purchase record. - page_buying_enabled: bool + ``credit_micros_granted`` is in micro-USD (1_000_000 = $1.00). + """ + + id: uuid.UUID + stripe_checkout_session_id: str + stripe_payment_intent_id: str | None = None + quantity: int + credit_micros_granted: int + amount_total: int | None = None + currency: str | None = None + source: str = "checkout" + status: str + completed_at: datetime | None = None + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CreditPurchaseHistoryResponse(BaseModel): + """Response containing the user's credit purchases.""" + + purchases: list[CreditPurchaseRead] + + +class CreditStripeStatusResponse(BaseModel): + """Response describing credit-buying availability and current balance. + + ``credit_micros_balance`` is in micro-USD; the FE divides by 1_000_000 + to display USD. + """ + + credit_buying_enabled: bool + credit_micros_balance: int = 0 class PagePurchaseRead(BaseModel): - """Serialized page-purchase record for purchase history.""" + """Serialized legacy page-purchase record (read-only history).""" id: uuid.UUID stripe_checkout_session_id: str @@ -45,11 +77,52 @@ class PagePurchaseRead(BaseModel): class PagePurchaseHistoryResponse(BaseModel): - """Response containing the authenticated user's page purchases.""" + """Response containing the authenticated user's legacy page purchases.""" purchases: list[PagePurchaseRead] +class AutoReloadSettingsResponse(BaseModel): + """Auto-reload configuration + saved-card state for the settings UI. + + All ``*_micros`` fields are micro-USD (1_000_000 == $1.00). ``feature_enabled`` + reflects the server-side ``AUTO_RELOAD_ENABLED`` flag; when it is false the + UI should hide / disable the auto-reload controls entirely. + """ + + feature_enabled: bool + enabled: bool = False + threshold_micros: int | None = None + amount_micros: int | None = None + min_amount_micros: int + has_payment_method: bool = False + failed_at: datetime | None = None + + +class UpdateAutoReloadSettingsRequest(BaseModel): + """Update auto-reload preferences. + + Enabling requires a saved card (set up via /stripe/auto-reload/setup) plus a + positive threshold and an amount of at least ``AUTO_RELOAD_MIN_AMOUNT_MICROS``. + """ + + enabled: bool + threshold_micros: int | None = Field(default=None, ge=0) + amount_micros: int | None = Field(default=None, ge=0) + + +class CreateAutoReloadSetupSessionRequest(BaseModel): + """Request body for starting the save-a-card (SetupIntent) checkout.""" + + search_space_id: int = Field(ge=1) + + +class CreateAutoReloadSetupSessionResponse(BaseModel): + """Response containing the Stripe-hosted setup (save-card) checkout URL.""" + + checkout_url: str + + class StripeWebhookResponse(BaseModel): """Generic acknowledgement for Stripe webhook delivery.""" @@ -66,64 +139,6 @@ class FinalizeCheckoutResponse(BaseModel): endpoint until it sees ``completed`` or a final ``failed``. """ - purchase_type: str # "page_packs" | "premium_tokens" - status: str # PagePurchaseStatus / PremiumTokenPurchaseStatus value - pages_limit: int | None = None - pages_used: int | None = None - pages_granted: int | None = None - premium_credit_micros_limit: int | None = None - premium_credit_micros_used: int | None = None - premium_credit_micros_granted: int | None = None - - -class CreateTokenCheckoutSessionRequest(BaseModel): - """Request body for creating a premium token purchase checkout session.""" - - quantity: int = Field(ge=1, le=100) - search_space_id: int = Field(ge=1) - - -class CreateTokenCheckoutSessionResponse(BaseModel): - """Response containing the Stripe-hosted checkout URL.""" - - checkout_url: str - - -class TokenPurchaseRead(BaseModel): - """Serialized premium credit purchase record. - - ``credit_micros_granted`` is in micro-USD (1_000_000 = $1.00). The - schema name kept ``Token`` for API back-compat with pinned clients. - """ - - id: uuid.UUID - stripe_checkout_session_id: str - stripe_payment_intent_id: str | None = None - quantity: int - credit_micros_granted: int - amount_total: int | None = None - currency: str | None = None status: str - completed_at: datetime | None = None - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class TokenPurchaseHistoryResponse(BaseModel): - """Response containing the user's premium credit purchases.""" - - purchases: list[TokenPurchaseRead] - - -class TokenStripeStatusResponse(BaseModel): - """Response describing premium-credit-buying availability and balance. - - All ``premium_credit_micros_*`` fields are in micro-USD; the FE - divides by 1_000_000 to display USD. - """ - - token_buying_enabled: bool - premium_credit_micros_used: int = 0 - premium_credit_micros_limit: int = 0 - premium_credit_micros_remaining: int = 0 + credit_micros_balance: int = 0 + credit_micros_granted: int | None = None diff --git a/surfsense_backend/app/schemas/users.py b/surfsense_backend/app/schemas/users.py index 88d0a4f37..558463f57 100644 --- a/surfsense_backend/app/schemas/users.py +++ b/surfsense_backend/app/schemas/users.py @@ -4,8 +4,7 @@ from fastapi_users import schemas class UserRead(schemas.BaseUser[uuid.UUID]): - pages_limit: int - pages_used: int + credit_micros_balance: int display_name: str | None = None avatar_url: str | None = None diff --git a/surfsense_backend/app/services/auto_model_pin_service.py b/surfsense_backend/app/services/auto_model_pin_service.py index 9bbca8669..c9fd8c315 100644 --- a/surfsense_backend/app/services/auto_model_pin_service.py +++ b/surfsense_backend/app/services/auto_model_pin_service.py @@ -268,7 +268,7 @@ async def _is_premium_eligible( parsed = _to_uuid(user_id) if parsed is None: return False - usage = await TokenQuotaService.premium_get_usage(session, parsed) + usage = await TokenQuotaService.credit_get_usage(session, parsed) return bool(usage.allowed) diff --git a/surfsense_backend/app/services/auto_reload_service.py b/surfsense_backend/app/services/auto_reload_service.py new file mode 100644 index 000000000..9f5114a56 --- /dev/null +++ b/surfsense_backend/app/services/auto_reload_service.py @@ -0,0 +1,99 @@ +"""Debit-triggered credit auto-reload. + +``maybe_trigger_auto_reload`` is a cheap, best-effort pre-filter invoked after +every credit debit (ETL ``charge_credits`` and premium ``credit_finalize``). +When the wallet drops below the user's configured threshold it enqueues the +Celery task that performs the authoritative re-check and the off-session Stripe +charge. All real safety (row lock, cooldown, Stripe idempotency) lives in the +task — this function only avoids enqueuing work that obviously isn't needed. + +Everything here is gated behind ``config.AUTO_RELOAD_ENABLED``; when the flag is +off this module is inert. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select + +from app.config import config + +logger = logging.getLogger(__name__) + + +async def maybe_trigger_auto_reload(user_id: str) -> None: + """Enqueue an auto-reload charge if the user's balance fell below threshold. + + Best-effort: any failure is swallowed by the caller. Opens its own + short-lived session so it never interferes with the caller's transaction + (it always runs after the caller has already committed the debit). + """ + if not config.AUTO_RELOAD_ENABLED: + return + + from app.db import CreditPurchase, CreditPurchaseStatus, User, async_session_maker + + async with async_session_maker() as session: + user = ( + (await session.execute(select(User).where(User.id == user_id))) + .unique() + .scalar_one_or_none() + ) + if user is None or not user.auto_reload_enabled: + return + + if not (user.stripe_customer_id and user.auto_reload_payment_method_id): + return + + threshold = user.auto_reload_threshold_micros + amount = user.auto_reload_amount_micros + if not threshold or not amount: + return + + available = user.credit_micros_balance - user.credit_micros_reserved + if available >= threshold: + return + + # Cheap cooldown pre-check: skip if a recent auto-reload purchase exists + # or a recent attempt failed (avoids hammering a declined card). + cutoff = datetime.now(UTC) - timedelta( + minutes=max(config.AUTO_RELOAD_COOLDOWN_MINUTES, 0) + ) + if user.auto_reload_failed_at and user.auto_reload_failed_at >= cutoff: + return + recent = ( + await session.execute( + select(CreditPurchase.id) + .where( + CreditPurchase.user_id == user.id, + CreditPurchase.source == "auto_reload", + CreditPurchase.created_at >= cutoff, + CreditPurchase.status.in_( + [ + CreditPurchaseStatus.PENDING, + CreditPurchaseStatus.COMPLETED, + ] + ), + ) + .limit(1) + ) + ).first() + if recent is not None: + return + + # Enqueue outside the session. The task re-checks everything with a row + # lock before charging, so a benign race here only costs a no-op task run. + try: + from app.tasks.celery_tasks.auto_reload_task import ( + auto_reload_credits_task, + ) + + auto_reload_credits_task.delay(str(user_id)) + except Exception: + logger.warning( + "Failed to enqueue auto_reload_credits task for user %s", + user_id, + exc_info=True, + ) diff --git a/surfsense_backend/app/services/billable_calls.py b/surfsense_backend/app/services/billable_calls.py index 92ccd6a78..919c49a21 100644 --- a/surfsense_backend/app/services/billable_calls.py +++ b/surfsense_backend/app/services/billable_calls.py @@ -69,8 +69,8 @@ BillableSessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]] class QuotaInsufficientError(Exception): - """Raised when ``TokenQuotaService.premium_reserve`` denies a billable - call because the user has exhausted their premium credit pool. + """Raised when ``TokenQuotaService.credit_reserve`` denies a billable + call because the user has exhausted their credit wallet. The route handler should catch this and return HTTP 402 Payment Required (or the equivalent for the surface area). Outside of the HTTP @@ -83,17 +83,15 @@ class QuotaInsufficientError(Exception): self, *, usage_type: str, - used_micros: int, - limit_micros: int, + balance_micros: int, remaining_micros: int, ) -> None: self.usage_type = usage_type - self.used_micros = used_micros - self.limit_micros = limit_micros + self.balance_micros = balance_micros self.remaining_micros = remaining_micros super().__init__( - f"Premium credit exhausted for {usage_type}: " - f"used={used_micros} limit={limit_micros} remaining={remaining_micros} (micro-USD)" + f"Credit exhausted for {usage_type}: " + f"balance={balance_micros} remaining={remaining_micros} (micro-USD)" ) @@ -267,7 +265,7 @@ async def billable_call( ``TokenTrackingCallback`` populates the accumulator automatically. Raises: - QuotaInsufficientError: when premium and ``premium_reserve`` denies. + QuotaInsufficientError: when premium and ``credit_reserve`` denies. """ is_premium = billing_tier == "premium" session_factory = billable_session_factory or shielded_async_session @@ -310,7 +308,7 @@ async def billable_call( request_id = str(uuid4()) async with session_factory() as quota_session: - reserve_result = await TokenQuotaService.premium_reserve( + reserve_result = await TokenQuotaService.credit_reserve( db_session=quota_session, user_id=user_id, request_id=request_id, @@ -320,18 +318,16 @@ async def billable_call( if not reserve_result.allowed: logger.info( "[billable_call] reserve DENIED user=%s usage_type=%s " - "reserve=%d used=%d limit=%d remaining=%d", + "reserve=%d balance=%d remaining=%d", user_id, usage_type, reserve_micros, - reserve_result.used, - reserve_result.limit, + reserve_result.balance, reserve_result.remaining, ) raise QuotaInsufficientError( usage_type=usage_type, - used_micros=reserve_result.used, - limit_micros=reserve_result.limit, + balance_micros=reserve_result.balance, remaining_micros=reserve_result.remaining, ) @@ -352,14 +348,14 @@ async def billable_call( # BaseException so cancellation also releases. try: async with session_factory() as quota_session: - await TokenQuotaService.premium_release( + await TokenQuotaService.credit_release( db_session=quota_session, user_id=user_id, reserved_micros=reserve_micros, ) except Exception: logger.exception( - "[billable_call] premium_release failed for user=%s " + "[billable_call] credit_release failed for user=%s " "reserve_micros=%d (reservation will be GC'd by quota " "reconciliation if/when implemented)", user_id, @@ -380,7 +376,7 @@ async def billable_call( thread_id, ) async with session_factory() as quota_session: - final_result = await TokenQuotaService.premium_finalize( + final_result = await TokenQuotaService.credit_finalize( db_session=quota_session, user_id=user_id, request_id=request_id, @@ -389,26 +385,25 @@ async def billable_call( ) logger.info( "[billable_call] finalize user=%s usage_type=%s actual=%d " - "reserved=%d → used=%d/%d (remaining=%d)", + "reserved=%d → balance=%d (remaining=%d)", user_id, usage_type, actual_micros, reserve_micros, - final_result.used, - final_result.limit, + final_result.balance, final_result.remaining, ) except Exception as finalize_exc: # Last-ditch: if finalize itself fails, we must at least release # so the reservation doesn't leak. logger.exception( - "[billable_call] premium_finalize failed for user=%s; " + "[billable_call] credit_finalize failed for user=%s; " "attempting release", user_id, ) try: async with session_factory() as quota_session: - await TokenQuotaService.premium_release( + await TokenQuotaService.credit_release( db_session=quota_session, user_id=user_id, reserved_micros=reserve_micros, @@ -465,7 +460,7 @@ async def _resolve_agent_billing_for_search_space( so the same model bills for chat + downstream podcast/video. If the user is not premium-eligible, the pin service auto-restricts to free deployments — denial only happens later in - ``billable_call.premium_reserve`` if the pin really is premium and + ``billable_call.credit_reserve`` if the pin really is premium and credit ran out mid-flow. * ``thread_id`` is None: fallback to ``("free", "auto")``. Forward-compat for any future direct-API path; today both Celery tasks always pass diff --git a/surfsense_backend/app/services/page_limit_service.py b/surfsense_backend/app/services/etl_credit_service.py similarity index 67% rename from surfsense_backend/app/services/page_limit_service.py rename to surfsense_backend/app/services/etl_credit_service.py index 47fe07fc6..5c4ea4bbd 100644 --- a/surfsense_backend/app/services/page_limit_service.py +++ b/surfsense_backend/app/services/etl_credit_service.py @@ -1,5 +1,14 @@ """ -Service for managing user page limits for ETL services. +Service for charging the unified credit wallet for ETL document processing. + +Replaces the legacy ``PageLimitService`` page-quota model. Page counts are +still estimated the same way; they are now converted to USD micro-credits +(``config.MICROS_PER_PAGE`` per page, times a per-mode multiplier) and debited +from ``user.credit_micros_balance``. + +When ``config.ETL_CREDIT_BILLING_ENABLED`` is False (the default for +self-hosted / OSS installs) every check/charge is a no-op, preserving the prior +effectively-unlimited ETL behaviour. """ import os @@ -8,141 +17,125 @@ from pathlib import Path, PurePosixPath from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from app.config import config -class PageLimitExceededError(Exception): - """ - Exception raised when a user exceeds their page processing limit. - """ + +class InsufficientCreditsError(Exception): + """Raised when a user lacks enough credit to process a document.""" def __init__( self, - message: str = "Page limit exceeded. Please contact admin to increase limits for your account.", - pages_used: int = 0, - pages_limit: int = 0, - pages_to_add: int = 0, + message: str = "Insufficient credits to process this document. " + "Add more credits to continue.", + balance_micros: int = 0, + required_micros: int = 0, ): - self.pages_used = pages_used - self.pages_limit = pages_limit - self.pages_to_add = pages_to_add + self.balance_micros = balance_micros + self.required_micros = required_micros super().__init__(message) -class PageLimitService: - """Service for checking and updating user page limits.""" +class EtlCreditService: + """Checks and charges the credit wallet for ETL page processing.""" def __init__(self, session: AsyncSession): self.session = session - async def check_page_limit( - self, user_id: str, estimated_pages: int = 1 - ) -> tuple[bool, int, int]: + @staticmethod + def billing_enabled() -> bool: + return config.ETL_CREDIT_BILLING_ENABLED + + @staticmethod + def pages_to_micros(pages: int, multiplier: int = 1) -> int: + """Convert a (multiplied) page count to USD micro-credits.""" + return int(pages) * int(multiplier) * config.MICROS_PER_PAGE + + async def get_available_micros(self, user_id: str) -> int | None: + """Return spendable credit in micro-USD (``balance - reserved``). + + Returns ``None`` when ETL billing is disabled, which callers treat as + "unlimited" (no batch skipping, no blocking). """ - Check if user has enough pages remaining for processing. + if not config.ETL_CREDIT_BILLING_ENABLED: + return None - Args: - user_id: The user's ID - estimated_pages: Estimated number of pages to be processed - - Returns: - Tuple of (has_capacity, pages_used, pages_limit) - - Raises: - PageLimitExceededError: If user would exceed their page limit - """ from app.db import User - # Get user's current page usage result = await self.session.execute( - select(User.pages_used, User.pages_limit).where(User.id == user_id) + select(User.credit_micros_balance, User.credit_micros_reserved).where( + User.id == user_id + ) ) row = result.first() - if not row: raise ValueError(f"User with ID {user_id} not found") - pages_used, pages_limit = row + balance, reserved = row + return balance - reserved - # Check if adding estimated pages would exceed limit - if pages_used + estimated_pages > pages_limit: - raise PageLimitExceededError( - message=f"Processing this document would exceed your page limit. " - f"Used: {pages_used}/{pages_limit} pages. " - f"Document has approximately {estimated_pages} page(s). " - f"Please contact admin to increase limits for your account.", - pages_used=pages_used, - pages_limit=pages_limit, - pages_to_add=estimated_pages, + async def check_credits( + self, user_id: str, estimated_pages: int = 1, multiplier: int = 1 + ) -> None: + """Raise :class:`InsufficientCreditsError` if the user can't afford to + process ``estimated_pages`` (times ``multiplier``). + + No-op when ETL billing is disabled. + """ + if not config.ETL_CREDIT_BILLING_ENABLED: + return + + required = self.pages_to_micros(estimated_pages, multiplier) + available = await self.get_available_micros(user_id) + if available is None: + return + + if required > available: + raise InsufficientCreditsError( + message=( + "Processing this document would exceed your available " + f"credit. Available: ${available / 1_000_000:.2f}. " + f"This document costs about ${required / 1_000_000:.2f} " + f"({estimated_pages} page(s)). Add more credits to continue." + ), + balance_micros=available, + required_micros=required, ) - return True, pages_used, pages_limit + async def charge_credits( + self, user_id: str, pages: int, multiplier: int = 1 + ) -> int | None: + """Debit the credit wallet after successful processing. - async def update_page_usage( - self, user_id: str, pages_to_add: int, allow_exceed: bool = False - ) -> int: + The balance may dip slightly negative when the actual page count + exceeds the pre-check estimate (the document is already processed), + mirroring the prior ``allow_exceed=True`` semantics. + + Returns the new balance in micros, or ``None`` when billing is disabled. """ - Update user's page usage after successful processing. + if not config.ETL_CREDIT_BILLING_ENABLED: + return None - Args: - user_id: The user's ID - pages_to_add: Number of pages to add to usage - allow_exceed: If True, allows update even if it exceeds limit - (used when document was already processed after passing initial check) - - Returns: - New total pages_used value - - Raises: - PageLimitExceededError: If adding pages would exceed limit and allow_exceed is False - """ from app.db import User - # Get user result = await self.session.execute(select(User).where(User.id == user_id)) user = result.unique().scalar_one_or_none() - if not user: raise ValueError(f"User with ID {user_id} not found") - # Check if this would exceed limit (only if allow_exceed is False) - new_usage = user.pages_used + pages_to_add - if not allow_exceed and new_usage > user.pages_limit: - raise PageLimitExceededError( - message=f"Cannot update page usage. Would exceed limit. " - f"Current: {user.pages_used}/{user.pages_limit}, " - f"Trying to add: {pages_to_add}", - pages_used=user.pages_used, - pages_limit=user.pages_limit, - pages_to_add=pages_to_add, - ) - - # Update usage - user.pages_used = new_usage + cost = self.pages_to_micros(pages, multiplier) + user.credit_micros_balance -= cost await self.session.commit() await self.session.refresh(user) - return user.pages_used + # Best-effort: fire an auto-reload check if the balance dropped low. + try: + from app.services.auto_reload_service import maybe_trigger_auto_reload - async def get_page_usage(self, user_id: str) -> tuple[int, int]: - """ - Get user's current page usage and limit. + await maybe_trigger_auto_reload(user_id) + except Exception: + pass - Args: - user_id: The user's ID - - Returns: - Tuple of (pages_used, pages_limit) - """ - from app.db import User - - result = await self.session.execute( - select(User.pages_used, User.pages_limit).where(User.id == user_id) - ) - row = result.first() - - if not row: - raise ValueError(f"User with ID {user_id} not found") - - return row + return user.credit_micros_balance def estimate_pages_from_elements(self, elements: list) -> int: """ diff --git a/surfsense_backend/app/services/token_quota_service.py b/surfsense_backend/app/services/token_quota_service.py index 310c3eb5e..d32c18722 100644 --- a/surfsense_backend/app/services/token_quota_service.py +++ b/surfsense_backend/app/services/token_quota_service.py @@ -99,7 +99,18 @@ class QuotaStatus(StrEnum): class QuotaResult: - __slots__ = ("allowed", "limit", "remaining", "reserved", "status", "used") + # ``used``/``limit`` are used by the anonymous (Redis) token path. + # ``balance``/``remaining``/``reserved`` are used by the credit (Postgres) + # path, all in USD micro-units. ``remaining`` == spendable (balance - reserved). + __slots__ = ( + "allowed", + "balance", + "limit", + "remaining", + "reserved", + "status", + "used", + ) def __init__( self, @@ -109,6 +120,7 @@ class QuotaResult: limit: int, reserved: int = 0, remaining: int = 0, + balance: int = 0, ): self.allowed = allowed self.status = status @@ -116,6 +128,7 @@ class QuotaResult: self.limit = limit self.reserved = reserved self.remaining = remaining + self.balance = balance def to_dict(self) -> dict[str, Any]: return { @@ -125,6 +138,7 @@ class QuotaResult: "limit": self.limit, "reserved": self.reserved, "remaining": self.remaining, + "balance": self.balance, } @@ -505,19 +519,33 @@ class TokenQuotaService: # ------------------------------------------------------------------ @staticmethod - async def premium_reserve( + def _credit_status(balance: int) -> QuotaStatus: + """Map a spendable balance to OK / WARNING / BLOCKED. + + There is no longer a fixed ceiling, so WARNING fires on a low absolute + balance (``config.CREDIT_LOW_BALANCE_WARNING_MICROS``) instead of a + percentage of a limit. + """ + if balance <= 0: + return QuotaStatus.BLOCKED + if balance < config.CREDIT_LOW_BALANCE_WARNING_MICROS: + return QuotaStatus.WARNING + return QuotaStatus.OK + + @staticmethod + async def credit_reserve( db_session: AsyncSession, user_id: Any, request_id: str, reserve_micros: int, ) -> QuotaResult: - """Reserve ``reserve_micros`` (USD micro-units) from the user's - premium credit balance. + """Reserve ``reserve_micros`` (USD micro-units) from the user's credit + wallet. - ``QuotaResult.used``/``limit``/``reserved``/``remaining`` are - all in micro-USD on this code path; callers (chat stream, - token-status route, FE display) convert to dollars by dividing - by 1_000_000. + ``QuotaResult.balance``/``reserved``/``remaining`` are in micro-USD on + this code path; callers (chat stream, credit-status route, FE display) + convert to dollars by dividing by 1_000_000. ``remaining`` is the + spendable amount (``balance - reserved``). """ from app.db import User @@ -538,48 +566,41 @@ class TokenQuotaService: limit=0, ) - limit = user.premium_credit_micros_limit - used = user.premium_credit_micros_used - reserved = user.premium_credit_micros_reserved + balance = user.credit_micros_balance + reserved = user.credit_micros_reserved - effective = used + reserved + reserve_micros - if effective > limit: - remaining = max(0, limit - used - reserved) + # Block when the new hold would exceed the spendable balance. + if reserved + reserve_micros > balance: + remaining = max(0, balance - reserved) await db_session.rollback() return QuotaResult( allowed=False, status=QuotaStatus.BLOCKED, - used=used, - limit=limit, + used=0, + limit=balance, reserved=reserved, remaining=remaining, + balance=balance, ) - user.premium_credit_micros_reserved = reserved + reserve_micros + user.credit_micros_reserved = reserved + reserve_micros await db_session.commit() new_reserved = reserved + reserve_micros - remaining = max(0, limit - used - new_reserved) - warning_threshold = int(limit * 0.8) - - if (used + new_reserved) >= limit: - status = QuotaStatus.BLOCKED - elif (used + new_reserved) >= warning_threshold: - status = QuotaStatus.WARNING - else: - status = QuotaStatus.OK + remaining = max(0, balance - new_reserved) return QuotaResult( allowed=True, - status=status, - used=used, - limit=limit, + status=TokenQuotaService._credit_status(remaining), + used=0, + limit=balance, reserved=new_reserved, remaining=remaining, + balance=balance, ) @staticmethod - async def premium_finalize( + async def credit_finalize( db_session: AsyncSession, user_id: Any, request_id: str, @@ -587,7 +608,8 @@ class TokenQuotaService: reserved_micros: int, ) -> QuotaResult: """Settle the reservation: release ``reserved_micros`` and debit - ``actual_micros`` (the LiteLLM-reported provider cost in micro-USD). + ``actual_micros`` (the LiteLLM-reported provider cost in micro-USD) + from the balance. """ from app.db import User @@ -605,44 +627,42 @@ class TokenQuotaService: allowed=False, status=QuotaStatus.BLOCKED, used=0, limit=0 ) - user.premium_credit_micros_reserved = max( - 0, user.premium_credit_micros_reserved - reserved_micros - ) - user.premium_credit_micros_used = ( - user.premium_credit_micros_used + actual_micros + user.credit_micros_reserved = max( + 0, user.credit_micros_reserved - reserved_micros ) + user.credit_micros_balance = user.credit_micros_balance - actual_micros await db_session.commit() - limit = user.premium_credit_micros_limit - used = user.premium_credit_micros_used - reserved = user.premium_credit_micros_reserved - remaining = max(0, limit - used - reserved) + balance = user.credit_micros_balance + reserved = user.credit_micros_reserved + remaining = max(0, balance - reserved) - warning_threshold = int(limit * 0.8) - if used >= limit: - status = QuotaStatus.BLOCKED - elif used >= warning_threshold: - status = QuotaStatus.WARNING - else: - status = QuotaStatus.OK + # Best-effort auto-reload nudge after the debit settles. + try: + from app.services.auto_reload_service import maybe_trigger_auto_reload + + await maybe_trigger_auto_reload(user_id) + except Exception: + pass return QuotaResult( allowed=True, - status=status, - used=used, - limit=limit, + status=TokenQuotaService._credit_status(remaining), + used=0, + limit=balance, reserved=reserved, remaining=remaining, + balance=balance, ) @staticmethod - async def premium_release( + async def credit_release( db_session: AsyncSession, user_id: Any, reserved_micros: int, ) -> None: - """Release ``reserved_micros`` previously held by ``premium_reserve``. + """Release ``reserved_micros`` previously held by ``credit_reserve``. Used when a request fails before finalize (so the reservation doesn't leak credit). @@ -659,13 +679,13 @@ class TokenQuotaService: .scalar_one_or_none() ) if user is not None: - user.premium_credit_micros_reserved = max( - 0, user.premium_credit_micros_reserved - reserved_micros + user.credit_micros_reserved = max( + 0, user.credit_micros_reserved - reserved_micros ) await db_session.commit() @staticmethod - async def premium_get_usage( + async def credit_get_usage( db_session: AsyncSession, user_id: Any, ) -> QuotaResult: @@ -681,24 +701,16 @@ class TokenQuotaService: allowed=False, status=QuotaStatus.BLOCKED, used=0, limit=0 ) - limit = user.premium_credit_micros_limit - used = user.premium_credit_micros_used - reserved = user.premium_credit_micros_reserved - remaining = max(0, limit - used - reserved) - - warning_threshold = int(limit * 0.8) - if used >= limit: - status = QuotaStatus.BLOCKED - elif used >= warning_threshold: - status = QuotaStatus.WARNING - else: - status = QuotaStatus.OK + balance = user.credit_micros_balance + reserved = user.credit_micros_reserved + remaining = max(0, balance - reserved) return QuotaResult( - allowed=used < limit, - status=status, - used=used, - limit=limit, + allowed=remaining > 0, + status=TokenQuotaService._credit_status(remaining), + used=0, + limit=balance, reserved=reserved, remaining=remaining, + balance=balance, ) diff --git a/surfsense_backend/app/tasks/celery_tasks/auto_reload_task.py b/surfsense_backend/app/tasks/celery_tasks/auto_reload_task.py new file mode 100644 index 000000000..41a6f0b70 --- /dev/null +++ b/surfsense_backend/app/tasks/celery_tasks/auto_reload_task.py @@ -0,0 +1,296 @@ +"""Debit-triggered off-session credit auto-reload. + +Enqueued (best-effort) by ``auto_reload_service.maybe_trigger_auto_reload`` +after a credit debit drops the wallet below the user's threshold. This task is +the authoritative path: it re-checks eligibility under a row lock, enforces the +cooldown, then charges the saved card off-session via a Stripe PaymentIntent +(Stripe: charging a saved card off-session). + +Idempotency comes from three layers: +- a per-attempt CreditPurchase row created PENDING before the charge, +- a Stripe idempotency key derived from that row id, +- the ``payment_intent.*`` webhook backstop in ``stripe_routes`` that only + transitions PENDING rows. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from stripe import CardError, StripeClient, StripeError + +from app.celery_app import celery_app +from app.config import config +from app.db import CreditPurchase, CreditPurchaseStatus, User +from app.notifications.service import NotificationService +from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task + +logger = logging.getLogger(__name__) + +# 1_000_000 micro-USD == $1.00 == 100 cents, so cents = micros / 10_000. +_MICROS_PER_CENT = 10_000 + + +def _get_stripe_client() -> StripeClient | None: + if not config.STRIPE_SECRET_KEY: + logger.warning("Auto-reload skipped because STRIPE_SECRET_KEY is not set.") + return None + return StripeClient(config.STRIPE_SECRET_KEY) + + +def _card_error_payment_intent_id(exc: CardError) -> str | None: + """Pull the PaymentIntent id off a declined off-session charge. + + Per Stripe's off-session guide the failed intent is on ``exc.error.payment_intent``, + which may be a StripeObject or a plain dict depending on the SDK path. + """ + err = getattr(exc, "error", None) + pi = getattr(err, "payment_intent", None) if err is not None else None + if pi is None: + return None + if isinstance(pi, dict): + return pi.get("id") + return getattr(pi, "id", None) + + +@celery_app.task(name="auto_reload_credits") +def auto_reload_credits_task(user_id: str): + """Charge the user's saved card to top up credits when below threshold.""" + return run_async_celery_task(_auto_reload_credits, user_id) + + +async def _auto_reload_credits(user_id: str) -> None: + if not config.AUTO_RELOAD_ENABLED: + return + + stripe_client = _get_stripe_client() + if stripe_client is None: + return + + cooldown = timedelta(minutes=max(config.AUTO_RELOAD_COOLDOWN_MINUTES, 0)) + now = datetime.now(UTC) + cutoff = now - cooldown + + async with get_celery_session_maker()() as db_session: + # Lock the user row so concurrent debits/tasks can't double-charge. + user = ( + ( + await db_session.execute( + select(User) + .where(User.id == uuid.UUID(user_id)) + .with_for_update(of=User) + ) + ) + .unique() + .scalar_one_or_none() + ) + if user is None or not user.auto_reload_enabled: + return + + if not (user.stripe_customer_id and user.auto_reload_payment_method_id): + return + + threshold = user.auto_reload_threshold_micros + amount = user.auto_reload_amount_micros + if not threshold or not amount: + return + + available = user.credit_micros_balance - user.credit_micros_reserved + if available >= threshold: + # Another reload (or a refund/grant) already restored the balance. + return + + # Cooldown: skip if a recent auto-reload purchase or failure happened. + recent = ( + await db_session.execute( + select(CreditPurchase.id) + .where( + CreditPurchase.user_id == user.id, + CreditPurchase.source == "auto_reload", + CreditPurchase.created_at >= cutoff, + CreditPurchase.status.in_( + [ + CreditPurchaseStatus.PENDING, + CreditPurchaseStatus.COMPLETED, + ] + ), + ) + .limit(1) + ) + ).first() + if recent is not None: + return + if user.auto_reload_failed_at and user.auto_reload_failed_at >= cutoff: + return + + customer_id = user.stripe_customer_id + payment_method_id = user.auto_reload_payment_method_id + amount_cents = max(round(amount / _MICROS_PER_CENT), 1) + + # Create the PENDING purchase row first so its id seeds the Stripe + # idempotency key and the webhook backstop can find it. + purchase = CreditPurchase( + user_id=user.id, + stripe_checkout_session_id=f"auto_reload:{uuid.uuid4()}", + quantity=0, + credit_micros_granted=amount, + amount_total=amount_cents, + currency="usd", + source="auto_reload", + status=CreditPurchaseStatus.PENDING, + ) + db_session.add(purchase) + await db_session.flush() + purchase_id = purchase.id + await db_session.commit() + + # Charge off-session outside the user-row lock so the network call doesn't + # hold the row. The purchase row is the synchronization point now. + try: + payment_intent = stripe_client.v1.payment_intents.create( + params={ + "amount": amount_cents, + "currency": "usd", + "customer": customer_id, + "payment_method": payment_method_id, + "off_session": True, + "confirm": True, + "metadata": { + "user_id": str(user_id), + "purchase_type": "auto_reload", + "purchase_id": str(purchase_id), + }, + }, + options={"idempotency_key": f"auto_reload:{purchase_id}"}, + ) + except CardError as exc: + await _record_failure( + purchase_id, + user_id, + amount, + payment_intent_id=_card_error_payment_intent_id(exc), + reason=getattr(exc, "user_message", None) or "Your card was declined.", + ) + return + except StripeError: + logger.exception("Auto-reload charge failed for user %s", user_id) + await _record_failure( + purchase_id, + user_id, + amount, + payment_intent_id=None, + reason="We couldn't process the charge. Please try again.", + ) + return + + payment_intent_id = str(payment_intent.id) + pi_status = getattr(payment_intent, "status", None) + + async with get_celery_session_maker()() as db_session: + purchase = ( + await db_session.execute( + select(CreditPurchase) + .where(CreditPurchase.id == purchase_id) + .with_for_update() + ) + ).scalar_one_or_none() + if purchase is None: + return + purchase.stripe_payment_intent_id = payment_intent_id + + if pi_status == "succeeded": + if purchase.status != CreditPurchaseStatus.COMPLETED: + user = ( + ( + await db_session.execute( + select(User) + .where(User.id == purchase.user_id) + .with_for_update(of=User) + ) + ) + .unique() + .scalar_one() + ) + purchase.status = CreditPurchaseStatus.COMPLETED + purchase.completed_at = datetime.now(UTC) + user.credit_micros_balance = ( + user.credit_micros_balance + purchase.credit_micros_granted + ) + user.auto_reload_failed_at = None + await db_session.commit() + logger.info( + "Auto-reload succeeded for user %s (+%s micro-USD)", + user_id, + amount, + ) + return + + # Not succeeded synchronously (e.g. requires_action / processing). + # Leave the row PENDING; the payment_intent webhook reconciles it. + await db_session.commit() + logger.info( + "Auto-reload PaymentIntent %s for user %s is %s; awaiting webhook.", + payment_intent_id, + user_id, + pi_status, + ) + + +async def _record_failure( + purchase_id: uuid.UUID, + user_id: str, + amount_micros: int, + *, + payment_intent_id: str | None, + reason: str | None, +) -> None: + """Mark the purchase FAILED, stamp the user, and notify them.""" + async with get_celery_session_maker()() as db_session: + purchase = ( + await db_session.execute( + select(CreditPurchase) + .where(CreditPurchase.id == purchase_id) + .with_for_update() + ) + ).scalar_one_or_none() + if purchase is not None and purchase.status == CreditPurchaseStatus.PENDING: + purchase.status = CreditPurchaseStatus.FAILED + if payment_intent_id: + purchase.stripe_payment_intent_id = payment_intent_id + + user = ( + ( + await db_session.execute( + select(User) + .where(User.id == uuid.UUID(user_id)) + .with_for_update(of=User) + ) + ) + .unique() + .scalar_one_or_none() + ) + if user is not None: + user.auto_reload_failed_at = datetime.now(UTC) + # Disable so a declined card doesn't get retried every debit; the + # user re-enables from settings (which clears the failure flag). + user.auto_reload_enabled = False + + await db_session.commit() + + try: + await NotificationService.auto_reload_failed.notify_auto_reload_failed( + session=db_session, + user_id=uuid.UUID(user_id), + amount_micros=amount_micros, + payment_intent_id=payment_intent_id, + reason=reason, + ) + except Exception: + logger.warning( + "Failed to create auto_reload_failed notification for user %s", + user_id, + exc_info=True, + ) diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index d38014124..41e029a60 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -668,52 +668,52 @@ async def _process_file_upload( # Import here to avoid circular dependencies from fastapi import HTTPException - from app.services.page_limit_service import PageLimitExceededError + from app.services.etl_credit_service import InsufficientCreditsError - # Check if this is a page limit error (either direct or wrapped in HTTPException) - page_limit_error: PageLimitExceededError | None = None - if isinstance(e, PageLimitExceededError): - page_limit_error = e + # Check if this is an insufficient-credit error (either direct or + # wrapped in HTTPException) + credit_error: InsufficientCreditsError | None = None + if isinstance(e, InsufficientCreditsError): + credit_error = e elif ( isinstance(e, HTTPException) and e.__cause__ - and isinstance(e.__cause__, PageLimitExceededError) + and isinstance(e.__cause__, InsufficientCreditsError) ): - # HTTPException wraps the original PageLimitExceededError - page_limit_error = e.__cause__ - elif isinstance(e, HTTPException) and "page limit" in str(e.detail).lower(): - # Fallback: HTTPException with page limit message but no cause - page_limit_error = None # We don't have the details + # HTTPException wraps the original InsufficientCreditsError + credit_error = e.__cause__ + elif isinstance(e, HTTPException) and "credit" in str(e.detail).lower(): + # Fallback: HTTPException with credit message but no cause + credit_error = None # We don't have the details - # For page limit errors, create a dedicated page_limit_exceeded notification - if page_limit_error is not None: - error_message = str(page_limit_error) - # Create a dedicated page limit exceeded notification + # For insufficient-credit errors, create a dedicated notification + if credit_error is not None: + error_message = str(credit_error) + # Create a dedicated insufficient credits notification try: # First, mark the processing notification as failed await session.refresh(notification) await NotificationService.document_processing.notify_processing_completed( session=session, notification=notification, - error_message="Page limit exceeded", + error_message="Insufficient credits", ) - # Then create a separate page_limit_exceeded notification for better UX - await NotificationService.page_limit.notify_page_limit_exceeded( + # Then create a separate insufficient_credits notification for better UX + await NotificationService.insufficient_credits.notify_insufficient_credits( session=session, user_id=UUID(user_id), document_name=filename, document_type="FILE", search_space_id=search_space_id, - pages_used=page_limit_error.pages_used, - pages_limit=page_limit_error.pages_limit, - pages_to_add=page_limit_error.pages_to_add, + balance_micros=credit_error.balance_micros, + required_micros=credit_error.required_micros, ) except Exception as notif_error: logger.error( - f"Failed to create page limit notification: {notif_error!s}" + f"Failed to create insufficient credits notification: {notif_error!s}" ) - elif isinstance(e, HTTPException) and "page limit" in str(e.detail).lower(): + elif isinstance(e, HTTPException) and "credit" in str(e.detail).lower(): # HTTPException with page limit message but no detailed cause error_message = str(e.detail) try: @@ -984,18 +984,18 @@ async def _process_file_with_document( # Import here to avoid circular dependencies from fastapi import HTTPException - from app.services.page_limit_service import PageLimitExceededError + from app.services.etl_credit_service import InsufficientCreditsError - # Check if this is a page limit error - page_limit_error: PageLimitExceededError | None = None - if isinstance(e, PageLimitExceededError): - page_limit_error = e + # Check if this is an insufficient-credit error + credit_error: InsufficientCreditsError | None = None + if isinstance(e, InsufficientCreditsError): + credit_error = e elif ( isinstance(e, HTTPException) and e.__cause__ - and isinstance(e.__cause__, PageLimitExceededError) + and isinstance(e.__cause__, InsufficientCreditsError) ): - page_limit_error = e.__cause__ + credit_error = e.__cause__ # Mark document as failed (shows error in UI via Zero) error_message = str(e)[:500] @@ -1006,28 +1006,27 @@ async def _process_file_with_document( f"[_process_file_with_document] Document {document_id} marked as failed: {error_message[:100]}" ) - # Handle page limit errors with dedicated notification - if page_limit_error is not None: + # Handle insufficient-credit errors with dedicated notification + if credit_error is not None: try: await session.refresh(notification) await NotificationService.document_processing.notify_processing_completed( session=session, notification=notification, - error_message="Page limit exceeded", + error_message="Insufficient credits", ) - await NotificationService.page_limit.notify_page_limit_exceeded( + await NotificationService.insufficient_credits.notify_insufficient_credits( session=session, user_id=UUID(user_id), document_name=filename, document_type="FILE", search_space_id=search_space_id, - pages_used=page_limit_error.pages_used, - pages_limit=page_limit_error.pages_limit, - pages_to_add=page_limit_error.pages_to_add, + balance_micros=credit_error.balance_micros, + required_micros=credit_error.required_micros, ) except Exception as notif_error: logger.error( - f"Failed to create page limit notification: {notif_error!s}" + f"Failed to create insufficient credits notification: {notif_error!s}" ) else: # Update notification on failure diff --git a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py index 8b311576e..5c3b957ad 100644 --- a/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/podcast_tasks.py @@ -164,11 +164,9 @@ async def _generate_content_podcast( ) except QuotaInsufficientError as exc: logger.info( - "Podcast %s denied: out of premium credits " - "(used=%d/%d remaining=%d)", + "Podcast %s denied: out of credits (balance=%d remaining=%d)", podcast.id, - exc.used_micros, - exc.limit_micros, + exc.balance_micros, exc.remaining_micros, ) podcast.status = PodcastStatus.FAILED diff --git a/surfsense_backend/app/tasks/celery_tasks/stripe_reconciliation_task.py b/surfsense_backend/app/tasks/celery_tasks/stripe_reconciliation_task.py index ace6ef7ca..f1ed6c6b3 100644 --- a/surfsense_backend/app/tasks/celery_tasks/stripe_reconciliation_task.py +++ b/surfsense_backend/app/tasks/celery_tasks/stripe_reconciliation_task.py @@ -1,4 +1,4 @@ -"""Reconcile pending Stripe purchases that might miss webhook fulfillment.""" +"""Reconcile pending Stripe credit purchases that might miss webhook fulfillment.""" from __future__ import annotations @@ -11,10 +11,8 @@ from stripe import StripeClient, StripeError from app.celery_app import celery_app from app.config import config from app.db import ( - PagePurchase, - PagePurchaseStatus, - PremiumTokenPurchase, - PremiumTokenPurchaseStatus, + CreditPurchase, + CreditPurchaseStatus, ) from app.routes import stripe_routes from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task @@ -32,14 +30,14 @@ def get_stripe_client() -> StripeClient | None: return StripeClient(config.STRIPE_SECRET_KEY) -@celery_app.task(name="reconcile_pending_stripe_page_purchases") -def reconcile_pending_stripe_page_purchases_task(): - """Recover paid purchases that were left pending due to missed webhook handling.""" - return run_async_celery_task(_reconcile_pending_page_purchases) +@celery_app.task(name="reconcile_pending_stripe_credit_purchases") +def reconcile_pending_stripe_credit_purchases_task(): + """Recover paid credit purchases that were left pending due to missed webhook handling.""" + return run_async_celery_task(_reconcile_pending_credit_purchases) -async def _reconcile_pending_page_purchases() -> None: - """Reconcile stale pending page purchases against Stripe source of truth. +async def _reconcile_pending_credit_purchases() -> None: + """Reconcile stale pending credit purchases against Stripe source of truth. Stripe retries webhook delivery automatically, but best practice is to add an application-level reconciliation path in case all retries fail or the endpoint @@ -57,12 +55,12 @@ async def _reconcile_pending_page_purchases() -> None: pending_purchases = ( ( await db_session.execute( - select(PagePurchase) + select(CreditPurchase) .where( - PagePurchase.status == PagePurchaseStatus.PENDING, - PagePurchase.created_at <= cutoff, + CreditPurchase.status == CreditPurchaseStatus.PENDING, + CreditPurchase.created_at <= cutoff, ) - .order_by(PagePurchase.created_at.asc()) + .order_by(CreditPurchase.created_at.asc()) .limit(batch_size) ) ) @@ -72,13 +70,13 @@ async def _reconcile_pending_page_purchases() -> None: if not pending_purchases: logger.debug( - "Stripe reconciliation found no pending purchases older than %s minutes.", + "Stripe credit reconciliation found no pending purchases older than %s minutes.", lookback_minutes, ) return logger.info( - "Stripe reconciliation checking %s pending purchases (cutoff=%s, batch=%s).", + "Stripe credit reconciliation checking %s pending purchases (cutoff=%s, batch=%s).", len(pending_purchases), lookback_minutes, batch_size, @@ -96,7 +94,7 @@ async def _reconcile_pending_page_purchases() -> None: ) except StripeError: logger.exception( - "Stripe reconciliation failed to retrieve checkout session %s", + "Stripe credit reconciliation failed to retrieve checkout session %s", checkout_session_id, ) await db_session.rollback() @@ -107,119 +105,24 @@ async def _reconcile_pending_page_purchases() -> None: try: if payment_status in {"paid", "no_payment_required"}: - await stripe_routes._fulfill_completed_purchase( + await stripe_routes._fulfill_completed_credit_purchase( db_session, checkout_session ) fulfilled_count += 1 elif session_status == "expired": - await stripe_routes._mark_purchase_failed( + await stripe_routes._mark_credit_purchase_failed( db_session, str(checkout_session.id) ) failed_count += 1 except Exception: logger.exception( - "Stripe reconciliation failed while processing checkout session %s", + "Stripe credit reconciliation failed while processing checkout session %s", checkout_session_id, ) await db_session.rollback() logger.info( - "Stripe page reconciliation completed. fulfilled=%s failed=%s checked=%s", - fulfilled_count, - failed_count, - len(pending_purchases), - ) - - -@celery_app.task(name="reconcile_pending_stripe_token_purchases") -def reconcile_pending_stripe_token_purchases_task(): - """Recover paid token purchases that were left pending due to missed webhook handling.""" - return run_async_celery_task(_reconcile_pending_token_purchases) - - -async def _reconcile_pending_token_purchases() -> None: - """Reconcile stale pending token purchases against Stripe source of truth.""" - stripe_client = get_stripe_client() - if stripe_client is None: - return - - lookback_minutes = max(config.STRIPE_RECONCILIATION_LOOKBACK_MINUTES, 0) - batch_size = max(config.STRIPE_RECONCILIATION_BATCH_SIZE, 1) - cutoff = datetime.now(UTC) - timedelta(minutes=lookback_minutes) - - async with get_celery_session_maker()() as db_session: - pending_purchases = ( - ( - await db_session.execute( - select(PremiumTokenPurchase) - .where( - PremiumTokenPurchase.status - == PremiumTokenPurchaseStatus.PENDING, - PremiumTokenPurchase.created_at <= cutoff, - ) - .order_by(PremiumTokenPurchase.created_at.asc()) - .limit(batch_size) - ) - ) - .scalars() - .all() - ) - - if not pending_purchases: - logger.debug( - "Stripe token reconciliation found no pending purchases older than %s minutes.", - lookback_minutes, - ) - return - - logger.info( - "Stripe token reconciliation checking %s pending purchases (cutoff=%s, batch=%s).", - len(pending_purchases), - lookback_minutes, - batch_size, - ) - - fulfilled_count = 0 - failed_count = 0 - - for purchase in pending_purchases: - checkout_session_id = purchase.stripe_checkout_session_id - - try: - checkout_session = stripe_client.v1.checkout.sessions.retrieve( - checkout_session_id - ) - except StripeError: - logger.exception( - "Stripe token reconciliation failed to retrieve checkout session %s", - checkout_session_id, - ) - await db_session.rollback() - continue - - payment_status = getattr(checkout_session, "payment_status", None) - session_status = getattr(checkout_session, "status", None) - - try: - if payment_status in {"paid", "no_payment_required"}: - await stripe_routes._fulfill_completed_token_purchase( - db_session, checkout_session - ) - fulfilled_count += 1 - elif session_status == "expired": - await stripe_routes._mark_token_purchase_failed( - db_session, str(checkout_session.id) - ) - failed_count += 1 - except Exception: - logger.exception( - "Stripe token reconciliation failed while processing checkout session %s", - checkout_session_id, - ) - await db_session.rollback() - - logger.info( - "Stripe token reconciliation completed. fulfilled=%s failed=%s checked=%s", + "Stripe credit reconciliation completed. fulfilled=%s failed=%s checked=%s", fulfilled_count, failed_count, len(pending_purchases), diff --git a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py index 08f22140c..c6ce0b350 100644 --- a/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/video_presentation_tasks.py @@ -174,11 +174,10 @@ async def _generate_video_presentation( ) except QuotaInsufficientError as exc: logger.info( - "VideoPresentation %s denied: out of premium credits " - "(used=%d/%d remaining=%d)", + "VideoPresentation %s denied: out of credits " + "(balance=%d remaining=%d)", video_pres.id, - exc.used_micros, - exc.limit_micros, + exc.balance_micros, exc.remaining_micros, ) video_pres.status = VideoPresentationStatus.FAILED diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py index e33dca376..1e6097e53 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -85,11 +85,11 @@ from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( setup_connector_and_firecrawl, ) from app.tasks.chat.streaming.flows.shared.premium_quota import ( - PremiumReservation, - finalize_premium, - needs_premium_quota, - release_premium, - reserve_premium, + CreditReservation, + finalize_credit, + needs_credit_quota, + release_credit, + reserve_credit, ) from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( can_recover_provider_rate_limit, @@ -182,7 +182,7 @@ async def stream_new_chat( accumulator = start_turn() - premium_reservation: PremiumReservation | None = None + premium_reservation: CreditReservation | None = None busy_error_raised = False emit_stream_error = partial( @@ -259,8 +259,8 @@ async def stream_new_chat( yield streaming_service.format_done() return - if needs_premium_quota(agent_config, user_id): - premium_reservation = await reserve_premium( + if needs_credit_quota(agent_config, user_id): + premium_reservation = await reserve_credit( agent_config=agent_config, user_id=user_id, # type: ignore[arg-type] ) @@ -336,7 +336,7 @@ async def stream_new_chat( else: yield emit_stream_error( message=( - "Buy more tokens to continue with this model, or " + "Buy more credits to continue with this model, or " "switch to a free model" ), error_kind="premium_quota_exhausted", @@ -762,7 +762,7 @@ async def stream_new_chat( # sub-agent calls during a premium turn still contribute to the bill # (they're $0 in practice anyway). if premium_reservation is not None and user_id: - await finalize_premium( + await finalize_credit( reservation=premium_reservation, user_id=user_id, accumulator=accumulator, @@ -812,7 +812,7 @@ async def stream_new_chat( end_turn(str(chat_id)) if premium_reservation is not None and user_id: - await release_premium(reservation=premium_reservation, user_id=user_id) + await release_credit(reservation=premium_reservation, user_id=user_id) await close_session_and_clear_ai_responding(session, chat_id) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py index 6d0924850..e1552e79e 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py @@ -64,11 +64,11 @@ from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( setup_connector_and_firecrawl, ) from app.tasks.chat.streaming.flows.shared.premium_quota import ( - PremiumReservation, - finalize_premium, - needs_premium_quota, - release_premium, - reserve_premium, + CreditReservation, + finalize_credit, + needs_credit_quota, + release_credit, + reserve_credit, ) from app.tasks.chat.streaming.flows.shared.rate_limit_recovery import ( can_recover_provider_rate_limit, @@ -144,7 +144,7 @@ async def stream_resume_chat( accumulator = start_turn() - premium_reservation: PremiumReservation | None = None + premium_reservation: CreditReservation | None = None busy_error_raised = False emit_stream_error = partial( @@ -212,8 +212,8 @@ async def stream_resume_chat( "[stream_resume] LLM config loaded in %.3fs", time.perf_counter() - _t0 ) - if needs_premium_quota(agent_config, user_id): - premium_reservation = await reserve_premium( + if needs_credit_quota(agent_config, user_id): + premium_reservation = await reserve_credit( agent_config=agent_config, user_id=user_id, # type: ignore[arg-type] ) @@ -285,7 +285,7 @@ async def stream_resume_chat( else: yield emit_stream_error( message=( - "Buy more tokens to continue with this model, or " + "Buy more credits to continue with this model, or " "switch to a free model" ), error_kind="premium_quota_exhausted", @@ -544,7 +544,7 @@ async def stream_resume_chat( return if premium_reservation is not None and user_id: - await finalize_premium( + await finalize_credit( reservation=premium_reservation, user_id=user_id, accumulator=accumulator, @@ -584,7 +584,7 @@ async def stream_resume_chat( end_turn(str(chat_id)) if premium_reservation is not None and user_id: - await release_premium(reservation=premium_reservation, user_id=user_id) + await release_credit(reservation=premium_reservation, user_id=user_id) await close_session_and_clear_ai_responding(session, chat_id) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py index 6c08cb29f..232071394 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/premium_quota.py @@ -1,13 +1,12 @@ -"""Premium credit (USD micro-units) reserve / finalize / release lifecycle. +"""Credit wallet (USD micro-units) reserve / finalize / release lifecycle. -Both ``stream_new_chat`` and ``stream_resume_chat`` reserve premium credits up -front (so a single LLM call can't run away with the budget), then finalize the -actual provider cost reported by LiteLLM when the turn completes successfully, -or release the reservation on the cancellation / interrupted-without-finalize -paths. +Both ``stream_new_chat`` and ``stream_resume_chat`` reserve credits up front (so +a single LLM call can't run away with the budget), then finalize the actual +provider cost reported by LiteLLM when the turn completes successfully, or +release the reservation on the cancellation / interrupted-without-finalize paths. -State is held by the orchestrator as a simple ``PremiumReservation`` tuple -so reservation, fallback-on-denied, finalize, and release can all be reasoned +State is held by the orchestrator as a simple ``CreditReservation`` so +reservation, fallback-on-denied, finalize, and release can all be reasoned about from one place. """ @@ -27,8 +26,8 @@ if TYPE_CHECKING: @dataclass -class PremiumReservation: - """Active premium-credit reservation for one turn. +class CreditReservation: + """Active credit reservation for one turn. ``request_id`` is the per-reservation idempotency key (also passed to ``finalize``/``release`` so racing branches resolve to the same row). @@ -41,15 +40,15 @@ class PremiumReservation: allowed: bool -def needs_premium_quota(agent_config: AgentConfig | None, user_id: str | None) -> bool: +def needs_credit_quota(agent_config: AgentConfig | None, user_id: str | None) -> bool: return bool(agent_config is not None and user_id and agent_config.is_premium) -async def reserve_premium( +async def reserve_credit( *, agent_config: AgentConfig, user_id: str, -) -> PremiumReservation: +) -> CreditReservation: """Reserve estimated micros up front; returns the reservation handle.""" from app.services.token_quota_service import ( TokenQuotaService, @@ -68,22 +67,22 @@ async def reserve_premium( quota_reserve_tokens=agent_config.quota_reserve_tokens, ) async with shielded_async_session() as quota_session: - quota_result = await TokenQuotaService.premium_reserve( + quota_result = await TokenQuotaService.credit_reserve( db_session=quota_session, user_id=UUID(user_id), request_id=request_id, reserve_micros=reserve_amount_micros, ) - return PremiumReservation( + return CreditReservation( request_id=request_id, reserved_micros=reserve_amount_micros, allowed=quota_result.allowed, ) -async def finalize_premium( +async def finalize_credit( *, - reservation: PremiumReservation, + reservation: CreditReservation, user_id: str, accumulator: TokenAccumulator, ) -> None: @@ -96,7 +95,7 @@ async def finalize_premium( from app.services.token_quota_service import TokenQuotaService async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_finalize( + await TokenQuotaService.credit_finalize( db_session=quota_session, user_id=UUID(user_id), request_id=reservation.request_id, @@ -105,15 +104,15 @@ async def finalize_premium( ) except Exception: logging.getLogger(__name__).warning( - "Failed to finalize premium quota for user %s", + "Failed to finalize credit quota for user %s", user_id, exc_info=True, ) -async def release_premium( +async def release_credit( *, - reservation: PremiumReservation, + reservation: CreditReservation, user_id: str, ) -> None: """Release the reservation on cancellation paths; never raises.""" @@ -121,12 +120,12 @@ async def release_premium( from app.services.token_quota_service import TokenQuotaService async with shielded_async_session() as quota_session: - await TokenQuotaService.premium_release( + await TokenQuotaService.credit_release( db_session=quota_session, user_id=UUID(user_id), reserved_micros=reservation.reserved_micros, ) except Exception: logging.getLogger(__name__).warning( - "Failed to release premium quota for user %s", user_id + "Failed to release credit quota for user %s", user_id ) diff --git a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py index 7cd3e1613..9bf290d85 100644 --- a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py @@ -28,7 +28,7 @@ from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.exceptions import safe_exception_message from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.page_limit_service import PageLimitService +from app.services.etl_credit_service import EtlCreditService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -423,9 +423,8 @@ async def _index_full_scan( }, ) - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 page_limit_reached = False @@ -467,13 +466,17 @@ async def _index_full_scan( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): if not page_limit_reached: logger.warning( - "Page limit reached during Dropbox full scan, " + "Insufficient credits during Dropbox full scan, " "skipping remaining files" ) page_limit_reached = True @@ -498,9 +501,7 @@ async def _index_full_scan( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) indexed = renamed_count + batch_indexed logger.info( @@ -523,9 +524,8 @@ async def _index_selected_files( vision_llm=None, ) -> tuple[int, int, int, list[str]]: """Index user-selected files using the parallel pipeline.""" - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 files_to_download: list[dict] = [] @@ -560,12 +560,16 @@ async def _index_selected_files( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): display = file_name or file_path - errors.append(f"File '{display}': page limit would be exceeded") + errors.append(f"File '{display}': insufficient credits") continue batch_estimated_pages += file_pages @@ -586,9 +590,7 @@ async def _index_selected_files( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) return renamed_count + batch_indexed, skipped, unsupported_count, errors diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index b76f84bac..37de66ffd 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -41,7 +41,7 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.page_limit_service import PageLimitService +from app.services.etl_credit_service import EtlCreditService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -555,11 +555,11 @@ async def _process_single_file( return 1, 0, 0 return 0, 1, 0 - page_limit_service = PageLimitService(session) - estimated_pages = PageLimitService.estimate_pages_from_metadata( + etl_credit_service = EtlCreditService(session) + estimated_pages = EtlCreditService.estimate_pages_from_metadata( file_name, file.get("size") ) - await page_limit_service.check_page_limit(user_id, estimated_pages) + await etl_credit_service.check_credits(user_id, estimated_pages) markdown, drive_metadata, error = await download_and_extract_content( drive_client, file, vision_llm=vision_llm @@ -602,9 +602,7 @@ async def _process_single_file( continue await pipeline.index(document, connector_doc) - await page_limit_service.update_page_usage( - user_id, estimated_pages, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, estimated_pages) logger.info(f"Successfully indexed Google Drive file: {file_name}") return 1, 0, 0 @@ -713,9 +711,8 @@ async def _index_selected_files( Returns (indexed_count, skipped_count, unsupported_count, errors). """ - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 files_to_download: list[dict] = [] @@ -741,12 +738,16 @@ async def _index_selected_files( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): display = file_name or file_id - errors.append(f"File '{display}': page limit would be exceeded") + errors.append(f"File '{display}': insufficient credits") continue batch_estimated_pages += file_pages @@ -775,9 +776,7 @@ async def _index_selected_files( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) return renamed_count + batch_indexed, skipped, unsupported_count, errors @@ -820,9 +819,8 @@ async def _index_full_scan( # ------------------------------------------------------------------ # Phase 1 (serial): collect files, run skip checks, track renames # ------------------------------------------------------------------ - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 page_limit_reached = False @@ -877,13 +875,19 @@ async def _index_full_scan( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros( + batch_estimated_pages + file_pages + ) + > available_micros + ): if not page_limit_reached: logger.warning( - "Page limit reached during Google Drive full scan, " + "Insufficient credits during Google Drive full scan, " "skipping remaining files" ) page_limit_reached = True @@ -938,9 +942,7 @@ async def _index_full_scan( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) indexed = renamed_count + batch_indexed logger.info( @@ -996,9 +998,8 @@ async def _index_with_delta_sync( # ------------------------------------------------------------------ # Phase 1 (serial): handle removals, collect files for download # ------------------------------------------------------------------ - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 page_limit_reached = False @@ -1034,13 +1035,17 @@ async def _index_with_delta_sync( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): if not page_limit_reached: logger.warning( - "Page limit reached during Google Drive delta sync, " + "Insufficient credits during Google Drive delta sync, " "skipping remaining files" ) page_limit_reached = True @@ -1079,9 +1084,7 @@ async def _index_with_delta_sync( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) indexed = renamed_count + batch_indexed logger.info( diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index 1cd92dcf8..1a2d4b967 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -33,7 +33,7 @@ from app.db import ( from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.page_limit_service import PageLimitExceededError, PageLimitService +from app.services.etl_credit_service import EtlCreditService, InsufficientCreditsError from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker from app.utils.document_versioning import create_version_snapshot @@ -46,38 +46,38 @@ from .base import ( HeartbeatCallbackType = Callable[[int], Awaitable[None]] -def _estimate_pages_safe(page_limit_service: PageLimitService, file_path: str) -> int: +def _estimate_pages_safe(etl_credit_service: EtlCreditService, file_path: str) -> int: """Estimate page count with a file-size fallback.""" try: - return page_limit_service.estimate_pages_before_processing(file_path) + return etl_credit_service.estimate_pages_before_processing(file_path) except Exception: file_size = os.path.getsize(file_path) return max(1, file_size // (80 * 1024)) -async def _check_page_limit_or_skip( - page_limit_service: PageLimitService, +async def _check_credits_or_skip( + etl_credit_service: EtlCreditService, user_id: str, file_path: str, page_multiplier: int = 1, ) -> tuple[int, int]: - """Estimate pages and check the limit; raises PageLimitExceededError if over quota. + """Estimate pages and check credit; raises InsufficientCreditsError if unaffordable. Returns (estimated_pages, billable_pages). """ - estimated = _estimate_pages_safe(page_limit_service, file_path) + estimated = _estimate_pages_safe(etl_credit_service, file_path) billable = estimated * page_multiplier - await page_limit_service.check_page_limit(user_id, billable) + await etl_credit_service.check_credits(user_id, billable) return estimated, billable def _compute_final_pages( - page_limit_service: PageLimitService, + etl_credit_service: EtlCreditService, estimated_pages: int, content_length: int, ) -> int: """Return the final page count as max(estimated, actual).""" - actual = page_limit_service.estimate_pages_from_content_length(content_length) + actual = etl_credit_service.estimate_pages_from_content_length(content_length) return max(estimated_pages, actual) @@ -635,7 +635,7 @@ async def index_local_folder( skipped_count = 0 failed_count = 0 - page_limit_service = PageLimitService(session) + etl_credit_service = EtlCreditService(session) # ================================================================ # PHASE 1: Pre-filter files (mtime / content-hash), version changed @@ -694,12 +694,12 @@ async def index_local_folder( continue try: - estimated_pages, _billable = await _check_page_limit_or_skip( - page_limit_service, user_id, file_path_abs + estimated_pages, _billable = await _check_credits_or_skip( + etl_credit_service, user_id, file_path_abs ) - except PageLimitExceededError: + except InsufficientCreditsError: logger.warning( - f"Page limit exceeded, skipping: {file_path_abs}" + f"Insufficient credits, skipping: {file_path_abs}" ) failed_count += 1 continue @@ -730,12 +730,12 @@ async def index_local_folder( await create_version_snapshot(session, existing_document) else: try: - estimated_pages, _billable = await _check_page_limit_or_skip( - page_limit_service, user_id, file_path_abs + estimated_pages, _billable = await _check_credits_or_skip( + etl_credit_service, user_id, file_path_abs ) - except PageLimitExceededError: + except InsufficientCreditsError: logger.warning( - f"Page limit exceeded, skipping: {file_path_abs}" + f"Insufficient credits, skipping: {file_path_abs}" ) failed_count += 1 continue @@ -858,11 +858,9 @@ async def index_local_folder( est = mtime_info.get("estimated_pages", 1) content_len = mtime_info.get("content_length", 0) final_pages = _compute_final_pages( - page_limit_service, est, content_len - ) - await page_limit_service.update_page_usage( - user_id, final_pages, allow_exceed=True + etl_credit_service, est, content_len ) + await etl_credit_service.charge_credits(user_id, final_pages) else: failed_count += 1 @@ -1072,13 +1070,13 @@ async def _index_single_file( await session.commit() return 0, 0, None - page_limit_service = PageLimitService(session) + etl_credit_service = EtlCreditService(session) try: - estimated_pages, _billable = await _check_page_limit_or_skip( - page_limit_service, user_id, str(full_path) + estimated_pages, _billable = await _check_credits_or_skip( + etl_credit_service, user_id, str(full_path) ) - except PageLimitExceededError as e: - return 0, 1, f"Page limit exceeded: {e}" + except InsufficientCreditsError as e: + return 0, 1, f"Insufficient credits: {e}" try: content, content_hash = await _compute_file_content_hash( @@ -1142,11 +1140,9 @@ async def _index_single_file( if indexed: final_pages = _compute_final_pages( - page_limit_service, estimated_pages, len(content) - ) - await page_limit_service.update_page_usage( - user_id, final_pages, allow_exceed=True + etl_credit_service, estimated_pages, len(content) ) + await etl_credit_service.charge_credits(user_id, final_pages) await task_logger.log_task_success( log_entry, f"Single file indexed: {rel_path}", @@ -1299,7 +1295,7 @@ async def index_uploaded_files( await _set_indexing_flag(session, root_folder_id) - page_limit_service = PageLimitService(session) + etl_credit_service = EtlCreditService(session) pipeline = IndexingPipelineService(session) vision_llm_instance = None @@ -1345,14 +1341,14 @@ async def index_uploaded_files( continue try: - estimated_pages, _billable_pages = await _check_page_limit_or_skip( - page_limit_service, + estimated_pages, _billable_pages = await _check_credits_or_skip( + etl_credit_service, user_id, temp_path, page_multiplier=mode.page_multiplier, ) - except PageLimitExceededError: - logger.warning(f"Page limit exceeded, skipping: {relative_path}") + except InsufficientCreditsError: + logger.warning(f"Insufficient credits, skipping: {relative_path}") failed_count += 1 continue @@ -1425,12 +1421,10 @@ async def index_uploaded_files( if DocumentStatus.is_state(db_doc.status, DocumentStatus.READY): indexed_count += 1 final_pages = _compute_final_pages( - page_limit_service, estimated_pages, len(content) + etl_credit_service, estimated_pages, len(content) ) final_billable = final_pages * mode.page_multiplier - await page_limit_service.update_page_usage( - user_id, final_billable, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, final_billable) else: failed_count += 1 diff --git a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py index 3fd8a79f2..1a83551fb 100644 --- a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py @@ -28,7 +28,7 @@ from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.exceptions import safe_exception_message from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.page_limit_service import PageLimitService +from app.services.etl_credit_service import EtlCreditService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -318,9 +318,8 @@ async def _index_selected_files( vision_llm=None, ) -> tuple[int, int, int, list[str]]: """Index user-selected files using the parallel pipeline.""" - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 files_to_download: list[dict] = [] @@ -346,12 +345,16 @@ async def _index_selected_files( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): display = file_name or file_id - errors.append(f"File '{display}': page limit would be exceeded") + errors.append(f"File '{display}': insufficient credits") continue batch_estimated_pages += file_pages @@ -372,9 +375,7 @@ async def _index_selected_files( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) return renamed_count + batch_indexed, skipped, unsupported_count, errors @@ -413,9 +414,8 @@ async def _index_full_scan( }, ) - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 page_limit_reached = False @@ -448,13 +448,17 @@ async def _index_full_scan( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( file.get("name", ""), file.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): if not page_limit_reached: logger.warning( - "Page limit reached during OneDrive full scan, " + "Insufficient credits during OneDrive full scan, " "skipping remaining files" ) page_limit_reached = True @@ -479,9 +483,7 @@ async def _index_full_scan( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) indexed = renamed_count + batch_indexed logger.info( @@ -532,9 +534,8 @@ async def _index_with_delta_sync( logger.info(f"Processing {len(changes)} delta changes") - page_limit_service = PageLimitService(session) - pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) - remaining_quota = pages_limit - pages_used + etl_credit_service = EtlCreditService(session) + available_micros = await etl_credit_service.get_available_micros(user_id) batch_estimated_pages = 0 page_limit_reached = False @@ -571,13 +572,17 @@ async def _index_with_delta_sync( skipped += 1 continue - file_pages = PageLimitService.estimate_pages_from_metadata( + file_pages = EtlCreditService.estimate_pages_from_metadata( change.get("name", ""), change.get("size") ) - if batch_estimated_pages + file_pages > remaining_quota: + if ( + available_micros is not None + and EtlCreditService.pages_to_micros(batch_estimated_pages + file_pages) + > available_micros + ): if not page_limit_reached: logger.warning( - "Page limit reached during OneDrive delta sync, " + "Insufficient credits during OneDrive delta sync, " "skipping remaining files" ) page_limit_reached = True @@ -602,9 +607,7 @@ async def _index_with_delta_sync( pages_to_deduct = max( 1, batch_estimated_pages * batch_indexed // len(files_to_download) ) - await page_limit_service.update_page_usage( - user_id, pages_to_deduct, allow_exceed=True - ) + await etl_credit_service.charge_credits(user_id, pages_to_deduct) indexed = renamed_count + batch_indexed logger.info( diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index f6929b87c..a646b7aa6 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -79,10 +79,10 @@ async def _notify( # --------------------------------------------------------------------------- -def _estimate_pages_safe(page_limit_service, file_path: str) -> int: +def _estimate_pages_safe(etl_credit_service, file_path: str) -> int: """Estimate page count with a file-size fallback.""" try: - return page_limit_service.estimate_pages_before_processing(file_path) + return etl_credit_service.estimate_pages_before_processing(file_path) except Exception: file_size = os.path.getsize(file_path) return max(1, file_size // (80 * 1024)) @@ -185,11 +185,14 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None: """Route a document file to the configured ETL service via the unified pipeline.""" from app.etl_pipeline.etl_document import EtlRequest, ProcessingMode from app.etl_pipeline.etl_pipeline_service import EtlPipelineService - from app.services.page_limit_service import PageLimitExceededError, PageLimitService + from app.services.etl_credit_service import ( + EtlCreditService, + InsufficientCreditsError, + ) mode = ProcessingMode.coerce(ctx.processing_mode) - page_limit_service = PageLimitService(ctx.session) - estimated_pages = _estimate_pages_safe(page_limit_service, ctx.file_path) + etl_credit_service = EtlCreditService(ctx.session) + estimated_pages = _estimate_pages_safe(etl_credit_service, ctx.file_path) billable_pages = estimated_pages * mode.page_multiplier await ctx.task_logger.log_task_progress( @@ -204,16 +207,16 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None: ) try: - await page_limit_service.check_page_limit(ctx.user_id, billable_pages) - except PageLimitExceededError as e: + await etl_credit_service.check_credits(ctx.user_id, billable_pages) + except InsufficientCreditsError as e: await ctx.task_logger.log_task_failure( ctx.log_entry, - f"Page limit exceeded before processing: {ctx.filename}", + f"Insufficient credits before processing: {ctx.filename}", str(e), { - "error_type": "PageLimitExceeded", - "pages_used": e.pages_used, - "pages_limit": e.pages_limit, + "error_type": "InsufficientCredits", + "balance_micros": e.balance_micros, + "required_micros": e.required_micros, "estimated_pages": estimated_pages, "billable_pages": billable_pages, "processing_mode": mode.value, @@ -259,9 +262,7 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None: ) if result: - await page_limit_service.update_page_usage( - ctx.user_id, billable_pages, allow_exceed=True - ) + await etl_credit_service.charge_credits(ctx.user_id, billable_pages) if ctx.connector: await update_document_from_connector(result, ctx.connector, ctx.session) await ctx.task_logger.log_task_success( @@ -337,11 +338,11 @@ async def process_file_in_background( except Exception as e: await session.rollback() - from app.services.page_limit_service import PageLimitExceededError + from app.services.etl_credit_service import InsufficientCreditsError - if isinstance(e, PageLimitExceededError): + if isinstance(e, InsufficientCreditsError): error_message = str(e) - elif isinstance(e, HTTPException) and "page limit" in str(e.detail).lower(): + elif isinstance(e, HTTPException) and "credit" in str(e.detail).lower(): error_message = str(e.detail) else: error_message = f"Failed to process file: {filename}" @@ -414,12 +415,12 @@ async def _extract_file_content( ) if category == FileCategory.DOCUMENT: - from app.services.page_limit_service import PageLimitService + from app.services.etl_credit_service import EtlCreditService - page_limit_service = PageLimitService(session) - estimated_pages = _estimate_pages_safe(page_limit_service, file_path) + etl_credit_service = EtlCreditService(session) + estimated_pages = _estimate_pages_safe(etl_credit_service, file_path) billable_pages = estimated_pages * mode.page_multiplier - await page_limit_service.check_page_limit(user_id, billable_pages) + await etl_credit_service.check_credits(user_id, billable_pages) # Vision LLM is provided to the ETL pipeline for any file category # when the operator opts in. Image files run through it directly; @@ -524,12 +525,10 @@ async def process_file_in_background_with_document( ) if billable_pages > 0: - from app.services.page_limit_service import PageLimitService + from app.services.etl_credit_service import EtlCreditService - page_limit_service = PageLimitService(session) - await page_limit_service.update_page_usage( - user_id, billable_pages, allow_exceed=True - ) + etl_credit_service = EtlCreditService(session) + await etl_credit_service.charge_credits(user_id, billable_pages) await task_logger.log_task_success( log_entry, @@ -547,11 +546,11 @@ async def process_file_in_background_with_document( except Exception as e: await session.rollback() - from app.services.page_limit_service import PageLimitExceededError + from app.services.etl_credit_service import InsufficientCreditsError - if isinstance(e, PageLimitExceededError): + if isinstance(e, InsufficientCreditsError): error_message = str(e) - elif isinstance(e, HTTPException) and "page limit" in str(e.detail).lower(): + elif isinstance(e, HTTPException) and "credit" in str(e.detail).lower(): error_message = str(e.detail) else: error_message = f"Failed to process file: {filename}" diff --git a/surfsense_backend/app/zero_publication.py b/surfsense_backend/app/zero_publication.py index d2755d0a1..6d2ddf529 100644 --- a/surfsense_backend/app/zero_publication.py +++ b/surfsense_backend/app/zero_publication.py @@ -38,10 +38,7 @@ DOCUMENT_COLS = [ USER_COLS = [ "id", - "pages_limit", - "pages_used", - "premium_credit_micros_limit", - "premium_credit_micros_used", + "credit_micros_balance", ] AUTOMATION_RUN_COLS = [ diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index 13e3ab59c..812140be3 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -204,32 +204,34 @@ async def _cleanup_documents( # --------------------------------------------------------------------------- -# Page-limit helpers (direct DB for setup, API for verification) +# Credit-wallet helpers (direct DB for setup, API for verification) # --------------------------------------------------------------------------- -async def _get_user_page_usage(email: str) -> tuple[int, int]: +async def _get_user_credit(email: str) -> tuple[int, int]: conn = await asyncpg.connect(_ASYNCPG_URL) try: row = await conn.fetchrow( - 'SELECT pages_used, pages_limit FROM "user" WHERE email = $1', + "SELECT credit_micros_balance, credit_micros_reserved " + 'FROM "user" WHERE email = $1', email, ) assert row is not None, f"User {email!r} not found in database" - return row["pages_used"], row["pages_limit"] + return row["credit_micros_balance"], row["credit_micros_reserved"] finally: await conn.close() -async def _set_user_page_limits( - email: str, *, pages_used: int, pages_limit: int +async def _set_user_credit( + email: str, *, balance_micros: int, reserved_micros: int = 0 ) -> None: conn = await asyncpg.connect(_ASYNCPG_URL) try: await conn.execute( - 'UPDATE "user" SET pages_used = $1, pages_limit = $2 WHERE email = $3', - pages_used, - pages_limit, + 'UPDATE "user" SET credit_micros_balance = $1, ' + "credit_micros_reserved = $2 WHERE email = $3", + balance_micros, + reserved_micros, email, ) finally: @@ -237,23 +239,39 @@ async def _set_user_page_limits( @pytest.fixture -async def page_limits(): - """Manipulate the test user's page limits (direct DB for setup only). +async def credits(): + """Manipulate the test user's credit wallet (direct DB for setup only). - Automatically restores original values after each test. + Force-enables ETL credit billing for the duration of the test (it is off + by default for self-hosted/OSS, which would bypass all gating), and + automatically restores the original balance and billing flag afterwards. + + ``MICROS_PER_PAGE`` is exposed so callers can size balances by page count. """ - class _PageLimits: - async def set(self, *, pages_used: int, pages_limit: int) -> None: - await _set_user_page_limits( - TEST_EMAIL, pages_used=pages_used, pages_limit=pages_limit + class _Credits: + micros_per_page = app_config.MICROS_PER_PAGE + + async def set(self, *, balance_micros: int, reserved_micros: int = 0) -> None: + await _set_user_credit( + TEST_EMAIL, + balance_micros=balance_micros, + reserved_micros=reserved_micros, ) - original = await _get_user_page_usage(TEST_EMAIL) - yield _PageLimits() - await _set_user_page_limits( - TEST_EMAIL, pages_used=original[0], pages_limit=original[1] - ) + def pages(self, n: int) -> int: + return n * app_config.MICROS_PER_PAGE + + original_billing = app_config.ETL_CREDIT_BILLING_ENABLED + app_config.ETL_CREDIT_BILLING_ENABLED = True + original = await _get_user_credit(TEST_EMAIL) + try: + yield _Credits() + finally: + app_config.ETL_CREDIT_BILLING_ENABLED = original_billing + await _set_user_credit( + TEST_EMAIL, balance_micros=original[0], reserved_micros=original[1] + ) # --------------------------------------------------------------------------- diff --git a/surfsense_backend/tests/integration/document_upload/test_page_limits.py b/surfsense_backend/tests/integration/document_upload/test_etl_credits.py similarity index 67% rename from surfsense_backend/tests/integration/document_upload/test_page_limits.py rename to surfsense_backend/tests/integration/document_upload/test_etl_credits.py index 985fd7128..6a2972598 100644 --- a/surfsense_backend/tests/integration/document_upload/test_page_limits.py +++ b/surfsense_backend/tests/integration/document_upload/test_etl_credits.py @@ -1,14 +1,14 @@ """ -Integration tests for page-limit enforcement during document upload. +Integration tests for ETL credit enforcement during document upload. -These tests manipulate the test user's ``pages_used`` / ``pages_limit`` -columns directly in the database (setup only) and then exercise the upload -pipeline to verify that: +These tests manipulate the test user's ``credit_micros_balance`` column +directly in the database (setup only) and then exercise the upload pipeline +to verify that: - - Uploads are rejected *before* ETL when the limit is exhausted. - - ``pages_used`` increases after a successful upload (verified via API). - - A ``page_limit_exceeded`` notification is created on rejection. - - ``pages_used`` is not modified when a document fails processing. + - Uploads are rejected *before* ETL when the wallet can't cover the cost. + - The balance decreases after a successful upload (verified via API). + - An ``insufficient_credits`` notification is created on rejection. + - The balance is not modified when a document fails processing. All tests reuse the existing small fixtures (``sample.pdf``, ``sample.txt``) so no additional processing time is introduced. @@ -32,36 +32,37 @@ pytestmark = pytest.mark.integration # --------------------------------------------------------------------------- -# Helper: read pages_used through the public API +# Helper: read credit balance through the public API # --------------------------------------------------------------------------- -async def _get_pages_used(client: httpx.AsyncClient, headers: dict[str, str]) -> int: - """Fetch the current user's pages_used via the /users/me API.""" +async def _get_balance(client: httpx.AsyncClient, headers: dict[str, str]) -> int: + """Fetch the current user's credit_micros_balance via the /users/me API.""" resp = await client.get("/users/me", headers=headers) assert resp.status_code == 200, ( f"GET /users/me failed ({resp.status_code}): {resp.text}" ) - return resp.json()["pages_used"] + return resp.json()["credit_micros_balance"] # --------------------------------------------------------------------------- -# Test A: Successful upload increments pages_used +# Test A: Successful upload decrements the balance # --------------------------------------------------------------------------- -class TestPageUsageIncrementsOnSuccess: - """After a successful PDF upload the user's ``pages_used`` must grow.""" +class TestBalanceDecrementsOnSuccess: + """After a successful PDF upload the user's balance must shrink.""" - async def test_pages_used_increases_after_pdf_upload( + async def test_balance_decreases_after_pdf_upload( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=0, pages_limit=1000) + await credits.set(balance_micros=credits.pages(1000)) + before = await _get_balance(client, headers) resp = await upload_file( client, headers, "sample.pdf", search_space_id=search_space_id @@ -76,30 +77,28 @@ class TestPageUsageIncrementsOnSuccess: for did in doc_ids: assert statuses[did]["status"]["state"] == "ready" - used = await _get_pages_used(client, headers) - assert used > 0, "pages_used should have increased after successful processing" + after = await _get_balance(client, headers) + assert after < before, "balance should have dropped after successful processing" # --------------------------------------------------------------------------- -# Test B: Upload rejected when page limit is fully exhausted +# Test B: Upload rejected when the wallet is empty # --------------------------------------------------------------------------- -class TestUploadRejectedWhenLimitExhausted: - """ - When ``pages_used == pages_limit`` (zero remaining) the document - should reach ``failed`` status with a page-limit reason. - """ +class TestUploadRejectedWhenCreditExhausted: + """When the balance is zero the document should reach ``failed`` status + with an insufficient-credit reason.""" - async def test_pdf_fails_when_no_pages_remaining( + async def test_pdf_fails_when_no_credit_remaining( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=100, pages_limit=100) + await credits.set(balance_micros=0) resp = await upload_file( client, headers, "sample.pdf", search_space_id=search_space_id @@ -114,19 +113,19 @@ class TestUploadRejectedWhenLimitExhausted: for did in doc_ids: assert statuses[did]["status"]["state"] == "failed" reason = statuses[did]["status"].get("reason", "").lower() - assert "page limit" in reason, ( - f"Expected 'page limit' in failure reason, got: {reason!r}" + assert "credit" in reason, ( + f"Expected 'credit' in failure reason, got: {reason!r}" ) - async def test_pages_used_unchanged_after_limit_rejection( + async def test_balance_unchanged_after_rejection( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=50, pages_limit=50) + await credits.set(balance_micros=0) resp = await upload_file( client, headers, "sample.pdf", search_space_id=search_space_id @@ -139,30 +138,30 @@ class TestUploadRejectedWhenLimitExhausted: client, headers, doc_ids, search_space_id=search_space_id, timeout=300.0 ) - used = await _get_pages_used(client, headers) - assert used == 50, ( - f"pages_used should remain 50 after rejected upload, got {used}" + balance = await _get_balance(client, headers) + assert balance == 0, ( + f"balance should remain 0 after rejected upload, got {balance}" ) # --------------------------------------------------------------------------- -# Test C: Page-limit notification is created on rejection +# Test C: Insufficient-credits notification is created on rejection # --------------------------------------------------------------------------- -class TestPageLimitNotification: - """A ``page_limit_exceeded`` notification must be created when upload - is rejected due to the limit.""" +class TestInsufficientCreditsNotification: + """An ``insufficient_credits`` notification must be created when upload + is rejected due to an empty wallet.""" - async def test_page_limit_exceeded_notification_created( + async def test_insufficient_credits_notification_created( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=100, pages_limit=100) + await credits.set(balance_micros=0) resp = await upload_file( client, headers, "sample.pdf", search_space_id=search_space_id @@ -178,19 +177,18 @@ class TestPageLimitNotification: notifications = await get_notifications( client, headers, - type_filter="page_limit_exceeded", + type_filter="insufficient_credits", search_space_id=search_space_id, ) assert len(notifications) >= 1, ( - "Expected at least one page_limit_exceeded notification" + "Expected at least one insufficient_credits notification" ) latest = notifications[0] assert ( - "page limit" in latest["title"].lower() - or "page limit" in latest["message"].lower() + "credit" in latest["title"].lower() or "credit" in latest["message"].lower() ), ( - f"Notification should mention page limit: title={latest['title']!r}, " + f"Notification should mention credit: title={latest['title']!r}, " f"message={latest['message']!r}" ) @@ -210,9 +208,9 @@ class TestDocumentProcessingNotification: headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=0, pages_limit=1000) + await credits.set(balance_micros=credits.pages(1000)) resp = await upload_file( client, headers, "sample.txt", search_space_id=search_space_id @@ -242,23 +240,24 @@ class TestDocumentProcessingNotification: # --------------------------------------------------------------------------- -# Test E: pages_used unchanged when a document fails for non-limit reasons +# Test E: balance unchanged when a document fails for non-credit reasons # --------------------------------------------------------------------------- -class TestPagesUnchangedOnProcessingFailure: - """If a document fails during ETL (e.g. empty/corrupt file) rather than - a page-limit rejection, ``pages_used`` should remain unchanged.""" +class TestBalanceUnchangedOnProcessingFailure: + """If a document fails during ETL (e.g. empty/corrupt file) rather than a + credit rejection, the balance should remain unchanged.""" - async def test_pages_used_stable_on_etl_failure( + async def test_balance_stable_on_etl_failure( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=10, pages_limit=1000) + starting = credits.pages(1000) + await credits.set(balance_micros=starting) resp = await upload_file( client, headers, "empty.pdf", search_space_id=search_space_id @@ -274,28 +273,32 @@ class TestPagesUnchangedOnProcessingFailure: for did in doc_ids: assert statuses[did]["status"]["state"] == "failed" - used = await _get_pages_used(client, headers) - assert used == 10, f"pages_used should remain 10 after ETL failure, got {used}" + balance = await _get_balance(client, headers) + assert balance == starting, ( + f"balance should remain {starting} after ETL failure, got {balance}" + ) # --------------------------------------------------------------------------- -# Test F: Second upload rejected after first consumes remaining quota +# Test F: Second upload rejected after first consumes remaining credit # --------------------------------------------------------------------------- -class TestSecondUploadExceedsLimit: - """Upload one PDF successfully, consuming the quota, then verify a - second upload is rejected.""" +class TestSecondUploadExceedsCredit: + """Upload one PDF successfully, consuming the credit, then verify a second + upload is rejected.""" - async def test_second_upload_rejected_after_quota_consumed( + async def test_second_upload_rejected_after_credit_consumed( self, client: httpx.AsyncClient, headers: dict[str, str], search_space_id: int, cleanup_doc_ids: list[int], - page_limits, + credits, ): - await page_limits.set(pages_used=0, pages_limit=1) + # Exactly one page of credit: the first 1-page PDF fits, the second + # is rejected once the wallet hits zero. + await credits.set(balance_micros=credits.pages(1)) resp1 = await upload_file( client, headers, "sample.pdf", search_space_id=search_space_id @@ -327,6 +330,6 @@ class TestSecondUploadExceedsLimit: for did in second_ids: assert statuses2[did]["status"]["state"] == "failed" reason = statuses2[did]["status"].get("reason", "").lower() - assert "page limit" in reason, ( - f"Expected 'page limit' in failure reason, got: {reason!r}" + assert "credit" in reason, ( + f"Expected 'credit' in failure reason, got: {reason!r}" ) diff --git a/surfsense_backend/tests/integration/document_upload/test_stripe_page_purchases.py b/surfsense_backend/tests/integration/document_upload/test_stripe_credit_purchases.py similarity index 76% rename from surfsense_backend/tests/integration/document_upload/test_stripe_page_purchases.py rename to surfsense_backend/tests/integration/document_upload/test_stripe_credit_purchases.py index 143c9e252..e1955494d 100644 --- a/surfsense_backend/tests/integration/document_upload/test_stripe_page_purchases.py +++ b/surfsense_backend/tests/integration/document_upload/test_stripe_credit_purchases.py @@ -1,3 +1,10 @@ +"""Integration tests for Stripe credit-pack purchases. + +Buying credit packs tops up ``user.credit_micros_balance``. Legacy page-pack +buying has been removed; these tests exercise the credit checkout session, +webhook fulfillment (idempotent), and the reconciliation fallback. +""" + from __future__ import annotations from types import SimpleNamespace @@ -19,6 +26,8 @@ pytestmark = pytest.mark.integration _ASYNCPG_URL = TEST_DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://") +_CREDIT_MICROS_PER_UNIT = 1_000_000 + async def _execute(query: str, *args) -> None: conn = await asyncpg.connect(_ASYNCPG_URL) @@ -42,10 +51,12 @@ async def _get_user_id(email: str) -> str: return str(row["id"]) -async def _get_pages_limit(email: str) -> int: - row = await _fetchrow('SELECT pages_limit FROM "user" WHERE email = $1', email) +async def _get_balance(email: str) -> int: + row = await _fetchrow( + 'SELECT credit_micros_balance FROM "user" WHERE email = $1', email + ) assert row is not None, f"User {email!r} not found" - return row["pages_limit"] + return row["credit_micros_balance"] def _extract_access_token(response: httpx.Response) -> str | None: @@ -101,10 +112,23 @@ def headers(auth_token: str) -> dict[str, str]: @pytest.fixture(autouse=True) -async def _cleanup_page_purchases(): - await _execute("DELETE FROM page_purchases") +async def _cleanup_credit_purchases(): + await _execute("DELETE FROM credit_purchases") yield - await _execute("DELETE FROM page_purchases") + await _execute("DELETE FROM credit_purchases") + + +def _configure_credit_buying(monkeypatch) -> None: + monkeypatch.setattr(stripe_routes.config, "STRIPE_CREDIT_BUYING_ENABLED", True) + monkeypatch.setattr( + stripe_routes.config, "STRIPE_CREDIT_PRICE_ID", "price_credit_1" + ) + monkeypatch.setattr( + stripe_routes.config, "STRIPE_CREDIT_MICROS_PER_UNIT", _CREDIT_MICROS_PER_UNIT + ) + monkeypatch.setattr( + stripe_routes.config, "NEXT_FRONTEND_URL", "http://localhost:3000" + ) class _FakeCreateStripeClient: @@ -152,18 +176,19 @@ class _FakeReconciliationStripeClient: class TestStripeCheckoutSessionCreation: - async def test_get_status_reflects_backend_toggle( + async def test_credit_status_reflects_backend_toggle( self, client, headers, monkeypatch ): - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGE_BUYING_ENABLED", False) - disabled_response = await client.get("/api/v1/stripe/status", headers=headers) - assert disabled_response.status_code == 200, disabled_response.text - assert disabled_response.json() == {"page_buying_enabled": False} + monkeypatch.setattr(stripe_routes.config, "STRIPE_CREDIT_BUYING_ENABLED", False) + disabled = await client.get("/api/v1/stripe/credit-status", headers=headers) + assert disabled.status_code == 200, disabled.text + assert disabled.json()["credit_buying_enabled"] is False + assert "credit_micros_balance" in disabled.json() - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGE_BUYING_ENABLED", True) - enabled_response = await client.get("/api/v1/stripe/status", headers=headers) - assert enabled_response.status_code == 200, enabled_response.text - assert enabled_response.json() == {"page_buying_enabled": True} + monkeypatch.setattr(stripe_routes.config, "STRIPE_CREDIT_BUYING_ENABLED", True) + enabled = await client.get("/api/v1/stripe/credit-status", headers=headers) + assert enabled.status_code == 200, enabled.text + assert enabled.json()["credit_buying_enabled"] is True async def test_create_checkout_session_records_pending_purchase( self, @@ -182,14 +207,10 @@ class TestStripeCheckoutSessionCreation: fake_client = _FakeCreateStripeClient(checkout_session) monkeypatch.setattr(stripe_routes, "get_stripe_client", lambda: fake_client) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PRICE_ID", "price_pages_1000") - monkeypatch.setattr( - stripe_routes.config, "NEXT_FRONTEND_URL", "http://localhost:3000" - ) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGES_PER_UNIT", 1000) + _configure_credit_buying(monkeypatch) response = await client.post( - "/api/v1/stripe/create-checkout-session", + "/api/v1/stripe/create-credit-checkout-session", headers=headers, json={"quantity": 2, "search_space_id": search_space_id}, ) @@ -199,7 +220,7 @@ class TestStripeCheckoutSessionCreation: assert fake_client.last_params is not None assert fake_client.last_params["mode"] == "payment" assert fake_client.last_params["line_items"] == [ - {"price": "price_pages_1000", "quantity": 2} + {"price": "price_credit_1", "quantity": 2} ] assert ( fake_client.last_params["success_url"] @@ -210,19 +231,21 @@ class TestStripeCheckoutSessionCreation: fake_client.last_params["cancel_url"] == f"http://localhost:3000/dashboard/{search_space_id}/purchase-cancel" ) + assert fake_client.last_params["metadata"]["purchase_type"] == "credits" purchase = await _fetchrow( """ - SELECT quantity, pages_granted, status - FROM page_purchases + SELECT quantity, credit_micros_granted, status, source + FROM credit_purchases WHERE stripe_checkout_session_id = $1 """, checkout_session.id, ) assert purchase is not None assert purchase["quantity"] == 2 - assert purchase["pages_granted"] == 2000 + assert purchase["credit_micros_granted"] == 2 * _CREDIT_MICROS_PER_UNIT assert purchase["status"] == "PENDING" + assert purchase["source"] == "checkout" async def test_create_checkout_session_returns_503_when_buying_disabled( self, @@ -231,34 +254,34 @@ class TestStripeCheckoutSessionCreation: search_space_id: int, monkeypatch, ): - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGE_BUYING_ENABLED", False) + monkeypatch.setattr(stripe_routes.config, "STRIPE_CREDIT_BUYING_ENABLED", False) response = await client.post( - "/api/v1/stripe/create-checkout-session", + "/api/v1/stripe/create-credit-checkout-session", headers=headers, json={"quantity": 2, "search_space_id": search_space_id}, ) assert response.status_code == 503, response.text assert ( - response.json()["detail"] == "Page purchases are temporarily unavailable." + response.json()["detail"] == "Credit purchases are temporarily unavailable." ) - purchase_count = await _fetchrow("SELECT COUNT(*) AS count FROM page_purchases") - assert purchase_count is not None - assert purchase_count["count"] == 0 + count = await _fetchrow("SELECT COUNT(*) AS count FROM credit_purchases") + assert count is not None + assert count["count"] == 0 class TestStripeWebhookFulfillment: - async def test_webhook_grants_pages_once( + async def test_webhook_grants_credit_once( self, client, headers, search_space_id: int, - page_limits, + credits, monkeypatch, ): - await page_limits.set(pages_used=0, pages_limit=100) + await credits.set(balance_micros=5_000_000) checkout_session = SimpleNamespace( id="cs_test_webhook_123", @@ -270,21 +293,16 @@ class TestStripeWebhookFulfillment: create_client = _FakeCreateStripeClient(checkout_session) monkeypatch.setattr(stripe_routes, "get_stripe_client", lambda: create_client) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PRICE_ID", "price_pages_1000") - monkeypatch.setattr( - stripe_routes.config, "NEXT_FRONTEND_URL", "http://localhost:3000" - ) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGES_PER_UNIT", 1000) + _configure_credit_buying(monkeypatch) create_response = await client.post( - "/api/v1/stripe/create-checkout-session", + "/api/v1/stripe/create-credit-checkout-session", headers=headers, json={"quantity": 3, "search_space_id": search_space_id}, ) assert create_response.status_code == 200, create_response.text - initial_limit = await _get_pages_limit(TEST_EMAIL) - assert initial_limit == 100 + assert await _get_balance(TEST_EMAIL) == 5_000_000 user_id = await _get_user_id(TEST_EMAIL) webhook_checkout_session = SimpleNamespace( @@ -296,7 +314,8 @@ class TestStripeWebhookFulfillment: metadata={ "user_id": user_id, "quantity": "3", - "pages_per_unit": "1000", + "credit_micros_per_unit": str(_CREDIT_MICROS_PER_UNIT), + "purchase_type": "credits", }, ) event = SimpleNamespace( @@ -315,13 +334,12 @@ class TestStripeWebhookFulfillment: ) assert first_response.status_code == 200, first_response.text - updated_limit = await _get_pages_limit(TEST_EMAIL) - assert updated_limit == 3100 + assert await _get_balance(TEST_EMAIL) == 5_000_000 + 3 * _CREDIT_MICROS_PER_UNIT purchase = await _fetchrow( """ SELECT status, amount_total, currency, stripe_payment_intent_id - FROM page_purchases + FROM credit_purchases WHERE stripe_checkout_session_id = $1 """, checkout_session.id, @@ -339,7 +357,8 @@ class TestStripeWebhookFulfillment: ) assert second_response.status_code == 200, second_response.text - assert await _get_pages_limit(TEST_EMAIL) == 3100 + # Idempotent: a duplicate webhook does not double-grant. + assert await _get_balance(TEST_EMAIL) == 5_000_000 + 3 * _CREDIT_MICROS_PER_UNIT class TestStripeReconciliation: @@ -348,10 +367,10 @@ class TestStripeReconciliation: client, headers, search_space_id: int, - page_limits, + credits, monkeypatch, ): - await page_limits.set(pages_used=220, pages_limit=150) + await credits.set(balance_micros=1_000_000) checkout_session = SimpleNamespace( id="cs_test_reconcile_paid_123", @@ -363,19 +382,15 @@ class TestStripeReconciliation: create_client = _FakeCreateStripeClient(checkout_session) monkeypatch.setattr(stripe_routes, "get_stripe_client", lambda: create_client) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PRICE_ID", "price_pages_1000") - monkeypatch.setattr( - stripe_routes.config, "NEXT_FRONTEND_URL", "http://localhost:3000" - ) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGES_PER_UNIT", 1000) + _configure_credit_buying(monkeypatch) create_response = await client.post( - "/api/v1/stripe/create-checkout-session", + "/api/v1/stripe/create-credit-checkout-session", headers=headers, json={"quantity": 3, "search_space_id": search_space_id}, ) assert create_response.status_code == 200, create_response.text - assert await _get_pages_limit(TEST_EMAIL) == 150 + assert await _get_balance(TEST_EMAIL) == 1_000_000 reconciled_session = SimpleNamespace( id=checkout_session.id, @@ -402,15 +417,15 @@ class TestStripeReconciliation: 20, ) - await stripe_reconciliation_task._reconcile_pending_page_purchases() + await stripe_reconciliation_task._reconcile_pending_credit_purchases() assert reconcile_client.requested_ids == [checkout_session.id] - assert await _get_pages_limit(TEST_EMAIL) == 3220 + assert await _get_balance(TEST_EMAIL) == 1_000_000 + 3 * _CREDIT_MICROS_PER_UNIT purchase = await _fetchrow( """ SELECT status, amount_total, currency, stripe_payment_intent_id - FROM page_purchases + FROM credit_purchases WHERE stripe_checkout_session_id = $1 """, checkout_session.id, @@ -426,10 +441,10 @@ class TestStripeReconciliation: client, headers, search_space_id: int, - page_limits, + credits, monkeypatch, ): - await page_limits.set(pages_used=0, pages_limit=500) + await credits.set(balance_micros=500_000) checkout_session = SimpleNamespace( id="cs_test_reconcile_expired_123", @@ -441,14 +456,10 @@ class TestStripeReconciliation: create_client = _FakeCreateStripeClient(checkout_session) monkeypatch.setattr(stripe_routes, "get_stripe_client", lambda: create_client) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PRICE_ID", "price_pages_1000") - monkeypatch.setattr( - stripe_routes.config, "NEXT_FRONTEND_URL", "http://localhost:3000" - ) - monkeypatch.setattr(stripe_routes.config, "STRIPE_PAGES_PER_UNIT", 1000) + _configure_credit_buying(monkeypatch) create_response = await client.post( - "/api/v1/stripe/create-checkout-session", + "/api/v1/stripe/create-credit-checkout-session", headers=headers, json={"quantity": 1, "search_space_id": search_space_id}, ) @@ -479,14 +490,14 @@ class TestStripeReconciliation: 20, ) - await stripe_reconciliation_task._reconcile_pending_page_purchases() + await stripe_reconciliation_task._reconcile_pending_credit_purchases() - assert await _get_pages_limit(TEST_EMAIL) == 500 + assert await _get_balance(TEST_EMAIL) == 500_000 purchase = await _fetchrow( """ SELECT status - FROM page_purchases + FROM credit_purchases WHERE stripe_checkout_session_id = $1 """, checkout_session.id, diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 2cd378343..e37c34388 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -961,24 +961,37 @@ class TestDirectConvert: # ==================================================================== -# Tier 8: Page Limits (PL1-PL6) +# Tier 8: ETL Credits (CR1-CR6) # ==================================================================== -class TestPageLimits: +class TestEtlCredits: + @pytest.fixture(autouse=True) + def _enable_etl_billing(self, monkeypatch): + """Force ETL credit billing on (off by default for self-hosted/OSS).""" + from app.config import config + + monkeypatch.setattr(config, "ETL_CREDIT_BILLING_ENABLED", True) + + @staticmethod + def _micros(pages: int) -> int: + from app.config import config + + return pages * config.MICROS_PER_PAGE + @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl1_full_scan_increments_pages_used( + async def test_cr1_full_scan_debits_balance( self, db_session: AsyncSession, db_user: User, db_search_space: SearchSpace, tmp_path: Path, ): - """PL1: Successful full-scan sync increments user.pages_used.""" + """CR1: Successful full-scan sync debits user.credit_micros_balance.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 0 - db_user.pages_limit = 500 + starting = self._micros(500) + db_user.credit_micros_balance = starting await db_session.flush() (tmp_path / "note.md").write_text("# Hello World\n\nContent here.") @@ -995,21 +1008,22 @@ class TestPageLimits: assert count == 1 await db_session.refresh(db_user) - assert db_user.pages_used > 0, "pages_used should increase after indexing" + assert db_user.credit_micros_balance < starting, ( + "balance should drop after indexing" + ) @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl2_full_scan_blocked_when_limit_exhausted( + async def test_cr2_full_scan_blocked_when_credit_exhausted( self, db_session: AsyncSession, db_user: User, db_search_space: SearchSpace, tmp_path: Path, ): - """PL2: Full-scan skips file when page limit is exhausted.""" + """CR2: Full-scan skips file when the wallet is empty.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 100 - db_user.pages_limit = 100 + db_user.credit_micros_balance = 0 await db_session.flush() (tmp_path / "note.md").write_text("# Hello World\n\nContent here.") @@ -1025,21 +1039,23 @@ class TestPageLimits: assert count == 0 await db_session.refresh(db_user) - assert db_user.pages_used == 100, "pages_used should not change on rejection" + assert db_user.credit_micros_balance == 0, ( + "balance should not change on rejection" + ) @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl3_single_file_increments_pages_used( + async def test_cr3_single_file_debits_balance( self, db_session: AsyncSession, db_user: User, db_search_space: SearchSpace, tmp_path: Path, ): - """PL3: Single-file mode increments user.pages_used on success.""" + """CR3: Single-file mode debits balance on success.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 0 - db_user.pages_limit = 500 + starting = self._micros(500) + db_user.credit_micros_balance = starting await db_session.flush() (tmp_path / "note.md").write_text("# Hello World\n\nContent here.") @@ -1057,21 +1073,22 @@ class TestPageLimits: assert count == 1 await db_session.refresh(db_user) - assert db_user.pages_used > 0, "pages_used should increase after indexing" + assert db_user.credit_micros_balance < starting, ( + "balance should drop after indexing" + ) @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl4_single_file_blocked_when_limit_exhausted( + async def test_cr4_single_file_blocked_when_credit_exhausted( self, db_session: AsyncSession, db_user: User, db_search_space: SearchSpace, tmp_path: Path, ): - """PL4: Single-file mode skips file when page limit is exhausted.""" + """CR4: Single-file mode skips file when the wallet is empty.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 100 - db_user.pages_limit = 100 + db_user.credit_micros_balance = 0 await db_session.flush() (tmp_path / "note.md").write_text("# Hello World\n\nContent here.") @@ -1087,24 +1104,25 @@ class TestPageLimits: assert count == 0 assert err is not None - assert "page limit" in err.lower() + assert "credit" in err.lower() await db_session.refresh(db_user) - assert db_user.pages_used == 100, "pages_used should not change on rejection" + assert db_user.credit_micros_balance == 0, ( + "balance should not change on rejection" + ) @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl5_unchanged_resync_no_extra_pages( + async def test_cr5_unchanged_resync_no_extra_debit( self, db_session: AsyncSession, db_user: User, db_search_space: SearchSpace, tmp_path: Path, ): - """PL5: Re-syncing an unchanged file does not consume additional pages.""" + """CR5: Re-syncing an unchanged file does not consume additional credit.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 0 - db_user.pages_limit = 500 + db_user.credit_micros_balance = self._micros(500) await db_session.flush() (tmp_path / "note.md").write_text("# Hello\n\nSame content.") @@ -1119,8 +1137,8 @@ class TestPageLimits: assert count1 == 1 await db_session.refresh(db_user) - pages_after_first = db_user.pages_used - assert pages_after_first > 0 + balance_after_first = db_user.credit_micros_balance + assert balance_after_first < self._micros(500) count2, _, _, _ = await index_local_folder( session=db_session, @@ -1133,12 +1151,12 @@ class TestPageLimits: assert count2 == 0 await db_session.refresh(db_user) - assert db_user.pages_used == pages_after_first, ( - "pages_used should not increase for unchanged files" + assert db_user.credit_micros_balance == balance_after_first, ( + "balance should not change for unchanged files" ) @pytest.mark.usefixtures(*UNIFIED_FIXTURES) - async def test_pl6_batch_partial_page_limit_exhaustion( + async def test_cr6_batch_partial_credit_exhaustion( self, db_session: AsyncSession, db_user: User, @@ -1146,11 +1164,11 @@ class TestPageLimits: tmp_path: Path, patched_batch_sessions, ): - """PL6: Batch mode with a very low page limit: some files succeed, rest fail.""" + """CR6: Batch mode with a tiny balance: some files succeed, rest fail.""" from app.tasks.connector_indexers.local_folder_indexer import index_local_folder - db_user.pages_used = 0 - db_user.pages_limit = 1 + # Exactly one page of credit. + db_user.credit_micros_balance = self._micros(1) await db_session.flush() (tmp_path / "a.md").write_text("File A content") @@ -1171,12 +1189,13 @@ class TestPageLimits: ) assert count >= 1, "at least one file should succeed" - assert failed >= 1, "at least one file should fail due to page limit" + assert failed >= 1, "at least one file should fail due to insufficient credits" assert count + failed == 3 await db_session.refresh(db_user) - assert db_user.pages_used > 0 - assert db_user.pages_used <= db_user.pages_limit + 1 + # The wallet was drained by the successful file(s); it may dip slightly + # negative when the actual page count exceeds the pre-check estimate. + assert db_user.credit_micros_balance <= 0 # ==================================================================== diff --git a/surfsense_backend/tests/integration/notifications/test_page_limit_handler.py b/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py similarity index 52% rename from surfsense_backend/tests/integration/notifications/test_page_limit_handler.py rename to surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py index ab89d63c9..bdfa1b30c 100644 --- a/surfsense_backend/tests/integration/notifications/test_page_limit_handler.py +++ b/surfsense_backend/tests/integration/notifications/test_insufficient_credits_handler.py @@ -1,4 +1,4 @@ -"""Behavior guard for the page-limit notification handler.""" +"""Behavior guard for the insufficient-credits notification handler.""" from __future__ import annotations @@ -10,52 +10,50 @@ from app.notifications.service import NotificationService pytestmark = pytest.mark.integration -handler = NotificationService.page_limit +handler = NotificationService.insufficient_credits -async def test_page_limit_message_and_action( +async def test_insufficient_credits_message_and_action( db_session: AsyncSession, db_user: User, db_search_space: SearchSpace ): - """A page-limit notification states usage and carries an upgrade action link.""" - notification = await handler.notify_page_limit_exceeded( + """An insufficient-credits notification states cost and carries a buy-credits link.""" + notification = await handler.notify_insufficient_credits( session=db_session, user_id=db_user.id, document_name="short.pdf", document_type="FILE", search_space_id=db_search_space.id, - pages_used=95, - pages_limit=100, - pages_to_add=10, + balance_micros=250_000, + required_micros=1_000_000, ) - assert notification.type == "page_limit_exceeded" - assert notification.title == "Page limit exceeded: short.pdf" + assert notification.type == "insufficient_credits" + assert notification.title == "Insufficient credits: short.pdf" assert notification.message == ( - "This document has ~10 page(s) but you've used 95/100 pages. " - "Upgrade to process more documents." + "This document costs about $1.00 to process but you have " + "$0.25 of credit left. Add more credits to continue." ) assert notification.notification_metadata["status"] == "failed" - assert notification.notification_metadata["action_label"] == "Upgrade Plan" + assert notification.notification_metadata["action_label"] == "Buy credits" assert notification.notification_metadata["action_url"] == ( - f"/dashboard/{db_search_space.id}/more-pages" + f"/dashboard/{db_search_space.id}/buy-more" ) -async def test_page_limit_truncates_long_name( +async def test_insufficient_credits_truncates_long_name( db_session: AsyncSession, db_user: User, db_search_space: SearchSpace ): """A long document name is truncated in the notification title.""" long_name = "a" * 50 - notification = await handler.notify_page_limit_exceeded( + notification = await handler.notify_insufficient_credits( session=db_session, user_id=db_user.id, document_name=long_name, document_type="FILE", search_space_id=db_search_space.id, - pages_used=95, - pages_limit=100, - pages_to_add=10, + balance_micros=250_000, + required_micros=1_000_000, ) - assert notification.title == f"Page limit exceeded: {'a' * 40}..." + assert notification.title == f"Insufficient credits: {'a' * 40}..." diff --git a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py index b87d1be42..a74591169 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py @@ -272,22 +272,26 @@ def full_scan_mocks(mock_dropbox_client, monkeypatch): download_and_index_mock = AsyncMock(return_value=(0, 0)) monkeypatch.setattr(_mod, "_download_and_index", download_and_index_mock) - from app.services.page_limit_service import PageLimitService as _RealPLS + from app.services.etl_credit_service import EtlCreditService as _RealECS - mock_page_limit_instance = MagicMock() - mock_page_limit_instance.get_page_usage = AsyncMock(return_value=(0, 999_999)) - mock_page_limit_instance.update_page_usage = AsyncMock() + # get_available_micros -> None means "unlimited" (billing disabled), so no + # batch is gated and charge_credits is a no-op — matching the prior + # 999_999 page-limit intent for these parallel-processing tests. + mock_credit_instance = MagicMock() + mock_credit_instance.get_available_micros = AsyncMock(return_value=None) + mock_credit_instance.charge_credits = AsyncMock(return_value=None) - class _MockPageLimitService: + class _MockEtlCreditService: estimate_pages_from_metadata = staticmethod( - _RealPLS.estimate_pages_from_metadata + _RealECS.estimate_pages_from_metadata ) + pages_to_micros = staticmethod(_RealECS.pages_to_micros) def __init__(self, session): - self.get_page_usage = mock_page_limit_instance.get_page_usage - self.update_page_usage = mock_page_limit_instance.update_page_usage + self.get_available_micros = mock_credit_instance.get_available_micros + self.charge_credits = mock_credit_instance.charge_credits - monkeypatch.setattr(_mod, "PageLimitService", _MockPageLimitService) + monkeypatch.setattr(_mod, "EtlCreditService", _MockEtlCreditService) return { "dropbox_client": mock_dropbox_client, @@ -393,22 +397,23 @@ def selected_files_mocks(mock_dropbox_client, monkeypatch): download_and_index_mock = AsyncMock(return_value=(0, 0)) monkeypatch.setattr(_mod, "_download_and_index", download_and_index_mock) - from app.services.page_limit_service import PageLimitService as _RealPLS + from app.services.etl_credit_service import EtlCreditService as _RealECS - mock_page_limit_instance = MagicMock() - mock_page_limit_instance.get_page_usage = AsyncMock(return_value=(0, 999_999)) - mock_page_limit_instance.update_page_usage = AsyncMock() + mock_credit_instance = MagicMock() + mock_credit_instance.get_available_micros = AsyncMock(return_value=None) + mock_credit_instance.charge_credits = AsyncMock(return_value=None) - class _MockPageLimitService: + class _MockEtlCreditService: estimate_pages_from_metadata = staticmethod( - _RealPLS.estimate_pages_from_metadata + _RealECS.estimate_pages_from_metadata ) + pages_to_micros = staticmethod(_RealECS.pages_to_micros) def __init__(self, session): - self.get_page_usage = mock_page_limit_instance.get_page_usage - self.update_page_usage = mock_page_limit_instance.update_page_usage + self.get_available_micros = mock_credit_instance.get_available_micros + self.charge_credits = mock_credit_instance.charge_credits - monkeypatch.setattr(_mod, "PageLimitService", _MockPageLimitService) + monkeypatch.setattr(_mod, "EtlCreditService", _MockEtlCreditService) return { "dropbox_client": mock_dropbox_client, diff --git a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py b/surfsense_backend/tests/unit/connector_indexers/test_etl_credits.py similarity index 73% rename from surfsense_backend/tests/unit/connector_indexers/test_page_limits.py rename to surfsense_backend/tests/unit/connector_indexers/test_etl_credits.py index 66722ffd7..aca811ee9 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_etl_credits.py @@ -1,17 +1,22 @@ -"""Tests for page limit enforcement in connector indexers. +"""Tests for ETL credit enforcement in connector indexers. Covers: - A) PageLimitService.estimate_pages_from_metadata — pure function (no mocks) - B) Page-limit quota gating in _index_selected_files tested through the - real PageLimitService with a mock DB session (system boundary). + A) EtlCreditService.estimate_pages_from_metadata — pure function (no mocks) + B) Credit-wallet gating in the connector indexers, tested through the real + EtlCreditService with a mock DB session (system boundary). ETL credit + billing is force-enabled per-test so the gating path is exercised. Google Drive is the primary, with OneDrive/Dropbox smoke tests. + +Page estimates are converted to micro-USD at ``config.MICROS_PER_PAGE`` per +page and debited from ``user.credit_micros_balance``. """ from unittest.mock import AsyncMock, MagicMock import pytest -from app.services.page_limit_service import PageLimitService +from app.config import config +from app.services.etl_credit_service import EtlCreditService pytestmark = pytest.mark.unit @@ -20,8 +25,23 @@ _CONNECTOR_ID = 42 _SEARCH_SPACE_ID = 1 +def _micros(pages: int) -> int: + """Convert a page count to micro-USD using the configured rate.""" + return pages * config.MICROS_PER_PAGE + + +@pytest.fixture(autouse=True) +def _enable_etl_billing(monkeypatch): + """Force ETL credit billing on so the gating/charging path runs. + + It defaults to off (self-hosted/OSS), which would short-circuit + get_available_micros to None and bypass every check in this module. + """ + monkeypatch.setattr(config, "ETL_CREDIT_BILLING_ENABLED", True) + + # =================================================================== -# A) PageLimitService.estimate_pages_from_metadata — pure function +# A) EtlCreditService.estimate_pages_from_metadata — pure function # No mocks: it's a staticmethod with no I/O. # =================================================================== @@ -30,88 +50,91 @@ class TestEstimatePagesFromMetadata: """Vertical slices for the page estimation staticmethod.""" def test_pdf_100kb_returns_1(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", 100 * 1024) == 1 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", 100 * 1024) == 1 def test_pdf_500kb_returns_5(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", 500 * 1024) == 5 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", 500 * 1024) == 5 def test_pdf_1mb(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", 1024 * 1024) == 10 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", 1024 * 1024) == 10 def test_docx_50kb_returns_1(self): - assert PageLimitService.estimate_pages_from_metadata(".docx", 50 * 1024) == 1 + assert EtlCreditService.estimate_pages_from_metadata(".docx", 50 * 1024) == 1 def test_docx_200kb(self): - assert PageLimitService.estimate_pages_from_metadata(".docx", 200 * 1024) == 4 + assert EtlCreditService.estimate_pages_from_metadata(".docx", 200 * 1024) == 4 def test_pptx_uses_200kb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".pptx", 600 * 1024) == 3 + assert EtlCreditService.estimate_pages_from_metadata(".pptx", 600 * 1024) == 3 def test_xlsx_uses_100kb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".xlsx", 300 * 1024) == 3 + assert EtlCreditService.estimate_pages_from_metadata(".xlsx", 300 * 1024) == 3 def test_txt_uses_3000_bytes_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".txt", 9000) == 3 + assert EtlCreditService.estimate_pages_from_metadata(".txt", 9000) == 3 def test_image_always_returns_1(self): for ext in (".jpg", ".png", ".gif", ".webp"): - assert PageLimitService.estimate_pages_from_metadata(ext, 5_000_000) == 1 + assert EtlCreditService.estimate_pages_from_metadata(ext, 5_000_000) == 1 def test_audio_uses_1mb_per_page(self): assert ( - PageLimitService.estimate_pages_from_metadata(".mp3", 3 * 1024 * 1024) == 3 + EtlCreditService.estimate_pages_from_metadata(".mp3", 3 * 1024 * 1024) == 3 ) def test_video_uses_5mb_per_page(self): assert ( - PageLimitService.estimate_pages_from_metadata(".mp4", 15 * 1024 * 1024) == 3 + EtlCreditService.estimate_pages_from_metadata(".mp4", 15 * 1024 * 1024) == 3 ) def test_unknown_ext_uses_80kb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".xyz", 160 * 1024) == 2 + assert EtlCreditService.estimate_pages_from_metadata(".xyz", 160 * 1024) == 2 def test_zero_size_returns_1(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", 0) == 1 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", 0) == 1 def test_negative_size_returns_1(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", -500) == 1 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", -500) == 1 def test_minimum_is_always_1(self): - assert PageLimitService.estimate_pages_from_metadata(".pdf", 50) == 1 + assert EtlCreditService.estimate_pages_from_metadata(".pdf", 50) == 1 def test_epub_uses_50kb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".epub", 250 * 1024) == 5 + assert EtlCreditService.estimate_pages_from_metadata(".epub", 250 * 1024) == 5 # =================================================================== -# B) Page-limit enforcement in connector indexers -# System boundary mocked: DB session (for PageLimitService) +# B) Credit enforcement in connector indexers +# System boundary mocked: DB session (for EtlCreditService) # System boundary mocked: external API clients, download/ETL -# NOT mocked: PageLimitService itself (our own code) +# NOT mocked: EtlCreditService itself (our own code) # =================================================================== class _FakeUser: """Stands in for the User ORM model at the DB boundary.""" - def __init__(self, pages_used: int = 0, pages_limit: int = 100): - self.pages_used = pages_used - self.pages_limit = pages_limit + def __init__(self, balance_micros: int = 0, reserved_micros: int = 0): + self.credit_micros_balance = balance_micros + self.credit_micros_reserved = reserved_micros -def _make_page_limit_session(pages_used: int = 0, pages_limit: int = 100): - """Build a mock DB session that real PageLimitService can operate against. +def _make_credit_session(balance_micros: int = _micros(100), reserved_micros: int = 0): + """Build a mock DB session that the real EtlCreditService can operate against. Every ``session.execute()`` returns a result compatible with both - ``get_page_usage`` (.first() → tuple) and ``update_page_usage`` - (.unique().scalar_one_or_none() → User-like). + ``get_available_micros`` (.first() → ``(balance, reserved)``) and + ``charge_credits`` (.unique().scalar_one_or_none() → User-like). """ - fake_user = _FakeUser(pages_used, pages_limit) + fake_user = _FakeUser(balance_micros, reserved_micros) session = AsyncMock() def _make_result(*_args, **_kwargs): result = MagicMock() - result.first.return_value = (fake_user.pages_used, fake_user.pages_limit) + result.first.return_value = ( + fake_user.credit_micros_balance, + fake_user.credit_micros_reserved, + ) result.unique.return_value.scalar_one_or_none.return_value = fake_user return result @@ -138,7 +161,7 @@ def gdrive_selected_mocks(monkeypatch): """Mocks for Google Drive _index_selected_files — only system boundaries.""" import app.tasks.connector_indexers.google_drive_indexer as _mod - session, fake_user = _make_page_limit_session(0, 100) + session, fake_user = _make_credit_session(_micros(100)) get_file_results: dict[str, tuple[dict | None, str | None]] = {} @@ -183,12 +206,11 @@ async def _run_gdrive_selected(mocks, file_ids): ) -async def test_gdrive_files_within_quota_are_downloaded(gdrive_selected_mocks): - """Files whose cumulative estimated pages fit within remaining quota +async def test_gdrive_files_within_credit_are_downloaded(gdrive_selected_mocks): + """Files whose cumulative estimated cost fits within available credit are sent to _download_and_index.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(100) for fid in ("f1", "f2", "f3"): m["get_file_results"][fid] = ( @@ -207,11 +229,10 @@ async def test_gdrive_files_within_quota_are_downloaded(gdrive_selected_mocks): assert len(call_files) == 3 -async def test_gdrive_files_exceeding_quota_rejected(gdrive_selected_mocks): - """Files whose pages would exceed remaining quota are rejected.""" +async def test_gdrive_files_exceeding_credit_rejected(gdrive_selected_mocks): + """Files whose cost would exceed available credit are rejected.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 98 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(2) m["get_file_results"]["big"] = ( _make_gdrive_file("big", "huge.pdf", size=500 * 1024), @@ -224,14 +245,13 @@ async def test_gdrive_files_exceeding_quota_rejected(gdrive_selected_mocks): assert indexed == 0 assert len(errors) == 1 - assert "page limit" in errors[0].lower() + assert "insufficient credits" in errors[0].lower() -async def test_gdrive_quota_mix_partial_indexing(gdrive_selected_mocks): - """3rd file pushes over quota → only first two indexed.""" +async def test_gdrive_credit_mix_partial_indexing(gdrive_selected_mocks): + """3rd file pushes over available credit → only first two indexed.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 2 + m["fake_user"].credit_micros_balance = _micros(2) for fid in ("f1", "f2", "f3"): m["get_file_results"][fid] = ( @@ -250,11 +270,10 @@ async def test_gdrive_quota_mix_partial_indexing(gdrive_selected_mocks): assert {f["id"] for f in call_files} == {"f1", "f2"} -async def test_gdrive_proportional_page_deduction(gdrive_selected_mocks): - """Pages deducted are proportional to successfully indexed files.""" +async def test_gdrive_proportional_credit_deduction(gdrive_selected_mocks): + """Credit deducted is proportional to successfully indexed files.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(100) for fid in ("f1", "f2", "f3", "f4"): m["get_file_results"][fid] = ( @@ -268,14 +287,14 @@ async def test_gdrive_proportional_page_deduction(gdrive_selected_mocks): [("f1", "f1.xyz"), ("f2", "f2.xyz"), ("f3", "f3.xyz"), ("f4", "f4.xyz")], ) - assert m["fake_user"].pages_used == 2 + # 4 estimated pages, 2 of 4 indexed → deduct 2 pages. + assert m["fake_user"].credit_micros_balance == _micros(100) - _micros(2) async def test_gdrive_no_deduction_when_nothing_indexed(gdrive_selected_mocks): - """If batch_indexed == 0, user's pages_used stays unchanged.""" + """If batch_indexed == 0, the user's balance stays unchanged.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 5 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(95) m["get_file_results"]["f1"] = ( _make_gdrive_file("f1", "f1.xyz", size=80 * 1024), @@ -285,14 +304,13 @@ async def test_gdrive_no_deduction_when_nothing_indexed(gdrive_selected_mocks): await _run_gdrive_selected(m, [("f1", "f1.xyz")]) - assert m["fake_user"].pages_used == 5 + assert m["fake_user"].credit_micros_balance == _micros(95) -async def test_gdrive_zero_quota_rejects_all(gdrive_selected_mocks): - """When pages_used == pages_limit, every file is rejected.""" +async def test_gdrive_zero_credit_rejects_all(gdrive_selected_mocks): + """When the balance is exhausted, every file is rejected.""" m = gdrive_selected_mocks - m["fake_user"].pages_used = 100 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = 0 for fid in ("f1", "f2"): m["get_file_results"][fid] = ( @@ -317,7 +335,7 @@ async def test_gdrive_zero_quota_rejects_all(gdrive_selected_mocks): def gdrive_full_scan_mocks(monkeypatch): import app.tasks.connector_indexers.google_drive_indexer as _mod - session, fake_user = _make_page_limit_session(0, 100) + session, fake_user = _make_credit_session(_micros(100)) mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -364,10 +382,9 @@ async def _run_gdrive_full_scan(mocks, max_files=500): ) -async def test_gdrive_full_scan_skips_over_quota(gdrive_full_scan_mocks, monkeypatch): +async def test_gdrive_full_scan_skips_over_credit(gdrive_full_scan_mocks, monkeypatch): m = gdrive_full_scan_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 2 + m["fake_user"].credit_micros_balance = _micros(2) page_files = [ _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(5) @@ -391,8 +408,7 @@ async def test_gdrive_full_scan_deducts_after_indexing( gdrive_full_scan_mocks, monkeypatch ): m = gdrive_full_scan_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(100) page_files = [ _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(3) @@ -408,7 +424,7 @@ async def test_gdrive_full_scan_deducts_after_indexing( await _run_gdrive_full_scan(m) - assert m["fake_user"].pages_used == 3 + assert m["fake_user"].credit_micros_balance == _micros(100) - _micros(3) # --------------------------------------------------------------------------- @@ -416,10 +432,10 @@ async def test_gdrive_full_scan_deducts_after_indexing( # --------------------------------------------------------------------------- -async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): +async def test_gdrive_delta_sync_skips_over_credit(monkeypatch): import app.tasks.connector_indexers.google_drive_indexer as _mod - session, _ = _make_page_limit_session(0, 2) + session, _ = _make_credit_session(_micros(2)) changes = [ { @@ -471,7 +487,7 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): # =================================================================== -# C) OneDrive smoke tests — verify page limit wiring +# C) OneDrive smoke tests — verify credit wiring # =================================================================== @@ -489,7 +505,7 @@ def _make_onedrive_file(file_id: str, name: str, size: int = 80 * 1024) -> dict: def onedrive_selected_mocks(monkeypatch): import app.tasks.connector_indexers.onedrive_indexer as _mod - session, fake_user = _make_page_limit_session(0, 100) + session, fake_user = _make_credit_session(_micros(100)) get_file_results: dict[str, tuple[dict | None, str | None]] = {} @@ -531,11 +547,10 @@ async def _run_onedrive_selected(mocks, file_ids): ) -async def test_onedrive_over_quota_rejected(onedrive_selected_mocks): - """OneDrive: files exceeding quota produce errors, not downloads.""" +async def test_onedrive_over_credit_rejected(onedrive_selected_mocks): + """OneDrive: files exceeding available credit produce errors, not downloads.""" m = onedrive_selected_mocks - m["fake_user"].pages_used = 99 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(1) m["get_file_results"]["big"] = ( _make_onedrive_file("big", "huge.pdf", size=500 * 1024), @@ -548,14 +563,13 @@ async def test_onedrive_over_quota_rejected(onedrive_selected_mocks): assert indexed == 0 assert len(errors) == 1 - assert "page limit" in errors[0].lower() + assert "insufficient credits" in errors[0].lower() async def test_onedrive_deducts_after_success(onedrive_selected_mocks): - """OneDrive: pages_used increases after successful indexing.""" + """OneDrive: balance decreases after successful indexing.""" m = onedrive_selected_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(100) for fid in ("f1", "f2"): m["get_file_results"][fid] = ( @@ -566,11 +580,11 @@ async def test_onedrive_deducts_after_success(onedrive_selected_mocks): await _run_onedrive_selected(m, [("f1", "f1.xyz"), ("f2", "f2.xyz")]) - assert m["fake_user"].pages_used == 2 + assert m["fake_user"].credit_micros_balance == _micros(100) - _micros(2) # =================================================================== -# D) Dropbox smoke tests — verify page limit wiring +# D) Dropbox smoke tests — verify credit wiring # =================================================================== @@ -590,7 +604,7 @@ def _make_dropbox_file(file_path: str, name: str, size: int = 80 * 1024) -> dict def dropbox_selected_mocks(monkeypatch): import app.tasks.connector_indexers.dropbox_indexer as _mod - session, fake_user = _make_page_limit_session(0, 100) + session, fake_user = _make_credit_session(_micros(100)) get_file_results: dict[str, tuple[dict | None, str | None]] = {} @@ -632,11 +646,10 @@ async def _run_dropbox_selected(mocks, file_paths): ) -async def test_dropbox_over_quota_rejected(dropbox_selected_mocks): - """Dropbox: files exceeding quota produce errors, not downloads.""" +async def test_dropbox_over_credit_rejected(dropbox_selected_mocks): + """Dropbox: files exceeding available credit produce errors, not downloads.""" m = dropbox_selected_mocks - m["fake_user"].pages_used = 99 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(1) m["get_file_results"]["/huge.pdf"] = ( _make_dropbox_file("/huge.pdf", "huge.pdf", size=500 * 1024), @@ -649,14 +662,13 @@ async def test_dropbox_over_quota_rejected(dropbox_selected_mocks): assert indexed == 0 assert len(errors) == 1 - assert "page limit" in errors[0].lower() + assert "insufficient credits" in errors[0].lower() async def test_dropbox_deducts_after_success(dropbox_selected_mocks): - """Dropbox: pages_used increases after successful indexing.""" + """Dropbox: balance decreases after successful indexing.""" m = dropbox_selected_mocks - m["fake_user"].pages_used = 0 - m["fake_user"].pages_limit = 100 + m["fake_user"].credit_micros_balance = _micros(100) for name in ("f1.xyz", "f2.xyz"): path = f"/{name}" @@ -668,4 +680,4 @@ async def test_dropbox_deducts_after_success(dropbox_selected_mocks): await _run_dropbox_selected(m, [("/f1.xyz", "f1.xyz"), ("/f2.xyz", "f2.xyz")]) - assert m["fake_user"].pages_used == 2 + assert m["fake_user"].credit_micros_balance == _micros(100) - _micros(2) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py index 9a13e4525..4f61976a6 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py @@ -242,20 +242,28 @@ def _folder_dict(file_id: str, name: str) -> dict: } -def _make_page_limit_session(pages_used=0, pages_limit=999_999): - """Build a mock DB session that real PageLimitService can operate against.""" +def _make_page_limit_session(balance_micros=999_999_000, reserved_micros=0): + """Build a mock DB session that real EtlCreditService can operate against. + + ETL credit billing is disabled by default in tests, so get_available_micros + short-circuits to None ("unlimited") and these fields are unused; they're + provided for parity if a test opts into billing. + """ class _FakeUser: - def __init__(self, pu, pl): - self.pages_used = pu - self.pages_limit = pl + def __init__(self, balance, reserved): + self.credit_micros_balance = balance + self.credit_micros_reserved = reserved - fake_user = _FakeUser(pages_used, pages_limit) + fake_user = _FakeUser(balance_micros, reserved_micros) session = AsyncMock() def _make_result(*_a, **_kw): r = MagicMock() - r.first.return_value = (fake_user.pages_used, fake_user.pages_limit) + r.first.return_value = ( + fake_user.credit_micros_balance, + fake_user.credit_micros_reserved, + ) r.unique.return_value.scalar_one_or_none.return_value = fake_user return r diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_insufficient_credits.py b/surfsense_backend/tests/unit/notifications/service/messages/test_insufficient_credits.py new file mode 100644 index 000000000..c5366cce2 --- /dev/null +++ b/surfsense_backend/tests/unit/notifications/service/messages/test_insufficient_credits.py @@ -0,0 +1,38 @@ +"""Unit tests for insufficient-credits presentation logic.""" + +from __future__ import annotations + +import pytest + +from app.notifications.service.messages import insufficient_credits as msg + +pytestmark = pytest.mark.unit + + +def test_operation_id_encodes_search_space(): + """The operation id embeds the search space id.""" + assert msg.operation_id("doc.pdf", 9).startswith("insufficient_credits_9_") + + +def test_summary_title_and_message(): + """The summary states the document and the required/available credit.""" + title, message = msg.summary( + "short.pdf", balance_micros=250_000, required_micros=1_000_000 + ) + assert title == "Insufficient credits: short.pdf" + assert message == ( + "This document costs about $1.00 to process but you have " + "$0.25 of credit left. Add more credits to continue." + ) + + +def test_summary_clamps_negative_balance_to_zero(): + """A negative balance is clamped to $0.00 in the message.""" + _, message = msg.summary("doc.pdf", balance_micros=-5_000, required_micros=500_000) + assert "$0.00 of credit left" in message + + +def test_summary_truncates_long_name(): + """A long document name is truncated in the title.""" + title, _ = msg.summary("a" * 50, balance_micros=0, required_micros=1_000) + assert title == f"Insufficient credits: {'a' * 40}..." diff --git a/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py b/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py deleted file mode 100644 index 606e985f2..000000000 --- a/surfsense_backend/tests/unit/notifications/service/messages/test_page_limit.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Unit tests for page-limit presentation logic.""" - -from __future__ import annotations - -import pytest - -from app.notifications.service.messages import page_limit as msg - -pytestmark = pytest.mark.unit - - -def test_operation_id_encodes_search_space(): - """The operation id embeds the search space id.""" - assert msg.operation_id("doc.pdf", 9).startswith("page_limit_9_") - - -def test_summary_title_and_message(): - """The summary states the document and the used/limit page counts.""" - title, message = msg.summary( - "short.pdf", pages_used=95, pages_limit=100, pages_to_add=10 - ) - assert title == "Page limit exceeded: short.pdf" - assert message == ( - "This document has ~10 page(s) but you've used 95/100 pages. " - "Upgrade to process more documents." - ) - - -def test_summary_truncates_long_name(): - """A long document name is truncated in the title.""" - title, _ = msg.summary("a" * 50, pages_used=1, pages_limit=2, pages_to_add=1) - assert title == f"Page limit exceeded: {'a' * 40}..." diff --git a/surfsense_backend/tests/unit/observability/test_helpers.py b/surfsense_backend/tests/unit/observability/test_helpers.py index ae60c1939..f871d989d 100644 --- a/surfsense_backend/tests/unit/observability/test_helpers.py +++ b/surfsense_backend/tests/unit/observability/test_helpers.py @@ -33,8 +33,7 @@ def _disable_otel(monkeypatch: pytest.MonkeyPatch): ("generate_video_presentation", "generate"), ("generate_content_podcast", "generate"), ("cleanup_stale_indexing_notifications", "cleanup"), - ("reconcile_pending_stripe_page_purchases", "reconcile"), - ("reconcile_pending_stripe_token_purchases", "reconcile"), + ("reconcile_pending_stripe_credit_purchases", "reconcile"), ("check_periodic_schedules", "check"), ("ai_sort_search_space", "ai"), ("index_notion_pages", "index"), diff --git a/surfsense_backend/tests/unit/services/test_auto_model_pin_service.py b/surfsense_backend/tests/unit/services/test_auto_model_pin_service.py index d1af29aeb..5c5c90283 100644 --- a/surfsense_backend/tests/unit/services/test_auto_model_pin_service.py +++ b/surfsense_backend/tests/unit/services/test_auto_model_pin_service.py @@ -90,7 +90,7 @@ async def test_auto_first_turn_pins_one_model(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -138,7 +138,7 @@ async def test_premium_eligible_auto_prefers_premium_over_free(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -196,7 +196,7 @@ async def test_premium_eligible_auto_prefers_azure_gpt_5_4(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -232,11 +232,11 @@ async def test_next_turn_reuses_existing_pin(monkeypatch): async def _must_not_call(*_args, **_kwargs): raise AssertionError( - "premium_get_usage should not be called for valid pin reuse" + "credit_get_usage should not be called for valid pin reuse" ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _must_not_call, ) @@ -275,7 +275,7 @@ async def test_premium_eligible_auto_can_pin_premium(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -320,7 +320,7 @@ async def test_premium_ineligible_auto_pins_free_only(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -365,7 +365,7 @@ async def test_pinned_premium_stays_premium_after_quota_exhaustion(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -410,7 +410,7 @@ async def test_force_repin_free_switches_auto_premium_pin_to_free(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -470,7 +470,7 @@ async def test_invalid_pinned_config_repairs_with_new_pin(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -529,7 +529,7 @@ async def test_health_gated_config_is_excluded_from_selection(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -581,7 +581,7 @@ async def test_tier_a_locks_first_premium_user_skips_or(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -633,7 +633,7 @@ async def test_tier_a_falls_through_to_or_when_a_pool_empty_for_user(monkeypatch return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -686,7 +686,7 @@ async def test_top_k_picks_only_high_score_models(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -754,7 +754,7 @@ async def test_pin_reuse_survives_health_gating_for_existing_pin(monkeypatch): return _FakeQuotaResult(allowed=True) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _allowed, ) @@ -803,10 +803,10 @@ async def test_pin_reuse_regression_existing_healthy_pin(monkeypatch): ) async def _must_not_call(*_args, **_kwargs): - raise AssertionError("premium_get_usage should not run on pin reuse") + raise AssertionError("credit_get_usage should not run on pin reuse") monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _must_not_call, ) @@ -864,7 +864,7 @@ async def test_runtime_cooled_down_pin_is_not_reused(monkeypatch): return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) @@ -904,10 +904,10 @@ async def test_clearing_runtime_cooldown_restores_pin_reuse(monkeypatch): ) async def _must_not_call(*_args, **_kwargs): - raise AssertionError("premium_get_usage should not run on healthy pin reuse") + raise AssertionError("credit_get_usage should not run on healthy pin reuse") monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _must_not_call, ) @@ -962,7 +962,7 @@ async def test_auto_pin_repin_excludes_previous_config_on_runtime_retry(monkeypa return _FakeQuotaResult(allowed=False) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _blocked, ) diff --git a/surfsense_backend/tests/unit/services/test_auto_pin_image_aware.py b/surfsense_backend/tests/unit/services/test_auto_pin_image_aware.py index 0e19b80e4..3ca5c7a67 100644 --- a/surfsense_backend/tests/unit/services/test_auto_pin_image_aware.py +++ b/surfsense_backend/tests/unit/services/test_auto_pin_image_aware.py @@ -114,7 +114,7 @@ async def test_image_turn_filters_out_text_only_candidates(monkeypatch): [_text_only_cfg(-1), _vision_cfg(-2)], ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) @@ -146,7 +146,7 @@ async def test_image_turn_force_repins_stale_text_only_pin(monkeypatch): [_text_only_cfg(-1), _vision_cfg(-2)], ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) @@ -178,7 +178,7 @@ async def test_image_turn_reuses_existing_vision_pin(monkeypatch): [_text_only_cfg(-1), _vision_cfg(-2), _vision_cfg(-3, quality=70)], ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) @@ -209,7 +209,7 @@ async def test_image_turn_with_no_vision_candidates_raises(monkeypatch): [_text_only_cfg(-1), _text_only_cfg(-2)], ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) @@ -237,7 +237,7 @@ async def test_non_image_turn_keeps_text_only_in_pool(monkeypatch): [_text_only_cfg(-1)], ) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) @@ -271,7 +271,7 @@ async def test_image_turn_unannotated_cfg_resolves_via_helper(monkeypatch): } monkeypatch.setattr(config, "GLOBAL_LLM_CONFIGS", [cfg_unannotated_vision]) monkeypatch.setattr( - "app.services.auto_model_pin_service.TokenQuotaService.premium_get_usage", + "app.services.auto_model_pin_service.TokenQuotaService.credit_get_usage", _premium_allowed, ) diff --git a/surfsense_backend/tests/unit/services/test_billable_call.py b/surfsense_backend/tests/unit/services/test_billable_call.py index c820724ed..8e2c2f1da 100644 --- a/surfsense_backend/tests/unit/services/test_billable_call.py +++ b/surfsense_backend/tests/unit/services/test_billable_call.py @@ -38,11 +38,13 @@ class _FakeQuotaResult: used: int = 0, limit: int = 5_000_000, remaining: int = 5_000_000, + balance: int = 5_000_000, ) -> None: self.allowed = allowed self.used = used self.limit = limit self.remaining = remaining + self.balance = balance class _FakeSession: @@ -118,17 +120,17 @@ def _patch_isolation_layer( return object() monkeypatch.setattr( - "app.services.billable_calls.TokenQuotaService.premium_reserve", + "app.services.billable_calls.TokenQuotaService.credit_reserve", _fake_reserve, raising=False, ) monkeypatch.setattr( - "app.services.billable_calls.TokenQuotaService.premium_finalize", + "app.services.billable_calls.TokenQuotaService.credit_finalize", _fake_finalize, raising=False, ) monkeypatch.setattr( - "app.services.billable_calls.TokenQuotaService.premium_release", + "app.services.billable_calls.TokenQuotaService.credit_release", _fake_release, raising=False, ) @@ -201,9 +203,7 @@ async def test_premium_reserve_denied_raises_quota_insufficient(monkeypatch): spies = _patch_isolation_layer( monkeypatch, - reserve_result=_FakeQuotaResult( - allowed=False, used=5_000_000, limit=5_000_000, remaining=0 - ), + reserve_result=_FakeQuotaResult(allowed=False, balance=0, remaining=0), ) user_id = uuid4() @@ -220,8 +220,7 @@ async def test_premium_reserve_denied_raises_quota_insufficient(monkeypatch): err = exc_info.value assert err.usage_type == "image_generation" - assert err.used_micros == 5_000_000 - assert err.limit_micros == 5_000_000 + assert err.balance_micros == 0 assert err.remaining_micros == 0 # Reserve was attempted, but no finalize/release on a denied reserve # — the reservation never actually held credit. @@ -532,7 +531,7 @@ async def test_premium_video_denial_raises_quota_insufficient(monkeypatch): spies = _patch_isolation_layer( monkeypatch, reserve_result=_FakeQuotaResult( - allowed=False, used=4_500_000, limit=5_000_000, remaining=500_000 + allowed=False, balance=500_000, remaining=500_000 ), ) user_id = uuid4() @@ -552,6 +551,7 @@ async def test_premium_video_denial_raises_quota_insufficient(monkeypatch): err = exc_info.value assert err.usage_type == "video_presentation_generation" + assert err.balance_micros == 500_000 assert err.remaining_micros == 500_000 assert spies["reserve"][0]["reserve_micros"] == 1_000_000 assert spies["finalize"] == [] diff --git a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx index b4ec015b7..da92cbdac 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx @@ -1,48 +1,13 @@ "use client"; -import { useState } from "react"; -import { BuyPagesContent } from "@/components/settings/buy-pages-content"; -import { BuyTokensContent } from "@/components/settings/buy-tokens-content"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; - -const TABS = [ - { id: "pages", label: "Pages" }, - { id: "tokens", label: "Premium Credit" }, -] as const; - -type TabId = (typeof TABS)[number]["id"]; +import { BuyCreditsContent } from "@/components/settings/buy-credits-content"; export default function BuyMorePage() { - const [activeTab, setActiveTab] = useState("pages"); - return ( -
- { - setActiveTab(value as TabId); - }} - className="relative min-h-[37rem] w-full" - > - - {TABS.map((tab) => ( - - {tab.label} - - ))} - - - - - - - - - +
+
+ +
); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/earn-credits/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/earn-credits/page.tsx new file mode 100644 index 000000000..3ff4c3cf8 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/earn-credits/page.tsx @@ -0,0 +1,11 @@ +"use client"; + +import { EarnCreditsContent } from "@/components/settings/earn-credits-content"; + +export default function EarnCreditsPage() { + return ( +
+ +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/more-pages/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/more-pages/page.tsx index 4b3301b9f..46f1965d0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/more-pages/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/more-pages/page.tsx @@ -1,11 +1,18 @@ "use client"; -import { MorePagesContent } from "@/components/settings/more-pages-content"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect } from "react"; +// Legacy route kept as a redirect: older "insufficient credits" notifications +// and bookmarks may still point at /more-pages. export default function MorePagesPage() { - return ( -
- -
- ); + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params?.search_space_id ?? ""; + + useEffect(() => { + router.replace(`/dashboard/${searchSpaceId}/earn-credits`); + }, [router, searchSpaceId]); + + return null; } diff --git a/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx index 8eaec3e5a..a8a88c5a5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/purchase-success/page.tsx @@ -112,9 +112,7 @@ export default function PurchaseSuccessPage() { {state.kind === "still_pending" && "Your payment is still being processed by your bank. We'll apply your purchase as soon as it clears — usually within a few minutes. You can safely close this page."} {state.kind === "completed" && - (state.data.purchase_type === "page_packs" - ? `Added ${formatNumber(state.data.pages_granted ?? 0)} pages to your account.` - : `Added ${formatCredit(state.data.premium_credit_micros_granted ?? 0)} of premium credit to your account.`)} + `Added ${formatCredit(state.data.credit_micros_granted ?? 0)} of credit to your account.`} {state.kind === "failed" && "Stripe reported the checkout as failed or expired. Your card was not charged."} {state.kind === "error" && @@ -123,18 +121,9 @@ export default function PurchaseSuccessPage() { - {state.kind === "completed" && state.data.purchase_type === "page_packs" && ( + {state.kind === "completed" && (

- New balance: {formatNumber(state.data.pages_limit ?? 0)} total pages - {typeof state.data.pages_used === "number" - ? ` (${formatNumber((state.data.pages_limit ?? 0) - state.data.pages_used)} remaining)` - : ""} -

- )} - {state.kind === "completed" && state.data.purchase_type === "premium_tokens" && ( -

- New premium credit balance:{" "} - {formatCredit(state.data.premium_credit_micros_limit ?? 0)} + New credit balance: {formatCredit(state.data.credit_micros_balance ?? 0)}

)} {state.kind === "error" && ( @@ -146,7 +135,7 @@ export default function PurchaseSuccessPage() { Back to Dashboard @@ -154,10 +143,6 @@ export default function PurchaseSuccessPage() { ); } -function formatNumber(n: number): string { - return new Intl.NumberFormat("en-US").format(n); -} - function formatCredit(micros: number): string { const dollars = micros / 1_000_000; return new Intl.NumberFormat("en-US", { diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AutoReloadSettings.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AutoReloadSettings.tsx new file mode 100644 index 000000000..9411b06d1 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AutoReloadSettings.tsx @@ -0,0 +1,281 @@ +"use client"; + +import { useQuery as useZeroQuery } from "@rocicorp/zero/react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { AlertTriangle, CreditCard, RefreshCw } from "lucide-react"; +import { useParams, useRouter, useSearchParams } from "next/navigation"; +import { useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import { stripeApiService } from "@/lib/apis/stripe-api.service"; +import { AppError } from "@/lib/error"; +import { queries } from "@/zero/queries"; + +const microsToDollars = (micros: number | null | undefined): string => { + if (micros == null) return ""; + return (micros / 1_000_000).toString(); +}; + +const dollarsToMicros = (value: string): number | null => { + const trimmed = value.trim(); + if (trimmed === "") return null; + const dollars = Number(trimmed); + if (!Number.isFinite(dollars) || dollars < 0) return null; + return Math.round(dollars * 1_000_000); +}; + +const formatUsd = (micros: number) => `$${(Math.max(0, micros) / 1_000_000).toFixed(2)}`; + +export function AutoReloadSettings() { + const params = useParams(); + const router = useRouter(); + const searchParams = useSearchParams(); + const queryClient = useQueryClient(); + const searchSpaceId = Number(params?.search_space_id); + + const [enabled, setEnabled] = useState(false); + const [thresholdInput, setThresholdInput] = useState(""); + const [amountInput, setAmountInput] = useState(""); + const seededRef = useRef(false); + + const [me] = useZeroQuery(queries.user.me({})); + const balanceMicros = me?.creditMicrosBalance ?? 0; + + const { data: settings, isLoading } = useQuery({ + queryKey: ["auto-reload-settings"], + queryFn: () => stripeApiService.getAutoReloadSettings(), + }); + + // Seed the form once from the server, then let the user own the inputs. + useEffect(() => { + if (settings && !seededRef.current) { + seededRef.current = true; + setEnabled(settings.enabled); + setThresholdInput(microsToDollars(settings.threshold_micros)); + setAmountInput(microsToDollars(settings.amount_micros)); + } + }, [settings]); + + // Surface the result of the Stripe card-setup redirect. + useEffect(() => { + const setupResult = searchParams.get("auto_reload_setup"); + if (!setupResult) return; + if (setupResult === "success") { + toast.success("Card saved. You can now enable auto-reload."); + queryClient.invalidateQueries({ queryKey: ["auto-reload-settings"] }); + } else if (setupResult === "cancel") { + toast.info("Card setup canceled."); + } + // Strip the query param so refreshes don't re-toast. + router.replace(`/dashboard/${searchSpaceId}/user-settings/purchases`); + }, [searchParams, router, searchSpaceId, queryClient]); + + const setupMutation = useMutation({ + mutationFn: () => + stripeApiService.createAutoReloadSetupSession({ search_space_id: searchSpaceId }), + onSuccess: (response) => { + window.location.assign(response.checkout_url); + }, + onError: () => { + toast.error("Couldn't start card setup. Please try again."); + }, + }); + + const saveMutation = useMutation({ + mutationFn: stripeApiService.updateAutoReloadSettings, + onSuccess: (updated) => { + queryClient.setQueryData(["auto-reload-settings"], updated); + toast.success(updated.enabled ? "Auto-reload is on." : "Auto-reload settings saved."); + }, + onError: (error) => { + if (error instanceof AppError && error.message) { + toast.error(error.message); + return; + } + toast.error("Couldn't save auto-reload settings. Please try again."); + }, + }); + + if (isLoading || !settings) { + return ( +
+ +
+ ); + } + + // Server-side feature flag: hide the whole card when auto-reload is off. + if (!settings.feature_enabled) { + return null; + } + + const minAmountDollars = (settings.min_amount_micros / 1_000_000).toFixed(2); + const hasCard = settings.has_payment_method; + + const handleSave = () => { + if (!enabled) { + saveMutation.mutate({ + enabled: false, + threshold_micros: dollarsToMicros(thresholdInput), + amount_micros: dollarsToMicros(amountInput), + }); + return; + } + + const thresholdMicros = dollarsToMicros(thresholdInput); + const amountMicros = dollarsToMicros(amountInput); + + if (!thresholdMicros || thresholdMicros <= 0) { + toast.error("Enter a low-balance threshold greater than $0."); + return; + } + if (amountMicros == null || amountMicros < settings.min_amount_micros) { + toast.error(`Reload amount must be at least $${minAmountDollars}.`); + return; + } + + saveMutation.mutate({ + enabled: true, + threshold_micros: thresholdMicros, + amount_micros: amountMicros, + }); + }; + + return ( + + + + + Auto-reload + + + Automatically top up your credit balance when it drops below a threshold, using a saved + card. Current balance:{" "} + {formatUsd(balanceMicros)}. + + + + {settings.failed_at && ( + + + Last auto-reload failed + + Your saved card was declined and auto-reload was turned off. Update your card and + re-enable it below to keep topping up automatically. + + + )} + + {!hasCard ? ( +
+
+ + Add a card to enable automatic top-ups. +
+ +
+ ) : ( + <> +
+
+ +

+ Charge your saved card when the balance gets low. +

+
+ +
+ +
+
+ +
+ + $ + + setThresholdInput(e.target.value)} + disabled={!enabled} + placeholder="5" + /> +
+
+
+ +
+ + $ + + setAmountInput(e.target.value)} + disabled={!enabled} + placeholder="10" + /> +
+

Minimum ${minAmountDollars}.

+
+
+ +
+ + +
+ + )} + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent.tsx index cf73b5eba..081ce358e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent.tsx @@ -14,24 +14,24 @@ import { TableRow, } from "@/components/ui/table"; import type { + CreditPurchase, PagePurchase, - PagePurchaseStatus, - TokenPurchase, + PurchaseStatus, } from "@/contracts/types/stripe.types"; import { stripeApiService } from "@/lib/apis/stripe-api.service"; import { cn } from "@/lib/utils"; -type PurchaseKind = "pages" | "tokens"; +type PurchaseKind = "pages" | "credits"; type UnifiedPurchase = { id: string; kind: PurchaseKind; created_at: string; - status: PagePurchaseStatus; + status: PurchaseStatus; /** * Granted units. Interpretation depends on ``kind``: - * - ``"pages"`` — integer number of indexed pages. - * - ``"tokens"`` — integer micro-USD of credit (1_000_000 = $1.00). + * - ``"pages"`` — integer number of indexed pages (legacy history). + * - ``"credits"`` — integer micro-USD of credit (1_000_000 = $1.00). * The ``Granted`` column formats accordingly. */ granted: number; @@ -39,7 +39,7 @@ type UnifiedPurchase = { currency: string | null; }; -const STATUS_STYLES: Record = { +const STATUS_STYLES: Record = { completed: { label: "Completed", className: "bg-emerald-600 text-white border-transparent hover:bg-emerald-600", @@ -63,8 +63,8 @@ const KIND_META: Record< icon: FileText, iconClass: "text-sky-500", }, - tokens: { - label: "Premium Credit", + credits: { + label: "Credits", icon: Coins, iconClass: "text-amber-500", }, @@ -97,10 +97,10 @@ function normalizePagePurchase(p: PagePurchase): UnifiedPurchase { }; } -function normalizeTokenPurchase(p: TokenPurchase): UnifiedPurchase { +function normalizeCreditPurchase(p: CreditPurchase): UnifiedPurchase { return { id: p.id, - kind: "tokens", + kind: "credits", created_at: p.created_at, status: p.status, granted: p.credit_micros_granted, @@ -110,10 +110,10 @@ function normalizeTokenPurchase(p: TokenPurchase): UnifiedPurchase { } function formatGranted(p: UnifiedPurchase): string { - if (p.kind === "tokens") { + if (p.kind === "credits") { const dollars = p.granted / 1_000_000; - // Premium credit packs are always whole dollars at the moment, but - // future fractional grants (refunds, partial top-ups) shouldn't + // Credit packs are always whole dollars at the moment, but future + // fractional grants (refunds, partial top-ups, auto-reload) shouldn't // silently round to "$0". if (dollars >= 1) return `$${dollars.toFixed(2)} of credit`; if (dollars > 0) return `$${dollars.toFixed(3)} of credit`; @@ -127,26 +127,26 @@ export function PurchaseHistoryContent() { queries: [ { queryKey: ["stripe-purchases"], - queryFn: () => stripeApiService.getPurchases(), + queryFn: () => stripeApiService.getPagePurchases(), }, { - queryKey: ["stripe-token-purchases"], - queryFn: () => stripeApiService.getTokenPurchases(), + queryKey: ["stripe-credit-purchases"], + queryFn: () => stripeApiService.getCreditPurchases(), }, ], }); - const [pagesQuery, tokensQuery] = results; - const isLoading = pagesQuery.isLoading || tokensQuery.isLoading; + const [pagesQuery, creditsQuery] = results; + const isLoading = pagesQuery.isLoading || creditsQuery.isLoading; const purchases = useMemo(() => { const pagePurchases = pagesQuery.data?.purchases ?? []; - const tokenPurchases = tokensQuery.data?.purchases ?? []; + const creditPurchases = creditsQuery.data?.purchases ?? []; return [ ...pagePurchases.map(normalizePagePurchase), - ...tokenPurchases.map(normalizeTokenPurchase), + ...creditPurchases.map(normalizeCreditPurchase), ].sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); - }, [pagesQuery.data, tokensQuery.data]); + }, [pagesQuery.data, creditsQuery.data]); if (isLoading) { return ( @@ -162,7 +162,7 @@ export function PurchaseHistoryContent() {

No purchases yet

- Your page and premium credit purchases will appear here after checkout. + Your credit purchases will appear here after checkout.

); diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/purchases/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/purchases/page.tsx index 3fa08c278..b7468349b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/purchases/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/purchases/page.tsx @@ -1,5 +1,11 @@ +import { AutoReloadSettings } from "../components/AutoReloadSettings"; import { PurchaseHistoryContent } from "../components/PurchaseHistoryContent"; export default function Page() { - return ; + return ( +
+ + +
+ ); } diff --git a/surfsense_web/components/layout/index.ts b/surfsense_web/components/layout/index.ts index 67f161d1a..eb475e414 100644 --- a/surfsense_web/components/layout/index.ts +++ b/surfsense_web/components/layout/index.ts @@ -12,6 +12,7 @@ export type { export { ChatListItem, CreateSearchSpaceDialog, + CreditBalanceDisplay, Header, IconRail, LayoutShell, @@ -19,7 +20,6 @@ export { MobileSidebarTrigger, NavIcon, NavSection, - PageUsageDisplay, SearchSpaceAvatar, Sidebar, SidebarCollapseButton, diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 46f6ec8ae..549e6e7d7 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -186,40 +186,40 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid setStatusInboxItems(statusInbox.inboxItems); }, [statusInbox.inboxItems, setStatusInboxItems]); - // Track seen notification IDs to detect new page_limit_exceeded notifications - const seenPageLimitNotifications = useRef>(new Set()); + // Track seen notification IDs to detect new insufficient_credits notifications + const seenCreditNotifications = useRef>(new Set()); const isInitialLoad = useRef(true); - // Effect to show toast for new page_limit_exceeded notifications + // Effect to show toast for new insufficient_credits notifications useEffect(() => { if (statusInbox.loading) return; - const pageLimitNotifications = statusInbox.inboxItems.filter( - (item) => item.type === "page_limit_exceeded" + const creditNotifications = statusInbox.inboxItems.filter( + (item) => item.type === "insufficient_credits" ); if (isInitialLoad.current) { - for (const notification of pageLimitNotifications) { - seenPageLimitNotifications.current.add(notification.id); + for (const notification of creditNotifications) { + seenCreditNotifications.current.add(notification.id); } isInitialLoad.current = false; return; } - const newNotifications = pageLimitNotifications.filter( - (notification) => !seenPageLimitNotifications.current.has(notification.id) + const newNotifications = creditNotifications.filter( + (notification) => !seenCreditNotifications.current.has(notification.id) ); for (const notification of newNotifications) { - seenPageLimitNotifications.current.add(notification.id); + seenCreditNotifications.current.add(notification.id); toast.error(notification.title, { description: notification.message, duration: 8000, icon: , action: { - label: "Get More Pages", - onClick: () => router.push(`/dashboard/${searchSpaceId}/more-pages`), + label: "Buy credits", + onClick: () => router.push(`/dashboard/${searchSpaceId}/buy-more`), }, }); } @@ -696,6 +696,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const isAutomationsPage = pathname?.includes("/automations") === true; const useWorkspacePanel = pathname?.endsWith("/buy-more") === true || + pathname?.endsWith("/earn-credits") === true || pathname?.endsWith("/more-pages") === true || isUserSettingsPage || isSearchSpaceSettingsPage || diff --git a/surfsense_web/components/layout/types/layout.types.ts b/surfsense_web/components/layout/types/layout.types.ts index 1bb0a089e..1dfb51ca8 100644 --- a/surfsense_web/components/layout/types/layout.types.ts +++ b/surfsense_web/components/layout/types/layout.types.ts @@ -74,11 +74,6 @@ export interface ChatsSectionProps { searchSpaceId?: string; } -export interface PageUsageDisplayProps { - pagesUsed: number; - pagesLimit: number; -} - export interface SidebarUserProfileProps { user: User; searchSpaceId?: string; diff --git a/surfsense_web/components/layout/ui/index.ts b/surfsense_web/components/layout/ui/index.ts index 00b862082..85a47bea1 100644 --- a/surfsense_web/components/layout/ui/index.ts +++ b/surfsense_web/components/layout/ui/index.ts @@ -4,10 +4,10 @@ export { IconRail, NavIcon, SearchSpaceAvatar } from "./icon-rail"; export { LayoutShell } from "./shell"; export { ChatListItem, + CreditBalanceDisplay, MobileSidebar, MobileSidebarTrigger, NavSection, - PageUsageDisplay, Sidebar, SidebarCollapseButton, SidebarHeader, diff --git a/surfsense_web/components/layout/ui/sidebar/AuthenticatedPageUsageDisplay.tsx b/surfsense_web/components/layout/ui/sidebar/AuthenticatedPageUsageDisplay.tsx deleted file mode 100644 index ad31d50bb..000000000 --- a/surfsense_web/components/layout/ui/sidebar/AuthenticatedPageUsageDisplay.tsx +++ /dev/null @@ -1,15 +0,0 @@ -"use client"; - -import { useQuery } from "@rocicorp/zero/react"; -import { useIsAnonymous } from "@/contexts/anonymous-mode"; -import { queries } from "@/zero/queries"; -import { PageUsageDisplay } from "./PageUsageDisplay"; - -export function AuthenticatedPageUsageDisplay() { - const isAnonymous = useIsAnonymous(); - const [me] = useQuery(queries.user.me({})); - - if (isAnonymous || !me) return null; - - return ; -} diff --git a/surfsense_web/components/layout/ui/sidebar/CreditBalanceDisplay.tsx b/surfsense_web/components/layout/ui/sidebar/CreditBalanceDisplay.tsx new file mode 100644 index 000000000..1d45137fb --- /dev/null +++ b/surfsense_web/components/layout/ui/sidebar/CreditBalanceDisplay.tsx @@ -0,0 +1,55 @@ +"use client"; + +import { useQuery } from "@rocicorp/zero/react"; +import { useIsAnonymous } from "@/contexts/anonymous-mode"; +import { cn } from "@/lib/utils"; +import { queries } from "@/zero/queries"; + +// Show the low-balance warning state once the wallet drops below $0.50. +const LOW_BALANCE_WARNING_MICROS = 500_000; + +function formatUsd(micros: number): string { + // Clamp at $0.00 — the balance can dip slightly negative when the actual + // cost of a job exceeds the pre-charge estimate. + const dollars = Math.max(0, micros) / 1_000_000; + if (dollars >= 100) return `$${dollars.toFixed(0)}`; + if (dollars >= 1) return `$${dollars.toFixed(2)}`; + // Sub-dollar balances need extra precision so the user can still tell what + // is left ("$0.042 of credit") instead of rounding to "$0.00". + if (dollars > 0) return `$${dollars.toFixed(3)}`; + return "$0.00"; +} + +/** + * Unified credit-wallet balance shown in the sidebar. + * + * The single ``creditMicrosBalance`` replaces the former page-limit and + * premium-credit meters. Values come from Zero (live-replicated from Postgres) + * as integer micro-USD (1_000_000 == $1.00). A low-balance warning highlights + * the amount when it falls below $0.50 so the user knows to top up or enable + * auto-reload. + */ +export function CreditBalanceDisplay() { + const isAnonymous = useIsAnonymous(); + const [me] = useQuery(queries.user.me({})); + + if (isAnonymous || !me) return null; + + const balanceMicros = me.creditMicrosBalance ?? 0; + const isLow = balanceMicros < LOW_BALANCE_WARNING_MICROS; + + return ( +
+ Credits + + {formatUsd(balanceMicros)} + +
+ ); +} diff --git a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx index f757db70e..f6c7f3e15 100644 --- a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx @@ -49,7 +49,7 @@ import { isConnectorIndexingMetadata, isDocumentProcessingMetadata, isNewMentionMetadata, - isPageLimitExceededMetadata, + isInsufficientCreditsMetadata, } from "@/contracts/types/inbox.types"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import type { InboxItem } from "@/hooks/use-inbox"; @@ -291,7 +291,7 @@ export function InboxSidebarContent({ (item: InboxItem): boolean => { if (activeFilter === "unread") return !item.read; if (activeFilter === "errors") { - if (item.type === "page_limit_exceeded") return true; + if (item.type === "insufficient_credits") return true; const meta = item.metadata as Record | undefined; return typeof meta?.status === "string" && meta.status === "failed"; } @@ -397,8 +397,8 @@ export function InboxSidebarContent({ router.push(url); } } - } else if (item.type === "page_limit_exceeded") { - if (isPageLimitExceededMetadata(item.metadata)) { + } else if (item.type === "insufficient_credits") { + if (isInsufficientCreditsMetadata(item.metadata)) { const actionUrl = item.metadata.action_url; if (actionUrl) { onOpenChange(false); @@ -470,7 +470,7 @@ export function InboxSidebarContent({ ); } - if (item.type === "page_limit_exceeded") { + if (item.type === "insufficient_credits") { return (
diff --git a/surfsense_web/components/layout/ui/sidebar/PageUsageDisplay.tsx b/surfsense_web/components/layout/ui/sidebar/PageUsageDisplay.tsx deleted file mode 100644 index 3d011b762..000000000 --- a/surfsense_web/components/layout/ui/sidebar/PageUsageDisplay.tsx +++ /dev/null @@ -1,24 +0,0 @@ -"use client"; - -import { Progress } from "@/components/ui/progress"; - -interface PageUsageDisplayProps { - pagesUsed: number; - pagesLimit: number; -} - -export function PageUsageDisplay({ pagesUsed, pagesLimit }: PageUsageDisplayProps) { - const usagePercentage = (pagesUsed / pagesLimit) * 100; - - return ( -
-
- - {pagesUsed.toLocaleString()} / {pagesLimit.toLocaleString()} pages - - {usagePercentage.toFixed(0)}% -
- -
- ); -} diff --git a/surfsense_web/components/layout/ui/sidebar/PremiumTokenUsageDisplay.tsx b/surfsense_web/components/layout/ui/sidebar/PremiumTokenUsageDisplay.tsx deleted file mode 100644 index 983672d0b..000000000 --- a/surfsense_web/components/layout/ui/sidebar/PremiumTokenUsageDisplay.tsx +++ /dev/null @@ -1,49 +0,0 @@ -"use client"; - -import { useQuery } from "@rocicorp/zero/react"; -import { Progress } from "@/components/ui/progress"; -import { useIsAnonymous } from "@/contexts/anonymous-mode"; -import { queries } from "@/zero/queries"; - -/** - * Premium credit balance shown in the sidebar. - * - * Values come from Zero (live-replicated from Postgres) and are stored as - * integer micro-USD (1_000_000 == $1.00). We render in dollars because - * users top up at $1/pack and the credit gets debited at actual provider - * cost. - */ -export function PremiumTokenUsageDisplay() { - const isAnonymous = useIsAnonymous(); - const [me] = useQuery(queries.user.me({})); - - if (isAnonymous || !me) return null; - - const usagePercentage = Math.min( - (me.premiumCreditMicrosUsed / Math.max(me.premiumCreditMicrosLimit, 1)) * 100, - 100 - ); - - const formatUsd = (micros: number) => { - const dollars = micros / 1_000_000; - if (dollars >= 100) return `$${dollars.toFixed(0)}`; - if (dollars >= 1) return `$${dollars.toFixed(2)}`; - // Sub-dollar balances need extra precision so the bar still tells the - // user what's left ("$0.04 of credit") instead of rounding to "$0". - if (dollars > 0) return `$${dollars.toFixed(3)}`; - return "$0"; - }; - - return ( -
-
- - {formatUsd(me.premiumCreditMicrosUsed)} / {formatUsd(me.premiumCreditMicrosLimit)} of - credit - - {usagePercentage.toFixed(0)}% -
- -
- ); -} diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 6a4785d98..ee891d78b 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -1,6 +1,6 @@ "use client"; -import { CreditCard, Dot, SquarePen, Zap } from "lucide-react"; +import { CreditCard, SquarePen, Zap } from "lucide-react"; import Link from "next/link"; import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; @@ -13,10 +13,9 @@ import { useIsAnonymous } from "@/contexts/anonymous-mode"; import { cn } from "@/lib/utils"; import { SIDEBAR_MIN_WIDTH } from "../../hooks/useSidebarResize"; import type { ChatItem, NavItem, PageUsage, SearchSpace, User } from "../../types/layout.types"; -import { AuthenticatedPageUsageDisplay } from "./AuthenticatedPageUsageDisplay"; import { ChatListItem } from "./ChatListItem"; +import { CreditBalanceDisplay } from "./CreditBalanceDisplay"; import { NavSection } from "./NavSection"; -import { PremiumTokenUsageDisplay } from "./PremiumTokenUsageDisplay"; import { SidebarButton } from "./SidebarButton"; import { SidebarCollapseButton } from "./SidebarCollapseButton"; import { SidebarHeader } from "./SidebarHeader"; @@ -404,17 +403,16 @@ function SidebarUsageFooter({ return (
- - +
- Get Free Pages + Earn credits FREE @@ -427,12 +425,7 @@ function SidebarUsageFooter({ > - Buy More - - - $1/1k - - $1/1M + Buy credits
diff --git a/surfsense_web/components/layout/ui/sidebar/index.ts b/surfsense_web/components/layout/ui/sidebar/index.ts index e25149b06..e9256cc3a 100644 --- a/surfsense_web/components/layout/ui/sidebar/index.ts +++ b/surfsense_web/components/layout/ui/sidebar/index.ts @@ -3,8 +3,8 @@ export { ChatListItem } from "./ChatListItem"; export { DocumentsSidebar } from "./DocumentsSidebar"; export { InboxSidebar, InboxSidebarContent } from "./InboxSidebar"; export { MobileSidebar, MobileSidebarTrigger } from "./MobileSidebar"; +export { CreditBalanceDisplay } from "./CreditBalanceDisplay"; export { NavSection } from "./NavSection"; -export { PageUsageDisplay } from "./PageUsageDisplay"; export { Sidebar } from "./Sidebar"; export { SidebarCollapseButton } from "./SidebarCollapseButton"; export { SidebarHeader } from "./SidebarHeader"; diff --git a/surfsense_web/components/pricing/pricing-section.tsx b/surfsense_web/components/pricing/pricing-section.tsx index 46ceee694..014cebd66 100644 --- a/surfsense_web/components/pricing/pricing-section.tsx +++ b/surfsense_web/components/pricing/pricing-section.tsx @@ -14,11 +14,11 @@ const demoPlans = [ price: "0", yearlyPrice: "0", period: "", - billingText: "500 pages + $5 in premium credits included", + billingText: "$5 of credit included to start", features: [ "Self Hostable", - "500 pages included to start", - "$5 in premium credits for paid AI models and premium AI features", + "$5 of credit included to start", + "One credit balance for document processing and premium AI features", "Includes access to OpenAI text, audio and image models", "AI automations and agents: scheduled and event-triggered workflows", "Desktop app: Quick, General and Screenshot Assist plus local folder sync", @@ -38,7 +38,7 @@ const demoPlans = [ billingText: "No subscription, buy only when you need more", features: [ "Everything in Free", - "Buy 1,000-page packs or $1 in premium credits at $1 each", + "Buy credit in $1 packs — $1 buys $1 of credit, with optional auto-reload", "Use premium AI models like GPT-5.4, Claude Sonnet 4.6, Gemini 2.5 Pro & 100+ more via OpenRouter", "Connector write-back to Notion, Slack, Linear & Jira", "Priority support on Discord", @@ -84,32 +84,32 @@ interface FAQSection { const faqData: FAQSection[] = [ { - title: "Pages & Document Billing", + title: "Credits & Document Billing", items: [ { - question: 'What exactly is a "page" in SurfSense?', + question: "What are credits in SurfSense?", answer: - "A page is a simple billing unit that measures how much content you add to your knowledge base. For PDFs, one page equals one real PDF page. For other document types like Word, PowerPoint, and Excel files, pages are automatically estimated based on the file. Every file uses at least 1 page.", + "Credits are a single prepaid balance shown in dollars that powers everything in SurfSense — both document processing and premium AI features. New accounts start with $5 of credit. Your balance goes down as you use the product and back up when you top up or earn more, so there's just one number to keep an eye on.", }, { - question: "What are Basic and Premium processing modes?", + question: "How much does document processing cost?", answer: - "When uploading documents, you can choose between two processing modes. Basic mode uses standard extraction and costs 1 page credit per page, great for most documents. Premium processing mode uses advanced extraction optimized for complex financial, medical, and legal documents with intricate tables, layouts, and formatting. It costs 10 page credits per page and does not use your premium AI credits.", + "Document processing is billed per page out of your credit balance. For PDFs, one page equals one real PDF page; for other document types like Word, PowerPoint, and Excel files, pages are automatically estimated. Basic mode costs $0.001 per page and Premium mode costs $0.01 per page. Premium processing uses advanced extraction optimized for complex financial, medical, and legal documents with intricate tables and layouts. Every file uses at least 1 page.", }, { question: "How does the Pay As You Go plan work?", answer: - "There's no monthly subscription. When you need more pages, simply purchase 1,000-page packs at $1 each. Purchased pages are added to your account immediately so you can keep indexing right away. You only pay when you actually need more.", + "There's no monthly subscription. When you need more credit, simply buy $1 packs — $1 buys exactly $1 of credit. Purchased credit is added to your balance immediately so you can keep working right away. You only pay when you actually need more, and you can enable auto-reload to top up automatically.", }, { - question: "What happens if I run out of pages?", + question: "What happens if I run out of credit?", answer: - "SurfSense checks your remaining pages before processing each file. If you don't have enough, the upload is paused and you'll be notified. You can purchase additional page packs at any time to continue. For cloud connector syncs, a small overage may be allowed so your sync doesn't partially fail.", + "SurfSense checks your remaining credit before processing each file. If you don't have enough, the upload is paused and you'll be notified so you can buy more credit and continue. For cloud connector syncs, a small overage may be allowed so your sync doesn't partially fail.", }, { - question: "If I delete a document, do I get my pages back?", + question: "If I delete a document, do I get my credit back?", answer: - "No. Deleting a document removes it from your knowledge base, but the pages it used are not refunded. Pages track your total usage over time, not how much is currently stored. So be mindful of what you index. Once pages are spent, they're spent even if you later remove the document.", + "No. Deleting a document removes it from your knowledge base, but the credit it used is not refunded. Credit tracks your total usage over time, not how much is currently stored, so be mindful of what you index. Once credit is spent, it's spent even if you later remove the document.", }, ], }, @@ -117,49 +117,49 @@ const faqData: FAQSection[] = [ title: "File Types & Connectors", items: [ { - question: "Which file types count toward my page limit?", + question: "Which file types use credit?", answer: - "Page limits only apply to document files that need processing, including PDFs, Word documents (DOC, DOCX, ODT, RTF), presentations (PPT, PPTX, ODP), spreadsheets (XLS, XLSX, ODS), ebooks (EPUB), and images (JPG, PNG, TIFF, WebP, BMP). Plain text files, code files, Markdown, CSV, TSV, HTML, audio, and video files do not consume any pages.", + "Credit is only used for document files that need processing, including PDFs, Word documents (DOC, DOCX, ODT, RTF), presentations (PPT, PPTX, ODP), spreadsheets (XLS, XLSX, ODS), ebooks (EPUB), and images (JPG, PNG, TIFF, WebP, BMP). Plain text files, code files, Markdown, CSV, TSV, HTML, audio, and video files do not consume any credit.", }, { - question: "How are pages consumed?", + question: "How is credit consumed for documents?", answer: - "Pages are deducted whenever a document file is successfully indexed into your knowledge base, whether through direct uploads or file-based connector syncs (Google Drive, OneDrive, Dropbox, Local Folder). In Basic mode, each page costs 1 page credit; in Premium mode, each page costs 10 page credits. SurfSense checks your remaining credits before processing and only charges you after the file is indexed. Duplicate documents are automatically detected and won't cost you extra pages.", + "Credit is deducted whenever a document file is successfully indexed into your knowledge base, whether through direct uploads or file-based connector syncs (Google Drive, OneDrive, Dropbox, Local Folder). In Basic mode each page costs $0.001; in Premium mode each page costs $0.01. SurfSense checks your remaining credit before processing and only charges you after the file is indexed. Duplicate documents are automatically detected and won't cost you extra.", }, { - question: "Do connectors like Slack, Notion, or Gmail use pages?", + question: "Do connectors like Slack, Notion, or Gmail use credit?", answer: - "No. Connectors that work with structured text data like Slack, Discord, Notion, Confluence, Jira, Linear, ClickUp, GitHub, Gmail, Google Calendar, Microsoft Teams, Airtable, Elasticsearch, Web Crawler, BookStack, Obsidian, and Luma do not use pages at all. Page limits only apply to file-based connectors that need document processing, such as Google Drive, OneDrive, Dropbox, and Local Folder syncs.", + "No. Connectors that work with structured text data like Slack, Discord, Notion, Confluence, Jira, Linear, ClickUp, GitHub, Gmail, Google Calendar, Microsoft Teams, Airtable, Elasticsearch, Web Crawler, BookStack, Obsidian, and Luma do not use credit at all. Document-processing charges only apply to file-based connectors such as Google Drive, OneDrive, Dropbox, and Local Folder syncs.", }, ], }, { - title: "Premium Credits", + title: "Premium AI & Credit", items: [ { - question: 'What are "premium credits"?', + question: "How is credit used for premium AI?", answer: - "Premium credits are your USD balance for paid AI usage in SurfSense, including premium AI models like GPT-5.4, Claude Sonnet 4.6, and Gemini 2.5 Pro, plus premium AI features such as image generation, podcasts, and video presentations when they use paid models. Each request debits the actual USD provider cost, so cheaper and more expensive models bill proportionally.", + "The same credit balance pays for paid AI usage in SurfSense, including premium AI models like GPT-5.4, Claude Sonnet 4.6, and Gemini 2.5 Pro, plus premium AI features such as image generation, podcasts, and video presentations when they use paid models. Each request debits the actual USD provider cost, so cheaper and more expensive models bill proportionally.", }, { - question: "How many premium credits do I get for free?", + question: "How much credit do I get for free?", answer: - "Every registered SurfSense account starts with $5 in premium credits at no cost. Anonymous users (no login) get 500,000 free tokens across free models before creating an account. Once your included premium credits run out, you can top up at any time.", + "Every registered SurfSense account starts with $5 of credit at no cost. Anonymous users (no login) get 500,000 free tokens across free models before creating an account. Once your included credit runs out, you can top up at any time or earn more by completing tasks.", }, { - question: "How does buying premium credits work?", + question: "How does buying credit work?", answer: - "Premium credit top-ups are pay as you go, with no subscription. $1 buys $1 of credit, and your balance is spent at provider cost. Purchased credit is added to your account immediately. You can buy up to $100 at a time.", + "Top-ups are pay as you go, with no subscription. $1 buys $1 of credit, and your balance is spent at provider cost. Purchased credit is added to your account immediately, and you can buy up to $100 at a time. Enable auto-reload to top up automatically when your balance runs low.", }, { - question: "Are premium credits the same as page credits?", + question: "Is there a separate balance for documents and AI?", answer: - "No. Page credits pay for document indexing and file-based connector processing. Premium credits pay for paid AI usage, such as premium model chats and premium AI generation features. Premium document processing mode sounds similar, but it consumes page credits, not premium credits.", + "No. SurfSense uses one unified credit balance for everything — document indexing, file-based connector processing, premium model chats, and premium AI generation features all draw from the same wallet. Premium document processing mode simply costs more per page ($0.01 vs $0.001), but it's the same credit.", }, { - question: "What happens if I run out of premium credits?", + question: "What happens if I run out of credit?", answer: - "When your premium credit balance runs low, you'll see a warning. Once you run out, paid model requests and premium AI features are paused until you top up. You can still use non-premium models and features that do not consume premium credits.", + "When your credit balance runs low, you'll see a warning. Once you run out, paid model requests, premium AI features, and document processing are paused until you top up. You can still use non-premium models and features that do not consume credit.", }, ], }, @@ -174,7 +174,7 @@ const faqData: FAQSection[] = [ { question: "Do automations and agents cost extra?", answer: - "No. There is no separate subscription or add-on fee for automations. Agents use the same page credits and premium credits as the rest of SurfSense. Indexing documents consumes page credits, and premium AI model usage during a workflow consumes premium credits at provider cost. If a workflow only uses free models, it does not touch your premium credits.", + "No. There is no separate subscription or add-on fee for automations. Agents draw from the same unified credit balance as the rest of SurfSense. Indexing documents and premium AI model usage during a workflow both consume credit at provider cost. If a workflow only uses free models and indexes no documents, it does not touch your credit.", }, { question: "How do event-triggered automations work?", @@ -192,9 +192,9 @@ const faqData: FAQSection[] = [ title: "Self-Hosting", items: [ { - question: "Can I self-host SurfSense with unlimited pages and credit?", + question: "Can I self-host SurfSense with unlimited usage?", answer: - "Yes! When self-hosting, you have full control over your page and premium credit limits. The default self-hosted setup gives you effectively unlimited pages and premium credits, so you can index as much data and use as many AI queries as your infrastructure supports.", + "Yes! When self-hosting, you have full control over billing. The default self-hosted setup leaves document-processing credit billing off and gives you effectively unlimited credit, so you can index as much data and use as many AI queries as your infrastructure supports.", }, ], }, @@ -286,7 +286,7 @@ function PricingFAQ() { Frequently Asked Questions

- Everything you need to know about SurfSense pages, premium credits, and billing. + Everything you need to know about SurfSense credits and billing. Can't find what you need? Reach out at{" "} rohan@surfsense.com @@ -372,7 +372,7 @@ function PricingBasic() { diff --git a/surfsense_web/components/settings/buy-tokens-content.tsx b/surfsense_web/components/settings/buy-credits-content.tsx similarity index 72% rename from surfsense_web/components/settings/buy-tokens-content.tsx rename to surfsense_web/components/settings/buy-credits-content.tsx index 4b0605f28..cfc1fcc49 100644 --- a/surfsense_web/components/settings/buy-tokens-content.tsx +++ b/surfsense_web/components/settings/buy-credits-content.tsx @@ -7,46 +7,45 @@ import { useParams } from "next/navigation"; import { useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; -import { Progress } from "@/components/ui/progress"; import { Spinner } from "@/components/ui/spinner"; import { stripeApiService } from "@/lib/apis/stripe-api.service"; import { AppError } from "@/lib/error"; import { cn } from "@/lib/utils"; import { queries } from "@/zero/queries"; -// One pack = $1.00 of credit, stored as 1_000_000 micro-USD on the -// backend. Premium turns are debited at the actual provider cost -// reported by LiteLLM, so $1 of credit always buys $1 of provider -// usage at cost. +// One pack = $1.00 of credit, stored as 1_000_000 micro-USD on the backend. +// ETL page processing and premium turns are both debited from the same wallet +// at the actual cost, so $1 of credit always buys $1 of usage at cost. const CREDIT_PER_PACK_MICROS = 1_000_000; const PRICE_PER_PACK_USD = 1; const PRESET_MULTIPLIERS = [1, 2, 5, 10, 25, 50] as const; -const formatUsd = (micros: number, options?: { compact?: boolean }) => { - const dollars = micros / 1_000_000; - if (options?.compact && dollars >= 1) return `$${dollars.toFixed(2)}`; +const formatUsd = (micros: number) => { + // Clamp at $0.00 — the balance can dip slightly negative when actual cost + // exceeds the pre-charge estimate. + const dollars = Math.max(0, micros) / 1_000_000; if (dollars >= 100) return `$${dollars.toFixed(0)}`; if (dollars >= 1) return `$${dollars.toFixed(2)}`; if (dollars > 0) return `$${dollars.toFixed(3)}`; - return "$0"; + return "$0.00"; }; -export function BuyTokensContent() { +export function BuyCreditsContent() { const params = useParams(); const searchSpaceId = Number(params?.search_space_id); const [quantity, setQuantity] = useState(1); // Server config flag: stays on REST, not per-user. - const { data: tokenStatus } = useQuery({ - queryKey: ["token-status"], - queryFn: () => stripeApiService.getTokenStatus(), + const { data: creditStatus } = useQuery({ + queryKey: ["credit-status"], + queryFn: () => stripeApiService.getCreditStatus(), }); // Live per-user balance via Zero. const [me] = useZeroQuery(queries.user.me({})); const purchaseMutation = useMutation({ - mutationFn: stripeApiService.createTokenCheckoutSession, + mutationFn: stripeApiService.createCreditCheckoutSession, onSuccess: (response) => { window.location.assign(response.checkout_url); }, @@ -62,10 +61,10 @@ export function BuyTokensContent() { const totalCreditMicros = quantity * CREDIT_PER_PACK_MICROS; const totalPrice = quantity * PRICE_PER_PACK_USD; - if (tokenStatus && !tokenStatus.token_buying_enabled) { + if (creditStatus && !creditStatus.credit_buying_enabled) { return (

-

Buy Premium Credit

+

Buy Credits

Credit purchases are temporarily unavailable.

@@ -73,35 +72,23 @@ export function BuyTokensContent() { ); } - const used = me?.premiumCreditMicrosUsed ?? 0; - const limit = me?.premiumCreditMicrosLimit ?? 0; - // Mirrors the backend formula in stripe_routes.py (max(0, limit - used)). - const remaining = Math.max(0, limit - used); - const usagePercentage = me ? Math.min((used / Math.max(limit, 1)) * 100, 100) : 0; + const balanceMicros = me?.creditMicrosBalance ?? creditStatus?.credit_micros_balance ?? 0; return (
-

Buy Premium Credit

+

Buy Credits

$1 buys $1 of credit, billed at provider cost

- {me && ( -
-
- - {formatUsd(used)} / {formatUsd(limit)} of credit - - {usagePercentage.toFixed(0)}% -
- -

- {formatUsd(remaining)} of credit remaining -

+
+
+ Current balance + {formatUsd(balanceMicros)}
- )} +
diff --git a/surfsense_web/components/settings/buy-pages-content.tsx b/surfsense_web/components/settings/buy-pages-content.tsx deleted file mode 100644 index 82b8d8e2a..000000000 --- a/surfsense_web/components/settings/buy-pages-content.tsx +++ /dev/null @@ -1,148 +0,0 @@ -"use client"; - -import { useMutation, useQuery } from "@tanstack/react-query"; -import { Minus, Plus } from "lucide-react"; -import { useParams } from "next/navigation"; -import { useState } from "react"; -import { toast } from "sonner"; -import { Button } from "@/components/ui/button"; -import { Spinner } from "@/components/ui/spinner"; -import { stripeApiService } from "@/lib/apis/stripe-api.service"; -import { AppError } from "@/lib/error"; -import { cn } from "@/lib/utils"; - -const PAGE_PACK_SIZE = 1000; -const PRICE_PER_PACK_USD = 1; -const PRESET_MULTIPLIERS = [1, 2, 5, 10, 25, 50] as const; - -export function BuyPagesContent() { - const params = useParams(); - const [quantity, setQuantity] = useState(1); - const { data: stripeStatus } = useQuery({ - queryKey: ["stripe-status"], - queryFn: () => stripeApiService.getStatus(), - }); - - const purchaseMutation = useMutation({ - mutationFn: stripeApiService.createCheckoutSession, - onSuccess: (response) => { - window.location.assign(response.checkout_url); - }, - onError: (error) => { - if (error instanceof AppError && error.message) { - toast.error(error.message); - return; - } - toast.error("Failed to start checkout. Please try again."); - }, - }); - - const searchSpaceId = Number(params.search_space_id); - const hasValidSearchSpace = Number.isFinite(searchSpaceId) && searchSpaceId > 0; - const totalPages = quantity * PAGE_PACK_SIZE; - const totalPrice = quantity * PRICE_PER_PACK_USD; - - if (stripeStatus && !stripeStatus.page_buying_enabled) { - return ( -
-

Buy Pages

-

Page purchases are temporarily unavailable.

-
- ); - } - - const handleBuyNow = () => { - if (!hasValidSearchSpace) { - toast.error("Unable to determine the current workspace for checkout."); - return; - } - purchaseMutation.mutate({ - quantity, - search_space_id: searchSpaceId, - }); - }; - - return ( -
-
-

Buy Pages

-

$1 per 1,000 pages, pay as you go

-
- -
- {/* Stepper */} -
- - - {totalPages.toLocaleString()} - - -
- - {/* Quick-pick presets */} -
- {PRESET_MULTIPLIERS.map((m) => ( - - ))} -
- -
- - {totalPages.toLocaleString()} pages - - ${totalPrice} -
- - -

Secure checkout via Stripe

-
-
- ); -} diff --git a/surfsense_web/components/settings/more-pages-content.tsx b/surfsense_web/components/settings/earn-credits-content.tsx similarity index 79% rename from surfsense_web/components/settings/more-pages-content.tsx rename to surfsense_web/components/settings/earn-credits-content.tsx index e1b05f4d2..21b7f8a5b 100644 --- a/surfsense_web/components/settings/more-pages-content.tsx +++ b/surfsense_web/components/settings/earn-credits-content.tsx @@ -22,7 +22,14 @@ import { } from "@/lib/posthog/events"; import { cn } from "@/lib/utils"; -export function MorePagesContent() { +// Compact dollar label for a task's reward (e.g. "+$0.03"). +const formatRewardUsd = (micros: number) => { + const dollars = micros / 1_000_000; + if (dollars >= 1) return `+$${dollars.toFixed(2)}`; + return `+$${dollars.toFixed(2)}`; +}; + +export function EarnCreditsContent() { const params = useParams(); const queryClient = useQueryClient(); const searchSpaceId = params?.search_space_id ?? ""; @@ -35,11 +42,11 @@ export function MorePagesContent() { queryKey: ["incentive-tasks"], queryFn: () => incentiveTasksApiService.getTasks(), }); - const { data: stripeStatus } = useQuery({ - queryKey: ["stripe-status"], - queryFn: () => stripeApiService.getStatus(), + const { data: creditStatus } = useQuery({ + queryKey: ["credit-status"], + queryFn: () => stripeApiService.getCreditStatus(), }); - const pageBuyingEnabled = stripeStatus?.page_buying_enabled ?? true; + const creditBuyingEnabled = creditStatus?.credit_buying_enabled ?? true; const completeMutation = useMutation({ mutationFn: incentiveTasksApiService.completeTask, @@ -48,7 +55,7 @@ export function MorePagesContent() { toast.success(response.message); const task = data?.tasks.find((t) => t.task_type === taskType); if (task) { - trackIncentiveTaskCompleted(taskType, task.pages_reward); + trackIncentiveTaskCompleted(taskType, task.credit_micros_reward); } queryClient.invalidateQueries({ queryKey: ["incentive-tasks"] }); queryClient.invalidateQueries({ queryKey: USER_QUERY_KEY }); @@ -69,12 +76,14 @@ export function MorePagesContent() { return (
-

Get Free Pages

-

Earn bonus pages by completing tasks

+

Earn Credits

+

+ Earn bonus credits by completing tasks +

-

Earn Bonus Pages

+

Earn Bonus Credits

{isLoading ? (
{["github", "reddit", "discord"].map((task) => ( @@ -97,14 +106,16 @@ export function MorePagesContent() {
{task.completed ? ( ) : ( - +{task.pages_reward} + + {formatRewardUsd(task.credit_micros_reward)} + )}

Need more?

- {pageBuyingEnabled ? ( + {creditBuyingEnabled ? ( ) : (

- Page purchases are temporarily unavailable. + Credit purchases are temporarily unavailable.

)}
diff --git a/surfsense_web/contracts/types/auth.types.ts b/surfsense_web/contracts/types/auth.types.ts index 29a296c11..b630c461b 100644 --- a/surfsense_web/contracts/types/auth.types.ts +++ b/surfsense_web/contracts/types/auth.types.ts @@ -20,8 +20,7 @@ export const registerRequest = loginRequest.omit({ grant_type: true, username: t export const registerResponse = registerRequest.omit({ password: true }).extend({ id: z.string(), - pages_limit: z.number(), - pages_used: z.number(), + credit_micros_balance: z.number(), }); export type LoginRequest = z.infer; diff --git a/surfsense_web/contracts/types/inbox.types.ts b/surfsense_web/contracts/types/inbox.types.ts index b4cf01710..94e533809 100644 --- a/surfsense_web/contracts/types/inbox.types.ts +++ b/surfsense_web/contracts/types/inbox.types.ts @@ -11,7 +11,7 @@ export const inboxItemTypeEnum = z.enum([ "document_processing", "new_mention", "comment_reply", - "page_limit_exceeded", + "insufficient_credits", ]); /** @@ -116,15 +116,17 @@ export const commentReplyMetadata = z.object({ }); /** - * Page limit exceeded metadata schema + * Insufficient credits metadata schema. + * + * ``balance_micros`` / ``required_micros`` are integer micro-USD + * (1_000_000 == $1.00); the UI divides by 1M when displaying. */ -export const pageLimitExceededMetadata = baseInboxItemMetadata.extend({ +export const insufficientCreditsMetadata = baseInboxItemMetadata.extend({ document_name: z.string(), document_type: z.string(), - pages_used: z.number(), - pages_limit: z.number(), - pages_to_add: z.number(), - error_type: z.literal("page_limit_exceeded"), + balance_micros: z.number(), + required_micros: z.number(), + error_type: z.literal("insufficient_credits"), // Navigation target for frontend action_url: z.string(), action_label: z.string(), @@ -140,7 +142,7 @@ export const inboxItemMetadata = z.union([ documentProcessingMetadata, newMentionMetadata, commentReplyMetadata, - pageLimitExceededMetadata, + insufficientCreditsMetadata, baseInboxItemMetadata, ]); @@ -188,9 +190,9 @@ export const commentReplyInboxItem = inboxItem.extend({ metadata: commentReplyMetadata, }); -export const pageLimitExceededInboxItem = inboxItem.extend({ - type: z.literal("page_limit_exceeded"), - metadata: pageLimitExceededMetadata, +export const insufficientCreditsInboxItem = inboxItem.extend({ + type: z.literal("insufficient_credits"), + metadata: insufficientCreditsMetadata, }); // ============================================================================= @@ -341,12 +343,12 @@ export function isCommentReplyMetadata(metadata: unknown): metadata is CommentRe } /** - * Type guard for PageLimitExceededMetadata + * Type guard for InsufficientCreditsMetadata */ -export function isPageLimitExceededMetadata( +export function isInsufficientCreditsMetadata( metadata: unknown -): metadata is PageLimitExceededMetadata { - return pageLimitExceededMetadata.safeParse(metadata).success; +): metadata is InsufficientCreditsMetadata { + return insufficientCreditsMetadata.safeParse(metadata).success; } /** @@ -361,7 +363,7 @@ export function parseInboxItemMetadata( | DocumentProcessingMetadata | NewMentionMetadata | CommentReplyMetadata - | PageLimitExceededMetadata + | InsufficientCreditsMetadata | null { switch (type) { case "connector_indexing": { @@ -384,8 +386,8 @@ export function parseInboxItemMetadata( const result = commentReplyMetadata.safeParse(metadata); return result.success ? result.data : null; } - case "page_limit_exceeded": { - const result = pageLimitExceededMetadata.safeParse(metadata); + case "insufficient_credits": { + const result = insufficientCreditsMetadata.safeParse(metadata); return result.success ? result.data : null; } default: @@ -406,7 +408,7 @@ export type ConnectorDeletionMetadata = z.infer; export type NewMentionMetadata = z.infer; export type CommentReplyMetadata = z.infer; -export type PageLimitExceededMetadata = z.infer; +export type InsufficientCreditsMetadata = z.infer; export type InboxItemMetadata = z.infer; export type InboxItem = z.infer; export type ConnectorIndexingInboxItem = z.infer; @@ -414,7 +416,7 @@ export type ConnectorDeletionInboxItem = z.infer; export type NewMentionInboxItem = z.infer; export type CommentReplyInboxItem = z.infer; -export type PageLimitExceededInboxItem = z.infer; +export type InsufficientCreditsInboxItem = z.infer; // API Request/Response types export type GetNotificationsRequest = z.infer; diff --git a/surfsense_web/contracts/types/incentive-tasks.types.ts b/surfsense_web/contracts/types/incentive-tasks.types.ts index c45121c29..abe91d905 100644 --- a/surfsense_web/contracts/types/incentive-tasks.types.ts +++ b/surfsense_web/contracts/types/incentive-tasks.types.ts @@ -12,7 +12,8 @@ export const incentiveTaskInfo = z.object({ task_type: incentiveTaskTypeEnum, title: z.string(), description: z.string(), - pages_reward: z.number(), + // Reward in micro-USD (1_000_000 == $1.00) credited to the wallet. + credit_micros_reward: z.number(), action_url: z.string(), completed: z.boolean(), completed_at: z.string().nullable(), @@ -23,7 +24,7 @@ export const incentiveTaskInfo = z.object({ */ export const getIncentiveTasksResponse = z.object({ tasks: z.array(incentiveTaskInfo), - total_pages_earned: z.number(), + total_credit_micros_earned: z.number(), }); /** @@ -32,8 +33,8 @@ export const getIncentiveTasksResponse = z.object({ export const completeTaskSuccessResponse = z.object({ success: z.literal(true), message: z.string(), - pages_awarded: z.number(), - new_pages_limit: z.number(), + credit_micros_awarded: z.number(), + new_balance_micros: z.number(), }); /** diff --git a/surfsense_web/contracts/types/stripe.types.ts b/surfsense_web/contracts/types/stripe.types.ts index 35ec0cb17..fc8c63fd9 100644 --- a/surfsense_web/contracts/types/stripe.types.ts +++ b/surfsense_web/contracts/types/stripe.types.ts @@ -1,20 +1,49 @@ import { z } from "zod"; -export const pagePurchaseStatusEnum = z.enum(["pending", "completed", "failed"]); +export const purchaseStatusEnum = z.enum(["pending", "completed", "failed"]); -export const createCheckoutSessionRequest = z.object({ +// --------------------------------------------------------------------------- +// Credit purchases ($1 packs that top up credit_micros_balance) +// --------------------------------------------------------------------------- + +export const createCreditCheckoutSessionRequest = z.object({ quantity: z.number().int().min(1).max(100), search_space_id: z.number().int().min(1), }); -export const createCheckoutSessionResponse = z.object({ +export const createCreditCheckoutSessionResponse = z.object({ checkout_url: z.string(), }); -export const stripeStatusResponse = z.object({ - page_buying_enabled: z.boolean(), +// Credit balance availability + records. Unit is integer micro-USD +// (1_000_000 == $1.00); the FE divides by 1M when displaying. +export const creditStripeStatusResponse = z.object({ + credit_buying_enabled: z.boolean(), + credit_micros_balance: z.number().default(0), }); +export const creditPurchase = z.object({ + id: z.uuid(), + stripe_checkout_session_id: z.string(), + stripe_payment_intent_id: z.string().nullable(), + quantity: z.number(), + credit_micros_granted: z.number(), + amount_total: z.number().nullable(), + currency: z.string().nullable(), + source: z.string().default("checkout"), + status: purchaseStatusEnum, + completed_at: z.string().nullable(), + created_at: z.string(), +}); + +export const getCreditPurchasesResponse = z.object({ + purchases: z.array(creditPurchase), +}); + +// --------------------------------------------------------------------------- +// Legacy page purchases (read-only history; page buying is removed) +// --------------------------------------------------------------------------- + export const pagePurchase = z.object({ id: z.uuid(), stripe_checkout_session_id: z.string(), @@ -23,7 +52,7 @@ export const pagePurchase = z.object({ pages_granted: z.number(), amount_total: z.number().nullable(), currency: z.string().nullable(), - status: pagePurchaseStatusEnum, + status: purchaseStatusEnum, completed_at: z.string().nullable(), created_at: z.string(), }); @@ -32,70 +61,59 @@ export const getPagePurchasesResponse = z.object({ purchases: z.array(pagePurchase), }); -// Premium credit purchases -export const createTokenCheckoutSessionRequest = z.object({ - quantity: z.number().int().min(1).max(100), +// Response from /stripe/finalize-checkout (credit purchases only). +export const finalizeCheckoutResponse = z.object({ + status: purchaseStatusEnum, + credit_micros_balance: z.number().default(0), + credit_micros_granted: z.number().nullable().optional(), +}); + +// --------------------------------------------------------------------------- +// Auto-reload (off-session top-up when the balance drops below a threshold) +// All *_micros fields are integer micro-USD (1_000_000 == $1.00). +// --------------------------------------------------------------------------- + +export const autoReloadSettingsResponse = z.object({ + feature_enabled: z.boolean(), + enabled: z.boolean().default(false), + threshold_micros: z.number().nullable(), + amount_micros: z.number().nullable(), + min_amount_micros: z.number(), + has_payment_method: z.boolean().default(false), + failed_at: z.string().nullable(), +}); + +export const updateAutoReloadSettingsRequest = z.object({ + enabled: z.boolean(), + threshold_micros: z.number().int().min(0).nullable().optional(), + amount_micros: z.number().int().min(0).nullable().optional(), +}); + +export const createAutoReloadSetupSessionRequest = z.object({ search_space_id: z.number().int().min(1), }); -export const createTokenCheckoutSessionResponse = z.object({ +export const createAutoReloadSetupSessionResponse = z.object({ checkout_url: z.string(), }); -// Premium credit balance + purchase records. -// -// The unit is integer micro-USD (1_000_000 == $1.00). The schema names -// kept the ``Token`` prefix for API back-compat with pinned clients; -// the field names below are authoritative. -export const tokenStripeStatusResponse = z.object({ - token_buying_enabled: z.boolean(), - premium_credit_micros_used: z.number().default(0), - premium_credit_micros_limit: z.number().default(0), - premium_credit_micros_remaining: z.number().default(0), -}); +export type AutoReloadSettingsResponse = z.infer; +export type UpdateAutoReloadSettingsRequest = z.infer; +export type CreateAutoReloadSetupSessionRequest = z.infer< + typeof createAutoReloadSetupSessionRequest +>; +export type CreateAutoReloadSetupSessionResponse = z.infer< + typeof createAutoReloadSetupSessionResponse +>; -export const tokenPurchaseStatusEnum = pagePurchaseStatusEnum; - -export const tokenPurchase = z.object({ - id: z.uuid(), - stripe_checkout_session_id: z.string(), - stripe_payment_intent_id: z.string().nullable(), - quantity: z.number(), - credit_micros_granted: z.number(), - amount_total: z.number().nullable(), - currency: z.string().nullable(), - status: tokenPurchaseStatusEnum, - completed_at: z.string().nullable(), - created_at: z.string(), -}); - -export const getTokenPurchasesResponse = z.object({ - purchases: z.array(tokenPurchase), -}); - -// Response from /stripe/finalize-checkout. Either page or token fields -// are populated depending on purchase_type. -export const finalizeCheckoutResponse = z.object({ - purchase_type: z.enum(["page_packs", "premium_tokens"]), - status: pagePurchaseStatusEnum, - pages_limit: z.number().nullable().optional(), - pages_used: z.number().nullable().optional(), - pages_granted: z.number().nullable().optional(), - premium_credit_micros_limit: z.number().nullable().optional(), - premium_credit_micros_used: z.number().nullable().optional(), - premium_credit_micros_granted: z.number().nullable().optional(), -}); - -export type PagePurchaseStatus = z.infer; -export type CreateCheckoutSessionRequest = z.infer; -export type CreateCheckoutSessionResponse = z.infer; -export type StripeStatusResponse = z.infer; +export type PurchaseStatus = z.infer; +export type CreateCreditCheckoutSessionRequest = z.infer; +export type CreateCreditCheckoutSessionResponse = z.infer< + typeof createCreditCheckoutSessionResponse +>; +export type CreditStripeStatusResponse = z.infer; +export type CreditPurchase = z.infer; +export type GetCreditPurchasesResponse = z.infer; export type PagePurchase = z.infer; export type GetPagePurchasesResponse = z.infer; -export type CreateTokenCheckoutSessionRequest = z.infer; -export type CreateTokenCheckoutSessionResponse = z.infer; -export type TokenStripeStatusResponse = z.infer; -export type TokenPurchaseStatus = z.infer; -export type TokenPurchase = z.infer; -export type GetTokenPurchasesResponse = z.infer; export type FinalizeCheckoutResponse = z.infer; diff --git a/surfsense_web/contracts/types/user.types.ts b/surfsense_web/contracts/types/user.types.ts index 85fee49a8..706656064 100644 --- a/surfsense_web/contracts/types/user.types.ts +++ b/surfsense_web/contracts/types/user.types.ts @@ -6,8 +6,7 @@ export const user = z.object({ is_active: z.boolean(), is_superuser: z.boolean(), is_verified: z.boolean(), - pages_limit: z.number(), - pages_used: z.number(), + credit_micros_balance: z.number(), display_name: z.string().nullish(), avatar_url: z.string().nullish(), }); diff --git a/surfsense_web/hooks/use-inbox.ts b/surfsense_web/hooks/use-inbox.ts index e1070219a..860c0e01a 100644 --- a/surfsense_web/hooks/use-inbox.ts +++ b/surfsense_web/hooks/use-inbox.ts @@ -22,7 +22,7 @@ const CATEGORY_TYPES: Record = { "connector_indexing", "connector_deletion", "document_processing", - "page_limit_exceeded", + "insufficient_credits", ], }; diff --git a/surfsense_web/lib/apis/stripe-api.service.ts b/surfsense_web/lib/apis/stripe-api.service.ts index f119fbf6a..b2f5698fb 100644 --- a/surfsense_web/lib/apis/stripe-api.service.ts +++ b/surfsense_web/lib/apis/stripe-api.service.ts @@ -1,64 +1,50 @@ import { - type CreateCheckoutSessionRequest, - type CreateCheckoutSessionResponse, - type CreateTokenCheckoutSessionRequest, - type CreateTokenCheckoutSessionResponse, - createCheckoutSessionResponse, - createTokenCheckoutSessionResponse, + type AutoReloadSettingsResponse, + autoReloadSettingsResponse, + type CreateAutoReloadSetupSessionRequest, + type CreateAutoReloadSetupSessionResponse, + type CreateCreditCheckoutSessionRequest, + type CreateCreditCheckoutSessionResponse, + type CreditStripeStatusResponse, + createAutoReloadSetupSessionResponse, + createCreditCheckoutSessionResponse, + creditStripeStatusResponse, type FinalizeCheckoutResponse, finalizeCheckoutResponse, + type GetCreditPurchasesResponse, type GetPagePurchasesResponse, - type GetTokenPurchasesResponse, + getCreditPurchasesResponse, getPagePurchasesResponse, - getTokenPurchasesResponse, - type StripeStatusResponse, - stripeStatusResponse, - type TokenStripeStatusResponse, - tokenStripeStatusResponse, + type UpdateAutoReloadSettingsRequest, } from "@/contracts/types/stripe.types"; import { baseApiService } from "./base-api.service"; class StripeApiService { - createCheckoutSession = async ( - request: CreateCheckoutSessionRequest - ): Promise => { + createCreditCheckoutSession = async ( + request: CreateCreditCheckoutSessionRequest + ): Promise => { return baseApiService.post( - "/api/v1/stripe/create-checkout-session", - createCheckoutSessionResponse, - { - body: request, - } - ); - }; - - getPurchases = async (): Promise => { - return baseApiService.get("/api/v1/stripe/purchases", getPagePurchasesResponse); - }; - - getStatus = async (): Promise => { - return baseApiService.get("/api/v1/stripe/status", stripeStatusResponse); - }; - - createTokenCheckoutSession = async ( - request: CreateTokenCheckoutSessionRequest - ): Promise => { - return baseApiService.post( - "/api/v1/stripe/create-token-checkout-session", - createTokenCheckoutSessionResponse, + "/api/v1/stripe/create-credit-checkout-session", + createCreditCheckoutSessionResponse, { body: request } ); }; - getTokenStatus = async (): Promise => { - return baseApiService.get("/api/v1/stripe/token-status", tokenStripeStatusResponse); + getCreditStatus = async (): Promise => { + return baseApiService.get("/api/v1/stripe/credit-status", creditStripeStatusResponse); }; - getTokenPurchases = async (): Promise => { - return baseApiService.get("/api/v1/stripe/token-purchases", getTokenPurchasesResponse); + getCreditPurchases = async (): Promise => { + return baseApiService.get("/api/v1/stripe/credit-purchases", getCreditPurchasesResponse); + }; + + /** Legacy page-purchase history (read-only; page buying is removed). */ + getPagePurchases = async (): Promise => { + return baseApiService.get("/api/v1/stripe/purchases", getPagePurchasesResponse); }; /** - * Synchronously fulfil a checkout session from the success page. + * Synchronously fulfil a credit checkout session from the success page. * * Solves the race where the user lands on /purchase-success before * Stripe's checkout.session.completed webhook arrives. Idempotent — @@ -70,6 +56,30 @@ class StripeApiService { finalizeCheckoutResponse ); }; + + // --- Auto-reload -------------------------------------------------------- + + getAutoReloadSettings = async (): Promise => { + return baseApiService.get("/api/v1/stripe/auto-reload", autoReloadSettingsResponse); + }; + + updateAutoReloadSettings = async ( + request: UpdateAutoReloadSettingsRequest + ): Promise => { + return baseApiService.put("/api/v1/stripe/auto-reload", autoReloadSettingsResponse, { + body: request, + }); + }; + + createAutoReloadSetupSession = async ( + request: CreateAutoReloadSetupSessionRequest + ): Promise => { + return baseApiService.post( + "/api/v1/stripe/auto-reload/setup", + createAutoReloadSetupSessionResponse, + { body: request } + ); + }; } export const stripeApiService = new StripeApiService(); diff --git a/surfsense_web/lib/posthog/events.ts b/surfsense_web/lib/posthog/events.ts index 4dc644d5e..1f8875a6d 100644 --- a/surfsense_web/lib/posthog/events.ts +++ b/surfsense_web/lib/posthog/events.ts @@ -569,10 +569,10 @@ export function trackIncentivePageViewed() { safeCapture("incentive_page_viewed"); } -export function trackIncentiveTaskCompleted(taskType: string, pagesRewarded: number) { +export function trackIncentiveTaskCompleted(taskType: string, creditMicrosRewarded: number) { safeCapture("incentive_task_completed", { task_type: taskType, - pages_rewarded: pagesRewarded, + credit_micros_rewarded: creditMicrosRewarded, }); } diff --git a/surfsense_web/zero/schema/user.ts b/surfsense_web/zero/schema/user.ts index f483fa9b4..3b6c3ec92 100644 --- a/surfsense_web/zero/schema/user.ts +++ b/surfsense_web/zero/schema/user.ts @@ -3,18 +3,16 @@ import { number, string, table } from "@rocicorp/zero"; /** * Live-meter slice of the ``user`` table replicated through Zero. * - * ``premiumCreditMicrosLimit`` / ``premiumCreditMicrosUsed`` are stored - * as integer micro-USD (1_000_000 == $1.00). UI consumers divide by 1M - * when displaying. Sensitive fields (email, hashed_password, oauth, etc.) - * are intentionally omitted via the Postgres column-list publication so - * they never enter WAL replication. + * ``creditMicrosBalance`` is stored as integer micro-USD (1_000_000 == $1.00); + * UI consumers divide by 1M when displaying and clamp at $0.00 (the balance can + * dip slightly negative when actual cost exceeds the pre-charge estimate). + * Sensitive fields (email, hashed_password, oauth, etc.) are intentionally + * omitted via the Postgres column-list publication so they never enter WAL + * replication. */ export const userTable = table("user") .columns({ id: string(), - pagesLimit: number().from("pages_limit"), - pagesUsed: number().from("pages_used"), - premiumCreditMicrosLimit: number().from("premium_credit_micros_limit"), - premiumCreditMicrosUsed: number().from("premium_credit_micros_used"), + creditMicrosBalance: number().from("credit_micros_balance"), }) .primaryKey("id");