mirror of
https://github.com/dograh-hq/dograh.git
synced 2026-07-04 10:52:17 +02:00
feat(webhooks): durable retrying delivery for final webhooks (#478)
* feat(webhooks): durable retrying delivery for final webhooks
Final webhook nodes were fired inline with a single best-effort httpx POST
(run_integrations._execute_webhook_node). On a transient error the failure was
swallowed at three levels, so ARQ never retried and the final call report was
permanently lost -- leaving downstream receivers stuck (e.g. a CRM showing a
call as still "in conversation").
Replace the one-shot POST with a durable, idempotent delivery pipeline modelled
on the campaign retry pattern (persisted row + scheduled_for + bounded attempts):
- New webhook_deliveries table (WebhookDeliveryModel) is the source of truth.
Payload is rendered once and frozen so retries are deterministic; secrets are
not stored -- the credential is referenced by uuid and re-resolved at send time.
- run_integrations now persists a delivery row and enqueues deliver_webhook with
a deterministic ARQ job id instead of sending inline.
- deliver_webhook (new ARQ task) sends the request and:
* 2xx -> succeeded
* transient -> retry with capped exponential backoff (RequestError /
5xx / 408 / 425 / 429), up to max_attempts then dead_letter
* permanent 4xx -> dead_letter immediately (no pointless looping)
It is idempotent: a non-pending delivery is a no-op, so a duplicate enqueue or
sweeper re-injection can't double-send.
- sweep_webhook_deliveries cron (every 5 min) re-enqueues overdue pending
deliveries so nothing is lost to a worker restart / Redis flush.
- Stable X-Dograh-Delivery-Id / Workflow-Run-Id / Attempt headers let receivers
dedupe retried deliveries.
- enqueue_job now forwards ARQ job options (_job_id, _defer_by); failures log
repr(e) so empty-message errors like ConnectTimeout are diagnosable.
Config via DEFAULT_WEBHOOK_DELIVERY_CONFIG (env-overridable): max_attempts=5,
base_delay=30s, max_delay=600s, timeout=30s.
Tests cover payload rendering, persist+enqueue, success, transient retry,
retryable 5xx, permanent 4xx dead-letter, attempt exhaustion, and idempotency.
Migration verified to apply/rollback against Postgres; table/enum/indexes confirmed.
* fix(webhooks): atomic claim, safe success-recording, sweep paging, migration cleanup
Address review feedback on the webhook delivery pipeline:
- deliver_webhook now atomically claims a delivery (conditional UPDATE that
leases scheduled_for) before sending, so concurrent ARQ executions can't
double-send (the prior status=='pending' read was non-atomic).
- Recording success is moved out of the dead-letter try-block: if the receiver
accepted the payload (2xx) but the success DB-write fails, the row is left
pending for the sweeper to reconcile instead of being dead-lettered.
- The sweep keyset-paginates by id so a backlog over the page size is fully
drained, and logs the true re-enqueued total.
- Migration downgrade drops the enum via op.execute(DROP TYPE IF EXISTS ...)
instead of the deprecated op.get_bind().
* fix(webhooks): idempotent delivery creation and drop secret custom headers
Address the remaining review feedback:
- Add a (workflow_run_id, webhook_node_id) unique constraint and make
create_webhook_delivery a get-or-create returning (delivery, created). A
retried run_integrations now reuses the existing row instead of creating and
sending a duplicate final webhook; only a freshly-created row is enqueued.
- Stop persisting secret-looking custom headers (Authorization, X-API-Key,
Cookie, ...) in plaintext on the delivery row: they are dropped with a warning
pointing at the credential store (which is re-resolved securely at send time).
Non-secret custom headers are unaffected.
* fix(webhooks): harden idempotency key, secret-header match, sweep reclaim id
Address follow-up review feedback:
- webhook_node_id is now NOT NULL so a NULL can't slip past the
(workflow_run_id, webhook_node_id) unique constraint and create duplicates.
- Secret-header filtering matches normalized markers (auth/token/secret/cookie/
api-key/...) instead of an exact name list, catching variants like
X-Custom-Auth-Token while leaving benign headers (e.g. X-Idempotency-Key).
- The sweeper re-enqueues with a reclaim-specific job id (the lease timestamp)
so reconciling a delivered-but-unrecorded row isn't deduped against the
original attempt's already-completed ARQ job. The atomic claim still ensures
at most one send.
* fix(webhooks): scope delivery rows to workflow org
---------
Co-authored-by: Abhishek Kumar <abhishek@a6k.me>
This commit is contained in:
Tararais
GitHub
parent
3a770a6538
commit
fd0d144b08
12 changed files with 1360 additions and 141 deletions
119
api/alembic/versions/b7e3c9a1d2f4_add_webhook_deliveries.py
Normal file
119
api/alembic/versions/b7e3c9a1d2f4_add_webhook_deliveries.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""add webhook_deliveries
|
||||
|
||||
Durable, retrying outbound webhook delivery so a transient network error can't
|
||||
permanently drop a workflow's final webhook.
|
||||
|
||||
Revision ID: b7e3c9a1d2f4
|
||||
Revises: 91cc6ba3e1c7
|
||||
Create Date: 2026-06-28 19:40:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "b7e3c9a1d2f4"
|
||||
down_revision: Union[str, None] = "91cc6ba3e1c7"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"webhook_deliveries",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("delivery_uuid", sa.String(length=36), nullable=False),
|
||||
sa.Column("workflow_run_id", sa.Integer(), nullable=False),
|
||||
sa.Column("organization_id", sa.Integer(), nullable=False),
|
||||
sa.Column("webhook_name", sa.String(), nullable=True),
|
||||
sa.Column("endpoint_url", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"http_method",
|
||||
sa.String(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"payload",
|
||||
sa.JSON(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("custom_headers", sa.JSON(), nullable=True),
|
||||
sa.Column("credential_uuid", sa.String(length=36), nullable=True),
|
||||
sa.Column("webhook_node_id", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum(
|
||||
"pending",
|
||||
"succeeded",
|
||||
"dead_letter",
|
||||
name="webhook_delivery_status",
|
||||
),
|
||||
server_default=sa.text("'pending'"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"attempt_count",
|
||||
sa.Integer(),
|
||||
server_default=sa.text("0"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"max_attempts",
|
||||
sa.Integer(),
|
||||
server_default=sa.text("5"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("scheduled_for", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_status_code", sa.Integer(), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["workflow_run_id"], ["workflow_runs.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["organization_id"], ["organizations.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"workflow_run_id",
|
||||
"webhook_node_id",
|
||||
name="uq_webhook_deliveries_run_node",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_webhook_deliveries_delivery_uuid",
|
||||
"webhook_deliveries",
|
||||
["delivery_uuid"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"idx_webhook_deliveries_run",
|
||||
"webhook_deliveries",
|
||||
["workflow_run_id"],
|
||||
unique=False,
|
||||
)
|
||||
# Partial index for the sweeper's hot path: due pending deliveries.
|
||||
op.create_index(
|
||||
"idx_webhook_deliveries_pending_scheduled",
|
||||
"webhook_deliveries",
|
||||
["scheduled_for"],
|
||||
unique=False,
|
||||
postgresql_where=sa.text("status = 'pending'"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"idx_webhook_deliveries_pending_scheduled",
|
||||
table_name="webhook_deliveries",
|
||||
)
|
||||
op.drop_index("idx_webhook_deliveries_run", table_name="webhook_deliveries")
|
||||
op.drop_index(
|
||||
"ix_webhook_deliveries_delivery_uuid", table_name="webhook_deliveries"
|
||||
)
|
||||
op.drop_table("webhook_deliveries")
|
||||
op.execute("DROP TYPE IF EXISTS webhook_delivery_status")
|
||||
Loading…
Add table
Add a link
Reference in a new issue