mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
feat(telemetry): collect PostHog $exception error reports in CLI and daemon (#262)
* feat(telemetry): add node exception reporter * feat(telemetry): report node cli exceptions * feat(telemetry): add daemon exception reporter * feat(telemetry): report daemon exceptions * docs(telemetry): document error reports * fix(telemetry): pass redaction snapshots from node call sites * test(telemetry): verify prepared node exception payload * fix(telemetry): close daemon exception lifecycle gaps * test(telemetry): verify prepared daemon exception payload * test(telemetry): close error collection acceptance gaps * test(telemetry): close posthog exception acceptance gaps
This commit is contained in:
Andrey Avtomonov
GitHub
parent
c3d8cedb0b
commit
fb7b94b60e
36 changed files with 2870 additions and 140 deletions
|
|
@ -6,6 +6,8 @@ import argparse
|
|||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from types import TracebackType
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
|
@ -90,6 +92,41 @@ def _read_stdin_json() -> dict[str, Any]:
|
|||
return parsed
|
||||
|
||||
|
||||
def install_serve_http_exception_hooks(started_at: float) -> Callable[[], None]:
|
||||
original_hook = sys.excepthook
|
||||
|
||||
def hook(
|
||||
exc_type: type[BaseException],
|
||||
exc: BaseException,
|
||||
tb: TracebackType | None,
|
||||
) -> None:
|
||||
report_serve_http_crash(exc, started_at=started_at)
|
||||
original_hook(exc_type, exc, tb)
|
||||
|
||||
sys.excepthook = hook
|
||||
|
||||
def dispose() -> None:
|
||||
sys.excepthook = original_hook
|
||||
|
||||
return dispose
|
||||
|
||||
|
||||
def report_serve_http_crash(error: BaseException, *, started_at: float) -> None:
|
||||
from ktx_daemon.telemetry import report_exception
|
||||
from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once
|
||||
|
||||
report_exception(
|
||||
error,
|
||||
source="serve-http",
|
||||
handled=False,
|
||||
fatal=True,
|
||||
)
|
||||
emit_daemon_stopped_once(
|
||||
reason="crash",
|
||||
uptime_ms=max(0, (time.perf_counter() - started_at) * 1000),
|
||||
)
|
||||
|
||||
|
||||
def run_http_server(
|
||||
*,
|
||||
host: str,
|
||||
|
|
@ -102,15 +139,23 @@ def run_http_server(
|
|||
from ktx_daemon.app import create_app
|
||||
|
||||
started_at = time.perf_counter()
|
||||
uvicorn.run(
|
||||
create_app(
|
||||
enable_code_execution=enable_code_execution,
|
||||
telemetry_started_at=started_at,
|
||||
),
|
||||
host=host,
|
||||
port=port,
|
||||
log_level=log_level,
|
||||
)
|
||||
dispose_hooks = install_serve_http_exception_hooks(started_at)
|
||||
try:
|
||||
try:
|
||||
uvicorn.run(
|
||||
create_app(
|
||||
enable_code_execution=enable_code_execution,
|
||||
telemetry_started_at=started_at,
|
||||
),
|
||||
host=host,
|
||||
port=port,
|
||||
log_level=log_level,
|
||||
)
|
||||
except Exception as error:
|
||||
report_serve_http_crash(error, started_at=started_at)
|
||||
raise
|
||||
finally:
|
||||
dispose_hooks()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
|
|
@ -169,6 +214,14 @@ def main(argv: list[str] | None = None) -> int:
|
|||
sys.stderr.write(f"{error}\n")
|
||||
return 1
|
||||
except Exception as error:
|
||||
from ktx_daemon.telemetry import report_exception
|
||||
|
||||
report_exception(
|
||||
error,
|
||||
source=str(args.command),
|
||||
handled=True,
|
||||
fatal=False,
|
||||
)
|
||||
sys.stderr.write(f"{type(error).__name__}: {error}\n")
|
||||
return 1
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ from contextlib import asynccontextmanager
|
|||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import Response
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from ktx_daemon import VERSION
|
||||
from ktx_daemon.code_execution import (
|
||||
|
|
@ -65,9 +65,11 @@ from ktx_daemon.table_identifier import (
|
|||
ParseTableIdentifierBatchResponse,
|
||||
parse_table_identifier_response,
|
||||
)
|
||||
from ktx_daemon.telemetry import track_telemetry_event
|
||||
from ktx_daemon.telemetry import report_exception, track_telemetry_event
|
||||
from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CREDENTIAL_KEYS = {"url", "password", "token", "api_key", "apikey", "auth_header"}
|
||||
|
||||
|
||||
class NumpyORJSONResponse(Response):
|
||||
|
|
@ -77,6 +79,36 @@ class NumpyORJSONResponse(Response):
|
|||
return dumps_numpy_json(content)
|
||||
|
||||
|
||||
def _route_source(request: Request) -> str:
|
||||
route = request.scope.get("route")
|
||||
path = getattr(route, "path", None)
|
||||
if isinstance(path, str) and path:
|
||||
return f"app:{path}"
|
||||
return f"app:{request.url.path}"
|
||||
|
||||
|
||||
def _secret_snapshot_from_payload(value: Any) -> list[str]:
|
||||
secrets: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
normalized_key = str(key).lower()
|
||||
if normalized_key in CREDENTIAL_KEYS and isinstance(child, str) and child:
|
||||
secrets.append(child)
|
||||
secrets.extend(_secret_snapshot_from_payload(child))
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
secrets.extend(_secret_snapshot_from_payload(child))
|
||||
return secrets
|
||||
|
||||
|
||||
async def _request_secret_snapshot(request: Request) -> list[str]:
|
||||
try:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
return []
|
||||
return _secret_snapshot_from_payload(payload)
|
||||
|
||||
|
||||
def create_app(
|
||||
*,
|
||||
embedding_provider: EmbeddingProvider | None = None,
|
||||
|
|
@ -104,12 +136,9 @@ def create_app(
|
|||
try:
|
||||
yield
|
||||
finally:
|
||||
track_telemetry_event(
|
||||
"daemon_stopped",
|
||||
{
|
||||
"reason": "request",
|
||||
"uptimeMs": max(0, (clock() - started_at) * 1000),
|
||||
},
|
||||
emit_daemon_stopped_once(
|
||||
reason="request",
|
||||
uptime_ms=max(0, (clock() - started_at) * 1000),
|
||||
)
|
||||
|
||||
app = FastAPI(
|
||||
|
|
@ -119,6 +148,25 @@ def create_app(
|
|||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def report_unhandled_exceptions(request: Request, call_next):
|
||||
redaction_secrets = await _request_secret_snapshot(request)
|
||||
try:
|
||||
return await call_next(request)
|
||||
except Exception as error:
|
||||
logger.exception("Unhandled daemon request failed: %s", error)
|
||||
report_exception(
|
||||
error,
|
||||
source=_route_source(request),
|
||||
handled=True,
|
||||
fatal=False,
|
||||
redaction_secrets=redaction_secrets,
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"detail": f"Daemon request failed: {error}"},
|
||||
)
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict[str, str]:
|
||||
response = {"status": "healthy"}
|
||||
|
|
@ -137,12 +185,6 @@ def create_app(
|
|||
except ValueError as error:
|
||||
logger.warning("Database introspection rejected: %s", error)
|
||||
raise HTTPException(status_code=400, detail=str(error)) from error
|
||||
except Exception as error:
|
||||
logger.exception("Database introspection failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Database introspection failed: {error}",
|
||||
) from error
|
||||
|
||||
@app.post("/embeddings/compute", response_model=ComputeEmbeddingResponse)
|
||||
async def embedding_compute(
|
||||
|
|
@ -156,12 +198,6 @@ def create_app(
|
|||
except ValueError as error:
|
||||
logger.warning("Embedding compute rejected: %s", error)
|
||||
raise HTTPException(status_code=400, detail=str(error)) from error
|
||||
except Exception as error:
|
||||
logger.exception("Embedding compute failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Embedding compute failed: {error}",
|
||||
) from error
|
||||
|
||||
@app.post(
|
||||
"/embeddings/compute-bulk",
|
||||
|
|
@ -178,12 +214,6 @@ def create_app(
|
|||
except ValueError as error:
|
||||
logger.warning("Bulk embedding compute rejected: %s", error)
|
||||
raise HTTPException(status_code=400, detail=str(error)) from error
|
||||
except Exception as error:
|
||||
logger.exception("Bulk embedding compute failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Bulk embedding compute failed: {error}",
|
||||
) from error
|
||||
|
||||
if enable_code_execution:
|
||||
|
||||
|
|
@ -193,29 +223,15 @@ def create_app(
|
|||
response_class=NumpyORJSONResponse,
|
||||
)
|
||||
async def code_execute(request: ExecuteCodeRequest) -> ExecuteCodeResponse:
|
||||
try:
|
||||
return execute_code_response(
|
||||
request,
|
||||
nest_api_url=None,
|
||||
auth_header=None,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.exception("Code execution failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Code execution failed: {error}",
|
||||
) from error
|
||||
return execute_code_response(
|
||||
request,
|
||||
nest_api_url=None,
|
||||
auth_header=None,
|
||||
)
|
||||
|
||||
@app.post("/lookml/parse", response_model=ParseLookMLResponse)
|
||||
async def lookml_parse(request: ParseLookMLRequest) -> ParseLookMLResponse:
|
||||
try:
|
||||
return parse_lookml_project(request)
|
||||
except Exception as error:
|
||||
logger.exception("LookML parsing failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"LookML parsing failed: {error}",
|
||||
) from error
|
||||
return parse_lookml_project(request)
|
||||
|
||||
@app.post(
|
||||
"/sql/parse-table-identifier",
|
||||
|
|
@ -224,40 +240,19 @@ def create_app(
|
|||
async def sql_parse_table_identifier(
|
||||
request: ParseTableIdentifierBatchRequest,
|
||||
) -> ParseTableIdentifierBatchResponse:
|
||||
try:
|
||||
return parse_table_identifier_response(request)
|
||||
except Exception as error:
|
||||
logger.exception("Table identifier parsing failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Table identifier parsing failed: {error}",
|
||||
) from error
|
||||
return parse_table_identifier_response(request)
|
||||
|
||||
@app.post("/sql/validate-read-only", response_model=ValidateReadOnlySqlResponse)
|
||||
async def sql_validate_read_only(
|
||||
request: ValidateReadOnlySqlRequest,
|
||||
) -> ValidateReadOnlySqlResponse:
|
||||
try:
|
||||
return validate_read_only_sql_response(request)
|
||||
except Exception as error:
|
||||
logger.exception("SQL read-only validation failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"SQL read-only validation failed: {error}",
|
||||
) from error
|
||||
return validate_read_only_sql_response(request)
|
||||
|
||||
@app.post("/sql/analyze-batch", response_model=AnalyzeSqlBatchResponse)
|
||||
async def sql_analyze_batch(
|
||||
request: AnalyzeSqlBatchRequest,
|
||||
) -> AnalyzeSqlBatchResponse:
|
||||
try:
|
||||
return analyze_sql_batch_response(request)
|
||||
except Exception as error:
|
||||
logger.exception("SQL batch analysis failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"SQL batch analysis failed: {error}",
|
||||
) from error
|
||||
return analyze_sql_batch_response(request)
|
||||
|
||||
@app.post(
|
||||
"/semantic-layer/generate-sources", response_model=GenerateSourcesResponse
|
||||
|
|
@ -265,14 +260,7 @@ def create_app(
|
|||
async def semantic_generate_sources(
|
||||
request: GenerateSourcesRequest,
|
||||
) -> GenerateSourcesResponse:
|
||||
try:
|
||||
return generate_sources_response(request)
|
||||
except Exception as error:
|
||||
logger.exception("Semantic source generation failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Semantic source generation failed: {error}",
|
||||
) from error
|
||||
return generate_sources_response(request)
|
||||
|
||||
@app.post("/semantic-layer/query", response_model=SemanticLayerQueryResponse)
|
||||
async def semantic_query(
|
||||
|
|
@ -283,12 +271,6 @@ def create_app(
|
|||
except ValueError as error:
|
||||
logger.warning("Semantic query rejected: %s", error)
|
||||
raise HTTPException(status_code=400, detail=str(error)) from error
|
||||
except Exception as error:
|
||||
logger.exception("Semantic query failed: %s", error)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Semantic layer query failed: {error}",
|
||||
) from error
|
||||
|
||||
@app.post("/semantic-layer/validate", response_model=ValidateSourcesResponse)
|
||||
async def semantic_validate(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
import time
|
||||
from typing import Any
|
||||
|
||||
from ktx_daemon.telemetry import error_class, track_telemetry_event
|
||||
from ktx_daemon.telemetry import error_class, report_exception, track_telemetry_event
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from semantic_layer.duplicate_check import validate_measure_duplicates
|
||||
from semantic_layer.engine import SemanticEngine
|
||||
|
|
@ -150,6 +150,13 @@ def query_semantic_layer(
|
|||
track_telemetry_event(
|
||||
"sql_gen_completed", sql_fields, project_id=request.project_id
|
||||
)
|
||||
report_exception(
|
||||
error,
|
||||
source="semantic-query",
|
||||
handled=True,
|
||||
fatal=False,
|
||||
project_id=request.project_id,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from ktx_daemon.telemetry.daemon_lifecycle import emit_daemon_stopped_once
|
||||
from ktx_daemon.telemetry.emitter import error_class, track_telemetry_event
|
||||
from ktx_daemon.telemetry.exception import report_exception
|
||||
|
||||
__all__ = ["error_class", "track_telemetry_event"]
|
||||
__all__ = [
|
||||
"emit_daemon_stopped_once",
|
||||
"error_class",
|
||||
"report_exception",
|
||||
"track_telemetry_event",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from ktx_daemon.telemetry.emitter import track_telemetry_event
|
||||
|
||||
StopReason = Literal["signal", "request", "crash"]
|
||||
|
||||
_daemon_stop_emitted = False
|
||||
|
||||
|
||||
def emit_daemon_stopped_once(*, reason: StopReason, uptime_ms: float) -> bool:
|
||||
global _daemon_stop_emitted
|
||||
if _daemon_stop_emitted:
|
||||
return False
|
||||
_daemon_stop_emitted = True
|
||||
track_telemetry_event(
|
||||
"daemon_stopped",
|
||||
{
|
||||
"reason": reason,
|
||||
"uptimeMs": max(0, uptime_ms),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def reset_daemon_lifecycle_for_tests() -> None:
|
||||
global _daemon_stop_emitted
|
||||
_daemon_stop_emitted = False
|
||||
156
python/ktx-daemon/src/ktx_daemon/telemetry/exception.py
Normal file
156
python/ktx-daemon/src/ktx_daemon/telemetry/exception.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ktx_daemon import VERSION
|
||||
from ktx_daemon.telemetry.emitter import POSTHOG_HOST, POSTHOG_PROJECT_API_KEY
|
||||
from ktx_daemon.telemetry.events import _common_envelope
|
||||
from ktx_daemon.telemetry.identity import load_telemetry_identity
|
||||
|
||||
_KTX_REPORTED_ATTR = "__ktx_posthog_exception_reported"
|
||||
|
||||
|
||||
def _debug_enabled(env: Mapping[str, str]) -> bool:
|
||||
return env.get("KTX_TELEMETRY_DEBUG") == "1"
|
||||
|
||||
|
||||
def _host(env: Mapping[str, str]) -> str:
|
||||
return env.get("KTX_TELEMETRY_ENDPOINT") or POSTHOG_HOST
|
||||
|
||||
|
||||
def _redact_static(value: str) -> str:
|
||||
patterns = [
|
||||
(
|
||||
r"([a-z][a-z0-9+.-]*://[^:\s/@]+:)([^@\s/]+)(@)",
|
||||
r"\1[redacted]\3",
|
||||
),
|
||||
(r"\b(password|pwd)=([^;&\s]+)", r"\1=[redacted]"),
|
||||
(r"\bAuthorization\s*:\s*[^\r\n,;]+", "Authorization: [redacted]"),
|
||||
(r"\bBearer\s+[A-Za-z0-9._~+/=-]+", "Bearer [redacted]"),
|
||||
(r"\b(api[_-]?key)\s*[:=]\s*([^\s,;]+)", r"\1=[redacted]"),
|
||||
(
|
||||
r"\b(KTX_[A-Z0-9_]*|[A-Z0-9_]*(?:TOKEN|SECRET))\s*[:=]\s*([^\s,;]+)",
|
||||
r"\1=[redacted]",
|
||||
),
|
||||
(r"([?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[^&\s]+", r"\1[redacted]"),
|
||||
]
|
||||
redacted = value
|
||||
for pattern, replacement in patterns:
|
||||
redacted = re.sub(pattern, replacement, redacted, flags=re.IGNORECASE)
|
||||
return redacted
|
||||
|
||||
|
||||
def _redact_text(value: str, secrets: Sequence[str]) -> str:
|
||||
redacted = value
|
||||
for secret in secrets:
|
||||
if secret:
|
||||
redacted = redacted.replace(secret, "[redacted]")
|
||||
return _redact_static(redacted)
|
||||
|
||||
|
||||
def _clone_exception(exception: BaseException, secrets: Sequence[str]) -> BaseException:
|
||||
redacted_args = [_redact_text(str(arg), secrets) for arg in exception.args]
|
||||
try:
|
||||
cloned = type(exception)(*redacted_args)
|
||||
except Exception:
|
||||
cloned = RuntimeError(_redact_text(str(exception), secrets))
|
||||
cloned.__traceback__ = exception.__traceback__
|
||||
cloned.__cause__ = (
|
||||
_clone_exception(exception.__cause__, secrets) if exception.__cause__ else None
|
||||
)
|
||||
cloned.__context__ = (
|
||||
_clone_exception(exception.__context__, secrets)
|
||||
if exception.__context__
|
||||
else None
|
||||
)
|
||||
return cloned
|
||||
|
||||
|
||||
def _should_skip_as_reported(exception: BaseException) -> bool:
|
||||
if getattr(exception, _KTX_REPORTED_ATTR, False):
|
||||
return True
|
||||
try:
|
||||
setattr(exception, _KTX_REPORTED_ATTR, True)
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _properties(*, source: str, handled: bool, fatal: bool) -> dict[str, Any]:
|
||||
return {
|
||||
**_common_envelope(),
|
||||
"daemonVersion": os.environ.get("KTX_DAEMON_VERSION", VERSION),
|
||||
"source": source,
|
||||
"handled": handled,
|
||||
"fatal": fatal,
|
||||
}
|
||||
|
||||
|
||||
def report_exception(
|
||||
exception: BaseException,
|
||||
*,
|
||||
source: str,
|
||||
handled: bool,
|
||||
fatal: bool,
|
||||
project_id: str | None = None,
|
||||
home_dir: Path | None = None,
|
||||
env: Mapping[str, str] | None = None,
|
||||
redaction_secrets: Sequence[str] | None = None,
|
||||
) -> None:
|
||||
source_env = env if env is not None else os.environ
|
||||
try:
|
||||
identity = load_telemetry_identity(home_dir=home_dir, env=source_env)
|
||||
if not identity.enabled or not identity.install_id:
|
||||
return
|
||||
|
||||
if _should_skip_as_reported(exception):
|
||||
return
|
||||
|
||||
properties = _properties(source=source, handled=handled, fatal=fatal)
|
||||
groups = {"project": project_id} if project_id else None
|
||||
safe_exception = _clone_exception(exception, redaction_secrets or [])
|
||||
|
||||
if _debug_enabled(source_env):
|
||||
sys.stderr.write(
|
||||
"[telemetry-exception] "
|
||||
+ json.dumps(
|
||||
{
|
||||
"distinctId": identity.install_id,
|
||||
"message": str(safe_exception),
|
||||
"properties": properties,
|
||||
"groups": groups,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return
|
||||
|
||||
if not POSTHOG_PROJECT_API_KEY.strip() or not _host(source_env).strip():
|
||||
return
|
||||
|
||||
from posthog import Posthog
|
||||
|
||||
client = Posthog(
|
||||
POSTHOG_PROJECT_API_KEY,
|
||||
host=_host(source_env),
|
||||
flush_at=1,
|
||||
flush_interval=0,
|
||||
sync_mode=True,
|
||||
timeout=1,
|
||||
)
|
||||
client.capture_exception(
|
||||
safe_exception,
|
||||
distinct_id=identity.install_id,
|
||||
properties=properties,
|
||||
groups=groups,
|
||||
)
|
||||
client.shutdown()
|
||||
except Exception:
|
||||
return
|
||||
Loading…
Add table
Add a link
Reference in a new issue