feat(backend): integrate PostHog analytics for enhanced observability

- Added PostHog configuration options to .env.example files for both Docker and Surfsense backend.
- Introduced PostHog dependency in pyproject.toml.
- Implemented analytics middleware to capture various events across the application, including user authentication, automation runs, and API requests.
- Enhanced existing routes and services to emit analytics events, providing insights into user interactions and system performance.
- Ensured graceful shutdown of analytics clients in worker processes and application lifecycles.

surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts

@@ -21,8 +21,6 @@ import { OAUTH_RESULT_COOKIE, parseOAuthCallbackResult } from "@/contracts/types
 import { authenticatedFetch } from "@/lib/auth-fetch";
 import { buildBackendUrl } from "@/lib/env-config";
 import {
-	trackConnectorConnected,
-	trackConnectorDeleted,
 	trackConnectorSetupFailure,
 	trackConnectorSetupStarted,
 	trackIndexWithDateRangeOpened,
@@ -296,11 +294,9 @@ export const useConnectorDialog = () => {
 				if (newConnector && oauthConnector) {
 					const connectorValidation = searchSourceConnector.safeParse(newConnector);
 					if (connectorValidation.success) {
-						trackConnectorConnected(
-							Number(workspaceId),
-							oauthConnector.connectorType,
-							newConnector.id
-						);
+						// connector_connected is now emitted server-side (after_insert
+						// listener in search_source_connectors_routes.py) — covers the
+						// OAuth callback redirect the frontend often never observes.
 
 						const isLiveConnector = LIVE_CONNECTOR_TYPES.has(oauthConnector.connectorType);
 
@@ -436,12 +432,7 @@ export const useConnectorDialog = () => {
 				if (connector) {
 					const connectorValidation = searchSourceConnector.safeParse(connector);
 					if (connectorValidation.success) {
-						// Track webcrawler connector connected
-						trackConnectorConnected(
-							Number(workspaceId),
-							EnumConnectorName.WEBCRAWLER_CONNECTOR,
-							connector.id
-						);
+						// connector_connected is now emitted server-side (after_insert).
 
 						const config = validateIndexingConfigState({
 							connectorType: EnumConnectorName.WEBCRAWLER_CONNECTOR,
@@ -540,8 +531,7 @@ export const useConnectorDialog = () => {
 							// Store connectingConnectorType before clearing it
 							const currentConnectorType = connectingConnectorType;
 
-							// Track connector connected event for non-OAuth connectors
-							trackConnectorConnected(Number(workspaceId), currentConnectorType, connector.id);
+							// connector_connected is now emitted server-side (after_insert).
 
 							// Find connector title from constants
 							const connectorInfo = OTHER_CONNECTORS.find(
@@ -1229,12 +1219,8 @@ export const useConnectorDialog = () => {
 					id: editingConnector.id,
 				});
 
-				// Track connector deleted event
-				trackConnectorDeleted(
-					Number(workspaceId),
-					editingConnector.connector_type,
-					editingConnector.id
-				);
+				// connector_deleted is now emitted server-side
+				// (search_source_connectors_routes.delete_search_source_connector).
 
 				toast.success(
 					editingConnector.connector_type === "MCP_CONNECTOR"

surfsense_web/components/layout/ui/dialogs/CreateWorkspaceDialog.tsx

@@ -28,7 +28,6 @@ import {
 } from "@/components/ui/form";
 import { Input } from "@/components/ui/input";
 import { Spinner } from "@/components/ui/spinner";
-import { trackWorkspaceCreated } from "@/lib/posthog/events";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { queryClient } from "@/lib/query-client/client";
 
@@ -68,7 +67,8 @@ export function CreateWorkspaceDialog({ open, onOpenChange }: CreateWorkspaceDia
 				description: values.description || "",
 			});
 
-			trackWorkspaceCreated(result.id, values.name);
+			// workspace_created is now emitted server-side (workspaces_routes.py)
+			// so PAT/MCP-created workspaces are also counted.
 
 			// Seed the gate's query so it resolves without a loader flash, and
 			// route straight to onboarding vs. new-chat on the first hop.

surfsense_web/components/settings/earn-credits-content.tsx

@@ -4,7 +4,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { Check, ExternalLink } from "lucide-react";
 import Link from "next/link";
 import { useParams } from "next/navigation";
-import { useEffect } from "react";
 import { toast } from "sonner";
 import { USER_QUERY_KEY } from "@/atoms/user/user-query.atoms";
 import { Button } from "@/components/ui/button";
@@ -15,11 +14,7 @@ import { Spinner } from "@/components/ui/spinner";
 import type { IncentiveTaskInfo } from "@/contracts/types/incentive-tasks.types";
 import { incentiveTasksApiService } from "@/lib/apis/incentive-tasks-api.service";
 import { stripeApiService } from "@/lib/apis/stripe-api.service";
-import {
-	trackIncentivePageViewed,
-	trackIncentiveTaskClicked,
-	trackIncentiveTaskCompleted,
-} from "@/lib/posthog/events";
+import { trackIncentiveTaskClicked } from "@/lib/posthog/events";
 import { getWorkspaceIdParam } from "@/lib/route-params";
 import { cn } from "@/lib/utils";
 
@@ -35,9 +30,7 @@ export function EarnCreditsContent() {
 	const queryClient = useQueryClient();
 	const workspaceId = getWorkspaceIdParam(params) ?? "";
 
-	useEffect(() => {
-		trackIncentivePageViewed();
-	}, []);
+	// incentive_page_viewed removed — redundant with $pageview.
 
 	const { data, isLoading } = useQuery({
 		queryKey: ["incentive-tasks"],
@@ -51,13 +44,11 @@ export function EarnCreditsContent() {
 
 	const completeMutation = useMutation({
 		mutationFn: incentiveTasksApiService.completeTask,
-		onSuccess: (response, taskType) => {
+		onSuccess: (response) => {
 			if (response.success) {
 				toast.success(response.message);
-				const task = data?.tasks.find((t) => t.task_type === taskType);
-				if (task) {
-					trackIncentiveTaskCompleted(taskType, task.credit_micros_reward);
-				}
+				// incentive_task_completed is now emitted server-side
+				// (incentive_tasks_routes.complete_task) where credit is granted.
 				queryClient.invalidateQueries({ queryKey: ["incentive-tasks"] });
 				queryClient.invalidateQueries({ queryKey: USER_QUERY_KEY });
 			}

surfsense_web/components/sources/DocumentUploadTab.tsx

@@ -29,11 +29,7 @@ import { Switch } from "@/components/ui/switch";
 import type { ProcessingMode } from "@/contracts/types/document.types";
 import { useElectronAPI } from "@/hooks/use-platform";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
-import {
-	trackDocumentUploadFailure,
-	trackDocumentUploadStarted,
-	trackDocumentUploadSuccess,
-} from "@/lib/posthog/events";
+import { trackDocumentUploadStarted } from "@/lib/posthog/events";
 import {
 	getAcceptedFileTypes,
 	getSupportedExtensions,
@@ -380,13 +376,14 @@ export function DocumentUploadTab({
 				setUploadProgress(Math.round((uploaded / total) * 100));
 			}
 
-			trackDocumentUploadSuccess(Number(workspaceId), total);
+			// Ingestion outcome is now emitted server-side
+			// (document_processing_completed/_failed in document_tasks.py); the
+			// upload POST succeeding only means the file was accepted, not processed.
 			toast(t("upload_initiated"), { description: t("upload_initiated_desc") });
 			setFolderUpload(null);
 			onSuccess?.();
 		} catch (error) {
 			const message = error instanceof Error ? error.message : "Upload failed";
-			trackDocumentUploadFailure(Number(workspaceId), message);
 			toast(t("upload_error"), {
 				description: `${t("upload_error_desc")}: ${message}`,
 			});
@@ -421,7 +418,7 @@ export function DocumentUploadTab({
 				onSuccess: () => {
 					if (progressIntervalRef.current) clearInterval(progressIntervalRef.current);
 					setUploadProgress(100);
-					trackDocumentUploadSuccess(Number(workspaceId), files.length);
+					// Ingestion outcome now server-side (document_processing_*).
 					toast(t("upload_initiated"), { description: t("upload_initiated_desc") });
 					onSuccess?.();
 				},
@@ -429,7 +426,6 @@ export function DocumentUploadTab({
 					if (progressIntervalRef.current) clearInterval(progressIntervalRef.current);
 					setUploadProgress(0);
 					const message = error instanceof Error ? error.message : "Upload failed";
-					trackDocumentUploadFailure(Number(workspaceId), message);
 					toast(t("upload_error"), {
 						description: `${t("upload_error_desc")}: ${message}`,
 					});