From fc915fec133ff6574e9312e3d37d9cfa2b4c03bf Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:22:02 +0200
Subject: [PATCH 01/36] getdocuments reques / response zod schema

---
 .../contracts/types/document.types.ts         | 73 +++++++++++++++++++
 surfsense_web/contracts/types/index.ts        |  4 +
 .../lib/apis/documents-api.service.ts         |  0
 3 files changed, 77 insertions(+)
 create mode 100644 surfsense_web/contracts/types/document.types.ts
 create mode 100644 surfsense_web/lib/apis/documents-api.service.ts

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
new file mode 100644
index 000000000..3a6bdde03
--- /dev/null
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -0,0 +1,73 @@
+import { z } from "zod";
+import { paginationQueryParams } from ".";
+
+export const documentTypeEnum = z.enum([
+	"EXTENSION",
+	"CRAWLED_URL",
+	"FILE",
+	"SLACK_CONNECTOR",
+	"NOTION_CONNECTOR",
+	"YOUTUBE_VIDEO",
+	"GITHUB_CONNECTOR",
+	"LINEAR_CONNECTOR",
+	"DISCORD_CONNECTOR",
+	"JIRA_CONNECTOR",
+	"CONFLUENCE_CONNECTOR",
+	"CLICKUP_CONNECTOR",
+	"GOOGLE_CALENDAR_CONNECTOR",
+	"GOOGLE_GMAIL_CONNECTOR",
+	"AIRTABLE_CONNECTOR",
+	"LUMA_CONNECTOR",
+	"ELASTICSEARCH_CONNECTOR",
+]);
+
+export const document = z.object({
+	id: z.number(),
+	title: z.string(),
+	document_type: documentTypeEnum,
+	document_metadata: z.record(z.string(), z.any()),
+	content: z.string(),
+	created_at: z.string(),
+	search_space_id: z.number(),
+});
+
+export const extensionDocumentContent = z.object({
+	metadata: z.object({
+		BrowsingSessionId: z.string(),
+		VisitedWebPageURL: z.string(),
+		VisitedWebPageTitle: z.string(),
+		VisitedWebPageDateWithTimeInISOString: z.string(),
+		VisitedWebPageReffererURL: z.string(),
+		VisitedWebPageVisitDurationInMilliseconds: z.string(),
+	}),
+	pageContent: z.string(),
+});
+
+export const getDocumentsRequest = z.object({
+	queryParams: paginationQueryParams
+		.extend({
+			search_space_id: z.number().or(z.string()).optional(),
+			document_type: z.array(documentTypeEnum).optional(),
+		})
+		.nullish(),
+});
+
+export const getDocumentResponse = z.object({
+	items: z.array(document),
+	total: z.number(),
+});
+
+export const createDocumentRequest = document
+	.pick({ document_type: true, search_space_id: true })
+	.extend({
+		content: z.string().or(z.array(z.string())).or(z.array(extensionDocumentContent)),
+	});
+
+export const createDocumentResponse = z.object({
+	message: z.literal("Document created successfully"),
+});
+
+export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
+export type GetDocumentResponse = z.infer<typeof getDocumentResponse>;
+export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
+export type CreateDocumentResponse = z.infer<typeof createDocumentResponse>;
diff --git a/surfsense_web/contracts/types/index.ts b/surfsense_web/contracts/types/index.ts
index d00f7903f..e6f5da376 100644
--- a/surfsense_web/contracts/types/index.ts
+++ b/surfsense_web/contracts/types/index.ts
@@ -3,6 +3,10 @@ import { z } from "zod";
 export const paginationQueryParams = z.object({
 	limit: z.number().optional(),
 	skip: z.number().optional(),
+	// OR
+	page: z.number().optional(),
+	page_size: z.number().optional(),
+	// TODO: Normalize to page/page_size or limit/skip in backend
 });
 
 export type PaginationQueryParams = z.infer<typeof paginationQueryParams>;
diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
new file mode 100644
index 000000000..e69de29bb

From 02ea730ab86ca9c5f36bd9eac01a54b97577559e Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:23:08 +0200
Subject: [PATCH 02/36] createdocuments reques / response zod schema

---
 surfsense_web/contracts/types/document.types.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 3a6bdde03..df5514c1d 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -9,7 +9,6 @@ export const documentTypeEnum = z.enum([
 	"NOTION_CONNECTOR",
 	"YOUTUBE_VIDEO",
 	"GITHUB_CONNECTOR",
-	"LINEAR_CONNECTOR",
 	"DISCORD_CONNECTOR",
 	"JIRA_CONNECTOR",
 	"CONFLUENCE_CONNECTOR",
@@ -19,6 +18,7 @@ export const documentTypeEnum = z.enum([
 	"AIRTABLE_CONNECTOR",
 	"LUMA_CONNECTOR",
 	"ELASTICSEARCH_CONNECTOR",
+	"LINEAR_CONNECTOR",
 ]);
 
 export const document = z.object({

From e237caf91aeaf67b69818f6b7add1113edeafa8d Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:41:47 +0200
Subject: [PATCH 03/36] add upload document request / response zod schema

---
 .../contracts/types/document.types.ts         | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index df5514c1d..5720cd313 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -43,6 +43,9 @@ export const extensionDocumentContent = z.object({
 	pageContent: z.string(),
 });
 
+/**
+ * Get documents
+ */
 export const getDocumentsRequest = z.object({
 	queryParams: paginationQueryParams
 		.extend({
@@ -57,6 +60,9 @@ export const getDocumentResponse = z.object({
 	total: z.number(),
 });
 
+/**
+ * Create documents
+ */
 export const createDocumentRequest = document
 	.pick({ document_type: true, search_space_id: true })
 	.extend({
@@ -67,7 +73,52 @@ export const createDocumentResponse = z.object({
 	message: z.literal("Document created successfully"),
 });
 
+/**
+ * Upload documents
+ */
+export const uploadDocumentRequest = z.object({
+	files: z.array(z.instanceof(File)),
+	search_space_id: z.number(),
+});
+
+export const uploadDocumentResponse = z.object({
+	message: z.literal("Files uploaded for processing"),
+});
+
+/**
+ * Search documents
+ */
+export const searchDocumentsRequest = z.object({
+	queryParams: paginationQueryParams
+		.extend({
+			search_space_id: z.number().or(z.string()).optional(),
+			document_type: z.array(documentTypeEnum).optional(),
+			title: z.string().optional(),
+		})
+		.nullish(),
+});
+
+export const searchDocumentsResponse = z.object({
+	items: z.array(document),
+	total: z.number(),
+});
+
+/**
+ * Get document type counts
+ */
+export const getDocumentTypeCountsRequest = z.object({
+	queryParams: z
+		.object({
+			search_space_id: z.number().or(z.string()).optional(),
+		})
+		.nullish(),
+});
+
+export const getDocumentTypeCountsResponse = z.record(z.string(), z.number());
+
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentResponse = z.infer<typeof getDocumentResponse>;
 export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
 export type CreateDocumentResponse = z.infer<typeof createDocumentResponse>;
+export type UploadDocumentRequest = z.infer<typeof uploadDocumentRequest>;
+export type UploadDocumentResponse = z.infer<typeof uploadDocumentResponse>;

From e7693bd19f17194116f2fa70192ef42b58ca9b14 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:42:32 +0200
Subject: [PATCH 04/36] add seach documents request / response zod schema

---
 surfsense_web/contracts/types/document.types.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 5720cd313..ef8a293d9 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -122,3 +122,5 @@ export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
 export type CreateDocumentResponse = z.infer<typeof createDocumentResponse>;
 export type UploadDocumentRequest = z.infer<typeof uploadDocumentRequest>;
 export type UploadDocumentResponse = z.infer<typeof uploadDocumentResponse>;
+export type SearchDocumentsRequest = z.infer<typeof searchDocumentsRequest>;
+export type SearchDocumentsResponse = z.infer<typeof searchDocumentsResponse>;

From 6f3b34f38c0d85b07a356d26d222c84ba5f37156 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:43:47 +0200
Subject: [PATCH 05/36] add getdocuments types counts request / response zod
 schema

---
 surfsense_web/contracts/types/document.types.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index ef8a293d9..d6b19baa3 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -124,3 +124,5 @@ export type UploadDocumentRequest = z.infer<typeof uploadDocumentRequest>;
 export type UploadDocumentResponse = z.infer<typeof uploadDocumentResponse>;
 export type SearchDocumentsRequest = z.infer<typeof searchDocumentsRequest>;
 export type SearchDocumentsResponse = z.infer<typeof searchDocumentsResponse>;
+export type GetDocumentTypeCountsRequest = z.infer<typeof getDocumentTypeCountsRequest>;
+export type GetDocumentTypeCountsResponse = z.infer<typeof getDocumentTypeCountsResponse>;

From 557e736a53a8e7f76b1eb95370f23c5015f363ee Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:47:30 +0200
Subject: [PATCH 06/36] add get document by chunks request / response zod
 schema

---
 .../contracts/types/document.types.ts         | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index d6b19baa3..99416d1c4 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -43,6 +43,16 @@ export const extensionDocumentContent = z.object({
 	pageContent: z.string(),
 });
 
+export const documentWithChunks = document.extend({
+	chunks: z.array(
+		z.object({
+			id: z.number(),
+			content: z.string(),
+			created_at: z.string(),
+		})
+	),
+});
+
 /**
  * Get documents
  */
@@ -116,6 +126,15 @@ export const getDocumentTypeCountsRequest = z.object({
 
 export const getDocumentTypeCountsResponse = z.record(z.string(), z.number());
 
+/**
+ * Get document by chunk
+ */
+export const getDocumentByChunkRequest = z.object({
+	chunk_id: z.number(),
+});
+
+export const getDocumentByChunkResponse = documentWithChunks;
+
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentResponse = z.infer<typeof getDocumentResponse>;
 export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
@@ -126,3 +145,5 @@ export type SearchDocumentsRequest = z.infer<typeof searchDocumentsRequest>;
 export type SearchDocumentsResponse = z.infer<typeof searchDocumentsResponse>;
 export type GetDocumentTypeCountsRequest = z.infer<typeof getDocumentTypeCountsRequest>;
 export type GetDocumentTypeCountsResponse = z.infer<typeof getDocumentTypeCountsResponse>;
+export type GetDocumentByChunkRequest = z.infer<typeof getDocumentByChunkRequest>;
+export type GetDocumentByChunkResponse = z.infer<typeof getDocumentByChunkResponse>;

From 0fa969925a4497b921b90f5a24de15a6e8f73374 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:50:38 +0200
Subject: [PATCH 07/36] add get document  request / response zod schema

---
 surfsense_web/contracts/types/document.types.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 99416d1c4..c67f51743 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -65,11 +65,18 @@ export const getDocumentsRequest = z.object({
 		.nullish(),
 });
 
-export const getDocumentResponse = z.object({
+export const getDocumentsResponse = z.object({
 	items: z.array(document),
 	total: z.number(),
 });
 
+/**
+ * Get document
+ */
+export const getDocumentRequest = document.pick({ id: true });
+
+export const getDocumentResponse = document;
+
 /**
  * Create documents
  */
@@ -136,6 +143,8 @@ export const getDocumentByChunkRequest = z.object({
 export const getDocumentByChunkResponse = documentWithChunks;
 
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
+export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
+export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;
 export type GetDocumentResponse = z.infer<typeof getDocumentResponse>;
 export type CreateDocumentRequest = z.infer<typeof createDocumentRequest>;
 export type CreateDocumentResponse = z.infer<typeof createDocumentResponse>;

From dd652e36fc669fb2ca6345a35f934d3f2febe8d2 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:56:22 +0200
Subject: [PATCH 08/36] add update document  request / response zod schema

---
 surfsense_web/contracts/types/document.types.ts | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index c67f51743..e739b5f89 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -142,6 +142,16 @@ export const getDocumentByChunkRequest = z.object({
 
 export const getDocumentByChunkResponse = documentWithChunks;
 
+/**
+ * Update document
+ */
+export const updateDocumentRequest = z.object({
+	id: z.number(),
+	data: document.pick({ search_space_id: true, document_type: true, content: true }),
+});
+
+export const updateDocumentResponse = document;
+
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
 export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;
@@ -156,3 +166,5 @@ export type GetDocumentTypeCountsRequest = z.infer<typeof getDocumentTypeCountsR
 export type GetDocumentTypeCountsResponse = z.infer<typeof getDocumentTypeCountsResponse>;
 export type GetDocumentByChunkRequest = z.infer<typeof getDocumentByChunkRequest>;
 export type GetDocumentByChunkResponse = z.infer<typeof getDocumentByChunkResponse>;
+export type UpdateDocumentRequest = z.infer<typeof updateDocumentRequest>;
+export type UpdateDocumentResponse = z.infer<typeof updateDocumentResponse>;

From 6378d27bcbe3afbb6112b5c0575bdc146555ec7f Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 09:57:16 +0200
Subject: [PATCH 09/36] add delete document  request / response zod schema

---
 surfsense_web/contracts/types/document.types.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index e739b5f89..2168b0a21 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -152,6 +152,15 @@ export const updateDocumentRequest = z.object({
 
 export const updateDocumentResponse = document;
 
+/**
+ * Delete document
+ */
+export const deleteDocumentRequest = document.pick({ id: true });
+
+export const deleteDocumentResponse = z.object({
+	message: z.literal("Document deleted successfully"),
+});
+
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
 export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;
@@ -168,3 +177,5 @@ export type GetDocumentByChunkRequest = z.infer<typeof getDocumentByChunkRequest
 export type GetDocumentByChunkResponse = z.infer<typeof getDocumentByChunkResponse>;
 export type UpdateDocumentRequest = z.infer<typeof updateDocumentRequest>;
 export type UpdateDocumentResponse = z.infer<typeof updateDocumentResponse>;
+export type DeleteDocumentRequest = z.infer<typeof deleteDocumentRequest>;
+export type DeleteDocumentResponse = z.infer<typeof deleteDocumentResponse>;

From 5a887e9243a64daac0bd10eca37ded6e172e4332 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 11:00:42 +0200
Subject: [PATCH 10/36] add documents api service

---
 .../contracts/types/document.types.ts         |   2 +-
 surfsense_web/lib/apis/base-api.service.ts    |  20 ++
 .../lib/apis/documents-api.service.ts         | 263 ++++++++++++++++++
 3 files changed, 284 insertions(+), 1 deletion(-)

diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 2168b0a21..c5d777080 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -87,7 +87,7 @@ export const createDocumentRequest = document
 	});
 
 export const createDocumentResponse = z.object({
-	message: z.literal("Document created successfully"),
+	message: z.literal("Documents created successfully"),
 });
 
 /**
diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts
index 3013be70a..e4b20faa4 100644
--- a/surfsense_web/lib/apis/base-api.service.ts
+++ b/surfsense_web/lib/apis/base-api.service.ts
@@ -260,6 +260,26 @@ class BaseApiService {
 			responseType: ResponseType.BLOB,
 		});
 	}
+
+	async postFormData<T>(
+		url: string,
+		responseSchema?: z.ZodSchema<T>,
+		options?: Omit<RequestOptions, "method" | "responseType" | "body"> & { body: FormData }
+	) {
+		// Remove Content-Type from options headers if present
+		const { "Content-Type": _, ...headersWithoutContentType } = options?.headers ?? {};
+
+		return this.request(url, responseSchema, {
+			method: "POST",
+			...options,
+			headers: {
+				// Don't set Content-Type - let browser set it with multipart boundary
+				Authorization: `Bearer ${this.bearerToken}`,
+				...headersWithoutContentType,
+			},
+			responseType: ResponseType.JSON,
+		});
+	}
 }
 
 export const baseApiService = new BaseApiService(
diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index e69de29bb..6e67b6aa0 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -0,0 +1,263 @@
+import {
+	type CreateDocumentRequest,
+	createDocumentRequest,
+	createDocumentResponse,
+	type DeleteDocumentRequest,
+	deleteDocumentRequest,
+	deleteDocumentResponse,
+	type GetDocumentByChunkRequest,
+	type GetDocumentRequest,
+	type GetDocumentsRequest,
+	type GetDocumentTypeCountsRequest,
+	getDocumentByChunkRequest,
+	getDocumentByChunkResponse,
+	getDocumentRequest,
+	getDocumentResponse,
+	getDocumentsRequest,
+	getDocumentsResponse,
+	getDocumentTypeCountsRequest,
+	getDocumentTypeCountsResponse,
+	type SearchDocumentsRequest,
+	searchDocumentsRequest,
+	searchDocumentsResponse,
+	type UpdateDocumentRequest,
+	type UploadDocumentRequest,
+	updateDocumentRequest,
+	updateDocumentResponse,
+	uploadDocumentRequest,
+	uploadDocumentResponse,
+} from "@/contracts/types/document.types";
+import { ValidationError } from "../error";
+import { baseApiService } from "./base-api.service";
+
+class DocumentsApiService {
+	/**
+	 * Get a list of documents with optional filtering and pagination
+	 */
+	getDocuments = async (request: GetDocumentsRequest) => {
+		// Validate the request
+		const parsedRequest = getDocumentsRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Transform query params to be string values
+		const transformedQueryParams = parsedRequest.data.queryParams
+			? Object.fromEntries(
+					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
+						// Handle array values (document_type)
+						if (Array.isArray(v)) {
+							return [k, v.join(",")];
+						}
+						return [k, String(v)];
+					})
+				)
+			: undefined;
+
+		const queryParams = transformedQueryParams
+			? new URLSearchParams(transformedQueryParams).toString()
+			: "";
+
+		return baseApiService.get(`/api/v1/documents?${queryParams}`, getDocumentsResponse);
+	};
+
+	/**
+	 * Get a single document by ID
+	 */
+	getDocument = async (request: GetDocumentRequest) => {
+		// Validate the request
+		const parsedRequest = getDocumentRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.get(`/api/v1/documents/${request.id}`, getDocumentResponse);
+	};
+
+	/**
+	 * Create documents (extension, crawled URL, or YouTube video)
+	 */
+	createDocument = async (request: CreateDocumentRequest) => {
+		// Validate the request
+		const parsedRequest = createDocumentRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.post(`/api/v1/documents`, createDocumentResponse, {
+			body: parsedRequest.data,
+		});
+	};
+
+	/**
+	 * Upload document files
+	 */
+	uploadDocument = async (request: UploadDocumentRequest) => {
+		// Validate the request
+		const parsedRequest = uploadDocumentRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Create FormData for file upload
+		const formData = new FormData();
+		parsedRequest.data.files.forEach((file) => {
+			formData.append("files", file);
+		});
+		formData.append("search_space_id", String(parsedRequest.data.search_space_id));
+
+		return baseApiService.postFormData(`/api/v1/documents/fileupload`, uploadDocumentResponse, {
+			body: formData,
+		});
+	};
+
+	/**
+	 * Search documents by title
+	 */
+	searchDocuments = async (request: SearchDocumentsRequest) => {
+		// Validate the request
+		const parsedRequest = searchDocumentsRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Transform query params to be string values
+		const transformedQueryParams = parsedRequest.data.queryParams
+			? Object.fromEntries(
+					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
+						// Handle array values (document_type)
+						if (Array.isArray(v)) {
+							return [k, v.join(",")];
+						}
+						return [k, String(v)];
+					})
+				)
+			: undefined;
+
+		const queryParams = transformedQueryParams
+			? new URLSearchParams(transformedQueryParams).toString()
+			: "";
+
+		return baseApiService.get(`/api/v1/documents/search?${queryParams}`, searchDocumentsResponse);
+	};
+
+	/**
+	 * Get document type counts
+	 */
+	getDocumentTypeCounts = async (request: GetDocumentTypeCountsRequest) => {
+		// Validate the request
+		const parsedRequest = getDocumentTypeCountsRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		// Transform query params to be string values
+		const transformedQueryParams = parsedRequest.data.queryParams
+			? Object.fromEntries(
+					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => [k, String(v)])
+				)
+			: undefined;
+
+		const queryParams = transformedQueryParams
+			? new URLSearchParams(transformedQueryParams).toString()
+			: "";
+
+		return baseApiService.get(
+			`/api/v1/documents/type-counts?${queryParams}`,
+			getDocumentTypeCountsResponse
+		);
+	};
+
+	/**
+	 * Get document by chunk ID (includes all chunks)
+	 */
+	getDocumentByChunk = async (request: GetDocumentByChunkRequest) => {
+		// Validate the request
+		const parsedRequest = getDocumentByChunkRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.get(
+			`/api/v1/documents/by-chunk/${request.chunk_id}`,
+			getDocumentByChunkResponse
+		);
+	};
+
+	/**
+	 * Update a document
+	 */
+	updateDocument = async (request: UpdateDocumentRequest) => {
+		// Validate the request
+		const parsedRequest = updateDocumentRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		const { id, data } = parsedRequest.data;
+
+		return baseApiService.put(`/api/v1/documents/${id}`, updateDocumentResponse, {
+			body: data,
+		});
+	};
+
+	/**
+	 * Delete a document
+	 */
+	deleteDocument = async (request: DeleteDocumentRequest) => {
+		// Validate the request
+		const parsedRequest = deleteDocumentRequest.safeParse(request);
+
+		if (!parsedRequest.success) {
+			console.error("Invalid request:", parsedRequest.error);
+
+			// Format a user friendly error message
+			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
+			throw new ValidationError(`Invalid request: ${errorMessage}`);
+		}
+
+		return baseApiService.delete(`/api/v1/documents/${request.id}`, deleteDocumentResponse);
+	};
+}
+
+export const documentsApiService = new DocumentsApiService();

From ed07699ffce873c5c14f48fb7493b07006411541 Mon Sep 17 00:00:00 2001
From: thierryverse <thierryuniverse@gmail.com>
Date: Thu, 20 Nov 2025 11:33:37 +0200
Subject: [PATCH 11/36] add documents queries & mutations atoms

---
 .../documents/document-mutation.atoms.ts      | 115 ++++++++++++++++++
 .../atoms/documents/document-query.atoms.ts   |  86 +++++++++++++
 surfsense_web/atoms/documents/ui.atoms.ts     |   7 ++
 surfsense_web/lib/query-client/cache-keys.ts  |   8 ++
 4 files changed, 216 insertions(+)
 create mode 100644 surfsense_web/atoms/documents/document-mutation.atoms.ts
 create mode 100644 surfsense_web/atoms/documents/document-query.atoms.ts
 create mode 100644 surfsense_web/atoms/documents/ui.atoms.ts

diff --git a/surfsense_web/atoms/documents/document-mutation.atoms.ts b/surfsense_web/atoms/documents/document-mutation.atoms.ts
new file mode 100644
index 000000000..ec7bad237
--- /dev/null
+++ b/surfsense_web/atoms/documents/document-mutation.atoms.ts
@@ -0,0 +1,115 @@
+import { atomWithMutation } from "jotai-tanstack-query";
+import { toast } from "sonner";
+import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom";
+import type {
+	CreateDocumentRequest,
+	DeleteDocumentRequest,
+	GetDocumentsResponse,
+	UpdateDocumentRequest,
+	UploadDocumentRequest,
+} from "@/contracts/types/document.types";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { queryClient } from "@/lib/query-client/client";
+import { globalDocumentsQueryParamsAtom } from "./ui.atoms";
+
+export const createDocumentMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
+
+	return {
+		mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: CreateDocumentRequest) => {
+			return documentsApiService.createDocument(request);
+		},
+
+		onSuccess: () => {
+			toast.success("Document created successfully");
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
+			});
+		},
+	};
+});
+
+export const uploadDocumentMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
+
+	return {
+		mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: UploadDocumentRequest) => {
+			return documentsApiService.uploadDocument(request);
+		},
+
+		onSuccess: () => {
+			toast.success("Files uploaded for processing");
+		},
+	};
+});
+
+export const updateDocumentMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
+
+	return {
+		mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+		enabled: !!searchSpaceId,
+		mutationFn: async (request: UpdateDocumentRequest) => {
+			return documentsApiService.updateDocument(request);
+		},
+
+		onSuccess: (_, request: UpdateDocumentRequest) => {
+			toast.success("Document updated successfully");
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.document(String(request.id)),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
+			});
+		},
+	};
+});
+
+export const deleteDocumentMutationAtom = atomWithMutation((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	const authToken = localStorage.getItem("surfsense_bearer_token");
+	const documentsQueryParams = get(globalDocumentsQueryParamsAtom);
+
+	return {
+		mutationKey: cacheKeys.documents.globalQueryParams(documentsQueryParams),
+		enabled: !!searchSpaceId && !!authToken,
+		mutationFn: async (request: DeleteDocumentRequest) => {
+			return documentsApiService.deleteDocument(request);
+		},
+
+		onSuccess: (_, request: DeleteDocumentRequest) => {
+			toast.success("Document deleted successfully");
+			queryClient.setQueryData(
+				cacheKeys.documents.globalQueryParams(documentsQueryParams),
+				(oldData: GetDocumentsResponse | undefined) => {
+					if (!oldData) return oldData;
+					return {
+						...oldData,
+						items: oldData.items.filter((doc) => doc.id !== request.id),
+						total: oldData.total - 1,
+					};
+				}
+			);
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.document(String(request.id)),
+			});
+			queryClient.invalidateQueries({
+				queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
+			});
+		},
+	};
+});
diff --git a/surfsense_web/atoms/documents/document-query.atoms.ts b/surfsense_web/atoms/documents/document-query.atoms.ts
new file mode 100644
index 000000000..d1b5466a3
--- /dev/null
+++ b/surfsense_web/atoms/documents/document-query.atoms.ts
@@ -0,0 +1,86 @@
+import { atomWithQuery } from "jotai-tanstack-query";
+import { activeSearchSpaceIdAtom } from "@/atoms/seach-spaces/seach-space-queries.atom";
+import type { SearchDocumentsRequest } from "@/contracts/types/document.types";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { globalDocumentsQueryParamsAtom } from "./ui.atoms";
+
+export const documentsAtom = atomWithQuery((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+	const queryParams = get(globalDocumentsQueryParamsAtom);
+
+	return {
+		queryKey: cacheKeys.documents.globalQueryParams(queryParams),
+		enabled: !!searchSpaceId,
+		queryFn: async () => {
+			return documentsApiService.getDocuments({
+				queryParams: queryParams,
+			});
+		},
+	};
+});
+
+export const getDocumentAtom = (documentId: number) =>
+	atomWithQuery(() => {
+		return {
+			queryKey: cacheKeys.documents.document(String(documentId)),
+			enabled: !!documentId,
+			queryFn: async () => {
+				if (!documentId) {
+					throw new Error("No active document id found");
+				}
+
+				return documentsApiService.getDocument({ id: documentId });
+			},
+		};
+	});
+
+export const documentTypeCountsAtom = atomWithQuery((get) => {
+	const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+	return {
+		queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
+		enabled: !!searchSpaceId,
+		queryFn: async () => {
+			return documentsApiService.getDocumentTypeCounts({
+				queryParams: {
+					search_space_id: searchSpaceId ?? undefined,
+				},
+			});
+		},
+	};
+});
+
+export const getDocumentByChunkAtom = (chunkId: number) =>
+	atomWithQuery(() => {
+		return {
+			queryKey: cacheKeys.documents.byChunk(String(chunkId)),
+			enabled: !!chunkId,
+			queryFn: async () => {
+				if (!chunkId) {
+					throw new Error("No active chunk id found");
+				}
+
+				return documentsApiService.getDocumentByChunk({ chunk_id: chunkId });
+			},
+		};
+	});
+
+export const searchDocumentsAtom = (request: SearchDocumentsRequest) =>
+	atomWithQuery((get) => {
+		const searchSpaceId = get(activeSearchSpaceIdAtom);
+
+		return {
+			queryKey: cacheKeys.documents.globalQueryParams(request.queryParams),
+			enabled: !!searchSpaceId,
+			queryFn: async () => {
+				return documentsApiService.searchDocuments({
+					...request,
+					queryParams: {
+						...request.queryParams,
+						search_space_id: searchSpaceId ?? undefined,
+					},
+				});
+			},
+		};
+	});
diff --git a/surfsense_web/atoms/documents/ui.atoms.ts b/surfsense_web/atoms/documents/ui.atoms.ts
new file mode 100644
index 000000000..33740e9c7
--- /dev/null
+++ b/surfsense_web/atoms/documents/ui.atoms.ts
@@ -0,0 +1,7 @@
+import { atom } from "jotai";
+import type { GetDocumentsRequest } from "@/contracts/types/document.types";
+
+export const globalDocumentsQueryParamsAtom = atom<GetDocumentsRequest["queryParams"]>({
+	page_size: 10,
+	page: 0,
+});
diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts
index 12b73bc35..813ba62cf 100644
--- a/surfsense_web/lib/query-client/cache-keys.ts
+++ b/surfsense_web/lib/query-client/cache-keys.ts
@@ -1,4 +1,5 @@
 import type { GetChatsRequest } from "@/contracts/types/chat.types";
+import type { GetDocumentsRequest } from "@/contracts/types/document.types";
 import type { GetPodcastsRequest } from "@/contracts/types/podcast.types";
 
 export const cacheKeys = {
@@ -11,6 +12,13 @@ export const cacheKeys = {
 		globalQueryParams: (queries: GetPodcastsRequest["queryParams"]) =>
 			["podcasts", ...(queries ? Object.values(queries) : [])] as const,
 	},
+	documents: {
+		globalQueryParams: (queries: GetDocumentsRequest["queryParams"]) =>
+			["documents", ...(queries ? Object.values(queries) : [])] as const,
+		document: (documentId: string) => ["document", documentId] as const,
+		typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const,
+		byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const,
+	},
 	auth: {
 		user: ["auth", "user"] as const,
 	},

From 1d8cfbb143017abcc5fdb5996fae4e494ae9999a Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Tue, 2 Dec 2025 10:44:13 +0200
Subject: [PATCH 12/36] fix (migration 40): Add an 'if not exists' guard

---
 .../40_move_llm_preferences_to_searchspace.py | 74 ++++++++++---------
 1 file changed, 38 insertions(+), 36 deletions(-)

diff --git a/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py b/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
index 1067cffcc..5f6ccb852 100644
--- a/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
+++ b/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
@@ -1,19 +1,6 @@
-"""Move LLM preferences from user-level to search space level
-
-Revision ID: 40
-Revises: 39
-Create Date: 2024-11-27
-
-This migration moves LLM preferences (long_context_llm_id, fast_llm_id, strategic_llm_id)
-from the user_search_space_preferences table to the searchspaces table itself.
-
-This change supports the RBAC model where LLM preferences are shared by all members
-of a search space, rather than being per-user.
-"""
-
 import sqlalchemy as sa
-
 from alembic import op
+from sqlalchemy import inspect
 
 # revision identifiers, used by Alembic.
 revision = "40"
@@ -23,26 +10,32 @@ depends_on = None
 
 
 def upgrade():
-    # Add LLM preference columns to searchspaces table
-    op.add_column(
-        "searchspaces",
-        sa.Column("long_context_llm_id", sa.Integer(), nullable=True),
-    )
-    op.add_column(
-        "searchspaces",
-        sa.Column("fast_llm_id", sa.Integer(), nullable=True),
-    )
-    op.add_column(
-        "searchspaces",
-        sa.Column("strategic_llm_id", sa.Integer(), nullable=True),
-    )
+    conn = op.get_bind()
+    inspector = inspect(conn)
 
-    # Migrate existing preferences from user_search_space_preferences to searchspaces
-    # We take the owner's preferences (the user who created the search space)
-    connection = op.get_bind()
+    existing_cols = {col["name"] for col in inspector.get_columns("searchspaces")}
 
-    # Get all search spaces and their owner's preferences
-    connection.execute(
+    # Add columns only if they don't already exist
+    if "long_context_llm_id" not in existing_cols:
+        op.add_column(
+            "searchspaces",
+            sa.Column("long_context_llm_id", sa.Integer(), nullable=True),
+        )
+
+    if "fast_llm_id" not in existing_cols:
+        op.add_column(
+            "searchspaces",
+            sa.Column("fast_llm_id", sa.Integer(), nullable=True),
+        )
+
+    if "strategic_llm_id" not in existing_cols:
+        op.add_column(
+            "searchspaces",
+            sa.Column("strategic_llm_id", sa.Integer(), nullable=True),
+        )
+
+    # Migrate existing data
+    conn.execute(
         sa.text("""
             UPDATE searchspaces ss
             SET 
@@ -57,7 +50,16 @@ def upgrade():
 
 
 def downgrade():
-    # Remove LLM preference columns from searchspaces table
-    op.drop_column("searchspaces", "strategic_llm_id")
-    op.drop_column("searchspaces", "fast_llm_id")
-    op.drop_column("searchspaces", "long_context_llm_id")
+    conn = op.get_bind()
+    inspector = inspect(conn)
+    existing_cols = {col["name"] for col in inspector.get_columns("searchspaces")}
+
+    # Drop columns only if they exist
+    if "strategic_llm_id" in existing_cols:
+        op.drop_column("searchspaces", "strategic_llm_id")
+
+    if "fast_llm_id" in existing_cols:
+        op.drop_column("searchspaces", "fast_llm_id")
+
+    if "long_context_llm_id" in existing_cols:
+        op.drop_column("searchspaces", "long_context_llm_id")

From 509ef211fa3c68099931a4f9af8fececa99ca661 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Tue, 2 Dec 2025 10:46:47 +0200
Subject: [PATCH 13/36] fix (migration 43): Add an 'if not exists' guard

---
 .../43_add_blocknote_fields_to_documents.py   | 85 ++++++++++---------
 1 file changed, 46 insertions(+), 39 deletions(-)

diff --git a/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py b/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
index 32e7780eb..ee9230a44 100644
--- a/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
+++ b/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
@@ -16,6 +16,7 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
 from alembic import op
+from sqlalchemy import inspect
 
 # revision identifiers, used by Alembic.
 revision: str = "43"
@@ -25,51 +26,57 @@ depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
-    """Upgrade schema - Add BlockNote fields and trigger population task."""
+    """Upgrade schema - Add BlockNote fields (idempotent)."""
 
-    # Add the columns
-    op.add_column(
-        "documents",
-        sa.Column(
-            "blocknote_document", postgresql.JSONB(astext_type=sa.Text()), nullable=True
-        ),
-    )
-    op.add_column(
-        "documents",
-        sa.Column(
-            "content_needs_reindexing",
-            sa.Boolean(),
-            nullable=False,
-            server_default=sa.false(),
-        ),
-    )
-    op.add_column(
-        "documents",
-        sa.Column("last_edited_at", sa.TIMESTAMP(timezone=True), nullable=True),
-    )
+    conn = op.get_bind()
+    inspector = inspect(conn)
+    existing_cols = {c["name"] for c in inspector.get_columns("documents")}
 
-    # Trigger the Celery task to populate blocknote_document for existing documents
-    try:
-        from app.tasks.celery_tasks.blocknote_migration_tasks import (
-            populate_blocknote_for_documents_task,
+    # Add blocknote_document (JSONB) if doest not exist
+    if "blocknote_document" not in existing_cols:
+        op.add_column(
+            "documents",
+            sa.Column(
+                "blocknote_document",
+                postgresql.JSONB(astext_type=sa.Text()),
+                nullable=True,
+            ),
         )
 
-        # Queue the task to run asynchronously
-        populate_blocknote_for_documents_task.apply_async()
-        print(
-            "✓ Queued Celery task to populate blocknote_document for existing documents"
+    # Add content_needs_reindexing (boolean) if doest not exist
+    if "content_needs_reindexing" not in existing_cols:
+        op.add_column(
+            "documents",
+            sa.Column(
+                "content_needs_reindexing",
+                sa.Boolean(),
+                nullable=False,
+                server_default=sa.false(),
+            ),
         )
-    except Exception as e:
-        # If Celery is not available or task queueing fails, log but don't fail the migration
-        print(f"⚠ Warning: Could not queue blocknote population task: {e}")
-        print("  You can manually trigger it later with:")
-        print(
-            "  celery -A app.celery_app call app.tasks.celery_tasks.blocknote_migration_tasks.populate_blocknote_for_documents_task"
+
+    # Add last_edited_at (timestamp with tz) if doest not exist
+    if "last_edited_at" not in existing_cols:
+        op.add_column(
+            "documents",
+            sa.Column("last_edited_at", sa.TIMESTAMP(timezone=True), nullable=True),
         )
 
+    # NOTE: We intentionally do NOT import or queue Celery tasks here.
+    # Running background jobs during migrations causes hard-to-debug failures.
+    # After running migrations, trigger the backfill task manually (instructions below).
+
 
 def downgrade() -> None:
-    """Downgrade schema - Remove BlockNote fields."""
-    op.drop_column("documents", "last_edited_at")
-    op.drop_column("documents", "content_needs_reindexing")
-    op.drop_column("documents", "blocknote_document")
+    """Downgrade schema - Remove BlockNote fields (only if present)."""
+
+    conn = op.get_bind()
+    inspector = inspect(conn)
+    existing_cols = {c["name"] for c in inspector.get_columns("documents")}
+
+    if "last_edited_at" in existing_cols:
+        op.drop_column("documents", "last_edited_at")
+    if "content_needs_reindexing" in existing_cols:
+        op.drop_column("documents", "content_needs_reindexing")
+    if "blocknote_document" in existing_cols:
+        op.drop_column("documents", "blocknote_document")

From 6c43e8aa0966c501bde8deac3f10f7c17a0d92b2 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Tue, 2 Dec 2025 20:40:59 +0000
Subject: [PATCH 14/36] fix (migration1): add 'if not exixst' guard

---
 .gitignore                                    |  3 +-
 .../versions/1_add_github_connector_enum.py   | 72 +++++++++++--------
 2 files changed, 43 insertions(+), 32 deletions(-)

diff --git a/.gitignore b/.gitignore
index 342c0b258..3228f0dfe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@
 ./surfsense_backend/podcasts/
 .env
 node_modules/
-.ruff_cache/
\ No newline at end of file
+.ruff_cache/
+.venv
\ No newline at end of file
diff --git a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
index 235908b1f..a359f8962 100644
--- a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
+++ b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
@@ -2,17 +2,12 @@
 
 Revision ID: 1
 Revises:
-
 """
 
 from collections.abc import Sequence
 
 from alembic import op
 
-# Import pgvector if needed for other types, though not for this ENUM change
-# import pgvector
-
-
 # revision identifiers, used by Alembic.
 revision: str = "1"
 down_revision: str | None = None
@@ -21,10 +16,24 @@ depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
+    # Ensure the enum type exists
+    op.execute(
+        """
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'searchsourceconnectortype') THEN
+        CREATE TYPE searchsourceconnectortype AS ENUM(
+            'SERPER_API', 
+            'TAVILY_API', 
+            'SLACK_CONNECTOR', 
+            'NOTION_CONNECTOR'
+        );
+    END IF;
+END$$;
+"""
+    )
 
-    # Manually add the command to add the enum value
-    # Note: It's generally better to let autogenerate handle this, but we're bypassing it
+    # Add the new enum value if it doesn't exist
     op.execute(
         """
 DO $$
@@ -43,30 +52,31 @@ END$$;
 """
     )
 
-    # Pass for the rest, as autogenerate didn't run to add other schema details
-    pass
-    # ### end Alembic commands ###
-
 
 def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
+    # Removing an enum value safely requires recreating the type
+    op.execute(
+        """
+DO $$
+BEGIN
+    -- Rename existing type
+    ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old;
 
-    # Downgrading removal of an enum value is complex and potentially dangerous
-    # if the value is in use. Often omitted or requires manual SQL based on context.
-    # For now, we'll just pass. If you needed to reverse this, you'd likely
-    # have to manually check if 'GITHUB_CONNECTOR' is used in the table
-    # and then potentially recreate the type without it.
-    op.execute(
-        "ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old"
-    )
-    op.execute(
-        "CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR')"
-    )
-    op.execute(
-        "ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
-        "connector_type::text::searchsourceconnectortype"
-    )
-    op.execute("DROP TYPE searchsourceconnectortype_old")
+    -- Create new type without GITHUB_CONNECTOR
+    CREATE TYPE searchsourceconnectortype AS ENUM(
+        'SERPER_API', 
+        'TAVILY_API', 
+        'SLACK_CONNECTOR', 
+        'NOTION_CONNECTOR'
+    );
 
-    pass
-    # ### end Alembic commands ###
+    -- Update table columns to use new type
+    ALTER TABLE search_source_connectors
+    ALTER COLUMN connector_type TYPE searchsourceconnectortype
+    USING connector_type::text::searchsourceconnectortype;
+
+    -- Drop old type
+    DROP TYPE searchsourceconnectortype_old;
+END$$;
+"""
+    )

From 9cd3756c2fe39677b26e6274d3e7e2e0b5f31e46 Mon Sep 17 00:00:00 2001
From: "Thierry CH." <bakerathierry@gmail.com>
Date: Wed, 3 Dec 2025 23:43:34 +0200
Subject: [PATCH 15/36] Refactor request validation in getDocuments method

---
 surfsense_web/lib/apis/documents-api.service.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index 6e67b6aa0..90ccf40c5 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -35,13 +35,13 @@ class DocumentsApiService {
 	 * Get a list of documents with optional filtering and pagination
 	 */
 	getDocuments = async (request: GetDocumentsRequest) => {
-		// Validate the request
+	
 		const parsedRequest = getDocumentsRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
+			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From ccea51084ce13446b0b68ee0f3c0a4e9b22db8d9 Mon Sep 17 00:00:00 2001
From: "Thierry CH." <bakerathierry@gmail.com>
Date: Wed, 3 Dec 2025 23:44:00 +0200
Subject: [PATCH 16/36] Refactor request validation in getDocument method

---
 surfsense_web/lib/apis/documents-api.service.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index 90ccf40c5..d6cfee459 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -70,13 +70,13 @@ class DocumentsApiService {
 	 * Get a single document by ID
 	 */
 	getDocument = async (request: GetDocumentRequest) => {
-		// Validate the request
+		
 		const parsedRequest = getDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
+			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From 3ce1ec822597a5b854c18ae663564b47080129fc Mon Sep 17 00:00:00 2001
From: "Thierry CH." <bakerathierry@gmail.com>
Date: Wed, 3 Dec 2025 23:44:20 +0200
Subject: [PATCH 17/36] Clean up comments in createDocument method

Removed unnecessary comments in createDocument function.
---
 surfsense_web/lib/apis/documents-api.service.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index d6cfee459..2aed750e0 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -88,13 +88,13 @@ class DocumentsApiService {
 	 * Create documents (extension, crawled URL, or YouTube video)
 	 */
 	createDocument = async (request: CreateDocumentRequest) => {
-		// Validate the request
+		
 		const parsedRequest = createDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
+			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From 8bf7957827f2150fd5fd03f85a61f5698a61be57 Mon Sep 17 00:00:00 2001
From: "Thierry CH." <bakerathierry@gmail.com>
Date: Wed, 3 Dec 2025 23:44:52 +0200
Subject: [PATCH 18/36] Improve request validation for uploadDocument

Refactor error handling for document upload requests.
---
 surfsense_web/lib/apis/documents-api.service.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index 2aed750e0..a03c554d2 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -108,13 +108,13 @@ class DocumentsApiService {
 	 * Upload document files
 	 */
 	uploadDocument = async (request: UploadDocumentRequest) => {
-		// Validate the request
+		
 		const parsedRequest = uploadDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
+			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From 074c71aec314074cc0a4f2d684dba07a7478e67a Mon Sep 17 00:00:00 2001
From: "Thierry CH." <bakerathierry@gmail.com>
Date: Wed, 3 Dec 2025 23:45:16 +0200
Subject: [PATCH 19/36] Improve request validation in searchDocuments

Refactor error handling for searchDocuments method.
---
 surfsense_web/lib/apis/documents-api.service.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index a03c554d2..554b40b60 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -135,13 +135,13 @@ class DocumentsApiService {
 	 * Search documents by title
 	 */
 	searchDocuments = async (request: SearchDocumentsRequest) => {
-		// Validate the request
+		
 		const parsedRequest = searchDocumentsRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			// Format a user friendly error message
+			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From f92b1ab4d560d3ceb755ce5d4be263a84b804feb Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 11:08:29 +0000
Subject: [PATCH 20/36] chore: add .pnpm-store to .gitignore

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 3228f0dfe..cb6d28b4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,5 @@
 .env
 node_modules/
 .ruff_cache/
-.venv
\ No newline at end of file
+.venv
+.pnpm-store

From 013bb1547dac46b018a6c1faab699ee02454e51f Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 11:39:13 +0000
Subject: [PATCH 21/36] feat: add useQuery implementation with centralized
 cache key management in SourceDetailSheet

---
 surfsense_web/components/chat/SourceDetailSheet.tsx | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx
index e28e35070..b7cb3afac 100644
--- a/surfsense_web/components/chat/SourceDetailSheet.tsx
+++ b/surfsense_web/components/chat/SourceDetailSheet.tsx
@@ -3,6 +3,9 @@
 import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react";
 import type React from "react";
 import { type ReactNode, useEffect, useRef, useState } from "react";
+import { useQuery } from "@tanstack/react-query";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { MarkdownViewer } from "@/components/markdown-viewer";
 import { Button } from "@/components/ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
@@ -51,6 +54,14 @@ export function SourceDetailSheet({
 	const highlightedChunkRef = useRef<HTMLDivElement>(null);
 	const [summaryOpen, setSummaryOpen] = useState(false);
 
+	// Add useQuery to fetch document by chunk
+	const { data: queryDocument, isLoading: queryLoading, error: queryError } = useQuery({
+		queryKey: cacheKeys.documents.byChunk(chunkId.toString()),
+		queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }),
+		enabled: !!chunkId && open,
+		staleTime: 5 * 60 * 1000, // 5 minutes
+	});
+
 	// Check if this is a source type that should render directly from node
 	const isDirectRenderSource =
 		sourceType === "TAVILY_API" ||

From 9bff7a619f16b7860be26f69ac63d9ae534bdc39 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 11:40:50 +0000
Subject: [PATCH 22/36] refactor: use descriptive variable names for useQuery
 destructuring in SourceDetailSheet

---
 surfsense_web/components/chat/SourceDetailSheet.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx
index b7cb3afac..e028d6a8e 100644
--- a/surfsense_web/components/chat/SourceDetailSheet.tsx
+++ b/surfsense_web/components/chat/SourceDetailSheet.tsx
@@ -55,7 +55,7 @@ export function SourceDetailSheet({
 	const [summaryOpen, setSummaryOpen] = useState(false);
 
 	// Add useQuery to fetch document by chunk
-	const { data: queryDocument, isLoading: queryLoading, error: queryError } = useQuery({
+	const { data: documentFromChunkQuery, isLoading: isLoadingDocumentFromChunk, error: documentFetchError } = useQuery({
 		queryKey: cacheKeys.documents.byChunk(chunkId.toString()),
 		queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }),
 		enabled: !!chunkId && open,

From ecb48ddb03f2f6ac54bbfa792a63ccba72f41f44 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 11:52:32 +0000
Subject: [PATCH 23/36] refactor: replace useDocumentByChunk with useQuery
 implementation in SourceDetailSheet

- Remove useDocumentByChunk hook dependency
- Use useQuery with centralized cache keys and descriptive variable names
- Replace all document, loading, and error references with new implementation
- Remove manual fetch/clear effects as useQuery handles lifecycle automatically
---
 .../components/chat/SourceDetailSheet.tsx      | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx
index e028d6a8e..0e48043b5 100644
--- a/surfsense_web/components/chat/SourceDetailSheet.tsx
+++ b/surfsense_web/components/chat/SourceDetailSheet.tsx
@@ -18,7 +18,6 @@ import {
 	SheetTitle,
 } from "@/components/ui/sheet";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useDocumentByChunk } from "@/hooks/use-document-by-chunk";
 import { cn } from "@/lib/utils";
 
 interface SourceDetailSheetProps {
@@ -49,13 +48,12 @@ export function SourceDetailSheet({
 	url,
 	children,
 }: SourceDetailSheetProps) {
-	const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk();
 	const chunksContainerRef = useRef<HTMLDivElement>(null);
 	const highlightedChunkRef = useRef<HTMLDivElement>(null);
 	const [summaryOpen, setSummaryOpen] = useState(false);
 
 	// Add useQuery to fetch document by chunk
-	const { data: documentFromChunkQuery, isLoading: isLoadingDocumentFromChunk, error: documentFetchError } = useQuery({
+	const { data: document, isLoading: isDocumentByChunkFetching, error: documentByChunkFetchingError } = useQuery({
 		queryKey: cacheKeys.documents.byChunk(chunkId.toString()),
 		queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }),
 		enabled: !!chunkId && open,
@@ -69,14 +67,6 @@ export function SourceDetailSheet({
 		sourceType === "SEARXNG_API" ||
 		sourceType === "BAIDU_SEARCH_API";
 
-	useEffect(() => {
-		if (open && chunkId && !isDirectRenderSource) {
-			fetchDocumentByChunk(chunkId);
-		} else if (!open && !isDirectRenderSource) {
-			clearDocument();
-		}
-	}, [open, chunkId, isDirectRenderSource, fetchDocumentByChunk, clearDocument]);
-
 	useEffect(() => {
 		// Scroll to highlighted chunk when document loads
 		if (document && highlightedChunkRef.current && chunksContainerRef.current) {
@@ -111,15 +101,15 @@ export function SourceDetailSheet({
 					</SheetDescription>
 				</SheetHeader>
 
-				{!isDirectRenderSource && loading && (
+				{!isDirectRenderSource && isDocumentByChunkFetching && (
 					<div className="flex items-center justify-center h-64 px-6">
 						<Loader2 className="h-8 w-8 animate-spin text-muted-foreground" />
 					</div>
 				)}
 
-				{!isDirectRenderSource && error && (
+				{!isDirectRenderSource && documentByChunkFetchingError && (
 					<div className="flex items-center justify-center h-64 px-6">
-						<p className="text-sm text-destructive">{error}</p>
+						<p className="text-sm text-destructive">{documentByChunkFetchingError.message || "Failed to load document"}</p>
 					</div>
 				)}
 

From 7f80c9c4086b21bcaa756082346613bf20a847e6 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 12:37:12 +0000
Subject: [PATCH 24/36] feat: migrate document upload to use jotai + tanstack
 query mutation atom

- Replace imperative fetch API with uploadDocumentMutationAtom in DocumentUploadTab
- Maintain backward compatibility with existing UI behavior
- Remove unused document query atoms from document-query.atoms.ts
- Use mutation state (isPending) for loading state management
- Preserve progress animation and error handling
---
 .../atoms/documents/document-query.atoms.ts   | 49 -----------
 .../components/sources/DocumentUploadTab.tsx  | 81 +++++++++----------
 2 files changed, 36 insertions(+), 94 deletions(-)

diff --git a/surfsense_web/atoms/documents/document-query.atoms.ts b/surfsense_web/atoms/documents/document-query.atoms.ts
index d1b5466a3..8caeafb3d 100644
--- a/surfsense_web/atoms/documents/document-query.atoms.ts
+++ b/surfsense_web/atoms/documents/document-query.atoms.ts
@@ -20,21 +20,6 @@ export const documentsAtom = atomWithQuery((get) => {
 	};
 });
 
-export const getDocumentAtom = (documentId: number) =>
-	atomWithQuery(() => {
-		return {
-			queryKey: cacheKeys.documents.document(String(documentId)),
-			enabled: !!documentId,
-			queryFn: async () => {
-				if (!documentId) {
-					throw new Error("No active document id found");
-				}
-
-				return documentsApiService.getDocument({ id: documentId });
-			},
-		};
-	});
-
 export const documentTypeCountsAtom = atomWithQuery((get) => {
 	const searchSpaceId = get(activeSearchSpaceIdAtom);
 
@@ -50,37 +35,3 @@ export const documentTypeCountsAtom = atomWithQuery((get) => {
 		},
 	};
 });
-
-export const getDocumentByChunkAtom = (chunkId: number) =>
-	atomWithQuery(() => {
-		return {
-			queryKey: cacheKeys.documents.byChunk(String(chunkId)),
-			enabled: !!chunkId,
-			queryFn: async () => {
-				if (!chunkId) {
-					throw new Error("No active chunk id found");
-				}
-
-				return documentsApiService.getDocumentByChunk({ chunk_id: chunkId });
-			},
-		};
-	});
-
-export const searchDocumentsAtom = (request: SearchDocumentsRequest) =>
-	atomWithQuery((get) => {
-		const searchSpaceId = get(activeSearchSpaceIdAtom);
-
-		return {
-			queryKey: cacheKeys.documents.globalQueryParams(request.queryParams),
-			enabled: !!searchSpaceId,
-			queryFn: async () => {
-				return documentsApiService.searchDocuments({
-					...request,
-					queryParams: {
-						...request.queryParams,
-						search_space_id: searchSpaceId ?? undefined,
-					},
-				});
-			},
-		};
-	});
diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx
index c51522a74..7ad82d124 100644
--- a/surfsense_web/components/sources/DocumentUploadTab.tsx
+++ b/surfsense_web/components/sources/DocumentUploadTab.tsx
@@ -1,5 +1,6 @@
 "use client";
 
+import { useAtom } from "jotai";
 import { CheckCircle2, FileType, Info, Loader2, Tag, Upload, X } from "lucide-react";
 import { AnimatePresence, motion } from "motion/react";
 import { useRouter } from "next/navigation";
@@ -7,6 +8,7 @@ import { useTranslations } from "next-intl";
 import { useCallback, useState } from "react";
 import { useDropzone } from "react-dropzone";
 import { toast } from "sonner";
+import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
 
 import { Alert, AlertDescription } from "@/components/ui/alert";
 import { Badge } from "@/components/ui/badge";
@@ -14,7 +16,6 @@ import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Progress } from "@/components/ui/progress";
 import { Separator } from "@/components/ui/separator";
-import { getAuthHeaders } from "@/lib/auth-utils";
 import { GridPattern } from "./GridPattern";
 
 interface DocumentUploadTabProps {
@@ -25,9 +26,12 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) {
 	const t = useTranslations("upload_documents");
 	const router = useRouter();
 	const [files, setFiles] = useState<File[]>([]);
-	const [isUploading, setIsUploading] = useState(false);
 	const [uploadProgress, setUploadProgress] = useState(0);
 
+	// Use the uploadDocumentMutationAtom
+	const [uploadDocumentMutation] = useAtom(uploadDocumentMutationAtom);
+	const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation;
+
 	const audioFileTypes = {
 		"audio/mpeg": [".mp3", ".mpeg", ".mpga"],
 		"audio/mp4": [".mp4", ".m4a"],
@@ -148,53 +152,40 @@ export function DocumentUploadTab({ searchSpaceId }: DocumentUploadTabProps) {
 	};
 
 	const handleUpload = async () => {
-		setIsUploading(true);
 		setUploadProgress(0);
 
-		const formData = new FormData();
-		files.forEach((file) => {
-			formData.append("files", file);
-		});
-		formData.append("search_space_id", searchSpaceId);
+		// Create a progress interval to simulate progress
+		const progressInterval = setInterval(() => {
+			setUploadProgress((prev) => {
+				if (prev >= 90) return prev;
+				return prev + Math.random() * 10;
+			});
+		}, 200);
 
-		try {
-			const progressInterval = setInterval(() => {
-				setUploadProgress((prev) => {
-					if (prev >= 90) return prev;
-					return prev + Math.random() * 10;
-				});
-			}, 200);
-
-			const response = await fetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/fileupload`,
-				{
-					method: "POST",
-					headers: getAuthHeaders(),
-					body: formData,
-				}
-			);
-
-			clearInterval(progressInterval);
-			setUploadProgress(100);
-
-			if (!response.ok) {
-				throw new Error("Upload failed");
+		// Use the mutation to upload documents
+		uploadDocuments(
+			{
+				files,
+				search_space_id: Number(searchSpaceId),
+			},
+			{
+				onSuccess: () => {
+					clearInterval(progressInterval);
+					setUploadProgress(100);
+					toast(t("upload_initiated"), {
+						description: t("upload_initiated_desc"),
+					});
+					router.push(`/dashboard/${searchSpaceId}/documents`);
+				},
+				onError: (error: any) => {
+					clearInterval(progressInterval);
+					setUploadProgress(0);
+					toast(t("upload_error"), {
+						description: `${t("upload_error_desc")}: ${error.message || "Upload failed"}`,
+					});
+				},
 			}
-
-			await response.json();
-
-			toast(t("upload_initiated"), {
-				description: t("upload_initiated_desc"),
-			});
-
-			router.push(`/dashboard/${searchSpaceId}/documents`);
-		} catch (error: any) {
-			setIsUploading(false);
-			setUploadProgress(0);
-			toast(t("upload_error"), {
-				description: `${t("upload_error_desc")}: ${error.message}`,
-			});
-		}
+		);
 	};
 
 	const getTotalFileSize = () => {

From 034e42e15ec28598be342b27deb19f02c46f15d0 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 12:38:51 +0000
Subject: [PATCH 25/36] refactor: update SourceDetailSheet to use useQuery with
 centralized cache keys

- Replace useDocumentByChunk hook with useQuery implementation
- Use descriptive variable names (isDocumentByChunkFetching, documentByChunkFetchingError)
- Integrate with centralized cache key management
- Update all loading and error state references
- Add 5-minute stale time for document queries
---
 .../components/chat/SourceDetailSheet.tsx        | 16 +++++++++++-----
 surfsense_web/lib/apis/base-api.service.ts       | 15 +++++++++++++--
 surfsense_web/lib/apis/documents-api.service.ts  | 10 ----------
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx
index 0e48043b5..1b227b8df 100644
--- a/surfsense_web/components/chat/SourceDetailSheet.tsx
+++ b/surfsense_web/components/chat/SourceDetailSheet.tsx
@@ -1,11 +1,9 @@
 "use client";
 
+import { useQuery } from "@tanstack/react-query";
 import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react";
 import type React from "react";
 import { type ReactNode, useEffect, useRef, useState } from "react";
-import { useQuery } from "@tanstack/react-query";
-import { documentsApiService } from "@/lib/apis/documents-api.service";
-import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { MarkdownViewer } from "@/components/markdown-viewer";
 import { Button } from "@/components/ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
@@ -18,6 +16,8 @@ import {
 	SheetTitle,
 } from "@/components/ui/sheet";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { cn } from "@/lib/utils";
 
 interface SourceDetailSheetProps {
@@ -53,7 +53,11 @@ export function SourceDetailSheet({
 	const [summaryOpen, setSummaryOpen] = useState(false);
 
 	// Add useQuery to fetch document by chunk
-	const { data: document, isLoading: isDocumentByChunkFetching, error: documentByChunkFetchingError } = useQuery({
+	const {
+		data: document,
+		isLoading: isDocumentByChunkFetching,
+		error: documentByChunkFetchingError,
+	} = useQuery({
 		queryKey: cacheKeys.documents.byChunk(chunkId.toString()),
 		queryFn: () => documentsApiService.getDocumentByChunk({ chunk_id: chunkId }),
 		enabled: !!chunkId && open,
@@ -109,7 +113,9 @@ export function SourceDetailSheet({
 
 				{!isDirectRenderSource && documentByChunkFetchingError && (
 					<div className="flex items-center justify-center h-64 px-6">
-						<p className="text-sm text-destructive">{documentByChunkFetchingError.message || "Failed to load document"}</p>
+						<p className="text-sm text-destructive">
+							{documentByChunkFetchingError.message || "Failed to load document"}
+						</p>
 					</div>
 				)}
 
diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts
index e4b20faa4..e84e43be2 100644
--- a/surfsense_web/lib/apis/base-api.service.ts
+++ b/surfsense_web/lib/apis/base-api.service.ts
@@ -58,7 +58,6 @@ class BaseApiService {
 			 */
 			const defaultOptions: RequestOptions = {
 				headers: {
-					"Content-Type": "application/json",
 					Authorization: `Bearer ${this.bearerToken || ""}`,
 				},
 				method: "GET",
@@ -211,8 +210,11 @@ class BaseApiService {
 		options?: Omit<RequestOptions, "method" | "responseType">
 	) {
 		return this.request(url, responseSchema, {
-			...options,
 			method: "GET",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			...options,
 			responseType: ResponseType.JSON,
 		});
 	}
@@ -224,6 +226,9 @@ class BaseApiService {
 	) {
 		return this.request(url, responseSchema, {
 			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
 			...options,
 			responseType: ResponseType.JSON,
 		});
@@ -236,6 +241,9 @@ class BaseApiService {
 	) {
 		return this.request(url, responseSchema, {
 			method: "PUT",
+			headers: {
+				"Content-Type": "application/json",
+			},
 			...options,
 			responseType: ResponseType.JSON,
 		});
@@ -248,6 +256,9 @@ class BaseApiService {
 	) {
 		return this.request(url, responseSchema, {
 			method: "DELETE",
+			headers: {
+				"Content-Type": "application/json",
+			},
 			...options,
 			responseType: ResponseType.JSON,
 		});
diff --git a/surfsense_web/lib/apis/documents-api.service.ts b/surfsense_web/lib/apis/documents-api.service.ts
index 554b40b60..5a82da439 100644
--- a/surfsense_web/lib/apis/documents-api.service.ts
+++ b/surfsense_web/lib/apis/documents-api.service.ts
@@ -35,13 +35,11 @@ class DocumentsApiService {
 	 * Get a list of documents with optional filtering and pagination
 	 */
 	getDocuments = async (request: GetDocumentsRequest) => {
-	
 		const parsedRequest = getDocumentsRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
@@ -70,13 +68,11 @@ class DocumentsApiService {
 	 * Get a single document by ID
 	 */
 	getDocument = async (request: GetDocumentRequest) => {
-		
 		const parsedRequest = getDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
@@ -88,13 +84,11 @@ class DocumentsApiService {
 	 * Create documents (extension, crawled URL, or YouTube video)
 	 */
 	createDocument = async (request: CreateDocumentRequest) => {
-		
 		const parsedRequest = createDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
@@ -108,13 +102,11 @@ class DocumentsApiService {
 	 * Upload document files
 	 */
 	uploadDocument = async (request: UploadDocumentRequest) => {
-		
 		const parsedRequest = uploadDocumentRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}
@@ -135,13 +127,11 @@ class DocumentsApiService {
 	 * Search documents by title
 	 */
 	searchDocuments = async (request: SearchDocumentsRequest) => {
-		
 		const parsedRequest = searchDocumentsRequest.safeParse(request);
 
 		if (!parsedRequest.success) {
 			console.error("Invalid request:", parsedRequest.error);
 
-			
 			const errorMessage = parsedRequest.error.errors.map((err) => err.message).join(", ");
 			throw new ValidationError(`Invalid request: ${errorMessage}`);
 		}

From 5ad9e718d1d1666ab597de90817841299fa6a0b7 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 13:02:50 +0000
Subject: [PATCH 26/36] refactor: migrate YouTube tab to use jotai + tanstack
 query mutation

- Replace imperative authenticatedFetch with createDocumentMutationAtom
- Use mutation's isPending state instead of local isSubmitting state
- Maintain backward compatibility with existing UI behavior
- Keep all validation, error handling, and toast notifications
- Remove dependency on auth-utils for API calls
---
 .../components/sources/YouTubeTab.tsx         | 70 ++++++++-----------
 1 file changed, 31 insertions(+), 39 deletions(-)

diff --git a/surfsense_web/components/sources/YouTubeTab.tsx b/surfsense_web/components/sources/YouTubeTab.tsx
index 8301f51cc..0a51d0dca 100644
--- a/surfsense_web/components/sources/YouTubeTab.tsx
+++ b/surfsense_web/components/sources/YouTubeTab.tsx
@@ -7,7 +7,9 @@ import { motion } from "motion/react";
 import { useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useState } from "react";
+import { useAtom } from "jotai";
 import { toast } from "sonner";
+import { createDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
 
 import { Button } from "@/components/ui/button";
 import {
@@ -19,7 +21,6 @@ import {
 	CardTitle,
 } from "@/components/ui/card";
 import { Label } from "@/components/ui/label";
-import { authenticatedFetch } from "@/lib/auth-utils";
 
 const youtubeRegex =
 	/^(https:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/)([a-zA-Z0-9_-]{11})$/;
@@ -33,9 +34,12 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) {
 	const router = useRouter();
 	const [videoTags, setVideoTags] = useState<TagType[]>([]);
 	const [activeTagIndex, setActiveTagIndex] = useState<number | null>(null);
-	const [isSubmitting, setIsSubmitting] = useState(false);
 	const [error, setError] = useState<string | null>(null);
 
+	// Use the createDocumentMutationAtom
+	const [createDocumentMutation] = useAtom(createDocumentMutationAtom);
+	const { mutate: createYouTubeDocument, isPending: isSubmitting } = createDocumentMutation;
+
 	const isValidYoutubeUrl = (url: string): boolean => {
 		return youtubeRegex.test(url);
 	};
@@ -58,47 +62,35 @@ export function YouTubeTab({ searchSpaceId }: YouTubeTabProps) {
 		}
 
 		setError(null);
-		setIsSubmitting(true);
 
-		try {
-			toast(t("processing_toast"), {
-				description: t("processing_toast_desc"),
-			});
+		toast(t("processing_toast"), {
+			description: t("processing_toast_desc"),
+		});
 
-			const videoUrls = videoTags.map((tag) => tag.text);
+		const videoUrls = videoTags.map((tag) => tag.text);
 
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents`,
-				{
-					method: "POST",
-					headers: { "Content-Type": "application/json" },
-					body: JSON.stringify({
-						document_type: "YOUTUBE_VIDEO",
-						content: videoUrls,
-						search_space_id: parseInt(searchSpaceId),
-					}),
-				}
-			);
-
-			if (!response.ok) {
-				throw new Error("Failed to process YouTube videos");
+		// Use the mutation to create YouTube documents
+		createYouTubeDocument(
+			{
+				document_type: "YOUTUBE_VIDEO",
+				content: videoUrls,
+				search_space_id: parseInt(searchSpaceId),
+			},
+			{
+				onSuccess: () => {
+					toast(t("success_toast"), {
+						description: t("success_toast_desc"),
+					});
+					router.push(`/dashboard/${searchSpaceId}/documents`);
+				},
+				onError: (error: any) => {
+					setError(error.message || t("error_generic"));
+					toast(t("error_toast"), {
+						description: `${t("error_toast_desc")}: ${error.message || "Failed to process YouTube videos"}`,
+					});
+				},
 			}
-
-			await response.json();
-
-			toast(t("success_toast"), {
-				description: t("success_toast_desc"),
-			});
-
-			router.push(`/dashboard/${searchSpaceId}/documents`);
-		} catch (error: any) {
-			setError(error.message || t("error_generic"));
-			toast(t("error_toast"), {
-				description: `${t("error_toast_desc")}: ${error.message}`,
-			});
-		} finally {
-			setIsSubmitting(false);
-		}
+		);
 	};
 
 	const handleAddTag = (text: string) => {

From e5d1598270891f339f8411ef528c0643a8bae1c3 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 13:12:15 +0000
Subject: [PATCH 27/36] refactor: remove unused use-document-by-chunk hook

- Delete use-document-by-chunk.ts file
- Remove export from hooks index
- Hook has been replaced with useQuery implementation in SourceDetailSheet
---
 surfsense_web/hooks/index.ts                 |   1 -
 surfsense_web/hooks/use-document-by-chunk.ts | 106 -------------------
 2 files changed, 107 deletions(-)
 delete mode 100644 surfsense_web/hooks/use-document-by-chunk.ts

diff --git a/surfsense_web/hooks/index.ts b/surfsense_web/hooks/index.ts
index a244609a2..2cea293e8 100644
--- a/surfsense_web/hooks/index.ts
+++ b/surfsense_web/hooks/index.ts
@@ -1,4 +1,3 @@
-export * from "./use-document-by-chunk";
 export * from "./use-logs";
 export * from "./use-rbac";
 export * from "./use-search-source-connectors";
diff --git a/surfsense_web/hooks/use-document-by-chunk.ts b/surfsense_web/hooks/use-document-by-chunk.ts
deleted file mode 100644
index 630e810a2..000000000
--- a/surfsense_web/hooks/use-document-by-chunk.ts
+++ /dev/null
@@ -1,106 +0,0 @@
-"use client";
-import { useCallback, useState } from "react";
-import { toast } from "sonner";
-import { authenticatedFetch } from "@/lib/auth-utils";
-
-export interface Chunk {
-	id: number;
-	content: string;
-	document_id: number;
-	created_at: string;
-}
-
-export interface DocumentWithChunks {
-	id: number;
-	title: string;
-	document_type: DocumentType;
-	document_metadata: any;
-	content: string;
-	created_at: string;
-	search_space_id: number;
-	chunks: Chunk[];
-}
-
-export type DocumentType =
-	| "EXTENSION"
-	| "CRAWLED_URL"
-	| "SLACK_CONNECTOR"
-	| "NOTION_CONNECTOR"
-	| "FILE"
-	| "YOUTUBE_VIDEO"
-	| "GITHUB_CONNECTOR"
-	| "LINEAR_CONNECTOR"
-	| "DISCORD_CONNECTOR"
-	| "JIRA_CONNECTOR"
-	| "CONFLUENCE_CONNECTOR"
-	| "CLICKUP_CONNECTOR"
-	| "GOOGLE_CALENDAR_CONNECTOR"
-	| "GOOGLE_GMAIL_CONNECTOR"
-	| "LUMA_CONNECTOR"
-	| "ELASTICSEARCH_CONNECTOR";
-
-export function useDocumentByChunk() {
-	const [document, setDocument] = useState<DocumentWithChunks | null>(null);
-	const [loading, setLoading] = useState(false);
-	const [error, setError] = useState<string | null>(null);
-
-	const fetchDocumentByChunk = useCallback(async (chunkId: number) => {
-		try {
-			setLoading(true);
-			setError(null);
-			setDocument(null);
-
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/by-chunk/${chunkId}`,
-				{
-					headers: { "Content-Type": "application/json" },
-					method: "GET",
-				}
-			);
-
-			if (!response.ok) {
-				const errorText = await response.text();
-				let errorMessage = "Failed to fetch document";
-
-				try {
-					const errorData = JSON.parse(errorText);
-					errorMessage = errorData.detail || errorMessage;
-				} catch {
-					// If parsing fails, use default message
-				}
-
-				if (response.status === 404) {
-					errorMessage = "Chunk not found or you don't have access to it";
-				}
-
-				toast.error(errorMessage);
-				throw new Error(errorMessage);
-			}
-
-			const data: DocumentWithChunks = await response.json();
-			setDocument(data);
-			setError(null);
-			return data;
-		} catch (err: any) {
-			const errorMessage = err.message || "Failed to fetch document";
-			setError(errorMessage);
-			console.error("Error fetching document by chunk:", err);
-			throw err;
-		} finally {
-			setLoading(false);
-		}
-	}, []);
-
-	const clearDocument = useCallback(() => {
-		setDocument(null);
-		setError(null);
-	}, []);
-
-	return {
-		document,
-		loading,
-		error,
-		fetchDocumentByChunk,
-		clearDocument,
-	};
-}

From bccbd6533376a62a8fe27995c7d8fd2292115211 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Thu, 4 Dec 2025 13:40:08 +0000
Subject: [PATCH 28/36] refactor: migrate document types fetch to jotai +
 tanstack query

- Replace useDocumentTypes hook with documentTypeCountsAtom in ChatInputGroup
- Replace useDocumentTypes hook with documentTypeCountsAtom in researcher page
- Delete obsolete use-document-types.ts hook
- Transform atom response to maintain backward compatibility
---
 .../researcher/[[...chat_id]]/page.tsx        | 18 +++-
 .../components/chat/ChatInputGroup.tsx        | 24 +++--
 surfsense_web/hooks/use-document-types.ts     | 90 -------------------
 3 files changed, 32 insertions(+), 100 deletions(-)
 delete mode 100644 surfsense_web/hooks/use-document-types.ts

diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx
index 1a9a607fb..a9d8b9649 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx
@@ -1,15 +1,15 @@
 "use client";
 
 import { type CreateMessage, type Message, useChat } from "@ai-sdk/react";
-import { useAtomValue } from "jotai";
+import { useAtom, useAtomValue } from "jotai";
 import { useParams, useRouter } from "next/navigation";
 import { useEffect, useMemo, useRef } from "react";
 import { createChatMutationAtom, updateChatMutationAtom } from "@/atoms/chats/chat-mutation.atoms";
 import { activeChatAtom } from "@/atoms/chats/chat-query.atoms";
 import { activeChatIdAtom } from "@/atoms/chats/ui.atoms";
+import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
 import ChatInterface from "@/components/chat/ChatInterface";
 import { useChatState } from "@/hooks/use-chat";
-import { useDocumentTypes } from "@/hooks/use-document-types";
 import type { Document } from "@/hooks/use-documents";
 import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
 
@@ -46,7 +46,19 @@ export default function ResearcherPage() {
 	});
 
 	// Fetch all available sources (document types + live search connectors)
-	const { documentTypes } = useDocumentTypes(Number(search_space_id));
+	// Use the documentTypeCountsAtom for fetching document types
+	const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom);
+	const { data: documentTypeCountsData } = documentTypeCountsQuery;
+
+	// Transform the response into the expected format
+	const documentTypes = useMemo(() => {
+		if (!documentTypeCountsData) return [];
+		return Object.entries(documentTypeCountsData).map(([type, count]) => ({
+			type,
+			count,
+		}));
+	}, [documentTypeCountsData]);
+
 	const { connectors: searchConnectors } = useSearchSourceConnectors(
 		false,
 		Number(search_space_id)
diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx
index 7a76c4d56..4e63048d1 100644
--- a/surfsense_web/components/chat/ChatInputGroup.tsx
+++ b/surfsense_web/components/chat/ChatInputGroup.tsx
@@ -3,7 +3,9 @@
 import { ChatInput } from "@llamaindex/chat-ui";
 import { Brain, Check, FolderOpen, Minus, Plus, PlusCircle, Zap } from "lucide-react";
 import { useParams, useRouter } from "next/navigation";
-import React, { Suspense, useCallback, useState } from "react";
+import React, { Suspense, useCallback, useState, useMemo } from "react";
+import { useAtom } from "jotai";
+import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
 import { DocumentsDataTable } from "@/components/chat/DocumentsDataTable";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -25,7 +27,6 @@ import {
 } from "@/components/ui/select";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { useDocumentTypes } from "@/hooks/use-document-types";
 import type { Document } from "@/hooks/use-documents";
 import { useGlobalLLMConfigs, useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs";
 import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors";
@@ -118,11 +119,20 @@ const ConnectorSelector = React.memo(
 		const router = useRouter();
 		const [isOpen, setIsOpen] = useState(false);
 
-		// Fetch immediately (not lazy) so the button can show the correct count
-		const { documentTypes, isLoading, isLoaded, fetchDocumentTypes } = useDocumentTypes(
-			Number(search_space_id),
-			false
-		);
+		// Use the documentTypeCountsAtom for fetching document types
+		const [documentTypeCountsQuery] = useAtom(documentTypeCountsAtom);
+		const { data: documentTypeCountsData, isLoading, refetch: fetchDocumentTypes } = documentTypeCountsQuery;
+
+		// Transform the response into the expected format
+		const documentTypes = useMemo(() => {
+			if (!documentTypeCountsData) return [];
+			return Object.entries(documentTypeCountsData).map(([type, count]) => ({
+				type,
+				count,
+			}));
+		}, [documentTypeCountsData]);
+
+		const isLoaded = !!documentTypeCountsData;
 
 		// Fetch live search connectors immediately (non-indexable)
 		const {
diff --git a/surfsense_web/hooks/use-document-types.ts b/surfsense_web/hooks/use-document-types.ts
deleted file mode 100644
index 21c9eb6fe..000000000
--- a/surfsense_web/hooks/use-document-types.ts
+++ /dev/null
@@ -1,90 +0,0 @@
-import { useCallback, useEffect, useState } from "react";
-import { authenticatedFetch } from "@/lib/auth-utils";
-
-export interface DocumentTypeCount {
-	type: string;
-	count: number;
-}
-
-/**
- * Hook to fetch document type counts from the API
- * @param searchSpaceId - The search space ID to filter document types
- * @param lazy - If true, types won't be fetched on mount
- */
-export const useDocumentTypes = (searchSpaceId?: number, lazy: boolean = false) => {
-	const [documentTypes, setDocumentTypes] = useState<DocumentTypeCount[]>([]);
-	const [isLoading, setIsLoading] = useState(!lazy);
-	const [isLoaded, setIsLoaded] = useState(false);
-	const [error, setError] = useState<Error | null>(null);
-
-	const fetchDocumentTypes = useCallback(
-		async (spaceId?: number) => {
-			if (isLoaded && lazy) return;
-
-			try {
-				setIsLoading(true);
-				setError(null);
-
-				// Build URL with optional search_space_id query parameter
-				const url = new URL(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts`
-				);
-				if (spaceId !== undefined) {
-					url.searchParams.append("search_space_id", spaceId.toString());
-				}
-
-				const response = await authenticatedFetch(url.toString(), {
-					method: "GET",
-					headers: { "Content-Type": "application/json" },
-				});
-
-				if (!response.ok) {
-					throw new Error(`Failed to fetch document types: ${response.statusText}`);
-				}
-
-				const data = await response.json();
-
-				// Convert the object to an array of DocumentTypeCount
-				const typeCounts: DocumentTypeCount[] = Object.entries(data).map(([type, count]) => ({
-					type,
-					count: count as number,
-				}));
-
-				setDocumentTypes(typeCounts);
-				setIsLoaded(true);
-
-				return typeCounts;
-			} catch (err) {
-				setError(err instanceof Error ? err : new Error("An unknown error occurred"));
-				console.error("Error fetching document types:", err);
-			} finally {
-				setIsLoading(false);
-			}
-		},
-		[isLoaded, lazy]
-	);
-
-	useEffect(() => {
-		if (!lazy) {
-			fetchDocumentTypes(searchSpaceId);
-		}
-	}, [lazy, fetchDocumentTypes, searchSpaceId]);
-
-	// Function to refresh the document types
-	const refreshDocumentTypes = useCallback(
-		async (spaceId?: number) => {
-			setIsLoaded(false);
-			await fetchDocumentTypes(spaceId !== undefined ? spaceId : searchSpaceId);
-		},
-		[fetchDocumentTypes, searchSpaceId]
-	);
-
-	return {
-		documentTypes,
-		isLoading,
-		isLoaded,
-		error,
-		fetchDocumentTypes,
-		refreshDocumentTypes,
-	};
-};

From 42e10bbe556d329685d9d22cccf80356062935c4 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 08:41:29 +0000
Subject: [PATCH 29/36] feat: add useQuery implementation for document fetching
 in DocumentsDataTable with 3-minute stale time

---
 .../components/chat/DocumentsDataTable.tsx    | 126 ++++++++++--------
 .../contracts/types/document.types.ts         |   5 +-
 surfsense_web/lib/query-client/cache-keys.ts  |   1 +
 3 files changed, 77 insertions(+), 55 deletions(-)

diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx
index 331c1b404..48c8b130e 100644
--- a/surfsense_web/components/chat/DocumentsDataTable.tsx
+++ b/surfsense_web/components/chat/DocumentsDataTable.tsx
@@ -10,6 +10,7 @@ import {
 import { ArrowUpDown, Calendar, FileText, Filter, Plus, Search } from "lucide-react";
 import { useRouter } from "next/navigation";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useQuery } from "@tanstack/react-query";
 import { Button } from "@/components/ui/button";
 import { Checkbox } from "@/components/ui/checkbox";
 import { Input } from "@/components/ui/input";
@@ -32,6 +33,9 @@ import {
 } from "@/components/ui/table";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
 import { type Document, type DocumentType, useDocuments } from "@/hooks/use-documents";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { DocumentTypeEnum } from "@/contracts/types/document.types";
 
 interface DocumentsDataTableProps {
 	searchSpaceId: number;
@@ -182,18 +186,62 @@ export function DocumentsDataTable({
 	const [sorting, setSorting] = useState<SortingState>([]);
 	const [search, setSearch] = useState("");
 	const debouncedSearch = useDebounced(search, 300);
-	const [documentTypeFilter, setDocumentTypeFilter] = useState<string[]>([]);
+	const [documentTypeFilter, setDocumentTypeFilter] = useState<DocumentTypeEnum[]>([]);
 	const [pageIndex, setPageIndex] = useState(0);
 	const [pageSize, setPageSize] = useState(10);
 	const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
 
+	const fetchQueryParams = useMemo(
+		() => ({
+			search_space_id: searchSpaceId,
+			page: pageIndex ,
+			page_size: pageSize,
+			...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }),
+		}),
+		[searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch]
+	);
+
+	const searchQueryParams = useMemo(() => {
+		return {
+			...fetchQueryParams,
+			title : debouncedSearch,
+		}
+	},[debouncedSearch])
+
+	// Use query for fetching documents
+	const {
+		data: documents,
+		isLoading: isDocumentsLoading,
+	} = useQuery({
+		queryKey: cacheKeys.documents.withQueryParams(fetchQueryParams),
+		queryFn: () => documentsApiService.getDocuments({ queryParams : fetchQueryParams }),
+		staleTime: 3 * 60 * 1000, // 3 minutes
+		enabled: !!searchSpaceId && !debouncedSearch.trim(), 
+	});
+
+	// Seaching
+		const {
+		data: searchedDocuments,
+		isLoading: isSearchedDocumentsLoading,
+	} = useQuery({
+		queryKey: cacheKeys.documents.withQueryParams(searchQueryParams),
+		queryFn: () => documentsApiService.searchDocuments({ queryParams : searchQueryParams }),
+		staleTime: 3 * 60 * 1000, // 3 minutes
+		enabled: !!searchSpaceId && !!debouncedSearch.trim(),
+	});
+
 	// Use server-side pagination, search, and filtering
-	const { documents, total, loading, fetchDocuments, searchDocuments, getDocumentTypeCounts } =
+	const { getDocumentTypeCounts } =
 		useDocuments(searchSpaceId, {
 			page: pageIndex,
 			pageSize: pageSize,
 		});
 
+	// Use query data when not searching, otherwise use hook data
+	const actualDocuments = debouncedSearch.trim() ? searchedDocuments?.items|| [] : documents?.items|| [];
+	const actualTotal = debouncedSearch.trim() ? searchedDocuments?.total || 0 : documents?.total || 0;
+	const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading;
+
 	// Fetch document type counts on mount
 	useEffect(() => {
 		if (searchSpaceId && getDocumentTypeCounts) {
@@ -201,34 +249,6 @@ export function DocumentsDataTable({
 		}
 	}, [searchSpaceId, getDocumentTypeCounts]);
 
-	// Refetch when pagination changes or when search/filters change
-	useEffect(() => {
-		if (searchSpaceId) {
-			if (debouncedSearch.trim()) {
-				searchDocuments?.(
-					debouncedSearch,
-					pageIndex,
-					pageSize,
-					documentTypeFilter.length > 0 ? documentTypeFilter : undefined
-				);
-			} else {
-				fetchDocuments?.(
-					pageIndex,
-					pageSize,
-					documentTypeFilter.length > 0 ? documentTypeFilter : undefined
-				);
-			}
-		}
-	}, [
-		pageIndex,
-		pageSize,
-		debouncedSearch,
-		documentTypeFilter,
-		searchSpaceId,
-		fetchDocuments,
-		searchDocuments,
-	]);
-
 	// Memoize initial row selection to prevent infinite loops
 	const initialRowSelection = useMemo(() => {
 		if (!initialSelectedDocuments.length) return {};
@@ -272,14 +292,14 @@ export function DocumentsDataTable({
 
 	// Update the selected documents map when row selection changes
 	useEffect(() => {
-		if (!documents || documents.length === 0) return;
+		if (!actualDocuments || actualDocuments.length === 0) return;
 
 		setSelectedDocumentsMap((prev) => {
 			const newMap = new Map(prev);
 			let hasChanges = false;
 
 			// Process only current page documents
-			for (const doc of documents) {
+			for (const doc of actualDocuments) {
 				const docId = doc.id;
 				const isSelected = rowSelection[docId.toString()];
 				const wasInMap = newMap.has(docId);
@@ -319,14 +339,14 @@ export function DocumentsDataTable({
 	}, [selectedDocumentsArray, onSelectionChange]);
 
 	const table = useReactTable({
-		data: documents || [],
+		data: actualDocuments || [],
 		columns,
 		getRowId: (row) => row.id.toString(),
 		onSortingChange: setSorting,
 		getCoreRowModel: getCoreRowModel(),
 		onRowSelectionChange: setRowSelection,
 		manualPagination: true,
-		pageCount: Math.ceil(total / pageSize),
+		pageCount: Math.ceil(actualTotal / pageSize),
 		state: { sorting, rowSelection, pagination: { pageIndex, pageSize } },
 	});
 
@@ -344,7 +364,7 @@ export function DocumentsDataTable({
 		setRowSelection(newSelection);
 	}, [table, rowSelection]);
 
-	const handleToggleType = useCallback((type: string, checked: boolean) => {
+	const handleToggleType = useCallback((type: DocumentTypeEnum, checked: boolean) => {
 		setDocumentTypeFilter((prev) => {
 			if (checked) {
 				return [...prev, type];
@@ -358,7 +378,7 @@ export function DocumentsDataTable({
 
 	// Get available document types from type counts (memoized)
 	const availableTypes = useMemo(() => {
-		const types = Object.keys(typeCounts);
+		const types = Object.keys(typeCounts) as DocumentTypeEnum[];
 		return types.length > 0 ? types.sort() : [];
 	}, [typeCounts]);
 
@@ -435,7 +455,7 @@ export function DocumentsDataTable({
 				<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3">
 					<div className="flex flex-col sm:flex-row sm:items-center gap-2">
 						<span className="text-sm text-muted-foreground whitespace-nowrap">
-							{selectedCount} selected {loading && "· Loading..."}
+							{selectedCount} selected {actualLoading && "· Loading..."}
 						</span>
 						<div className="hidden sm:block h-4 w-px bg-border mx-2" />
 						<div className="flex items-center gap-2 flex-wrap">
@@ -453,7 +473,7 @@ export function DocumentsDataTable({
 								size="sm"
 								onClick={handleSelectPage}
 								className="text-xs sm:text-sm"
-								disabled={loading}
+								disabled={actualLoading}
 							>
 								Select Page
 							</Button>
@@ -490,7 +510,7 @@ export function DocumentsDataTable({
 			{/* Table Container */}
 			<div className="border rounded-lg flex-1 min-h-0 overflow-hidden bg-background">
 				<div className="overflow-auto h-full">
-					{loading ? (
+					{actualLoading ? (
 						<div className="flex items-center justify-center h-full">
 							<div className="text-center space-y-2">
 								<div className="animate-spin h-8 w-8 border-2 border-primary border-t-transparent rounded-full mx-auto" />
@@ -561,31 +581,31 @@ export function DocumentsDataTable({
 			{/* Footer Pagination */}
 			<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 text-xs sm:text-sm text-muted-foreground border-t pt-3 md:pt-4 flex-shrink-0">
 				<div className="text-center sm:text-left">
-					Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, total)} of{" "}
-					{total} documents
+					Showing {pageIndex * pageSize + 1} to {Math.min((pageIndex + 1) * pageSize, actualTotal)} of{" "}
+					{actualTotal} documents
 				</div>
 				<div className="flex items-center justify-center sm:justify-end space-x-2">
 					<Button
 						variant="outline"
-						size="sm"
-						onClick={() => setPageIndex((p) => Math.max(0, p - 1))}
-						disabled={pageIndex === 0 || loading}
-						className="text-xs sm:text-sm"
+					size="sm"
+					onClick={() => setPageIndex((p) => Math.max(0, p - 1))}
+					disabled={pageIndex === 0 || actualLoading}
+					className="text-xs sm:text-sm"
 					>
 						Previous
 					</Button>
 					<div className="flex items-center space-x-1 text-xs sm:text-sm">
 						<span>Page</span>
-						<strong>{pageIndex + 1}</strong>
-						<span>of</span>
-						<strong>{Math.ceil(total / pageSize)}</strong>
-					</div>
+					<strong>{pageIndex + 1}</strong>
+					<span>of</span>
+					<strong>{Math.ceil(actualTotal / pageSize)}</strong>
+				</div>
 					<Button
 						variant="outline"
-						size="sm"
-						onClick={() => setPageIndex((p) => p + 1)}
-						disabled={pageIndex >= Math.ceil(total / pageSize) - 1 || loading}
-						className="text-xs sm:text-sm"
+					size="sm"
+					onClick={() => setPageIndex((p) => p + 1)}
+					disabled={pageIndex >= Math.ceil(actualTotal / pageSize) - 1 || actualLoading}
+					className="text-xs sm:text-sm"
 					>
 						Next
 					</Button>
diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index c5d777080..941ed5052 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -60,7 +60,7 @@ export const getDocumentsRequest = z.object({
 	queryParams: paginationQueryParams
 		.extend({
 			search_space_id: z.number().or(z.string()).optional(),
-			document_type: z.array(documentTypeEnum).optional(),
+			document_types: z.array(documentTypeEnum).optional(),
 		})
 		.nullish(),
 });
@@ -109,7 +109,7 @@ export const searchDocumentsRequest = z.object({
 	queryParams: paginationQueryParams
 		.extend({
 			search_space_id: z.number().or(z.string()).optional(),
-			document_type: z.array(documentTypeEnum).optional(),
+			document_types: z.array(documentTypeEnum).optional(),
 			title: z.string().optional(),
 		})
 		.nullish(),
@@ -179,3 +179,4 @@ export type UpdateDocumentRequest = z.infer<typeof updateDocumentRequest>;
 export type UpdateDocumentResponse = z.infer<typeof updateDocumentResponse>;
 export type DeleteDocumentRequest = z.infer<typeof deleteDocumentRequest>;
 export type DeleteDocumentResponse = z.infer<typeof deleteDocumentResponse>;
+export type DocumentTypeEnum = z.infer<typeof documentTypeEnum>
diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts
index 813ba62cf..72fac6dc5 100644
--- a/surfsense_web/lib/query-client/cache-keys.ts
+++ b/surfsense_web/lib/query-client/cache-keys.ts
@@ -15,6 +15,7 @@ export const cacheKeys = {
 	documents: {
 		globalQueryParams: (queries: GetDocumentsRequest["queryParams"]) =>
 			["documents", ...(queries ? Object.values(queries) : [])] as const,
+		withQueryParams :(queries: GetDocumentsRequest["queryParams"]) => ["documents-with-queries", ...(queries ? Object.values(queries) : [])] as const,
 		document: (documentId: string) => ["document", documentId] as const,
 		typeCounts: (searchSpaceId?: string) => ["documents", "type-counts", searchSpaceId] as const,
 		byChunk: (chunkId: string) => ["documents", "by-chunk", chunkId] as const,

From e7982ead4ea43bcc14b1984f612c0012181becaa Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 09:10:04 +0000
Subject: [PATCH 30/36] feat: replace getDocumentTypeCounts with useQuery
 implementation (10min stale time)

---
 .../atoms/documents/document-query.atoms.ts   |  1 +
 .../components/chat/DocumentsDataTable.tsx    | 29 +++++++------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/surfsense_web/atoms/documents/document-query.atoms.ts b/surfsense_web/atoms/documents/document-query.atoms.ts
index 8caeafb3d..acdff9aaf 100644
--- a/surfsense_web/atoms/documents/document-query.atoms.ts
+++ b/surfsense_web/atoms/documents/document-query.atoms.ts
@@ -26,6 +26,7 @@ export const documentTypeCountsAtom = atomWithQuery((get) => {
 	return {
 		queryKey: cacheKeys.documents.typeCounts(searchSpaceId ?? undefined),
 		enabled: !!searchSpaceId,
+		staleTime: 10 * 60 * 1000, // 10 minutes
 		queryFn: async () => {
 			return documentsApiService.getDocumentTypeCounts({
 				queryParams: {
diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx
index 48c8b130e..e8298245b 100644
--- a/surfsense_web/components/chat/DocumentsDataTable.tsx
+++ b/surfsense_web/components/chat/DocumentsDataTable.tsx
@@ -36,6 +36,8 @@ import { type Document, type DocumentType, useDocuments } from "@/hooks/use-docu
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { DocumentTypeEnum } from "@/contracts/types/document.types";
+import { useAtomValue } from "jotai";
+import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
 
 interface DocumentsDataTableProps {
 	searchSpaceId: number;
@@ -189,7 +191,7 @@ export function DocumentsDataTable({
 	const [documentTypeFilter, setDocumentTypeFilter] = useState<DocumentTypeEnum[]>([]);
 	const [pageIndex, setPageIndex] = useState(0);
 	const [pageSize, setPageSize] = useState(10);
-	const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
+	const {data : typeCounts } = useAtomValue(documentTypeCountsAtom);
 
 	const fetchQueryParams = useMemo(
 		() => ({
@@ -203,10 +205,13 @@ export function DocumentsDataTable({
 
 	const searchQueryParams = useMemo(() => {
 		return {
-			...fetchQueryParams,
+			search_space_id: searchSpaceId,
+			page: pageIndex ,
+			page_size: pageSize,
+			...(documentTypeFilter.length > 0 && { document_types: documentTypeFilter }),
 			title : debouncedSearch,
 		}
-	},[debouncedSearch])
+	},[debouncedSearch, searchSpaceId, pageIndex, pageSize, documentTypeFilter, debouncedSearch])
 
 	// Use query for fetching documents
 	const {
@@ -230,25 +235,13 @@ export function DocumentsDataTable({
 		enabled: !!searchSpaceId && !!debouncedSearch.trim(),
 	});
 
-	// Use server-side pagination, search, and filtering
-	const { getDocumentTypeCounts } =
-		useDocuments(searchSpaceId, {
-			page: pageIndex,
-			pageSize: pageSize,
-		});
+
 
 	// Use query data when not searching, otherwise use hook data
 	const actualDocuments = debouncedSearch.trim() ? searchedDocuments?.items|| [] : documents?.items|| [];
 	const actualTotal = debouncedSearch.trim() ? searchedDocuments?.total || 0 : documents?.total || 0;
 	const actualLoading = debouncedSearch.trim() ? isSearchedDocumentsLoading : isDocumentsLoading;
 
-	// Fetch document type counts on mount
-	useEffect(() => {
-		if (searchSpaceId && getDocumentTypeCounts) {
-			getDocumentTypeCounts().then(setTypeCounts);
-		}
-	}, [searchSpaceId, getDocumentTypeCounts]);
-
 	// Memoize initial row selection to prevent infinite loops
 	const initialRowSelection = useMemo(() => {
 		if (!initialSelectedDocuments.length) return {};
@@ -378,7 +371,7 @@ export function DocumentsDataTable({
 
 	// Get available document types from type counts (memoized)
 	const availableTypes = useMemo(() => {
-		const types = Object.keys(typeCounts) as DocumentTypeEnum[];
+		const types = typeCounts ? Object.keys(typeCounts) as DocumentTypeEnum[] : [];
 		return types.length > 0 ? types.sort() : [];
 	}, [typeCounts]);
 
@@ -428,7 +421,7 @@ export function DocumentsDataTable({
 												className="flex grow justify-between gap-2 font-normal text-sm cursor-pointer"
 											>
 												<span>{type.replace(/_/g, " ")}</span>
-												<span className="text-xs text-muted-foreground">{typeCounts[type]}</span>
+												<span className="text-xs text-muted-foreground">{typeCounts?.[type]}</span>
 											</Label>
 										</div>
 									))}

From b8d629da71d8ab8ad08a52d20ff3699b1dc234f5 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 09:49:03 +0000
Subject: [PATCH 31/36] fix: update DocumentsFilters type definitions and
 complete migration cleanup

- Updated type definitions in DocumentsFilters to use DocumentTypeEnum
- Fixed type consistency across activeTypes and onToggleType props
- Final cleanup of DocumentsTable migration to React Query
---
 .../(manage)/components/DocumentsFilters.tsx  |  11 +-
 .../documents/(manage)/page.tsx               | 120 +++++++-----
 .../components/chat/DocumentsDataTable.tsx    |   2 +-
 surfsense_web/hooks/use-documents.ts          | 184 +-----------------
 4 files changed, 84 insertions(+), 233 deletions(-)

diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx
index 978cdf219..a0f079e7d 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx
@@ -28,6 +28,7 @@ import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
 import type { ColumnVisibility } from "./types";
+import { DocumentTypeEnum } from "@/contracts/types/document.types";
 
 const fadeInScale: Variants = {
 	hidden: { opacity: 0, scale: 0.95 },
@@ -46,13 +47,13 @@ export function DocumentsFilters({
 	columnVisibility,
 	onToggleColumn,
 }: {
-	typeCounts: Record<string, number>;
+	typeCounts: Record<DocumentTypeEnum, number>;
 	selectedIds: Set<number>;
 	onSearch: (v: string) => void;
 	searchValue: string;
 	onBulkDelete: () => Promise<void>;
-	onToggleType: (type: string, checked: boolean) => void;
-	activeTypes: string[];
+	onToggleType: (type: DocumentTypeEnum, checked: boolean) => void;
+	activeTypes: DocumentTypeEnum[];
 	columnVisibility: ColumnVisibility;
 	onToggleColumn: (id: keyof ColumnVisibility, checked: boolean) => void;
 }) {
@@ -61,7 +62,7 @@ export function DocumentsFilters({
 	const inputRef = useRef<HTMLInputElement>(null);
 
 	const uniqueTypes = useMemo(() => {
-		return Object.keys(typeCountsRecord).sort();
+		return Object.keys(typeCountsRecord).sort() as DocumentTypeEnum[];
 	}, [typeCountsRecord]);
 
 	const typeCounts = useMemo(() => {
@@ -156,7 +157,7 @@ export function DocumentsFilters({
 								<div className="text-xs font-medium text-muted-foreground">Filters</div>
 								<div className="space-y-3">
 									<AnimatePresence>
-										{uniqueTypes.map((value, i) => (
+										{uniqueTypes.map((value : DocumentTypeEnum, i) => (
 											<motion.div
 												key={value}
 												className="flex items-center gap-2"
diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
index da07ca4af..151bb54ce 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@@ -5,6 +5,11 @@ import { useParams } from "next/navigation";
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useId, useMemo, useState } from "react";
 import { toast } from "sonner";
+import { useQuery } from "@tanstack/react-query";
+import { useAtomValue } from "jotai";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
+import { cacheKeys } from "@/lib/query-client/cache-keys";
+import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
 
 import { useDocuments } from "@/hooks/use-documents";
 
@@ -12,6 +17,7 @@ import { DocumentsFilters } from "./components/DocumentsFilters";
 import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
 import { PaginationControls } from "./components/PaginationControls";
 import type { ColumnVisibility, Document } from "./components/types";
+import { DocumentTypeEnum } from "@/contracts/types/document.types";
 
 function useDebounced<T>(value: T, delay = 250) {
 	const [debounced, setDebounced] = useState(value);
@@ -30,7 +36,7 @@ export default function DocumentsTable() {
 
 	const [search, setSearch] = useState("");
 	const debouncedSearch = useDebounced(search, 250);
-	const [activeTypes, setActiveTypes] = useState<string[]>([]);
+	const [activeTypes, setActiveTypes] = useState<DocumentTypeEnum[]>([]);
 	const [columnVisibility, setColumnVisibility] = useState<ColumnVisibility>({
 		title: true,
 		document_type: true,
@@ -42,63 +48,80 @@ export default function DocumentsTable() {
 	const [sortKey, setSortKey] = useState<SortKey>("title");
 	const [sortDesc, setSortDesc] = useState(false);
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
-	const [typeCounts, setTypeCounts] = useState<Record<string, number>>({});
+	const {data: typeCounts} = useAtomValue(documentTypeCountsAtom) ;
+
+	// Build query parameters for fetching documents
+	const queryParams = useMemo(
+		() => ({
+			search_space_id: searchSpaceId,
+			page: pageIndex,
+			page_size: pageSize,
+			...(activeTypes.length > 0 && { document_types: activeTypes }),
+		}),
+		[searchSpaceId, pageIndex, pageSize, activeTypes]
+	);
+
+	// Build search query parameters
+	const searchQueryParams = useMemo(
+		() => ({
+			search_space_id: searchSpaceId,
+			page: pageIndex,
+			page_size: pageSize,
+			title: debouncedSearch.trim(),
+			...(activeTypes.length > 0 && { document_types: activeTypes }),
+		}),
+		[searchSpaceId, pageIndex, pageSize, activeTypes, debouncedSearch]
+	);
+
+	// Use query for fetching documents
+	const {
+		data: documentsResponse,
+		isLoading: isDocumentsLoading,
+		refetch: refetchDocuments,
+	} = useQuery({
+		queryKey: cacheKeys.documents.globalQueryParams(queryParams),
+		queryFn: () => documentsApiService.getDocuments({ queryParams }),
+		staleTime: 3 * 60 * 1000, // 3 minutes
+		enabled: !!searchSpaceId && !debouncedSearch.trim(),
+	});
+
+	// Use query for searching documents
+	const {
+		data: searchResponse,
+		isLoading: isSearchLoading,
+		refetch: refetchSearch,
+	} = useQuery({
+		queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
+		queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
+		staleTime: 3 * 60 * 1000, // 3 minutes
+		enabled: !!searchSpaceId && !!debouncedSearch.trim(),
+	});
+
+	// Extract documents and total based on search state
+	const documents = debouncedSearch.trim() 
+		? searchResponse?.items || [] 
+		: documentsResponse?.items || [];
+	const total = debouncedSearch.trim() 
+		? searchResponse?.total || 0 
+		: documentsResponse?.total || 0;
+	const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
 
 	// Use server-side pagination, search, and filtering
 	const {
-		documents,
-		total,
-		loading,
 		error,
-		fetchDocuments,
-		searchDocuments,
 		deleteDocument,
-		getDocumentTypeCounts,
 	} = useDocuments(searchSpaceId, {
 		page: pageIndex,
 		pageSize: pageSize,
 	});
 
-	// Fetch document type counts on mount and when search space changes
-	useEffect(() => {
-		if (searchSpaceId && getDocumentTypeCounts) {
-			getDocumentTypeCounts().then(setTypeCounts);
-		}
-	}, [searchSpaceId, getDocumentTypeCounts]);
-
-	// Refetch when pagination changes or when search/filters change
-	useEffect(() => {
-		if (searchSpaceId) {
-			if (debouncedSearch.trim()) {
-				// Use search endpoint if there's a search query
-				searchDocuments?.(
-					debouncedSearch,
-					pageIndex,
-					pageSize,
-					activeTypes.length > 0 ? activeTypes : undefined
-				);
-			} else {
-				// Use regular fetch if no search
-				fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined);
-			}
-		}
-	}, [
-		pageIndex,
-		pageSize,
-		debouncedSearch,
-		activeTypes,
-		searchSpaceId,
-		fetchDocuments,
-		searchDocuments,
-	]);
-
 	// Display server-filtered results directly
 	const displayDocs = documents || [];
 	const displayTotal = total;
 	const pageStart = pageIndex * pageSize;
 	const pageEnd = Math.min(pageStart + pageSize, displayTotal);
 
-	const onToggleType = (type: string, checked: boolean) => {
+	const onToggleType = (type: DocumentTypeEnum, checked: boolean) => {
 		setActiveTypes((prev) => (checked ? [...prev, type] : prev.filter((t) => t !== type)));
 		setPageIndex(0);
 	};
@@ -109,16 +132,11 @@ export default function DocumentsTable() {
 
 	const refreshCurrentView = useCallback(async () => {
 		if (debouncedSearch.trim()) {
-			await searchDocuments?.(
-				debouncedSearch,
-				pageIndex,
-				pageSize,
-				activeTypes.length > 0 ? activeTypes : undefined
-			);
+			await refetchSearch();
 		} else {
-			await fetchDocuments?.(pageIndex, pageSize, activeTypes.length > 0 ? activeTypes : undefined);
+			await refetchDocuments();
 		}
-	}, [debouncedSearch, pageIndex, pageSize, activeTypes, searchDocuments, fetchDocuments]);
+	}, [debouncedSearch, refetchSearch, refetchDocuments]);
 
 	const onBulkDelete = async () => {
 		if (selectedIds.size === 0) {
@@ -159,7 +177,7 @@ export default function DocumentsTable() {
 			className="w-full px-6 py-4"
 		>
 			<DocumentsFilters
-				typeCounts={typeCounts}
+				typeCounts={typeCounts ?? {}}
 				selectedIds={selectedIds}
 				onSearch={setSearch}
 				searchValue={search}
diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx
index e8298245b..671479da3 100644
--- a/surfsense_web/components/chat/DocumentsDataTable.tsx
+++ b/surfsense_web/components/chat/DocumentsDataTable.tsx
@@ -32,7 +32,7 @@ import {
 	TableRow,
 } from "@/components/ui/table";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { type Document, type DocumentType, useDocuments } from "@/hooks/use-documents";
+import { type Document, type DocumentType } from "@/hooks/use-documents";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { DocumentTypeEnum } from "@/contracts/types/document.types";
diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts
index b5c349091..0a00c7cd9 100644
--- a/surfsense_web/hooks/use-documents.ts
+++ b/surfsense_web/hooks/use-documents.ts
@@ -2,7 +2,6 @@
 import { useCallback, useEffect, useState } from "react";
 import { toast } from "sonner";
 import { authenticatedFetch } from "@/lib/auth-utils";
-import { normalizeListResponse } from "@/lib/pagination";
 
 export interface Document {
 	id: number;
@@ -43,142 +42,8 @@ export interface UseDocumentsOptions {
 export function useDocuments(searchSpaceId: number, options?: UseDocumentsOptions | boolean) {
 	// Support both old boolean API and new options API for backward compatibility
 	const opts = typeof options === "boolean" ? { lazy: options } : options || {};
-	const { page, pageSize = 300, lazy = false, documentTypes } = opts;
 
-	const [documents, setDocuments] = useState<Document[]>([]);
-	const [total, setTotal] = useState(0);
-	const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode
 	const [error, setError] = useState<string | null>(null);
-	const [isLoaded, setIsLoaded] = useState(false); // Memoization flag
-
-	const fetchDocuments = useCallback(
-		async (fetchPage?: number, fetchPageSize?: number, fetchDocumentTypes?: string[]) => {
-			if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode
-
-			try {
-				setLoading(true);
-
-				// Build query params
-				const params = new URLSearchParams({
-					search_space_id: searchSpaceId.toString(),
-				});
-
-				// Use passed parameters or fall back to state/options
-				const effectivePage = fetchPage !== undefined ? fetchPage : page;
-				const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize;
-				const effectiveDocumentTypes =
-					fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes;
-
-				if (effectivePage !== undefined) {
-					params.append("page", effectivePage.toString());
-				}
-				if (effectivePageSize !== undefined) {
-					params.append("page_size", effectivePageSize.toString());
-				}
-				if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) {
-					params.append("document_types", effectiveDocumentTypes.join(","));
-				}
-
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?${params.toString()}`,
-					{ method: "GET" }
-				);
-
-				if (!response.ok) {
-					toast.error("Failed to fetch documents");
-					throw new Error("Failed to fetch documents");
-				}
-
-				const data = await response.json();
-				const normalized = normalizeListResponse<Document>(data);
-				setDocuments(normalized.items);
-				setTotal(normalized.total);
-				setError(null);
-				setIsLoaded(true);
-			} catch (err: any) {
-				setError(err.message || "Failed to fetch documents");
-				console.error("Error fetching documents:", err);
-			} finally {
-				setLoading(false);
-			}
-		},
-		[searchSpaceId, page, pageSize, documentTypes, isLoaded, lazy]
-	);
-
-	useEffect(() => {
-		if (!lazy && searchSpaceId) {
-			fetchDocuments();
-		}
-	}, [searchSpaceId, lazy, fetchDocuments]);
-
-	// Function to refresh the documents list
-	const refreshDocuments = useCallback(async () => {
-		setIsLoaded(false); // Reset memoization flag to allow refetch
-		await fetchDocuments();
-	}, [fetchDocuments]);
-
-	// Function to search documents by title
-	const searchDocuments = useCallback(
-		async (
-			searchQuery: string,
-			fetchPage?: number,
-			fetchPageSize?: number,
-			fetchDocumentTypes?: string[]
-		) => {
-			if (!searchQuery.trim()) {
-				// If search is empty, fetch all documents
-				return fetchDocuments(fetchPage, fetchPageSize, fetchDocumentTypes);
-			}
-
-			try {
-				setLoading(true);
-
-				// Build query params
-				const params = new URLSearchParams({
-					search_space_id: searchSpaceId.toString(),
-					title: searchQuery,
-				});
-
-				// Use passed parameters or fall back to state/options
-				const effectivePage = fetchPage !== undefined ? fetchPage : page;
-				const effectivePageSize = fetchPageSize !== undefined ? fetchPageSize : pageSize;
-				const effectiveDocumentTypes =
-					fetchDocumentTypes !== undefined ? fetchDocumentTypes : documentTypes;
-
-				if (effectivePage !== undefined) {
-					params.append("page", effectivePage.toString());
-				}
-				if (effectivePageSize !== undefined) {
-					params.append("page_size", effectivePageSize.toString());
-				}
-				if (effectiveDocumentTypes && effectiveDocumentTypes.length > 0) {
-					params.append("document_types", effectiveDocumentTypes.join(","));
-				}
-
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/search?${params.toString()}`,
-					{ method: "GET" }
-				);
-
-				if (!response.ok) {
-					toast.error("Failed to search documents");
-					throw new Error("Failed to search documents");
-				}
-
-				const data = await response.json();
-				const normalized = normalizeListResponse<Document>(data);
-				setDocuments(normalized.items);
-				setTotal(normalized.total);
-				setError(null);
-			} catch (err: any) {
-				setError(err.message || "Failed to search documents");
-				console.error("Error searching documents:", err);
-			} finally {
-				setLoading(false);
-			}
-		},
-		[searchSpaceId, page, pageSize, documentTypes, fetchDocuments]
-	);
 
 	// Function to delete a document
 	const deleteDocument = useCallback(
@@ -195,52 +60,19 @@ export function useDocuments(searchSpaceId: number, options?: UseDocumentsOption
 				}
 
 				toast.success("Document deleted successfully");
-				// Update the local state after successful deletion
-				setDocuments(documents.filter((doc) => doc.id !== documentId));
-				return true;
-			} catch (err: any) {
-				toast.error(err.message || "Failed to delete document");
-				console.error("Error deleting document:", err);
-				return false;
-			}
+			// Note: The caller should handle refetching the documents list
+			return true;
+		} catch (err: any) {
+			toast.error(err.message || "Failed to delete document");
+			console.error("Error deleting document:", err);
+			return false;
+		}
 		},
-		[documents]
+		[]
 	);
 
-	// Function to get document type counts
-	const getDocumentTypeCounts = useCallback(async () => {
-		try {
-			const params = new URLSearchParams({
-				search_space_id: searchSpaceId.toString(),
-			});
-
-			const response = await authenticatedFetch(
-				`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/type-counts?${params.toString()}`,
-				{ method: "GET" }
-			);
-
-			if (!response.ok) {
-				throw new Error("Failed to fetch document type counts");
-			}
-
-			const counts = await response.json();
-			return counts as Record<string, number>;
-		} catch (err: any) {
-			console.error("Error fetching document type counts:", err);
-			return {};
-		}
-	}, [searchSpaceId]);
-
 	return {
-		documents,
-		total,
-		loading,
 		error,
-		isLoaded,
-		fetchDocuments, // Manual fetch function for lazy mode
-		searchDocuments, // Search function
-		refreshDocuments,
 		deleteDocument,
-		getDocumentTypeCounts, // Get type counts function
 	};
 }

From 8140a5fb6e2905030162bf7f69834c62ca9ebc3a Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 09:54:19 +0000
Subject: [PATCH 32/36] fix: handle error states from useQuery in
 DocumentsTable

---
 .../dashboard/[search_space_id]/documents/(manage)/page.tsx   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
index 151bb54ce..9ed28dbcf 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@@ -78,6 +78,7 @@ export default function DocumentsTable() {
 		data: documentsResponse,
 		isLoading: isDocumentsLoading,
 		refetch: refetchDocuments,
+		error : documentsError
 	} = useQuery({
 		queryKey: cacheKeys.documents.globalQueryParams(queryParams),
 		queryFn: () => documentsApiService.getDocuments({ queryParams }),
@@ -90,6 +91,7 @@ export default function DocumentsTable() {
 		data: searchResponse,
 		isLoading: isSearchLoading,
 		refetch: refetchSearch,
+		error: searchError
 	} = useQuery({
 		queryKey: cacheKeys.documents.globalQueryParams(searchQueryParams),
 		queryFn: () => documentsApiService.searchDocuments({ queryParams: searchQueryParams }),
@@ -105,10 +107,10 @@ export default function DocumentsTable() {
 		? searchResponse?.total || 0 
 		: documentsResponse?.total || 0;
 	const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
+	const error = debouncedSearch.trim() ? searchError : documentsError
 
 	// Use server-side pagination, search, and filtering
 	const {
-		error,
 		deleteDocument,
 	} = useDocuments(searchSpaceId, {
 		page: pageIndex,

From ede7020b61408a47d06585b28d318c08c4436b76 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 10:03:17 +0000
Subject: [PATCH 33/36] refactor: replace imperative deleteDocument with
 mutation atom

- Replaced deleteDocument from useDocuments with deleteDocumentMutationAtom
- Removed unused useDocuments hook completely
- Updated DocumentsTable to use mutation atom for single and bulk deletes
- Maintains backward compatibility with existing delete functionality
---
 .../documents/(manage)/page.tsx               | 44 +++++++----
 surfsense_web/hooks/use-documents.ts          | 78 -------------------
 2 files changed, 31 insertions(+), 91 deletions(-)
 delete mode 100644 surfsense_web/hooks/use-documents.ts

diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
index 9ed28dbcf..d52b8fcf9 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@@ -6,12 +6,11 @@ import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useId, useMemo, useState } from "react";
 import { toast } from "sonner";
 import { useQuery } from "@tanstack/react-query";
-import { useAtomValue } from "jotai";
+import { useAtomValue, useSetAtom } from "jotai";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
-
-import { useDocuments } from "@/hooks/use-documents";
+import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms";
 
 import { DocumentsFilters } from "./components/DocumentsFilters";
 import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
@@ -50,6 +49,9 @@ export default function DocumentsTable() {
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
 	const {data: typeCounts} = useAtomValue(documentTypeCountsAtom) ;
 
+	// Set up the delete mutation
+	const deleteDocumentMutation = useSetAtom(deleteDocumentMutationAtom);
+
 	// Build query parameters for fetching documents
 	const queryParams = useMemo(
 		() => ({
@@ -109,14 +111,6 @@ export default function DocumentsTable() {
 	const loading = debouncedSearch.trim() ? isSearchLoading : isDocumentsLoading;
 	const error = debouncedSearch.trim() ? searchError : documentsError
 
-	// Use server-side pagination, search, and filtering
-	const {
-		deleteDocument,
-	} = useDocuments(searchSpaceId, {
-		page: pageIndex,
-		pageSize: pageSize,
-	});
-
 	// Display server-filtered results directly
 	const displayDocs = documents || [];
 	const displayTotal = total;
@@ -140,13 +134,37 @@ export default function DocumentsTable() {
 		}
 	}, [debouncedSearch, refetchSearch, refetchDocuments]);
 
+	// Create a delete function for single document deletion
+	const deleteDocument = useCallback(
+		async (id: number) => {
+			try {
+				await deleteDocumentMutation([{ id }]);
+				return true;
+			} catch (error) {
+				console.error("Failed to delete document:", error);
+				return false;
+			}
+		},
+		[deleteDocumentMutation]
+	);
+
 	const onBulkDelete = async () => {
 		if (selectedIds.size === 0) {
 			toast.error(t("no_rows_selected"));
 			return;
 		}
 		try {
-			const results = await Promise.all(Array.from(selectedIds).map((id) => deleteDocument?.(id)));
+			// Delete documents one by one using the mutation
+			const results = await Promise.all(
+				Array.from(selectedIds).map(async (id) => {
+					try {
+						await deleteDocumentMutation([{ id }]);
+						return true;
+					} catch {
+						return false;
+					}
+				})
+			);
 			const okCount = results.filter((r) => r === true).length;
 			if (okCount === selectedIds.size)
 				toast.success(t("delete_success_count", { count: okCount }));
@@ -198,7 +216,7 @@ export default function DocumentsTable() {
 				selectedIds={selectedIds}
 				setSelectedIds={setSelectedIds}
 				columnVisibility={columnVisibility}
-				deleteDocument={(id) => deleteDocument?.(id) ?? Promise.resolve(false)}
+				deleteDocument={deleteDocument}
 				sortKey={sortKey}
 				sortDesc={sortDesc}
 				onSortChange={(key) => {
diff --git a/surfsense_web/hooks/use-documents.ts b/surfsense_web/hooks/use-documents.ts
deleted file mode 100644
index 0a00c7cd9..000000000
--- a/surfsense_web/hooks/use-documents.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-"use client";
-import { useCallback, useEffect, useState } from "react";
-import { toast } from "sonner";
-import { authenticatedFetch } from "@/lib/auth-utils";
-
-export interface Document {
-	id: number;
-	title: string;
-	document_type: DocumentType;
-	document_metadata: any;
-	content: string;
-	created_at: string;
-	search_space_id: number;
-}
-
-export type DocumentType =
-	| "EXTENSION"
-	| "CRAWLED_URL"
-	| "SLACK_CONNECTOR"
-	| "NOTION_CONNECTOR"
-	| "FILE"
-	| "YOUTUBE_VIDEO"
-	| "GITHUB_CONNECTOR"
-	| "LINEAR_CONNECTOR"
-	| "DISCORD_CONNECTOR"
-	| "JIRA_CONNECTOR"
-	| "CONFLUENCE_CONNECTOR"
-	| "CLICKUP_CONNECTOR"
-	| "GOOGLE_CALENDAR_CONNECTOR"
-	| "GOOGLE_GMAIL_CONNECTOR"
-	| "AIRTABLE_CONNECTOR"
-	| "LUMA_CONNECTOR"
-	| "ELASTICSEARCH_CONNECTOR";
-
-export interface UseDocumentsOptions {
-	page?: number;
-	pageSize?: number;
-	lazy?: boolean;
-	documentTypes?: string[];
-}
-
-export function useDocuments(searchSpaceId: number, options?: UseDocumentsOptions | boolean) {
-	// Support both old boolean API and new options API for backward compatibility
-	const opts = typeof options === "boolean" ? { lazy: options } : options || {};
-
-	const [error, setError] = useState<string | null>(null);
-
-	// Function to delete a document
-	const deleteDocument = useCallback(
-		async (documentId: number) => {
-			try {
-				const response = await authenticatedFetch(
-					`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/${documentId}`,
-					{ method: "DELETE" }
-				);
-
-				if (!response.ok) {
-					toast.error("Failed to delete document");
-					throw new Error("Failed to delete document");
-				}
-
-				toast.success("Document deleted successfully");
-			// Note: The caller should handle refetching the documents list
-			return true;
-		} catch (err: any) {
-			toast.error(err.message || "Failed to delete document");
-			console.error("Error deleting document:", err);
-			return false;
-		}
-		},
-		[]
-	);
-
-	return {
-		error,
-		deleteDocument,
-	};
-}

From 55502579e141f485cbb95208374afff957dbda94 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 10:18:21 +0000
Subject: [PATCH 34/36] fix: minor type adjustments and cleanup

- Added missing Document type export
- Fixed typeCounts usage in DocumentsTable
- Minor formatting adjustments in DocumentsDataTable
---
 .../[search_space_id]/documents/(manage)/page.tsx    | 12 +++++-------
 surfsense_web/components/chat/DocumentsDataTable.tsx |  7 +++----
 surfsense_web/contracts/types/document.types.ts      |  1 +
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
index d52b8fcf9..539e71344 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx
@@ -6,7 +6,7 @@ import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useId, useMemo, useState } from "react";
 import { toast } from "sonner";
 import { useQuery } from "@tanstack/react-query";
-import { useAtomValue, useSetAtom } from "jotai";
+import { useAtomValue } from "jotai";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
 import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
@@ -15,7 +15,7 @@ import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.
 import { DocumentsFilters } from "./components/DocumentsFilters";
 import { DocumentsTableShell, type SortKey } from "./components/DocumentsTableShell";
 import { PaginationControls } from "./components/PaginationControls";
-import type { ColumnVisibility, Document } from "./components/types";
+import type { ColumnVisibility } from "./components/types";
 import { DocumentTypeEnum } from "@/contracts/types/document.types";
 
 function useDebounced<T>(value: T, delay = 250) {
@@ -48,9 +48,7 @@ export default function DocumentsTable() {
 	const [sortDesc, setSortDesc] = useState(false);
 	const [selectedIds, setSelectedIds] = useState<Set<number>>(new Set());
 	const {data: typeCounts} = useAtomValue(documentTypeCountsAtom) ;
-
-	// Set up the delete mutation
-	const deleteDocumentMutation = useSetAtom(deleteDocumentMutationAtom);
+	const {mutateAsync : deleteDocumentMutation} = useAtomValue(deleteDocumentMutationAtom);
 
 	// Build query parameters for fetching documents
 	const queryParams = useMemo(
@@ -138,7 +136,7 @@ export default function DocumentsTable() {
 	const deleteDocument = useCallback(
 		async (id: number) => {
 			try {
-				await deleteDocumentMutation([{ id }]);
+				await deleteDocumentMutation({ id });
 				return true;
 			} catch (error) {
 				console.error("Failed to delete document:", error);
@@ -158,7 +156,7 @@ export default function DocumentsTable() {
 			const results = await Promise.all(
 				Array.from(selectedIds).map(async (id) => {
 					try {
-						await deleteDocumentMutation([{ id }]);
+						await deleteDocumentMutation({ id });
 						return true;
 					} catch {
 						return false;
diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx
index 671479da3..53cee8299 100644
--- a/surfsense_web/components/chat/DocumentsDataTable.tsx
+++ b/surfsense_web/components/chat/DocumentsDataTable.tsx
@@ -32,10 +32,9 @@ import {
 	TableRow,
 } from "@/components/ui/table";
 import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
-import { type Document, type DocumentType } from "@/hooks/use-documents";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
 import { cacheKeys } from "@/lib/query-client/cache-keys";
-import { DocumentTypeEnum } from "@/contracts/types/document.types";
+import { Document, DocumentTypeEnum } from "@/contracts/types/document.types";
 import { useAtomValue } from "jotai";
 import { documentTypeCountsAtom } from "@/atoms/documents/document-query.atoms";
 
@@ -110,8 +109,8 @@ const columns: ColumnDef<Document>[] = [
 		cell: ({ row }) => {
 			const type = row.getValue("document_type") as DocumentType;
 			return (
-				<div className="flex items-center gap-2" title={type}>
-					<span className="text-primary">{getConnectorIcon(type)}</span>
+				<div className="flex items-center gap-2" title={String(type)}>
+					<span className="text-primary">{getConnectorIcon(String(type))}</span>
 				</div>
 			);
 		},
diff --git a/surfsense_web/contracts/types/document.types.ts b/surfsense_web/contracts/types/document.types.ts
index 941ed5052..ae755f5f3 100644
--- a/surfsense_web/contracts/types/document.types.ts
+++ b/surfsense_web/contracts/types/document.types.ts
@@ -161,6 +161,7 @@ export const deleteDocumentResponse = z.object({
 	message: z.literal("Document deleted successfully"),
 });
 
+export type Document = z.infer<typeof document>
 export type GetDocumentsRequest = z.infer<typeof getDocumentsRequest>;
 export type GetDocumentsResponse = z.infer<typeof getDocumentsResponse>;
 export type GetDocumentRequest = z.infer<typeof getDocumentRequest>;

From 1d493783639372096db0fe171e37bacde08a6593 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 10:37:58 +0000
Subject: [PATCH 35/36] fix: improve scroll behavior and add missing import in
 SourceDetailSheet

- Added useLayoutEffect import
- Fixed scroll to highlighted chunk behavior
- Added open dependency to useEffect for proper re-triggering
---
 surfsense_web/components/chat/SourceDetailSheet.tsx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx
index 1b227b8df..70b73fcf2 100644
--- a/surfsense_web/components/chat/SourceDetailSheet.tsx
+++ b/surfsense_web/components/chat/SourceDetailSheet.tsx
@@ -3,7 +3,7 @@
 import { useQuery } from "@tanstack/react-query";
 import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react";
 import type React from "react";
-import { type ReactNode, useEffect, useRef, useState } from "react";
+import { type ReactNode, useEffect, useLayoutEffect, useRef, useState } from "react";
 import { MarkdownViewer } from "@/components/markdown-viewer";
 import { Button } from "@/components/ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
@@ -73,7 +73,7 @@ export function SourceDetailSheet({
 
 	useEffect(() => {
 		// Scroll to highlighted chunk when document loads
-		if (document && highlightedChunkRef.current && chunksContainerRef.current) {
+		if (document) {
 			setTimeout(() => {
 				highlightedChunkRef.current?.scrollIntoView({
 					behavior: "smooth",
@@ -81,7 +81,7 @@ export function SourceDetailSheet({
 				});
 			}, 100);
 		}
-	}, [document]);
+	}, [document, open]);
 
 	const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => {
 		e.preventDefault();

From 6566ae209fbe0bc88eee5dae195fb596ec989ba2 Mon Sep 17 00:00:00 2001
From: CREDO23 <thierrybakera12@gmail.com>
Date: Mon, 8 Dec 2025 10:53:12 +0000
Subject: [PATCH 36/36] revert: restore alembic migrations to match
 upstream/dev

- Reverted alembic migration files to upstream/dev version
- Added new migration 44_add_bookstack_connector_enums.py from upstream
- Removed local modifications to migrations 1, 40, and 43
---
 .../versions/1_add_github_connector_enum.py   | 72 +++++++---------
 .../40_move_llm_preferences_to_searchspace.py | 74 ++++++++--------
 .../43_add_blocknote_fields_to_documents.py   | 85 +++++++++----------
 3 files changed, 106 insertions(+), 125 deletions(-)

diff --git a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
index a359f8962..235908b1f 100644
--- a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
+++ b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
@@ -2,12 +2,17 @@
 
 Revision ID: 1
 Revises:
+
 """
 
 from collections.abc import Sequence
 
 from alembic import op
 
+# Import pgvector if needed for other types, though not for this ENUM change
+# import pgvector
+
+
 # revision identifiers, used by Alembic.
 revision: str = "1"
 down_revision: str | None = None
@@ -16,24 +21,10 @@ depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
-    # Ensure the enum type exists
-    op.execute(
-        """
-DO $$
-BEGIN
-    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'searchsourceconnectortype') THEN
-        CREATE TYPE searchsourceconnectortype AS ENUM(
-            'SERPER_API', 
-            'TAVILY_API', 
-            'SLACK_CONNECTOR', 
-            'NOTION_CONNECTOR'
-        );
-    END IF;
-END$$;
-"""
-    )
+    # ### commands auto generated by Alembic - please adjust! ###
 
-    # Add the new enum value if it doesn't exist
+    # Manually add the command to add the enum value
+    # Note: It's generally better to let autogenerate handle this, but we're bypassing it
     op.execute(
         """
 DO $$
@@ -52,31 +43,30 @@ END$$;
 """
     )
 
+    # Pass for the rest, as autogenerate didn't run to add other schema details
+    pass
+    # ### end Alembic commands ###
+
 
 def downgrade() -> None:
-    # Removing an enum value safely requires recreating the type
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    # Downgrading removal of an enum value is complex and potentially dangerous
+    # if the value is in use. Often omitted or requires manual SQL based on context.
+    # For now, we'll just pass. If you needed to reverse this, you'd likely
+    # have to manually check if 'GITHUB_CONNECTOR' is used in the table
+    # and then potentially recreate the type without it.
     op.execute(
-        """
-DO $$
-BEGIN
-    -- Rename existing type
-    ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old;
-
-    -- Create new type without GITHUB_CONNECTOR
-    CREATE TYPE searchsourceconnectortype AS ENUM(
-        'SERPER_API', 
-        'TAVILY_API', 
-        'SLACK_CONNECTOR', 
-        'NOTION_CONNECTOR'
-    );
-
-    -- Update table columns to use new type
-    ALTER TABLE search_source_connectors
-    ALTER COLUMN connector_type TYPE searchsourceconnectortype
-    USING connector_type::text::searchsourceconnectortype;
-
-    -- Drop old type
-    DROP TYPE searchsourceconnectortype_old;
-END$$;
-"""
+        "ALTER TYPE searchsourceconnectortype RENAME TO searchsourceconnectortype_old"
     )
+    op.execute(
+        "CREATE TYPE searchsourceconnectortype AS ENUM('SERPER_API', 'TAVILY_API', 'SLACK_CONNECTOR', 'NOTION_CONNECTOR')"
+    )
+    op.execute(
+        "ALTER TABLE search_source_connectors ALTER COLUMN connector_type TYPE searchsourceconnectortype USING "
+        "connector_type::text::searchsourceconnectortype"
+    )
+    op.execute("DROP TYPE searchsourceconnectortype_old")
+
+    pass
+    # ### end Alembic commands ###
diff --git a/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py b/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
index 5f6ccb852..1067cffcc 100644
--- a/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
+++ b/surfsense_backend/alembic/versions/40_move_llm_preferences_to_searchspace.py
@@ -1,6 +1,19 @@
+"""Move LLM preferences from user-level to search space level
+
+Revision ID: 40
+Revises: 39
+Create Date: 2024-11-27
+
+This migration moves LLM preferences (long_context_llm_id, fast_llm_id, strategic_llm_id)
+from the user_search_space_preferences table to the searchspaces table itself.
+
+This change supports the RBAC model where LLM preferences are shared by all members
+of a search space, rather than being per-user.
+"""
+
 import sqlalchemy as sa
+
 from alembic import op
-from sqlalchemy import inspect
 
 # revision identifiers, used by Alembic.
 revision = "40"
@@ -10,32 +23,26 @@ depends_on = None
 
 
 def upgrade():
-    conn = op.get_bind()
-    inspector = inspect(conn)
+    # Add LLM preference columns to searchspaces table
+    op.add_column(
+        "searchspaces",
+        sa.Column("long_context_llm_id", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "searchspaces",
+        sa.Column("fast_llm_id", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "searchspaces",
+        sa.Column("strategic_llm_id", sa.Integer(), nullable=True),
+    )
 
-    existing_cols = {col["name"] for col in inspector.get_columns("searchspaces")}
+    # Migrate existing preferences from user_search_space_preferences to searchspaces
+    # We take the owner's preferences (the user who created the search space)
+    connection = op.get_bind()
 
-    # Add columns only if they don't already exist
-    if "long_context_llm_id" not in existing_cols:
-        op.add_column(
-            "searchspaces",
-            sa.Column("long_context_llm_id", sa.Integer(), nullable=True),
-        )
-
-    if "fast_llm_id" not in existing_cols:
-        op.add_column(
-            "searchspaces",
-            sa.Column("fast_llm_id", sa.Integer(), nullable=True),
-        )
-
-    if "strategic_llm_id" not in existing_cols:
-        op.add_column(
-            "searchspaces",
-            sa.Column("strategic_llm_id", sa.Integer(), nullable=True),
-        )
-
-    # Migrate existing data
-    conn.execute(
+    # Get all search spaces and their owner's preferences
+    connection.execute(
         sa.text("""
             UPDATE searchspaces ss
             SET 
@@ -50,16 +57,7 @@ def upgrade():
 
 
 def downgrade():
-    conn = op.get_bind()
-    inspector = inspect(conn)
-    existing_cols = {col["name"] for col in inspector.get_columns("searchspaces")}
-
-    # Drop columns only if they exist
-    if "strategic_llm_id" in existing_cols:
-        op.drop_column("searchspaces", "strategic_llm_id")
-
-    if "fast_llm_id" in existing_cols:
-        op.drop_column("searchspaces", "fast_llm_id")
-
-    if "long_context_llm_id" in existing_cols:
-        op.drop_column("searchspaces", "long_context_llm_id")
+    # Remove LLM preference columns from searchspaces table
+    op.drop_column("searchspaces", "strategic_llm_id")
+    op.drop_column("searchspaces", "fast_llm_id")
+    op.drop_column("searchspaces", "long_context_llm_id")
diff --git a/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py b/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
index ee9230a44..32e7780eb 100644
--- a/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
+++ b/surfsense_backend/alembic/versions/43_add_blocknote_fields_to_documents.py
@@ -16,7 +16,6 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
 from alembic import op
-from sqlalchemy import inspect
 
 # revision identifiers, used by Alembic.
 revision: str = "43"
@@ -26,57 +25,51 @@ depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
-    """Upgrade schema - Add BlockNote fields (idempotent)."""
+    """Upgrade schema - Add BlockNote fields and trigger population task."""
 
-    conn = op.get_bind()
-    inspector = inspect(conn)
-    existing_cols = {c["name"] for c in inspector.get_columns("documents")}
+    # Add the columns
+    op.add_column(
+        "documents",
+        sa.Column(
+            "blocknote_document", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+    )
+    op.add_column(
+        "documents",
+        sa.Column(
+            "content_needs_reindexing",
+            sa.Boolean(),
+            nullable=False,
+            server_default=sa.false(),
+        ),
+    )
+    op.add_column(
+        "documents",
+        sa.Column("last_edited_at", sa.TIMESTAMP(timezone=True), nullable=True),
+    )
 
-    # Add blocknote_document (JSONB) if doest not exist
-    if "blocknote_document" not in existing_cols:
-        op.add_column(
-            "documents",
-            sa.Column(
-                "blocknote_document",
-                postgresql.JSONB(astext_type=sa.Text()),
-                nullable=True,
-            ),
+    # Trigger the Celery task to populate blocknote_document for existing documents
+    try:
+        from app.tasks.celery_tasks.blocknote_migration_tasks import (
+            populate_blocknote_for_documents_task,
         )
 
-    # Add content_needs_reindexing (boolean) if doest not exist
-    if "content_needs_reindexing" not in existing_cols:
-        op.add_column(
-            "documents",
-            sa.Column(
-                "content_needs_reindexing",
-                sa.Boolean(),
-                nullable=False,
-                server_default=sa.false(),
-            ),
+        # Queue the task to run asynchronously
+        populate_blocknote_for_documents_task.apply_async()
+        print(
+            "✓ Queued Celery task to populate blocknote_document for existing documents"
         )
-
-    # Add last_edited_at (timestamp with tz) if doest not exist
-    if "last_edited_at" not in existing_cols:
-        op.add_column(
-            "documents",
-            sa.Column("last_edited_at", sa.TIMESTAMP(timezone=True), nullable=True),
+    except Exception as e:
+        # If Celery is not available or task queueing fails, log but don't fail the migration
+        print(f"⚠ Warning: Could not queue blocknote population task: {e}")
+        print("  You can manually trigger it later with:")
+        print(
+            "  celery -A app.celery_app call app.tasks.celery_tasks.blocknote_migration_tasks.populate_blocknote_for_documents_task"
         )
 
-    # NOTE: We intentionally do NOT import or queue Celery tasks here.
-    # Running background jobs during migrations causes hard-to-debug failures.
-    # After running migrations, trigger the backfill task manually (instructions below).
-
 
 def downgrade() -> None:
-    """Downgrade schema - Remove BlockNote fields (only if present)."""
-
-    conn = op.get_bind()
-    inspector = inspect(conn)
-    existing_cols = {c["name"] for c in inspector.get_columns("documents")}
-
-    if "last_edited_at" in existing_cols:
-        op.drop_column("documents", "last_edited_at")
-    if "content_needs_reindexing" in existing_cols:
-        op.drop_column("documents", "content_needs_reindexing")
-    if "blocknote_document" in existing_cols:
-        op.drop_column("documents", "blocknote_document")
+    """Downgrade schema - Remove BlockNote fields."""
+    op.drop_column("documents", "last_edited_at")
+    op.drop_column("documents", "content_needs_reindexing")
+    op.drop_column("documents", "blocknote_document")