SurfSense/surfsense_web/lib/apis/documents-api.service.ts

import {
	type CreateDocumentRequest,
	createDocumentRequest,
	createDocumentResponse,
	type DeleteDocumentRequest,
	deleteDocumentRequest,
	deleteDocumentResponse,
	type GetDocumentByChunkRequest,
	type GetDocumentChunksRequest,
	type GetDocumentRequest,
	type GetDocumentsRequest,
	type GetDocumentsStatusRequest,
	type GetDocumentTypeCountsRequest,
	type GetSurfsenseDocsRequest,
	getDocumentByChunkRequest,
	getDocumentByChunkResponse,
	getDocumentChunksRequest,
	getDocumentChunksResponse,
	getDocumentRequest,
	getDocumentResponse,
	getDocumentsRequest,
	getDocumentsResponse,
	getDocumentsStatusRequest,
	getDocumentsStatusResponse,
	getDocumentTypeCountsRequest,
	getDocumentTypeCountsResponse,
	getSurfsenseDocsByChunkResponse,
	getSurfsenseDocsRequest,
	getSurfsenseDocsResponse,
	type SearchDocumentsRequest,
	type SearchDocumentTitlesRequest,
	searchDocumentsRequest,
	searchDocumentsResponse,
	searchDocumentTitlesRequest,
	searchDocumentTitlesResponse,
	type UpdateDocumentRequest,
	type UploadDocumentRequest,
	updateDocumentRequest,
	updateDocumentResponse,
	uploadDocumentRequest,
	uploadDocumentResponse,
} from "@/contracts/types/document.types";
import { folderListResponse } from "@/contracts/types/folder.types";
import { ValidationError } from "../error";
import { baseApiService } from "./base-api.service";

class DocumentsApiService {
	/**
	 * Get a list of documents with optional filtering and pagination
	 */
	getDocuments = async (request: GetDocumentsRequest) => {
		const parsedRequest = getDocumentsRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		// Transform query params to be string values
		const transformedQueryParams = parsedRequest.data.queryParams
			? Object.fromEntries(
					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
						// Handle array values (document_type)
						if (Array.isArray(v)) {
							return [k, v.join(",")];
						}
						return [k, String(v)];
					})
				)
			: undefined;

		const queryParams = transformedQueryParams
			? new URLSearchParams(transformedQueryParams).toString()
			: "";

		return baseApiService.get(`/api/v1/documents?${queryParams}`, getDocumentsResponse);
	};

	/**
	 * Get a single document by ID
	 */
	getDocument = async (request: GetDocumentRequest) => {
		const parsedRequest = getDocumentRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		return baseApiService.get(`/api/v1/documents/${request.id}`, getDocumentResponse);
	};

	/**
	 * Create documents (extension, crawled URL, or YouTube video)
	 */
	createDocument = async (request: CreateDocumentRequest) => {
		const parsedRequest = createDocumentRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		return baseApiService.post(`/api/v1/documents`, createDocumentResponse, {
			body: parsedRequest.data,
		});
	};

	/**
	 * Upload document files in batches to avoid proxy/LB timeouts.
	 * Files are split into chunks of UPLOAD_BATCH_SIZE and sent as separate
	 * requests. Results are aggregated into a single response.
	 */
	uploadDocument = async (request: UploadDocumentRequest) => {
		const parsedRequest = uploadDocumentRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		const { files, search_space_id, should_summarize } = parsedRequest.data;
		const UPLOAD_BATCH_SIZE = 5;

		const batches: File[][] = [];
		for (let i = 0; i < files.length; i += UPLOAD_BATCH_SIZE) {
			batches.push(files.slice(i, i + UPLOAD_BATCH_SIZE));
		}

		const allDocumentIds: number[] = [];
		const allDuplicateIds: number[] = [];
		let totalFiles = 0;
		let pendingFiles = 0;
		let skippedDuplicates = 0;

		for (const batch of batches) {
			const formData = new FormData();
			for (const file of batch) formData.append("files", file);
			formData.append("search_space_id", String(search_space_id));
			formData.append("should_summarize", String(should_summarize));

			const controller = new AbortController();
			const timeoutId = setTimeout(() => controller.abort(), 120_000);

			try {
				const result = await baseApiService.postFormData(
					`/api/v1/documents/fileupload`,
					uploadDocumentResponse,
					{ body: formData, signal: controller.signal }
				);

				allDocumentIds.push(...(result.document_ids ?? []));
				allDuplicateIds.push(...(result.duplicate_document_ids ?? []));
				totalFiles += result.total_files ?? batch.length;
				pendingFiles += result.pending_files ?? 0;
				skippedDuplicates += result.skipped_duplicates ?? 0;
			} finally {
				clearTimeout(timeoutId);
			}
		}

		return {
			message: "Files uploaded for processing" as const,
			document_ids: allDocumentIds,
			duplicate_document_ids: allDuplicateIds,
			total_files: totalFiles,
			pending_files: pendingFiles,
			skipped_duplicates: skippedDuplicates,
		};
	};

	/**
	 * Batch document status for async processing tracking
	 */
	getDocumentsStatus = async (request: GetDocumentsStatusRequest) => {
		const parsedRequest = getDocumentsStatusRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);
			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		const { search_space_id, document_ids } = parsedRequest.data.queryParams;
		const params = new URLSearchParams({
			search_space_id: String(search_space_id),
			document_ids: document_ids.join(","),
		});

		return baseApiService.get(
			`/api/v1/documents/status?${params.toString()}`,
			getDocumentsStatusResponse
		);
	};

	/**
	 * Search documents by title
	 */
	searchDocuments = async (request: SearchDocumentsRequest) => {
		const parsedRequest = searchDocumentsRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		// Transform query params to be string values
		const transformedQueryParams = parsedRequest.data.queryParams
			? Object.fromEntries(
					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => {
						// Handle array values (document_type)
						if (Array.isArray(v)) {
							return [k, v.join(",")];
						}
						return [k, String(v)];
					})
				)
			: undefined;

		const queryParams = transformedQueryParams
			? new URLSearchParams(transformedQueryParams).toString()
			: "";

		return baseApiService.get(`/api/v1/documents/search?${queryParams}`, searchDocumentsResponse);
	};

	/**
	 * Search document titles (lightweight, optimized for mention picker)
	 * Returns only id, title, document_type - no content or metadata
	 * @param request - The search request with query params
	 * @param signal - Optional AbortSignal for request cancellation
	 */
	searchDocumentTitles = async (request: SearchDocumentTitlesRequest, signal?: AbortSignal) => {
		const parsedRequest = searchDocumentTitlesRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		// Transform query params to be string values
		const transformedQueryParams = Object.fromEntries(
			Object.entries(parsedRequest.data.queryParams)
				.filter(([, v]) => v !== undefined)
				.map(([k, v]) => [k, String(v)])
		);

		const queryParams = new URLSearchParams(transformedQueryParams).toString();

		return baseApiService.get(
			`/api/v1/documents/search/titles?${queryParams}`,
			searchDocumentTitlesResponse,
			{ signal }
		);
	};

	/**
	 * Get document type counts
	 */
	getDocumentTypeCounts = async (request: GetDocumentTypeCountsRequest) => {
		// Validate the request
		const parsedRequest = getDocumentTypeCountsRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			// Format a user friendly error message
			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		// Transform query params to be string values
		const transformedQueryParams = parsedRequest.data.queryParams
			? Object.fromEntries(
					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => [k, String(v)])
				)
			: undefined;

		const queryParams = transformedQueryParams
			? new URLSearchParams(transformedQueryParams).toString()
			: "";

		return baseApiService.get(
			`/api/v1/documents/type-counts?${queryParams}`,
			getDocumentTypeCountsResponse
		);
	};

	/**
	 * Get document by chunk ID (includes a window of chunks around the cited one)
	 */
	getDocumentByChunk = async (request: GetDocumentByChunkRequest) => {
		const parsedRequest = getDocumentByChunkRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		const params = new URLSearchParams();
		if (request.chunk_window != null) {
			params.set("chunk_window", String(request.chunk_window));
		}
		const qs = params.toString();
		const url = `/api/v1/documents/by-chunk/${request.chunk_id}${qs ? `?${qs}` : ""}`;

		return baseApiService.get(url, getDocumentByChunkResponse);
	};

	/**
	 * Get paginated chunks for a document
	 */
	getDocumentChunks = async (request: GetDocumentChunksRequest) => {
		const parsedRequest = getDocumentChunksRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		const params = new URLSearchParams({
			page: String(parsedRequest.data.page),
			page_size: String(parsedRequest.data.page_size),
		});
		if (parsedRequest.data.start_offset != null) {
			params.set("start_offset", String(parsedRequest.data.start_offset));
		}

		return baseApiService.get(
			`/api/v1/documents/${parsedRequest.data.document_id}/chunks?${params}`,
			getDocumentChunksResponse
		);
	};

	/**
	 * Get Surfsense documentation by chunk ID
	 * Used for resolving [citation:doc-XXX] citations
	 */
	getSurfsenseDocByChunk = async (chunkId: number) => {
		return baseApiService.get(
			`/api/v1/surfsense-docs/by-chunk/${chunkId}`,
			getSurfsenseDocsByChunkResponse
		);
	};

	/**
	 * List all Surfsense documentation documents
	 * @param request - The request with query params
	 * @param signal - Optional AbortSignal for request cancellation
	 */
	getSurfsenseDocs = async (request: GetSurfsenseDocsRequest, signal?: AbortSignal) => {
		const parsedRequest = getSurfsenseDocsRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		// Transform query params to be string values
		const transformedQueryParams = parsedRequest.data.queryParams
			? Object.fromEntries(
					Object.entries(parsedRequest.data.queryParams).map(([k, v]) => [k, String(v)])
				)
			: undefined;

		const queryParams = transformedQueryParams
			? new URLSearchParams(transformedQueryParams).toString()
			: "";

		const url = `/api/v1/surfsense-docs?${queryParams}`;

		return baseApiService.get(url, getSurfsenseDocsResponse, { signal });
	};

	/**
	 * Update a document
	 */
	updateDocument = async (request: UpdateDocumentRequest) => {
		// Validate the request
		const parsedRequest = updateDocumentRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			// Format a user friendly error message
			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		const { id, data } = parsedRequest.data;

		return baseApiService.put(`/api/v1/documents/${id}`, updateDocumentResponse, {
			body: data,
		});
	};

	listDocumentVersions = async (documentId: number) => {
		return baseApiService.get(`/api/v1/documents/${documentId}/versions`);
	};

	getDocumentVersion = async (documentId: number, versionNumber: number) => {
		return baseApiService.get(`/api/v1/documents/${documentId}/versions/${versionNumber}`);
	};

	restoreDocumentVersion = async (documentId: number, versionNumber: number) => {
		return baseApiService.post(`/api/v1/documents/${documentId}/versions/${versionNumber}/restore`);
	};

	folderIndex = async (
		searchSpaceId: number,
		body: {
			folder_path: string;
			folder_name: string;
			search_space_id: number;
			exclude_patterns?: string[];
			file_extensions?: string[];
			root_folder_id?: number;
			enable_summary?: boolean;
		}
	) => {
		return baseApiService.post(`/api/v1/documents/folder-index`, undefined, { body });
	};

	folderIndexFiles = async (
		searchSpaceId: number,
		body: {
			folder_path: string;
			folder_name: string;
			search_space_id: number;
			target_file_paths: string[];
			root_folder_id?: number | null;
			enable_summary?: boolean;
		}
	) => {
		return baseApiService.post(`/api/v1/documents/folder-index-files`, undefined, { body });
	};

	folderMtimeCheck = async (body: {
		folder_name: string;
		search_space_id: number;
		files: { relative_path: string; mtime: number }[];
	}): Promise<{ files_to_upload: string[] }> => {
		return baseApiService.post(`/api/v1/documents/folder-mtime-check`, undefined, { body }) as unknown as { files_to_upload: string[] };
	};

	folderUploadFiles = async (
		files: File[],
		metadata: {
			folder_name: string;
			search_space_id: number;
			relative_paths: string[];
			root_folder_id?: number | null;
			enable_summary?: boolean;
		},
		signal?: AbortSignal,
	): Promise<{ message: string; status: string; root_folder_id: number; file_count: number }> => {
		const formData = new FormData();
		for (const file of files) {
			formData.append("files", file);
		}
		formData.append("folder_name", metadata.folder_name);
		formData.append("search_space_id", String(metadata.search_space_id));
		formData.append("relative_paths", JSON.stringify(metadata.relative_paths));
		if (metadata.root_folder_id != null) {
			formData.append("root_folder_id", String(metadata.root_folder_id));
		}
		formData.append("enable_summary", String(metadata.enable_summary ?? false));

		const totalSize = files.reduce((acc, f) => acc + f.size, 0);
		const timeoutMs = Math.min(Math.max((totalSize / (1024 * 1024)) * 5000, 30_000), 600_000);

		const controller = new AbortController();
		const timeoutId = setTimeout(() => controller.abort(), timeoutMs);

		if (signal) {
			signal.addEventListener("abort", () => controller.abort(), { once: true });
		}

		try {
			return await baseApiService.postFormData(
				`/api/v1/documents/folder-upload`,
				undefined,
				{ body: formData, signal: controller.signal },
			) as { message: string; status: string; root_folder_id: number; file_count: number };
		} finally {
			clearTimeout(timeoutId);
		}
	};

	folderNotifyUnlinked = async (body: {
		folder_name: string;
		search_space_id: number;
		root_folder_id: number | null;
		relative_paths: string[];
	}): Promise<{ deleted_count: number }> => {
		return baseApiService.post(`/api/v1/documents/folder-unlink`, undefined, { body }) as unknown as { deleted_count: number };
	};

	folderSyncFinalize = async (body: {
		folder_name: string;
		search_space_id: number;
		root_folder_id: number | null;
		all_relative_paths: string[];
	}): Promise<{ deleted_count: number }> => {
		return baseApiService.post(`/api/v1/documents/folder-sync-finalize`, undefined, { body }) as unknown as { deleted_count: number };
	};

	getWatchedFolders = async (searchSpaceId: number) => {
		return baseApiService.get(
			`/api/v1/documents/watched-folders?search_space_id=${searchSpaceId}`,
			folderListResponse
		);
	};

	/**
	 * Delete a document
	 */
	deleteDocument = async (request: DeleteDocumentRequest) => {
		// Validate the request
		const parsedRequest = deleteDocumentRequest.safeParse(request);

		if (!parsedRequest.success) {
			console.error("Invalid request:", parsedRequest.error);

			// Format a user friendly error message
			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
			throw new ValidationError(`Invalid request: ${errorMessage}`);
		}

		return baseApiService.delete(`/api/v1/documents/${request.id}`, deleteDocumentResponse);
	};
}

export const documentsApiService = new DocumentsApiService();