feat: add source_markdown column to documents and implement migration logic for existing records using a pure-Python BlockNote JSON to Markdown converter

This commit is contained in:
Anish Sarkar 2026-02-17 11:34:11 +05:30
parent f2a2872995
commit 8b497da130
22 changed files with 632 additions and 920 deletions

View file

@ -1,40 +0,0 @@
import { ServerBlockNoteEditor } from "@blocknote/server-util";
import { type NextRequest, NextResponse } from "next/server";
export async function POST(request: NextRequest) {
try {
const { markdown } = await request.json();
if (!markdown || typeof markdown !== "string") {
return NextResponse.json({ error: "Markdown string is required" }, { status: 400 });
}
// Log raw markdown input before conversion
// console.log(`\n${"=".repeat(80)}`);
// console.log("RAW MARKDOWN INPUT (BEFORE CONVERSION):");
// console.log("=".repeat(80));
// console.log(markdown);
// console.log(`${"=".repeat(80)}\n`);
// Create server-side editor instance
const editor = ServerBlockNoteEditor.create();
// Convert markdown directly to BlockNote blocks
const blocks = await editor.tryParseMarkdownToBlocks(markdown);
if (!blocks || blocks.length === 0) {
throw new Error("Markdown parsing returned no blocks");
}
return NextResponse.json({ blocknote_document: blocks });
} catch (error: any) {
console.error("Failed to convert markdown to BlockNote:", error);
return NextResponse.json(
{
error: "Failed to convert markdown to BlockNote blocks",
details: error.message,
},
{ status: 500 }
);
}
}

View file

@ -1,28 +0,0 @@
import { ServerBlockNoteEditor } from "@blocknote/server-util";
import { type NextRequest, NextResponse } from "next/server";
export async function POST(request: NextRequest) {
try {
const { blocknote_document } = await request.json();
if (!blocknote_document || !Array.isArray(blocknote_document)) {
return NextResponse.json({ error: "BlockNote document array is required" }, { status: 400 });
}
// Create server-side editor instance
const editor = ServerBlockNoteEditor.create();
// Convert BlockNote blocks to markdown
const markdown = await editor.blocksToMarkdownLossy(blocknote_document);
return NextResponse.json({
markdown,
});
} catch (error) {
console.error("Failed to convert BlockNote to markdown:", error);
return NextResponse.json(
{ error: "Failed to convert BlockNote blocks to markdown" },
{ status: 500 }
);
}
}

View file

@ -3,11 +3,11 @@
import { useAtom } from "jotai";
import { AlertCircle, ArrowLeft, FileText, Save } from "lucide-react";
import { motion } from "motion/react";
import dynamic from "next/dynamic";
import { useParams, useRouter } from "next/navigation";
import { useEffect, useMemo, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { hasUnsavedEditorChangesAtom, pendingEditorNavigationAtom } from "@/atoms/editor/ui.atoms";
import { BlockNoteEditor } from "@/components/DynamicBlockNoteEditor";
import {
AlertDialog,
AlertDialogAction,
@ -24,54 +24,28 @@ import { Spinner } from "@/components/ui/spinner";
import { notesApiService } from "@/lib/apis/notes-api.service";
import { authenticatedFetch, getBearerToken, redirectToLogin } from "@/lib/auth-utils";
// BlockNote types
type BlockNoteInlineContent =
| string
| { text?: string; type?: string; styles?: Record<string, unknown> };
interface BlockNoteBlock {
type: string;
content?: BlockNoteInlineContent[];
children?: BlockNoteBlock[];
props?: Record<string, unknown>;
}
type BlockNoteDocument = BlockNoteBlock[] | null | undefined;
// Dynamically import PlateEditor (uses 'use client' internally)
const PlateEditor = dynamic(
() => import("@/components/editor/plate-editor").then((mod) => ({ default: mod.PlateEditor })),
{ ssr: false, loading: () => <div className="flex items-center justify-center py-12"><Spinner size="xl" className="text-primary" /></div> }
);
interface EditorContent {
document_id: number;
title: string;
document_type?: string;
blocknote_document: BlockNoteDocument;
source_markdown: string;
updated_at: string | null;
}
// Helper function to extract title from BlockNote document
// Takes the text content from the first block (should be a heading for notes)
function extractTitleFromBlockNote(blocknoteDocument: BlockNoteDocument): string {
if (!blocknoteDocument || !Array.isArray(blocknoteDocument) || blocknoteDocument.length === 0) {
return "Untitled";
/** Extract title from markdown: first # heading, or first non-empty line. */
function extractTitleFromMarkdown(markdown: string | null | undefined): string {
if (!markdown) return "Untitled";
for (const line of markdown.split("\n")) {
const trimmed = line.trim();
if (trimmed.startsWith("# ")) return trimmed.slice(2).trim() || "Untitled";
if (trimmed) return trimmed.slice(0, 100);
}
const firstBlock = blocknoteDocument[0];
if (!firstBlock) {
return "Untitled";
}
// Extract text from block content
// BlockNote blocks have a content array with inline content
if (firstBlock.content && Array.isArray(firstBlock.content)) {
const textContent = firstBlock.content
.map((item: BlockNoteInlineContent) => {
if (typeof item === "string") return item;
if (typeof item === "object" && item?.text) return item.text;
return "";
})
.join("")
.trim();
return textContent || "Untitled";
}
return "Untitled";
}
@ -85,11 +59,14 @@ export default function EditorPage() {
const [document, setDocument] = useState<EditorContent | null>(null);
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [editorContent, setEditorContent] = useState<BlockNoteDocument>(null);
const [error, setError] = useState<string | null>(null);
const [hasUnsavedChanges, setHasUnsavedChanges] = useState(false);
const [showUnsavedDialog, setShowUnsavedDialog] = useState(false);
// Store the latest markdown from the editor
const markdownRef = useRef<string>("");
const initialLoadDone = useRef(false);
// Global state for cross-component communication
const [, setGlobalHasUnsavedChanges] = useAtom(hasUnsavedEditorChangesAtom);
const [pendingNavigation, setPendingNavigation] = useAtom(pendingEditorNavigationAtom);
@ -107,51 +84,46 @@ export default function EditorPage() {
};
}, [setGlobalHasUnsavedChanges, setPendingNavigation]);
// Handle pending navigation from sidebar (e.g., when user clicks "+" to create new note)
// Handle pending navigation from sidebar
useEffect(() => {
if (pendingNavigation) {
if (hasUnsavedChanges) {
// Show dialog to confirm navigation
setShowUnsavedDialog(true);
} else {
// No unsaved changes, navigate immediately
router.push(pendingNavigation);
setPendingNavigation(null);
}
}
}, [pendingNavigation, hasUnsavedChanges, router, setPendingNavigation]);
// Reset state when documentId changes (e.g., navigating from existing note to new note)
// Reset state when documentId changes
useEffect(() => {
setDocument(null);
setEditorContent(null);
setError(null);
setHasUnsavedChanges(false);
setLoading(true);
}, []);
initialLoadDone.current = false;
}, [documentId]);
// Fetch document content - DIRECT CALL TO FASTAPI
// Skip fetching if this is a new note
// Fetch document content
useEffect(() => {
async function fetchDocument() {
// For new notes, initialize with empty state
if (isNewNote) {
markdownRef.current = "";
setDocument({
document_id: 0,
title: "Untitled",
document_type: "NOTE",
blocknote_document: null,
source_markdown: "",
updated_at: null,
});
setEditorContent(null);
setLoading(false);
initialLoadDone.current = true;
return;
}
const token = getBearerToken();
if (!token) {
console.error("No auth token found");
// Redirect to login with current path saved
redirectToLogin();
return;
}
@ -166,29 +138,28 @@ export default function EditorPage() {
const errorData = await response
.json()
.catch(() => ({ detail: "Failed to fetch document" }));
const errorMessage = errorData.detail || "Failed to fetch document";
throw new Error(errorMessage);
throw new Error(errorData.detail || "Failed to fetch document");
}
const data = await response.json();
// Check if blocknote_document exists
if (!data.blocknote_document) {
const errorMsg =
"This document does not have BlockNote content. Please re-upload the document to enable editing.";
setError(errorMsg);
if (data.source_markdown === undefined || data.source_markdown === null) {
setError(
"This document does not have editable content. Please re-upload to enable editing."
);
setLoading(false);
return;
}
markdownRef.current = data.source_markdown;
setDocument(data);
setEditorContent(data.blocknote_document);
setError(null);
initialLoadDone.current = true;
} catch (error) {
console.error("Error fetching document:", error);
const errorMessage =
error instanceof Error ? error.message : "Failed to fetch document. Please try again.";
setError(errorMessage);
setError(
error instanceof Error ? error.message : "Failed to fetch document. Please try again."
);
} finally {
setLoading(false);
}
@ -199,29 +170,30 @@ export default function EditorPage() {
}
}, [documentId, params.search_space_id, isNewNote]);
// Track changes to mark as unsaved
useEffect(() => {
if (editorContent && document) {
setHasUnsavedChanges(true);
}
}, [editorContent, document]);
// Check if this is a NOTE type document
const isNote = isNewNote || document?.document_type === "NOTE";
// Extract title dynamically from editor content for notes, otherwise use document title
// Extract title dynamically from current markdown for notes
const displayTitle = useMemo(() => {
if (isNote && editorContent) {
return extractTitleFromBlockNote(editorContent);
if (isNote) {
return extractTitleFromMarkdown(markdownRef.current || document?.source_markdown);
}
return document?.title || "Untitled";
}, [isNote, editorContent, document?.title]);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [isNote, document?.title, document?.source_markdown, hasUnsavedChanges]);
// TODO: Maybe add Auto-save every 30 seconds - DIRECT CALL TO FASTAPI
// Handle markdown changes from the Plate editor
const handleMarkdownChange = useCallback(
(md: string) => {
markdownRef.current = md;
if (initialLoadDone.current) {
setHasUnsavedChanges(true);
}
},
[]
);
// Save and exit - DIRECT CALL TO FASTAPI
// For new notes, create the note first, then save
const handleSave = async () => {
// Save handler
const handleSave = useCallback(async () => {
const token = getBearerToken();
if (!token) {
toast.error("Please login to save");
@ -233,25 +205,26 @@ export default function EditorPage() {
setError(null);
try {
// If this is a new note, create it first
if (isNewNote) {
const title = extractTitleFromBlockNote(editorContent);
const currentMarkdown = markdownRef.current;
// Create the note first
if (isNewNote) {
const title = extractTitleFromMarkdown(currentMarkdown);
// Create the note
const note = await notesApiService.createNote({
search_space_id: searchSpaceId,
title: title,
blocknote_document: editorContent || undefined,
title,
source_markdown: currentMarkdown || undefined,
});
// If there's content, save it properly and trigger reindexing
if (editorContent) {
// If there's content, save & trigger reindexing
if (currentMarkdown) {
const response = await authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${note.id}/save`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ blocknote_document: editorContent }),
body: JSON.stringify({ source_markdown: currentMarkdown }),
}
);
@ -265,24 +238,15 @@ export default function EditorPage() {
setHasUnsavedChanges(false);
toast.success("Note created successfully! Reindexing in background...");
// Redirect to documents page after successful save
router.push(`/dashboard/${searchSpaceId}/documents`);
} else {
// Existing document - save normally
if (!editorContent) {
toast.error("No content to save");
setSaving(false);
return;
}
// Save blocknote_document and trigger reindexing in background
// Existing document — save
const response = await authenticatedFetch(
`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${params.search_space_id}/documents/${documentId}/save`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ blocknote_document: editorContent }),
body: JSON.stringify({ source_markdown: currentMarkdown }),
}
);
@ -295,8 +259,6 @@ export default function EditorPage() {
setHasUnsavedChanges(false);
toast.success("Document saved! Reindexing in background...");
// Redirect to documents page after successful save
router.push(`/dashboard/${searchSpaceId}/documents`);
}
} catch (error) {
@ -312,7 +274,7 @@ export default function EditorPage() {
} finally {
setSaving(false);
}
};
}, [isNewNote, searchSpaceId, documentId, params.search_space_id, router]);
const handleBack = () => {
if (hasUnsavedChanges) {
@ -324,11 +286,9 @@ export default function EditorPage() {
const handleConfirmLeave = () => {
setShowUnsavedDialog(false);
// Clear global unsaved state
setGlobalHasUnsavedChanges(false);
setHasUnsavedChanges(false);
// If there's a pending navigation (from sidebar), use that; otherwise go back to documents
if (pendingNavigation) {
router.push(pendingNavigation);
setPendingNavigation(null);
@ -339,7 +299,6 @@ export default function EditorPage() {
const handleCancelLeave = () => {
setShowUnsavedDialog(false);
// Clear pending navigation if user cancels
setPendingNavigation(null);
};
@ -356,7 +315,7 @@ export default function EditorPage() {
);
}
if (error) {
if (error && !document) {
return (
<div className="flex items-center justify-center min-h-[400px] p-6">
<motion.div
@ -465,11 +424,13 @@ export default function EditorPage() {
</motion.div>
)}
<div className="max-w-4xl mx-auto">
<BlockNoteEditor
key={documentId} // Force re-mount when document changes
initialContent={isNewNote ? undefined : editorContent}
onChange={setEditorContent}
useTitleBlock={isNote}
<PlateEditor
key={documentId}
markdown={document?.source_markdown ?? ""}
onMarkdownChange={handleMarkdownChange}
onSave={handleSave}
hasUnsavedChanges={hasUnsavedChanges}
isSaving={saving}
/>
</div>
</div>

View file

@ -1,213 +0,0 @@
"use client";
import { useTheme } from "next-themes";
import { useEffect, useMemo, useRef } from "react";
import "@blocknote/core/fonts/inter.css";
import "@blocknote/mantine/style.css";
import { BlockNoteView } from "@blocknote/mantine";
import { useCreateBlockNote } from "@blocknote/react";
interface BlockNoteEditorProps {
initialContent?: any;
onChange?: (content: any) => void;
useTitleBlock?: boolean; // Whether to use first block as title (Notion-style)
}
// Helper to ensure first block is a heading for title
function ensureTitleBlock(content: any[] | undefined): any[] {
if (!content || content.length === 0) {
// Return empty heading block for new notes
return [
{
type: "heading",
props: { level: 1 },
content: [],
children: [],
},
];
}
// If first block is not a heading, convert it to one
const firstBlock = content[0];
if (firstBlock?.type !== "heading") {
// Extract text from first block
let titleText = "";
if (firstBlock?.content && Array.isArray(firstBlock.content)) {
titleText = firstBlock.content
.map((item: any) => {
if (typeof item === "string") return item;
if (item?.text) return item.text;
return "";
})
.join("")
.trim();
}
// Create heading block with extracted text
const titleBlock = {
type: "heading",
props: { level: 1 },
content: titleText
? [
{
type: "text",
text: titleText,
styles: {},
},
]
: [],
children: [],
};
// Replace first block with heading, keep rest
return [titleBlock, ...content.slice(1)];
}
return content;
}
export default function BlockNoteEditor({
initialContent,
onChange,
useTitleBlock = false,
}: BlockNoteEditorProps) {
const { resolvedTheme } = useTheme();
// Track the initial content to prevent re-initialization
const initialContentRef = useRef<any>(null);
const isInitializedRef = useRef(false);
// Prepare initial content - ensure first block is a heading if useTitleBlock is true
const preparedInitialContent = useMemo(() => {
if (initialContentRef.current !== null) {
return undefined; // Already initialized
}
if (initialContent === undefined) {
// New note - create empty heading block
return useTitleBlock
? [
{
type: "heading",
props: { level: 1 },
content: [],
children: [],
},
]
: undefined;
}
// Existing note - ensure first block is heading
return useTitleBlock ? ensureTitleBlock(initialContent) : initialContent;
}, [initialContent, useTitleBlock]);
// Creates a new editor instance - only use initialContent on first render
const editor = useCreateBlockNote({
initialContent: initialContentRef.current === null ? preparedInitialContent : undefined,
});
// Store initial content on first render only
useEffect(() => {
if (preparedInitialContent !== undefined && initialContentRef.current === null) {
initialContentRef.current = preparedInitialContent;
isInitializedRef.current = true;
} else if (preparedInitialContent === undefined && initialContentRef.current === null) {
// Mark as initialized even when initialContent is undefined (for new notes)
isInitializedRef.current = true;
}
}, [preparedInitialContent]);
// Call onChange when document changes (but don't update from props)
useEffect(() => {
if (!onChange || !editor) return;
// For new notes (no initialContent), we need to wait for editor to be ready
// Use a small delay to ensure editor is fully initialized
if (!isInitializedRef.current) {
const timer = setTimeout(() => {
isInitializedRef.current = true;
}, 100);
return () => clearTimeout(timer);
}
const handleChange = () => {
onChange(editor.document);
};
// Subscribe to document changes
const unsubscribe = editor.onChange(handleChange);
// Also call onChange once with current document to capture initial state
// This ensures we capture content even if user doesn't make changes
if (editor.document) {
onChange(editor.document);
}
return () => {
unsubscribe();
};
}, [editor, onChange]);
// Determine theme for BlockNote with custom dark mode background
const blockNoteTheme = useMemo(() => {
if (resolvedTheme === "dark") {
// Custom dark theme - only override editor background, let BlockNote handle the rest
return {
colors: {
editor: {
background: "#0A0A0A", // Custom dark background
},
},
};
}
return "light" as const;
}, [resolvedTheme]);
// Renders the editor instance
return (
<div className="bn-container">
<style>{`
@media (max-width: 640px) {
.bn-container .bn-editor {
padding-inline: 12px !important;
}
/* Heading Level 1 (Title) */
.bn-container [data-content-type="heading"][data-level="1"] {
font-size: 1.75rem !important;
line-height: 1.2 !important;
margin-top: 1rem !important;
margin-bottom: 0.5rem !important;
}
/* Heading Level 2 */
.bn-container [data-content-type="heading"][data-level="2"] {
font-size: 1.5rem !important;
line-height: 1.2 !important;
margin-top: 0.875rem !important;
margin-bottom: 0.375rem !important;
}
/* Heading Level 3 */
.bn-container [data-content-type="heading"][data-level="3"] {
font-size: 1.25rem !important;
line-height: 1.2 !important;
margin-top: 0.75rem !important;
margin-bottom: 0.25rem !important;
}
/* Paragraphs and regular content */
.bn-container .bn-block-content {
font-size: 0.9375rem !important;
line-height: 1.5 !important;
}
/* Adjust lists */
.bn-container ul,
.bn-container ol {
padding-left: 1.25rem !important;
}
}
`}</style>
<BlockNoteView editor={editor} theme={blockNoteTheme} />
</div>
);
}

View file

@ -1,6 +0,0 @@
"use client";
import dynamic from "next/dynamic";
// Dynamically import BlockNote editor with SSR disabled
export const BlockNoteEditor = dynamic(() => import("./BlockNoteEditor"), { ssr: false });

View file

@ -6,7 +6,7 @@ import { baseApiService } from "./base-api.service";
const createNoteRequest = z.object({
search_space_id: z.number(),
title: z.string().min(1),
blocknote_document: z.array(z.any()).optional(),
source_markdown: z.string().optional(),
});
const createNoteResponse = z.object({
@ -82,12 +82,12 @@ class NotesApiService {
throw new ValidationError(`Invalid request: ${errorMessage}`);
}
const { search_space_id, title, blocknote_document } = parsedRequest.data;
const { search_space_id, title, source_markdown } = parsedRequest.data;
// Send both title and blocknote_document in request body
// Send both title and source_markdown in request body
const body = {
title,
...(blocknote_document && { blocknote_document }),
...(source_markdown !== undefined && { source_markdown }),
};
return baseApiService.post(

View file

@ -7,7 +7,6 @@ const withNextIntl = createNextIntlPlugin("./i18n/request.ts");
const nextConfig: NextConfig = {
output: "standalone",
// Disable StrictMode for BlockNote compatibility with React 19/Next 15
reactStrictMode: false,
typescript: {
ignoreBuildErrors: true,
@ -20,9 +19,6 @@ const nextConfig: NextConfig = {
},
],
},
// Mark BlockNote server packages as external
serverExternalPackages: ["@blocknote/server-util"],
// Turbopack config (used during `next dev --turbopack`)
turbopack: {
rules: {
@ -33,13 +29,8 @@ const nextConfig: NextConfig = {
},
},
// Configure webpack to handle blocknote packages + SVGR
webpack: (config, { isServer }) => {
if (isServer) {
// Don't bundle these packages on the server
config.externals = [...(config.externals || []), "@blocknote/server-util"];
}
// Configure webpack (SVGR)
webpack: (config) => {
// SVGR: import *.svg as React components
const fileLoaderRule = config.module.rules.find((rule: any) => rule.test?.test?.(".svg"));
config.module.rules.push(

View file

@ -26,10 +26,6 @@
"@assistant-ui/react": "^0.11.53",
"@assistant-ui/react-ai-sdk": "^1.1.20",
"@assistant-ui/react-markdown": "^0.11.9",
"@blocknote/core": "^0.45.0",
"@blocknote/mantine": "^0.45.0",
"@blocknote/react": "^0.45.0",
"@blocknote/server-util": "^0.45.0",
"@electric-sql/client": "^1.4.0",
"@electric-sql/pglite": "^0.3.14",
"@electric-sql/pglite-sync": "^0.4.0",