mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-05-19 18:35:18 +02:00
Fix library templates db syncing and pagination issues
This commit is contained in:
akhisud3195
parent
893ad87268
commit
af3adc797f
6 changed files with 147 additions and 102 deletions
|
|
@ -41,8 +41,15 @@ const CreateTemplateSchema = z.object({
|
|||
export async function listAssistantTemplates(request: z.infer<typeof ListTemplatesSchema>) {
|
||||
const user = await authCheck();
|
||||
|
||||
// Kick off best-effort library reconcile/seed on each UI fetch (non-blocking)
|
||||
try { void ensureLibraryTemplatesSeeded(); } catch {}
|
||||
// Throttle best-effort library reconcile/seed to avoid repeated bursts
|
||||
try {
|
||||
const last = (globalThis as any).__lastLibrarySeedAt as number | undefined;
|
||||
const now = Date.now();
|
||||
if (!last || now - last > 60_000) { // at most once per minute
|
||||
(globalThis as any).__lastLibrarySeedAt = now;
|
||||
void ensureLibraryTemplatesSeeded();
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const params = ListTemplatesSchema.parse(request);
|
||||
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ export async function ensureLibraryTemplatesSeeded(): Promise<void> {
|
|||
|
||||
const name = (tpl as any).name || prebuiltKey;
|
||||
|
||||
// check if already present (by name + authorName Rowboat and special tag)
|
||||
const existing = await collection.findOne({ name, authorName: "Rowboat", tags: { $in: [ `prebuilt:${prebuiltKey}`, "__library__" ] } });
|
||||
if (existing) {
|
||||
// Skip updating existing templates - we use original JSON at runtime
|
||||
continue;
|
||||
}
|
||||
|
||||
// Upsert to avoid race-condition duplicates
|
||||
const filter = {
|
||||
authorName: "Rowboat",
|
||||
source: 'library',
|
||||
tags: { $all: ["__library__", `prebuilt:${prebuiltKey}`] },
|
||||
} as const;
|
||||
const doc = {
|
||||
name,
|
||||
description: (tpl as any).description || "",
|
||||
|
|
@ -49,35 +48,64 @@ export async function ensureLibraryTemplatesSeeded(): Promise<void> {
|
|||
thumbnailUrl: undefined,
|
||||
source: 'library' as const,
|
||||
} as const;
|
||||
|
||||
await collection.insertOne(doc as any);
|
||||
await collection.updateOne(
|
||||
filter as any,
|
||||
{ $setOnInsert: doc } as any,
|
||||
{ upsert: true } as any
|
||||
);
|
||||
}
|
||||
|
||||
// Simple reconcile: delete library templates that are no longer present in prebuilt files
|
||||
// Strong reconcile: ensure DB exactly matches code exports
|
||||
try {
|
||||
const cursor = collection.find({
|
||||
const libCursor = collection.find({
|
||||
source: 'library',
|
||||
authorName: 'Rowboat',
|
||||
tags: { $in: ["__library__"] },
|
||||
}, { projection: { _id: 1, tags: 1, name: 1 } });
|
||||
}, { projection: { _id: 1, tags: 1, name: 1, publishedAt: 1 } });
|
||||
|
||||
const toDeleteIds: any[] = [];
|
||||
const removedNames: string[] = [];
|
||||
for await (const doc of cursor as any) {
|
||||
const prebuiltTag: string | undefined = (doc.tags || []).find((t: string) => t.startsWith('prebuilt:'));
|
||||
const key = prebuiltTag ? prebuiltTag.replace('prebuilt:', '') : undefined;
|
||||
if (!key || !currentPrebuiltKeys.has(key)) {
|
||||
toDeleteIds.push(doc._id);
|
||||
if (doc.name) removedNames.push(doc.name);
|
||||
type DocLite = { _id: any; tags?: string[]; name?: string; publishedAt?: string };
|
||||
const keyToDocs = new Map<string, DocLite[]>();
|
||||
const orphans: any[] = [];
|
||||
const orphanNames: string[] = [];
|
||||
|
||||
for await (const doc of libCursor as any as AsyncIterable<DocLite>) {
|
||||
const prebuiltTag = (doc.tags || []).find(t => typeof t === 'string' && t.startsWith('prebuilt:'));
|
||||
if (!prebuiltTag) {
|
||||
orphans.push(doc._id);
|
||||
if (doc.name) orphanNames.push(doc.name);
|
||||
continue;
|
||||
}
|
||||
const key = prebuiltTag.replace('prebuilt:', '');
|
||||
if (!currentPrebuiltKeys.has(key)) {
|
||||
orphans.push(doc._id);
|
||||
if (doc.name) orphanNames.push(doc.name);
|
||||
continue;
|
||||
}
|
||||
const arr = keyToDocs.get(key) || [];
|
||||
arr.push(doc);
|
||||
keyToDocs.set(key, arr);
|
||||
}
|
||||
|
||||
if (toDeleteIds.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: toDeleteIds } } as any);
|
||||
console.log(`[PrebuiltTemplates] Reconciled by deleting ${toDeleteIds.length} removed templates:`, removedNames);
|
||||
// Delete orphans (no key or key not in code)
|
||||
if (orphans.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: orphans } } as any);
|
||||
console.log(`[PrebuiltTemplates] Reconciled by deleting ${orphans.length} orphans/removed templates:`, orphanNames);
|
||||
}
|
||||
|
||||
// For each key, keep newest by publishedAt; delete others
|
||||
const dupRemovals: any[] = [];
|
||||
for (const [key, docs] of keyToDocs.entries()) {
|
||||
if (docs.length <= 1) continue;
|
||||
const sorted = [...docs].sort((a, b) => String(b.publishedAt || '').localeCompare(String(a.publishedAt || '')));
|
||||
const extras = sorted.slice(1).map(d => d._id);
|
||||
dupRemovals.push(...extras);
|
||||
}
|
||||
if (dupRemovals.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: dupRemovals } } as any);
|
||||
console.log(`[PrebuiltTemplates] De-duplicated ${dupRemovals.length} duplicate templates`);
|
||||
}
|
||||
} catch (reconcileErr) {
|
||||
console.error('[PrebuiltTemplates] Reconcile (delete missing) failed:', reconcileErr);
|
||||
console.error('[PrebuiltTemplates] Reconcile (strict sync) failed:', reconcileErr);
|
||||
}
|
||||
} catch (err) {
|
||||
// best-effort seed; do not throw to avoid breaking requests
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ import customerSupport from './customer-support.json';
|
|||
import githubIssueToSlack from './github-issue-to-slack.json';
|
||||
import githubPrToSlack from './github-pr-to-slack.json';
|
||||
import eisenhowerEmailOrganizer from './eisenhower-email-organizer.json';
|
||||
import test3 from './test3.json';
|
||||
import test4 from './test4.json';
|
||||
import test5 from './test5.json';
|
||||
import test6 from './test6.json';
|
||||
|
||||
// Keep keys consistent with prior file basenames to avoid breaking links.
|
||||
export const prebuiltTemplates = {
|
||||
|
|
@ -24,5 +28,9 @@ export const prebuiltTemplates = {
|
|||
'GitHub Issue to Slack': githubIssueToSlack,
|
||||
'GitHub PR to Slack': githubPrToSlack,
|
||||
'Eisenhower Email Organizer': eisenhowerEmailOrganizer,
|
||||
'Test 3': test3,
|
||||
'Test 4': test4,
|
||||
'Test 5': test5,
|
||||
'Test 6': test6,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -106,67 +106,55 @@ export function BuildAssistantSection() {
|
|||
return Array.from(uniqueToolsMap.values()).filter(tool => tool.logo); // Only show tools with logos like ToolkitCard
|
||||
};
|
||||
|
||||
const fetchLibraryTemplatesPage = useCallback(async (cursor?: string | null, limit: number = 20) => {
|
||||
setTemplatesLoading(true);
|
||||
setTemplatesError(null);
|
||||
try {
|
||||
const data = await listAssistantTemplates({
|
||||
source: 'library',
|
||||
limit,
|
||||
cursor: cursor || undefined
|
||||
});
|
||||
setTemplates(prev => cursor ? [...prev, ...data.items] : data.items);
|
||||
setTemplatesCursor(data.nextCursor || null);
|
||||
} catch (error) {
|
||||
console.error('Error fetching library templates:', error);
|
||||
setTemplatesError(error instanceof Error ? error.message : 'Failed to load templates');
|
||||
} finally {
|
||||
setTemplatesLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const fetchCommunityTemplatesPage = useCallback(async (cursor?: string | null, limit: number = 20) => {
|
||||
setCommunityTemplatesLoading(true);
|
||||
setCommunityTemplatesError(null);
|
||||
try {
|
||||
const data = await listAssistantTemplates({
|
||||
source: 'community',
|
||||
limit,
|
||||
cursor: cursor || undefined
|
||||
});
|
||||
setCommunityTemplates(prev => cursor ? [...prev, ...data.items] : data.items);
|
||||
setCommunityCursor(data.nextCursor || null);
|
||||
} catch (error) {
|
||||
console.error('Error fetching community templates:', error);
|
||||
setCommunityTemplatesError(error instanceof Error ? error.message : 'Failed to load community templates');
|
||||
} finally {
|
||||
setCommunityTemplatesLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Ensure we have at least `targetCount` items loaded for a given type
|
||||
const ensureTemplatesLoaded = useCallback(async (type: 'prebuilt' | 'community', targetCount: number) => {
|
||||
const current = type === 'prebuilt' ? templates.length : communityTemplates.length;
|
||||
const cursor = type === 'prebuilt' ? templatesCursor : communityCursor;
|
||||
if (current >= targetCount) return;
|
||||
// Fetch pages until we meet or exceed target or run out of pages
|
||||
// Use page size equal to remaining needed but capped reasonably
|
||||
let needed = targetCount - current;
|
||||
let nextCursor = cursor;
|
||||
while (needed > 0 && (nextCursor !== null || current === 0)) {
|
||||
const pageSize = Math.min(Math.max(needed, 12), 30);
|
||||
if (type === 'prebuilt') {
|
||||
await fetchLibraryTemplatesPage(nextCursor, pageSize);
|
||||
nextCursor = templatesCursor; // will be updated by set state; slight lag acceptable
|
||||
} else {
|
||||
await fetchCommunityTemplatesPage(nextCursor, pageSize);
|
||||
nextCursor = communityCursor;
|
||||
// Utility: append unique by id (prevents duplicates when paginating)
|
||||
const appendUniqueById = useCallback((prev: any[], next: any[]) => {
|
||||
const seen = new Set(prev.map(i => i.id));
|
||||
const merged = [...prev];
|
||||
for (const item of next) {
|
||||
if (!seen.has(item.id)) {
|
||||
merged.push(item);
|
||||
seen.add(item.id);
|
||||
}
|
||||
// Update needed based on latest lengths
|
||||
needed = targetCount - (type === 'prebuilt' ? templates.length : communityTemplates.length);
|
||||
if (nextCursor === null) break;
|
||||
}
|
||||
}, [templates.length, communityTemplates.length, templatesCursor, communityCursor, fetchLibraryTemplatesPage, fetchCommunityTemplatesPage]);
|
||||
return merged;
|
||||
}, []);
|
||||
|
||||
// Clean, single loader: load pages for 'library' or 'community' until target count
|
||||
const loadTemplatesToCount = useCallback(async (source: 'library' | 'community', targetCount: number) => {
|
||||
const setLoading = source === 'library' ? setTemplatesLoading : setCommunityTemplatesLoading;
|
||||
const setError = source === 'library' ? setTemplatesError : setCommunityTemplatesError;
|
||||
const getItems = () => (source === 'library' ? templates : communityTemplates);
|
||||
const setItems = source === 'library' ? setTemplates : setCommunityTemplates;
|
||||
const getCursor = () => (source === 'library' ? templatesCursor : communityCursor);
|
||||
const setCursor = source === 'library' ? setTemplatesCursor : setCommunityCursor;
|
||||
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
let items = getItems();
|
||||
let cursor = getCursor();
|
||||
while (items.length < targetCount && (cursor !== null || items.length === 0)) {
|
||||
const pageSize = Math.min(Math.max(targetCount - items.length, 12), 30);
|
||||
const data = await listAssistantTemplates({ source, limit: pageSize, cursor: cursor || undefined });
|
||||
items = appendUniqueById(items, data.items);
|
||||
setItems(items);
|
||||
cursor = data.nextCursor || null;
|
||||
setCursor(cursor);
|
||||
if (!cursor) break;
|
||||
}
|
||||
} catch (error) {
|
||||
const msg = error instanceof Error ? error.message : 'Failed to load templates';
|
||||
setError(msg);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [templates, communityTemplates, templatesCursor, communityCursor, appendUniqueById]);
|
||||
|
||||
// Adapter used by UI: map 'prebuilt' to 'library'
|
||||
const ensureTemplatesLoaded = useCallback(async (type: 'prebuilt' | 'community', targetCount: number) => {
|
||||
const source = type === 'prebuilt' ? 'library' : 'community';
|
||||
await loadTemplatesToCount(source, targetCount);
|
||||
}, [loadTemplatesToCount]);
|
||||
|
||||
// Handle template selection
|
||||
const handleTemplateSelect = async (template: any) => {
|
||||
|
|
@ -633,8 +621,8 @@ export function BuildAssistantSection() {
|
|||
error={templatesError || communityTemplatesError}
|
||||
onTemplateClick={handleTemplateSelect}
|
||||
onRetry={() => {
|
||||
fetchLibraryTemplatesPage(undefined, 12);
|
||||
fetchCommunityTemplatesPage(undefined, 12);
|
||||
loadTemplatesToCount('library', 12);
|
||||
loadTemplatesToCount('community', 12);
|
||||
}}
|
||||
loadingItemId={loadingTemplateId}
|
||||
onLike={handleTemplateLike}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue