mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-06-09 19:45:17 +02:00
Fix library templates db syncing and pagination issues
This commit is contained in:
akhisud3195
parent
893ad87268
commit
af3adc797f
6 changed files with 147 additions and 102 deletions
|
|
@ -41,8 +41,15 @@ const CreateTemplateSchema = z.object({
|
|||
export async function listAssistantTemplates(request: z.infer<typeof ListTemplatesSchema>) {
|
||||
const user = await authCheck();
|
||||
|
||||
// Kick off best-effort library reconcile/seed on each UI fetch (non-blocking)
|
||||
try { void ensureLibraryTemplatesSeeded(); } catch {}
|
||||
// Throttle best-effort library reconcile/seed to avoid repeated bursts
|
||||
try {
|
||||
const last = (globalThis as any).__lastLibrarySeedAt as number | undefined;
|
||||
const now = Date.now();
|
||||
if (!last || now - last > 60_000) { // at most once per minute
|
||||
(globalThis as any).__lastLibrarySeedAt = now;
|
||||
void ensureLibraryTemplatesSeeded();
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const params = ListTemplatesSchema.parse(request);
|
||||
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ export async function ensureLibraryTemplatesSeeded(): Promise<void> {
|
|||
|
||||
const name = (tpl as any).name || prebuiltKey;
|
||||
|
||||
// check if already present (by name + authorName Rowboat and special tag)
|
||||
const existing = await collection.findOne({ name, authorName: "Rowboat", tags: { $in: [ `prebuilt:${prebuiltKey}`, "__library__" ] } });
|
||||
if (existing) {
|
||||
// Skip updating existing templates - we use original JSON at runtime
|
||||
continue;
|
||||
}
|
||||
|
||||
// Upsert to avoid race-condition duplicates
|
||||
const filter = {
|
||||
authorName: "Rowboat",
|
||||
source: 'library',
|
||||
tags: { $all: ["__library__", `prebuilt:${prebuiltKey}`] },
|
||||
} as const;
|
||||
const doc = {
|
||||
name,
|
||||
description: (tpl as any).description || "",
|
||||
|
|
@ -49,35 +48,64 @@ export async function ensureLibraryTemplatesSeeded(): Promise<void> {
|
|||
thumbnailUrl: undefined,
|
||||
source: 'library' as const,
|
||||
} as const;
|
||||
|
||||
await collection.insertOne(doc as any);
|
||||
await collection.updateOne(
|
||||
filter as any,
|
||||
{ $setOnInsert: doc } as any,
|
||||
{ upsert: true } as any
|
||||
);
|
||||
}
|
||||
|
||||
// Simple reconcile: delete library templates that are no longer present in prebuilt files
|
||||
// Strong reconcile: ensure DB exactly matches code exports
|
||||
try {
|
||||
const cursor = collection.find({
|
||||
const libCursor = collection.find({
|
||||
source: 'library',
|
||||
authorName: 'Rowboat',
|
||||
tags: { $in: ["__library__"] },
|
||||
}, { projection: { _id: 1, tags: 1, name: 1 } });
|
||||
}, { projection: { _id: 1, tags: 1, name: 1, publishedAt: 1 } });
|
||||
|
||||
const toDeleteIds: any[] = [];
|
||||
const removedNames: string[] = [];
|
||||
for await (const doc of cursor as any) {
|
||||
const prebuiltTag: string | undefined = (doc.tags || []).find((t: string) => t.startsWith('prebuilt:'));
|
||||
const key = prebuiltTag ? prebuiltTag.replace('prebuilt:', '') : undefined;
|
||||
if (!key || !currentPrebuiltKeys.has(key)) {
|
||||
toDeleteIds.push(doc._id);
|
||||
if (doc.name) removedNames.push(doc.name);
|
||||
type DocLite = { _id: any; tags?: string[]; name?: string; publishedAt?: string };
|
||||
const keyToDocs = new Map<string, DocLite[]>();
|
||||
const orphans: any[] = [];
|
||||
const orphanNames: string[] = [];
|
||||
|
||||
for await (const doc of libCursor as any as AsyncIterable<DocLite>) {
|
||||
const prebuiltTag = (doc.tags || []).find(t => typeof t === 'string' && t.startsWith('prebuilt:'));
|
||||
if (!prebuiltTag) {
|
||||
orphans.push(doc._id);
|
||||
if (doc.name) orphanNames.push(doc.name);
|
||||
continue;
|
||||
}
|
||||
const key = prebuiltTag.replace('prebuilt:', '');
|
||||
if (!currentPrebuiltKeys.has(key)) {
|
||||
orphans.push(doc._id);
|
||||
if (doc.name) orphanNames.push(doc.name);
|
||||
continue;
|
||||
}
|
||||
const arr = keyToDocs.get(key) || [];
|
||||
arr.push(doc);
|
||||
keyToDocs.set(key, arr);
|
||||
}
|
||||
|
||||
if (toDeleteIds.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: toDeleteIds } } as any);
|
||||
console.log(`[PrebuiltTemplates] Reconciled by deleting ${toDeleteIds.length} removed templates:`, removedNames);
|
||||
// Delete orphans (no key or key not in code)
|
||||
if (orphans.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: orphans } } as any);
|
||||
console.log(`[PrebuiltTemplates] Reconciled by deleting ${orphans.length} orphans/removed templates:`, orphanNames);
|
||||
}
|
||||
|
||||
// For each key, keep newest by publishedAt; delete others
|
||||
const dupRemovals: any[] = [];
|
||||
for (const [key, docs] of keyToDocs.entries()) {
|
||||
if (docs.length <= 1) continue;
|
||||
const sorted = [...docs].sort((a, b) => String(b.publishedAt || '').localeCompare(String(a.publishedAt || '')));
|
||||
const extras = sorted.slice(1).map(d => d._id);
|
||||
dupRemovals.push(...extras);
|
||||
}
|
||||
if (dupRemovals.length > 0) {
|
||||
await collection.deleteMany({ _id: { $in: dupRemovals } } as any);
|
||||
console.log(`[PrebuiltTemplates] De-duplicated ${dupRemovals.length} duplicate templates`);
|
||||
}
|
||||
} catch (reconcileErr) {
|
||||
console.error('[PrebuiltTemplates] Reconcile (delete missing) failed:', reconcileErr);
|
||||
console.error('[PrebuiltTemplates] Reconcile (strict sync) failed:', reconcileErr);
|
||||
}
|
||||
} catch (err) {
|
||||
// best-effort seed; do not throw to avoid breaking requests
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ import customerSupport from './customer-support.json';
|
|||
import githubIssueToSlack from './github-issue-to-slack.json';
|
||||
import githubPrToSlack from './github-pr-to-slack.json';
|
||||
import eisenhowerEmailOrganizer from './eisenhower-email-organizer.json';
|
||||
import test3 from './test3.json';
|
||||
import test4 from './test4.json';
|
||||
import test5 from './test5.json';
|
||||
import test6 from './test6.json';
|
||||
|
||||
// Keep keys consistent with prior file basenames to avoid breaking links.
|
||||
export const prebuiltTemplates = {
|
||||
|
|
@ -24,5 +28,9 @@ export const prebuiltTemplates = {
|
|||
'GitHub Issue to Slack': githubIssueToSlack,
|
||||
'GitHub PR to Slack': githubPrToSlack,
|
||||
'Eisenhower Email Organizer': eisenhowerEmailOrganizer,
|
||||
'Test 3': test3,
|
||||
'Test 4': test4,
|
||||
'Test 5': test5,
|
||||
'Test 6': test6,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -106,67 +106,55 @@ export function BuildAssistantSection() {
|
|||
return Array.from(uniqueToolsMap.values()).filter(tool => tool.logo); // Only show tools with logos like ToolkitCard
|
||||
};
|
||||
|
||||
const fetchLibraryTemplatesPage = useCallback(async (cursor?: string | null, limit: number = 20) => {
|
||||
setTemplatesLoading(true);
|
||||
setTemplatesError(null);
|
||||
try {
|
||||
const data = await listAssistantTemplates({
|
||||
source: 'library',
|
||||
limit,
|
||||
cursor: cursor || undefined
|
||||
});
|
||||
setTemplates(prev => cursor ? [...prev, ...data.items] : data.items);
|
||||
setTemplatesCursor(data.nextCursor || null);
|
||||
} catch (error) {
|
||||
console.error('Error fetching library templates:', error);
|
||||
setTemplatesError(error instanceof Error ? error.message : 'Failed to load templates');
|
||||
} finally {
|
||||
setTemplatesLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const fetchCommunityTemplatesPage = useCallback(async (cursor?: string | null, limit: number = 20) => {
|
||||
setCommunityTemplatesLoading(true);
|
||||
setCommunityTemplatesError(null);
|
||||
try {
|
||||
const data = await listAssistantTemplates({
|
||||
source: 'community',
|
||||
limit,
|
||||
cursor: cursor || undefined
|
||||
});
|
||||
setCommunityTemplates(prev => cursor ? [...prev, ...data.items] : data.items);
|
||||
setCommunityCursor(data.nextCursor || null);
|
||||
} catch (error) {
|
||||
console.error('Error fetching community templates:', error);
|
||||
setCommunityTemplatesError(error instanceof Error ? error.message : 'Failed to load community templates');
|
||||
} finally {
|
||||
setCommunityTemplatesLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Ensure we have at least `targetCount` items loaded for a given type
|
||||
const ensureTemplatesLoaded = useCallback(async (type: 'prebuilt' | 'community', targetCount: number) => {
|
||||
const current = type === 'prebuilt' ? templates.length : communityTemplates.length;
|
||||
const cursor = type === 'prebuilt' ? templatesCursor : communityCursor;
|
||||
if (current >= targetCount) return;
|
||||
// Fetch pages until we meet or exceed target or run out of pages
|
||||
// Use page size equal to remaining needed but capped reasonably
|
||||
let needed = targetCount - current;
|
||||
let nextCursor = cursor;
|
||||
while (needed > 0 && (nextCursor !== null || current === 0)) {
|
||||
const pageSize = Math.min(Math.max(needed, 12), 30);
|
||||
if (type === 'prebuilt') {
|
||||
await fetchLibraryTemplatesPage(nextCursor, pageSize);
|
||||
nextCursor = templatesCursor; // will be updated by set state; slight lag acceptable
|
||||
} else {
|
||||
await fetchCommunityTemplatesPage(nextCursor, pageSize);
|
||||
nextCursor = communityCursor;
|
||||
// Utility: append unique by id (prevents duplicates when paginating)
|
||||
const appendUniqueById = useCallback((prev: any[], next: any[]) => {
|
||||
const seen = new Set(prev.map(i => i.id));
|
||||
const merged = [...prev];
|
||||
for (const item of next) {
|
||||
if (!seen.has(item.id)) {
|
||||
merged.push(item);
|
||||
seen.add(item.id);
|
||||
}
|
||||
// Update needed based on latest lengths
|
||||
needed = targetCount - (type === 'prebuilt' ? templates.length : communityTemplates.length);
|
||||
if (nextCursor === null) break;
|
||||
}
|
||||
}, [templates.length, communityTemplates.length, templatesCursor, communityCursor, fetchLibraryTemplatesPage, fetchCommunityTemplatesPage]);
|
||||
return merged;
|
||||
}, []);
|
||||
|
||||
// Clean, single loader: load pages for 'library' or 'community' until target count
|
||||
const loadTemplatesToCount = useCallback(async (source: 'library' | 'community', targetCount: number) => {
|
||||
const setLoading = source === 'library' ? setTemplatesLoading : setCommunityTemplatesLoading;
|
||||
const setError = source === 'library' ? setTemplatesError : setCommunityTemplatesError;
|
||||
const getItems = () => (source === 'library' ? templates : communityTemplates);
|
||||
const setItems = source === 'library' ? setTemplates : setCommunityTemplates;
|
||||
const getCursor = () => (source === 'library' ? templatesCursor : communityCursor);
|
||||
const setCursor = source === 'library' ? setTemplatesCursor : setCommunityCursor;
|
||||
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
let items = getItems();
|
||||
let cursor = getCursor();
|
||||
while (items.length < targetCount && (cursor !== null || items.length === 0)) {
|
||||
const pageSize = Math.min(Math.max(targetCount - items.length, 12), 30);
|
||||
const data = await listAssistantTemplates({ source, limit: pageSize, cursor: cursor || undefined });
|
||||
items = appendUniqueById(items, data.items);
|
||||
setItems(items);
|
||||
cursor = data.nextCursor || null;
|
||||
setCursor(cursor);
|
||||
if (!cursor) break;
|
||||
}
|
||||
} catch (error) {
|
||||
const msg = error instanceof Error ? error.message : 'Failed to load templates';
|
||||
setError(msg);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [templates, communityTemplates, templatesCursor, communityCursor, appendUniqueById]);
|
||||
|
||||
// Adapter used by UI: map 'prebuilt' to 'library'
|
||||
const ensureTemplatesLoaded = useCallback(async (type: 'prebuilt' | 'community', targetCount: number) => {
|
||||
const source = type === 'prebuilt' ? 'library' : 'community';
|
||||
await loadTemplatesToCount(source, targetCount);
|
||||
}, [loadTemplatesToCount]);
|
||||
|
||||
// Handle template selection
|
||||
const handleTemplateSelect = async (template: any) => {
|
||||
|
|
@ -633,8 +621,8 @@ export function BuildAssistantSection() {
|
|||
error={templatesError || communityTemplatesError}
|
||||
onTemplateClick={handleTemplateSelect}
|
||||
onRetry={() => {
|
||||
fetchLibraryTemplatesPage(undefined, 12);
|
||||
fetchCommunityTemplatesPage(undefined, 12);
|
||||
loadTemplatesToCount('library', 12);
|
||||
loadTemplatesToCount('community', 12);
|
||||
}}
|
||||
loadingItemId={loadingTemplateId}
|
||||
onLike={handleTemplateLike}
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ export function UnifiedTemplatesSection({
|
|||
const [currentUserId, setCurrentUserId] = useState<string | null>(null);
|
||||
const [confirmOpen, setConfirmOpen] = useState(false);
|
||||
const [pendingDeleteItem, setPendingDeleteItem] = useState<TemplateItem | null>(null);
|
||||
const [isLoadingMore, setIsLoadingMore] = useState<boolean>(false);
|
||||
|
||||
// Row-based pagination state (lock to 3 columns per row)
|
||||
const [columns, setColumns] = useState<number>(3);
|
||||
// Row-based pagination: paginate in rows of 3 regardless of screen size
|
||||
const [rowsShown, setRowsShown] = useState<number>(4);
|
||||
|
||||
// Track if user has interacted with likes to prevent ALL re-sorting
|
||||
|
|
@ -175,10 +175,7 @@ export function UnifiedTemplatesSection({
|
|||
return filtered;
|
||||
}, [allTemplates, searchQuery, selectedType, selectedCategories, sortBy, hasUserInteractedWithLikes, originalOrder]);
|
||||
|
||||
// Lock columns to 3 at all times to ensure consistent rows and pagination
|
||||
useEffect(() => {
|
||||
setColumns(3);
|
||||
}, []);
|
||||
// No-op: pagination decoupled from display columns
|
||||
|
||||
// Reset rowsShown and allow re-sorting when filters/sort change
|
||||
useEffect(() => {
|
||||
|
|
@ -188,9 +185,11 @@ export function UnifiedTemplatesSection({
|
|||
setOriginalOrder(new Map());
|
||||
}, [searchQuery, selectedType, selectedCategories, sortBy]);
|
||||
|
||||
const itemsPerRow = Math.max(columns, 1);
|
||||
const itemsPerRow = 3; // paginate by 3 items per row irrespective of viewport
|
||||
const visibleCount = rowsShown * itemsPerRow;
|
||||
const hasMore = filteredTemplates.length > visibleCount;
|
||||
// Show "View more" when there are more items than visible OR
|
||||
// when we filled the current page and can load more
|
||||
const hasMore = filteredTemplates.length > visibleCount || (!!onLoadMore && filteredTemplates.length >= visibleCount);
|
||||
const remainingItems = Math.max(filteredTemplates.length - visibleCount, 0);
|
||||
const remainingRows = Math.ceil(remainingItems / itemsPerRow);
|
||||
|
||||
|
|
@ -405,7 +404,7 @@ export function UnifiedTemplatesSection({
|
|||
<div className="text-sm text-gray-600 dark:text-gray-400">
|
||||
Showing {Math.min(visibleCount, filteredTemplates.length)} of {filteredTemplates.length} template{filteredTemplates.length !== 1 ? 's' : ''} ({rowsShown} row{rowsShown !== 1 ? 's' : ''})
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-4">
|
||||
<div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
|
||||
{visibleTemplates.map((item) => (
|
||||
<AssistantCard
|
||||
key={`${item.type}-${item.id}`}
|
||||
|
|
@ -436,15 +435,24 @@ export function UnifiedTemplatesSection({
|
|||
<div className="flex items-center justify-center pt-2">
|
||||
<button
|
||||
onClick={async () => {
|
||||
const target = (rowsShown + 4) * itemsPerRow;
|
||||
if (isLoadingMore) return;
|
||||
setIsLoadingMore(true);
|
||||
const nextRows = rowsShown + 4; // paginate in 4-row blocks
|
||||
const target = nextRows * itemsPerRow;
|
||||
if (onLoadMore) {
|
||||
await onLoadMore(selectedType, target);
|
||||
try {
|
||||
await onLoadMore(selectedType, target);
|
||||
} catch (_e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
setRowsShown(prev => prev + 4);
|
||||
setRowsShown(nextRows);
|
||||
setIsLoadingMore(false);
|
||||
}}
|
||||
className="text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300 text-sm font-medium underline-offset-2 hover:underline"
|
||||
disabled={isLoadingMore}
|
||||
className={`text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300 text-sm font-medium underline-offset-2 hover:underline ${isLoadingMore ? 'opacity-60 cursor-not-allowed' : ''}`}
|
||||
>
|
||||
View more
|
||||
{isLoadingMore ? 'Loading…' : 'View more'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,13 @@ export class MongoDBAssistantTemplatesRepository {
|
|||
}
|
||||
|
||||
const skip = cursor ? parseInt(cursor) : 0;
|
||||
const results = await this.collection.find(query).sort({ publishedAt: -1 }).skip(skip).limit(limit).toArray();
|
||||
// Stable sort: newest first, with _id as tiebreaker to ensure deterministic pages
|
||||
const results = await this.collection
|
||||
.find(query)
|
||||
.sort({ publishedAt: -1, _id: -1 } as any)
|
||||
.skip(skip)
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
const items = results.map(r => ({ ...r, id: r._id.toString() }));
|
||||
const nextCursor = results.length === limit ? (skip + limit).toString() : null;
|
||||
return { items, nextCursor } as any;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue