apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-05-31 19:15:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

move image description to debug message

Browse source
This commit is contained in:
arkml 2025-09-19 18:28:00 +05:30
parent 527afff601
commit b5f5301761
2 changed files with 64 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

64
apps/rowboat/app/projects/[projectId]/playground/components/chat.tsx
View file

@ -152,11 +152,24 @@ export function Chat({
}
}, []);
function handleUserMessage(prompt: string) {
const updatedMessages: z.infer<typeof Message>[] = [...messages, {
role: 'user',
content: prompt,
}];
function handleUserMessage(prompt: string, imageDebug?: { url: string; description?: string | null }) {
// Insert an internal-only debug message with image URL/markdown (if provided),
// then the actual user message last so streaming triggers correctly.
const debugMessages: z.infer<typeof Message>[] = imageDebug ? [{
role: 'assistant',
content: `Image Description\n\nURL: ${imageDebug.url}\n\n${imageDebug.description ? imageDebug.description : ''}`.trim(),
agentName: 'Image Description',
responseType: 'internal',
} as any] : [];
const updatedMessages: z.infer<typeof Message>[] = [
...messages,
...debugMessages,
{
role: 'user',
content: prompt,
} as any,
];
setMessages(updatedMessages);
setError(null);
setIsLastInteracted(true);
@ -229,9 +242,46 @@ export function Chat({
}
// set up a cached turn
// Merge-at-send: if the immediately preceding message is our internal
// Image Description debug message, append its details (URL/markdown)
// to the outgoing user message content, without changing the UI.
const last = messages[messages.length - 1];
let mergedContent = (typeof last?.content === 'string' ? last.content : '') || '';
if (messages.length >= 2) {
const prev = messages[messages.length - 2] as any;
const isImageDebug = prev && prev.role === 'assistant' && prev.responseType === 'internal' && prev.agentName === 'Image Description' && typeof prev.content === 'string';
if (isImageDebug) {
// Expect prev.content to have: "Image Description\n\nURL: <url>\n\n<markdown>"
// Extract URL and markdown blocks for a clean append
const content = prev.content as string;
let url: string | undefined;
let markdown: string | undefined;
const urlMatch = content.match(/URL:\s*(\S+)/i);
if (urlMatch) url = urlMatch[1];
// markdown is whatever comes after the blank line following URL
const parts = content.split(/\n\n/);
if (parts.length >= 3) {
markdown = parts.slice(2).join('\n\n').trim();
}
const appendSections: string[] = [];
if (url) appendSections.push(`The user uploaded an image. URL: ${url}`);
if (markdown) appendSections.push(`Image description (markdown):\n\n${markdown}`);
if (appendSections.length > 0) {
mergedContent = [mergedContent, appendSections.join('\n\n')]
.filter(Boolean)
.join('\n\n');
}
}
}
const messagesToSend: z.infer<typeof Message>[] = [{
role: 'user',
content: mergedContent,
} as any];
const response = await createCachedTurn({
conversationId: conversationId.current,
messages: messages.slice(-1), // only send the last message
messages: messagesToSend, // send merged content only
});
if (ignore) {
return;
@ -500,4 +550,4 @@ export function Chat({
/>
</div>
);
}
}

15
apps/rowboat/components/common/compose-box-playground.tsx
View file

@ -3,7 +3,7 @@ import { Textarea } from '@/components/ui/textarea';
import { Button, Spinner } from "@heroui/react";
interface ComposeBoxPlaygroundProps {
handleUserMessage: (message: string) => void;
handleUserMessage: (message: string, imageDebug?: { url: string; description?: string | null }) => void;
messages: any[];
loading: boolean;
disabled?: boolean;
@ -41,21 +41,20 @@ export function ComposeBoxPlayground({
if (!text && !pendingImage) {
return;
}
// Only include the user's typed text; omit image URL/markdown from user message
const parts: string[] = [];
if (text) parts.push(text);
if (pendingImage?.url) {
parts.push(`The user uploaded an image. URL: ${pendingImage.url}`);
if (pendingImage.description) {
parts.push(`Image description (markdown):\n\n${pendingImage.description}`);
}
}
const prompt = parts.join('\n\n');
// Build optional debug payload to render as internal-only message in debug view
const imageDebug = pendingImage?.url
? { url: pendingImage.url, description: pendingImage.description ?? null }
: undefined;
setInput('');
if (pendingImage?.previewSrc) {
try { URL.revokeObjectURL(pendingImage.previewSrc); } catch {}
}
setPendingImage(null);
handleUserMessage(prompt);
handleUserMessage(prompt, imageDebug);
}
const handleInputKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {