diff --git a/apps/rowboat/app/api/uploaded-images/describe/route.ts b/apps/rowboat/app/api/uploaded-images/describe/route.ts new file mode 100644 index 00000000..06b37765 --- /dev/null +++ b/apps/rowboat/app/api/uploaded-images/describe/route.ts @@ -0,0 +1,77 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { S3Client, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; +import { GoogleGenerativeAI } from '@google/generative-ai'; + +export async function POST(request: NextRequest) { + try { + const { id } = await request.json(); + if (!id || typeof id !== 'string') { + return NextResponse.json({ error: 'id is required' }, { status: 400 }); + } + + const bucket = process.env.RAG_UPLOADS_S3_BUCKET || ''; + if (!bucket) { + return NextResponse.json({ error: 'S3 bucket not configured' }, { status: 500 }); + } + + const region = process.env.RAG_UPLOADS_S3_REGION || 'us-east-1'; + const s3 = new S3Client({ + region, + credentials: process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY ? { + accessKeyId: process.env.AWS_ACCESS_KEY_ID as string, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY as string, + } : undefined, + }); + + const last2 = id.slice(-2).padStart(2, '0'); + const dirA = last2.charAt(0); + const dirB = last2.charAt(1); + const baseKey = `uploaded_images/${dirA}/${dirB}/${id}`; + const exts = ['.png', '.jpg', '.webp', '.bin']; + let foundExt: string | null = null; + for (const ext of exts) { + try { + await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: `${baseKey}${ext}` })); + foundExt = ext; break; + } catch {} + } + if (!foundExt) { + return NextResponse.json({ error: 'Not found' }, { status: 404 }); + } + + const key = `${baseKey}${foundExt}`; + const resp = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + const contentType = resp.ContentType || 'application/octet-stream'; + const body = resp.Body as any; + const chunks: Uint8Array[] = []; + await new Promise((resolve, reject) => { + body.on('data', (c: Uint8Array) => chunks.push(c)); + body.on('end', () => resolve()); + body.on('error', reject); + }); + const buf = Buffer.concat(chunks); + + let descriptionMarkdown: string | null = null; + try { + const apiKey = process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY || ''; + if (apiKey) { + const genAI = new GoogleGenerativeAI(apiKey); + const model = genAI.getGenerativeModel({ model: 'gemini-2.5-flash' }); + const prompt = 'Describe this image in concise, high-quality Markdown. Focus on key objects, text, layout, style, colors, and any notable details. Do not include extra commentary or instructions.'; + const result = await model.generateContent([ + { inlineData: { data: buf.toString('base64'), mimeType: contentType } }, + prompt, + ]); + descriptionMarkdown = result.response?.text?.() || null; + } + } catch (e) { + console.warn('Gemini description failed', e); + } + + return NextResponse.json({ id, description: descriptionMarkdown }); + } catch (e) { + console.error('describe error', e); + return NextResponse.json({ error: 'Failed to describe' }, { status: 500 }); + } +} + diff --git a/apps/rowboat/app/api/uploaded-images/upload-url/route.ts b/apps/rowboat/app/api/uploaded-images/upload-url/route.ts new file mode 100644 index 00000000..86c2537e --- /dev/null +++ b/apps/rowboat/app/api/uploaded-images/upload-url/route.ts @@ -0,0 +1,47 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import crypto from 'crypto'; + +export async function POST(request: NextRequest) { + try { + const bucket = process.env.RAG_UPLOADS_S3_BUCKET || ''; + if (!bucket) { + return NextResponse.json({ error: 'S3 bucket not configured' }, { status: 500 }); + } + + const { mimeType } = await request.json(); + if (!mimeType || typeof mimeType !== 'string') { + return NextResponse.json({ error: 'mimeType is required' }, { status: 400 }); + } + + const ext = mimeType === 'image/jpeg' ? '.jpg' + : mimeType === 'image/webp' ? '.webp' + : mimeType === 'image/png' ? '.png' + : '.bin'; + + const id = crypto.randomUUID(); + const last2 = id.slice(-2).padStart(2, '0'); + const dirA = last2.charAt(0); + const dirB = last2.charAt(1); + const key = `uploaded_images/${dirA}/${dirB}/${id}${ext}`; + + const region = process.env.RAG_UPLOADS_S3_REGION || 'us-east-1'; + const s3 = new S3Client({ + region, + credentials: process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY ? { + accessKeyId: process.env.AWS_ACCESS_KEY_ID as string, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY as string, + } : undefined, + }); + + const command = new PutObjectCommand({ Bucket: bucket, Key: key, ContentType: mimeType }); + const uploadUrl = await getSignedUrl(s3, command, { expiresIn: 600 }); + + return NextResponse.json({ id, key, uploadUrl, url: `/api/uploaded-images/${id}`, mimeType }); + } catch (e) { + console.error('upload-url error', e); + return NextResponse.json({ error: 'Failed to create upload URL' }, { status: 500 }); + } +} + diff --git a/apps/rowboat/components/common/compose-box-playground.tsx b/apps/rowboat/components/common/compose-box-playground.tsx index fe9f6396..39a30320 100644 --- a/apps/rowboat/components/common/compose-box-playground.tsx +++ b/apps/rowboat/components/common/compose-box-playground.tsx @@ -87,22 +87,73 @@ export function ComposeBoxPlayground({ } const controller = new AbortController(); uploadAbortRef.current = controller; - const form = new FormData(); - form.append('file', file); - const res = await fetch('/api/uploaded-images', { - method: 'POST', - body: form, - signal: controller.signal, - }); - if (!res.ok) { - throw new Error(`Upload failed: ${res.status}`); - } - const data = await res.json(); - const url: string | undefined = data?.url; - if (!url) throw new Error('No URL returned'); - // Only apply state if request wasn't aborted/dismissed - if (uploadAbortRef.current === controller) { - setPendingImage({ url, previewSrc, mimeType: data?.mimeType, description: data?.description ?? null }); + let usedFallback = false; + try { + // 1) Request a presigned S3 upload URL + const urlRes = await fetch('/api/uploaded-images/upload-url', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ mimeType: file.type }), + signal: controller.signal, + }); + if (!urlRes.ok) throw new Error(`Failed to get upload URL: ${urlRes.status}`); + const urlData = await urlRes.json(); + const uploadUrl: string | undefined = urlData?.uploadUrl; + const imageId: string | undefined = urlData?.id; + const imageUrl: string | undefined = urlData?.url; + if (!uploadUrl || !imageId || !imageUrl) throw new Error('Invalid upload URL response'); + + // 2) Upload the file directly to S3 + const putRes = await fetch(uploadUrl, { + method: 'PUT', + headers: { 'Content-Type': file.type }, + body: file, + signal: controller.signal, + }); + if (!putRes.ok) throw new Error(`Failed to upload image: ${putRes.status}`); + + // 3) Update local state with URL (description pending) + if (uploadAbortRef.current === controller) { + setPendingImage({ url: imageUrl, previewSrc, mimeType: file.type, description: undefined }); + } + + // 4) Ask server to generate description from S3 image + const descRes = await fetch('/api/uploaded-images/describe', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: imageId }), + signal: controller.signal, + }); + if (descRes.ok) { + const descData = await descRes.json(); + const description: string | null = descData?.description ?? null; + if (uploadAbortRef.current === controller) { + setPendingImage({ url: imageUrl, previewSrc, mimeType: file.type, description }); + } + } else { + // If description fails, still allow sending + if (uploadAbortRef.current === controller) { + setPendingImage({ url: imageUrl, previewSrc, mimeType: file.type, description: null }); + } + } + } catch (err) { + // On local, S3 may be unconfigured. Fallback to legacy temp upload endpoint. + if (err?.name === 'AbortError') throw err; + usedFallback = true; + const form = new FormData(); + form.append('file', file); + const res = await fetch('/api/uploaded-images', { + method: 'POST', + body: form, + signal: controller.signal, + }); + if (!res.ok) throw new Error(`Upload failed: ${res.status}`); + const data = await res.json(); + const url: string | undefined = data?.url; + if (!url) throw new Error('No URL returned'); + if (uploadAbortRef.current === controller) { + setPendingImage({ url, previewSrc, mimeType: data?.mimeType || file.type, description: data?.description ?? null }); + } } } catch (e: any) { if (e?.name === 'AbortError') {