From 2360dbd0fadb2c620cb4fdf06e0cfb433636d850 Mon Sep 17 00:00:00 2001 From: Fsocietyhhh <1211904451@qq.com> Date: Tue, 2 Jun 2026 11:49:25 -0700 Subject: [PATCH 1/3] Composer: multi-image attachments (image fusion + chat vision) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gateway image2image already accepted `image: string | string[]` (cap 4 for OpenAI, 3 for Google; see blockrun/src/app/api/v1/images/image2image/ route.ts:28) but the web composer was hardcoded to a single slot — clicking the + again replaced the picked image instead of appending. This lets the user attach up to the per-model cap. The + button: - accepts multiple files in one pick (multi-attribute when cap > 1) - appends to the existing list, never replaces - hides once the cap is reached - shows '2/4' counter in its title attribute Per-mode/per-model caps via maxAttachmentsFor(): - music: 0 (button never shown) - video: 1 (first attachment used as seed; rest dropped at send time) - image: maxImageFusionFor(model) — 4 for openai/*, 3 for google/*, 1 else - chat: 4 (matches the highest image cap for UX symmetry) Storage and wire: - ChatMessage gains `images?: string[]` for multi; single `image` kept for back-compat with legacy persisted conversations + assistant single outputs. userImages() helper flattens both shapes for every reader. - runChatWithTools maps every userImages(m) entry to its own image block in the Anthropic /v1/messages content array. - runMedia(image) sends `image: trimmed` (string when 1, string[] when 2+); video keeps single-seed behavior (first ref wins). UI: - Composer thumbnail strip switches from single .try-attach card to a .try-attach-row when there are 2+ images. - Bubble rendering switches to .try-msg-attach-row above 1 image. - Single-image rendering paths unchanged (zero visual regression for existing 1-attachment turns). --- src/app/globals.css | 29 +++++ src/components/try/FranklinChat.tsx | 110 ++++++++++++++----- src/hooks/use-franklin-chat.ts | 164 ++++++++++++++++++++++------ 3 files changed, 243 insertions(+), 60 deletions(-) diff --git a/src/app/globals.css b/src/app/globals.css index f94ffae..6aa2b43 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -4349,6 +4349,35 @@ h2.dark-h, margin: 0 0 8px 2px; } +/* Horizontal row of attached input thumbnails — shown above the textarea + * when the user picks multiple reference images for image2image fusion or + * multi-image chat vision. Single-attachment turns reuse the .try-attach + * shape directly, so this row only kicks in at 2+ items. */ +.try-attach-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin: 2px 0 8px 2px; +} +.try-attach-row .try-attach { margin: 0; } + +/* Multi-image bubble inside the message stream — same row layout, no + * outer card chrome (each thumbnail keeps its own subtle border). */ +.try-msg-attach-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 8px; +} +.try-msg-attach-row img { + width: 110px; + height: 110px; + object-fit: cover; + border-radius: 12px; + border: 1px solid var(--border); + cursor: zoom-in; +} + /* Nav highlight for Try Franklin */ .nav-link-try { color: var(--gold-dim); } diff --git a/src/components/try/FranklinChat.tsx b/src/components/try/FranklinChat.tsx index 57a7b52..40433d2 100644 --- a/src/components/try/FranklinChat.tsx +++ b/src/components/try/FranklinChat.tsx @@ -13,7 +13,7 @@ import { GalleryPanel } from "./GalleryPanel"; import { WalletPanel } from "./WalletPanel"; import { SkillsPanel } from "./SkillsPanel"; import { CLIPanel } from "./CLIPanel"; -import { useFranklinChat } from "@/hooks/use-franklin-chat"; +import { useFranklinChat, maxAttachmentsFor } from "@/hooks/use-franklin-chat"; import { useChatHistory } from "@/hooks/use-chat-history"; import { useUsageStats } from "@/hooks/use-usage-stats"; import { useAuth } from "@/hooks/use-auth"; @@ -78,7 +78,10 @@ export function FranklinChat() { const busy = isBusy || !!activeMediaJob; const [input, setInput] = useState(""); - const [attachment, setAttachment] = useState(null); + // Multi-attachment: gateway image2image accepts up to 4 (OpenAI) / 3 (Google); + // chat-vision matches the same cap for UX symmetry. maxAttachmentsFor() returns + // the per-mode/per-model ceiling; the "+" button hides when reached. + const [attachments, setAttachments] = useState([]); const [lightbox, setLightbox] = useState(null); const [sidebarOpen, setSidebarOpen] = useState(true); const [view, setView] = useState("chat"); @@ -167,18 +170,31 @@ export function FranklinChat() { const messages = history.messages; const [attachError, setAttachError] = useState(null); + // Per-mode/per-model attachment ceiling — used for both the file input's + // accept gate and to hide the "+" button once the user has hit the limit. + // Falls back to chat-mode when no model is picked (no model = no upload). + const attachCap = maxAttachmentsFor(mode, model); const onPickFile = async (e: React.ChangeEvent) => { - const file = e.target.files?.[0]; + const files = Array.from(e.target.files ?? []); e.target.value = ""; - if (!file) return; + if (files.length === 0) return; setAttachError(null); + // Respect the per-model cap on every pick — if the user already attached + // 3 of 4 then drops 3 more in one go, append only the first 1 that fits. + const room = Math.max(0, attachCap - attachments.length); + const accepted = files.slice(0, room); + if (accepted.length === 0) return; try { // Downscale/re-encode large images before upload (see image-compress.ts). - setAttachment(await prepareImageForUpload(file)); + const prepared = await Promise.all(accepted.map((f) => prepareImageForUpload(f))); + setAttachments((cur) => [...cur, ...prepared].slice(0, attachCap)); } catch (err) { setAttachError(err instanceof Error ? err.message : "Could not load that image."); } }; + const removeAttachment = (idx: number) => { + setAttachments((cur) => cur.filter((_, i) => i !== idx)); + }; useEffect(() => { scrollRef.current?.scrollTo({ top: scrollRef.current.scrollHeight, behavior: "smooth" }); @@ -187,8 +203,8 @@ export function FranklinChat() { // Image/video always need a wallet; paid chat models too. // Image/video need a wallet; paid chat models too; and an attachment forces a // (paid) vision model, so it needs one as well. - const needsWallet = (mode !== "chat" || !selectedModel?.free || !!attachment) && !isConnected; - const canSend = (!!input.trim() || !!attachment) && !busy && !needsWallet; + const needsWallet = (mode !== "chat" || !selectedModel?.free || attachments.length > 0) && !isConnected; + const canSend = (!!input.trim() || attachments.length > 0) && !busy && !needsWallet; const suggestions = mode === "image" ? t.sugImage : mode === "video" ? t.sugVideo : mode === "music" ? t.sugMusic : t.sugChat; @@ -236,15 +252,18 @@ export function FranklinChat() { const submit = () => { if (!canSend) return; + // send() normalizes single → array internally; pass undefined when empty + // so it can short-circuit the empty-attachment branch cleanly. + const atts = attachments.length > 0 ? attachments : undefined; if (focus) { // Force the focused tool; bump unreliable free models to a tool-capable one. const m = model.startsWith("nvidia/") ? TOOL_FOCUS_MODEL : model; - send(input, attachment ?? undefined, "chat", m, focus); + send(input, atts, "chat", m, focus); } else { - send(input, attachment ?? undefined); + send(input, atts); } setInput(""); - setAttachment(null); + setAttachments([]); }; const placeholder = needsWallet @@ -405,12 +424,29 @@ export function FranklinChat() { ) : ( <> - {m.image && ( -
- {/* eslint-disable-next-line @next/next/no-img-element */} - attachment setLightbox(m.image!)} /> -
- )} + {/* Render every attached input image. Legacy single-image + messages set `image`; multi-attachment turns use + `images`. Both flatten through this same row so a + single attachment doesn't get the multi-row styling. */} + {(() => { + const imgs = m.images && m.images.length > 0 ? m.images : m.image ? [m.image] : []; + if (imgs.length === 0) return null; + return ( +
1 ? "try-msg-attach-row" : "try-msg-attach"}> + {imgs.map((src, idx) => ( + // eslint-disable-next-line @next/next/no-img-element + {`attachment setLightbox(src)} + /> + ))} +
+ ); + })()} {m.activity && } {m.reasoning && (
@@ -525,13 +561,21 @@ export function FranklinChat() { {/* Composer: textarea on top, tool row inside the same box */}
- {attachment && ( -
- {/* eslint-disable-next-line @next/next/no-img-element */} - attachment - + {attachments.length > 0 && ( +
+ {attachments.map((src, i) => ( +
+ {/* eslint-disable-next-line @next/next/no-img-element */} + {`attachment + +
+ ))}
)} {attachError &&
{attachError}
} @@ -556,18 +600,32 @@ export function FranklinChat() { type="file" accept="image/*" className="try-file-input" + // Allow multi-pick when the active model fuses multiple refs + // (OpenAI gpt-image-* / Google Nano Banana). Single-anchor + // modes get the simpler single-pick input behavior. + multiple={attachCap > 1} onChange={onPickFile} /> {/* Music ignores reference images, so don't offer an attach button there — it would accept an image, show a thumbnail, - and silently drop it from the request. */} - {mode !== "music" && ( + and silently drop it from the request. The button also + hides once the per-model cap is hit so users don't pick + a file that would be rejected. */} + {mode !== "music" && attachments.length < attachCap && ( diff --git a/src/hooks/use-franklin-chat.ts b/src/hooks/use-franklin-chat.ts index ea5cb9f..0066f11 100644 --- a/src/hooks/use-franklin-chat.ts +++ b/src/hooks/use-franklin-chat.ts @@ -38,6 +38,38 @@ const MUSIC_ENDPOINT = "/api/blockrun/v1/audio/generations"; // EDIT_SUPPORTED_MODELS). When a reference is attached we force gpt-image-2. const EDIT_SUPPORTED_IMAGE_MODELS = new Set(["openai/gpt-image-1", "openai/gpt-image-2"]); +// Per-provider multi-image fusion cap for image2image. Mirrors the gateway's +// MAX_IMAGES_BY_PREFIX in `blockrun/src/app/api/v1/images/image2image/route.ts` +// — OpenAI gpt-image-* fuses up to 4 anchors, Google Nano Banana up to 3, and +// every other model accepts a single reference. The composer hides the +// "+" button past this cap and gateway hard-rejects any attempt to exceed it. +const IMAGE_FUSION_MAX_BY_PREFIX: Record = { + "openai/": 4, + "google/": 3, +}; +function maxImageFusionFor(modelId: string): number { + const prefix = modelId.split("/")[0]; + return IMAGE_FUSION_MAX_BY_PREFIX[`${prefix}/`] ?? 1; +} + +// Vision-attachment cap for chat mode. Anthropic /v1/messages tolerates 20+ +// image blocks per turn and OpenAI vision similarly accepts many; the actual +// constraint is token budget + UX. 4 is a clean upper bound that matches the +// image2image canvas cap so the composer feels symmetric across modes. +const CHAT_VISION_MAX = 4; + +/** Max attachments the composer should accept for the given mode + model. + * Music mode is text-only; video uses a single seed frame; image and chat + * scale by provider. */ +export function maxAttachmentsFor(mode: ChatMode, modelId: string): number { + if (mode === "music") return 0; + if (mode === "video") return 1; + if (mode === "image") return maxImageFusionFor(modelId); + // chat — only meaningful when the active model is vision-capable; the UI + // gates the file button on that elsewhere. + return CHAT_VISION_MAX; +} + // Video models that accept a seed image (image-to-video). Sora 2 does not, so a // reference attached to it falls back to the default img2video model. const VIDEO_IMAGE_INPUT_MODELS = new Set([ @@ -180,13 +212,28 @@ export interface ChatMessage { role: "user" | "assistant"; content: string; kind?: "text" | "image" | "video" | "music"; + /** Single image, kept for back-compat with legacy single-attachment + * messages already persisted in GCS and for assistant-generated single + * image outputs. New multi-attachment user turns use `images` instead. */ image?: string; + /** Multi-image user attachments (canvas fusion / chat vision). When + * present and non-empty, `image` is ignored on render. */ + images?: string[]; video?: string; music?: string; reasoning?: string; activity?: ChatActivity; } +/** Read a user message's attached images, normalized to a flat array — covers + * the legacy single-`image` shape and the new `images[]` shape so callers + * don't have to branch. Returns [] when the message has no attachments. */ +function userImages(m: ChatMessage): string[] { + if (m.images && m.images.length > 0) return m.images; + if (m.image) return [m.image]; + return []; +} + // Vision-capable chat models (mirrors Franklin's src/router/vision.ts). Used to // auto-swap to a vision model when the user attaches an image to a text-only one. const VISION_MODELS = new Set([ @@ -547,16 +594,30 @@ export function useFranklinChat( }, []); const send = useCallback( - async (text: string, attachment?: string, modeOverride?: ChatMode, modelOverride?: string, forceTool?: string) => { + async ( + text: string, + attachments?: string | string[], + modeOverride?: ChatMode, + modelOverride?: string, + forceTool?: string, + ) => { const prompt = text.trim(); - if (!prompt && !attachment) return; + // Normalize the single-string back-compat shape to an array — callers + // that were passing one image still work; the new multi-pick composer + // passes an array directly. + const atts: string[] = Array.isArray(attachments) + ? attachments + : attachments + ? [attachments] + : []; + if (!prompt && atts.length === 0) return; const activeMode0 = modeOverride ?? mode; // Heavy media → detached, per-conversation background job. It doesn't take // the global chat lock, so other conversations stay usable while it runs. if (activeMode0 === "image" || activeMode0 === "video" || activeMode0 === "music") { const mediaConvId = ensureConvId(); if (mediaAbortRef.current[mediaConvId]) return; // already generating here - void runMedia(mediaConvId, activeMode0, prompt, attachment, modelOverride); + void runMedia(mediaConvId, activeMode0, prompt, atts, modelOverride); return; } if (inFlight.current) return; @@ -577,7 +638,10 @@ export function useFranklinChat( role: "user", content: prompt, kind: "text", - ...(attachment ? { image: attachment } : {}), + // Persist using the multi-shape when >1, single-shape for 1, so + // legacy single-image renderers and persisted conversations keep + // working without a migration. + ...(atts.length > 1 ? { images: atts } : atts.length === 1 ? { image: atts[0] } : {}), }; const history = [...msgRef.current, userMsg]; setMessages(history); @@ -639,7 +703,7 @@ export function useFranklinChat( setMessages(base); const m: ChatMode = lastKind === "image" ? "image" : lastKind === "video" ? "video" : lastKind === "music" ? "music" : "chat"; - void send(userMsg.content, userMsg.image, m); + void send(userMsg.content, userImages(userMsg), m); }, [send, setMessages]); // Paid request → parsed JSON (handles 402→sign→retry). POST by default; GET @@ -762,17 +826,22 @@ export function useFranklinChat( }; const apiMessages: ApiMsg[] = history .filter((m) => m.kind !== "video" && m.kind !== "music" && !(m.kind === "image" && m.role === "assistant")) - .map((m) => - m.role === "user" && m.image - ? { - role: "user", - content: [ - ...(m.content ? [{ type: "text", text: m.content }] : []), - toImageBlock(m.image), - ], - } - : { role: m.role, content: m.content }, - ); + .map((m) => { + // User message with one or more attached images → Anthropic-shape + // content array. userImages() normalizes the legacy single-image + // and new multi-image shapes so we render either uniformly. + const imgs = m.role === "user" ? userImages(m) : []; + if (imgs.length > 0) { + return { + role: "user", + content: [ + ...(m.content ? [{ type: "text", text: m.content }] : []), + ...imgs.map(toImageBlock), + ], + }; + } + return { role: m.role, content: m.content }; + }); // Accumulate a compact record of the run so it can collapse into a summary // ("searched N keywords · M sources") attached to the final answer. @@ -916,7 +985,7 @@ export function useFranklinChat( // Resolve "Auto" client-side, then vision-route: if the turn carries an // image and the chosen model can't see, swap to a vision-capable sibling. const base = modelOverride || chatModel; - const hasImage = history.some((m) => m.role === "user" && m.image); + const hasImage = history.some((m) => m.role === "user" && userImages(m).length > 0); const lastPrompt = [...history].reverse().find((m) => m.role === "user")?.content ?? ""; let effectiveModel = base === "blockrun/auto" ? resolveAuto(lastPrompt) : base; if (hasImage && !isVisionModel(effectiveModel)) effectiveModel = pickVisionSibling(effectiveModel); @@ -932,15 +1001,25 @@ export function useFranklinChat( convId: string, kind: "image" | "video" | "music", prompt: string, - reference?: string, + references?: string[], modelOverride?: string, ) { + const refs = references ?? []; + const firstRef = refs[0]; // video seed / single-ref fallback for legacy paths const abort = new AbortController(); mediaAbortRef.current[convId] = abort; setMediaJobs((p) => ({ ...p, [convId]: { kind, phase: "generating" } })); const setPhase = (phase: MediaJob["phase"]) => setMediaJobs((p) => ({ ...p, [convId]: { kind, phase } })); setMessagesRaw( - (m) => [...m, { role: "user", content: prompt, kind: "text", ...(reference ? { image: reference } : {}) }], + (m) => [ + ...m, + { + role: "user", + content: prompt, + kind: "text", + ...(refs.length > 1 ? { images: refs } : refs.length === 1 ? { image: refs[0] } : {}), + }, + ], convId, ); @@ -949,26 +1028,43 @@ export function useFranklinChat( let endpoint: string; let body: string; if (kind === "image") { - const useEdit = !!reference; + const useEdit = refs.length > 0; const baseModel = modelOverride ?? imageModel; model = useEdit && !EDIT_SUPPORTED_IMAGE_MODELS.has(baseModel) ? "openai/gpt-image-2" : baseModel; endpoint = useEdit ? IMAGE_EDIT_ENDPOINT : IMAGE_ENDPOINT; - // Edit endpoint is locked to 1024² (the only size every edit-capable - // model accepts); text-to-image uses the user's selected aspect ratio, - // validated against the active model's whitelist. - const validSize = IMAGE_MODEL_SIZES[model]?.some((s) => s.size === imageSize) - ? imageSize - : defaultSizeFor(model); - body = useEdit - ? JSON.stringify({ model, prompt, image: reference, size: "1024x1024", n: 1 }) - : JSON.stringify({ model, prompt, size: validSize, n: 1 }); + if (useEdit) { + // Per-provider fusion cap — defensive trim. Gateway also caps but a + // 400 round trip is wasted wallet RTT, so the client mirrors the + // limit here too. + const cap = maxImageFusionFor(model); + const trimmed = refs.slice(0, Math.max(1, cap)); + // The gateway accepts either a single string or a string[] under + // the `image` key; passing the multi shape only when we actually + // have multiple keeps single-image requests on the same wire format + // the route has always honored. + body = JSON.stringify({ + model, + prompt, + image: trimmed.length === 1 ? trimmed[0] : trimmed, + size: "1024x1024", + n: 1, + }); + } else { + // Text-to-image uses the user's selected aspect ratio, validated + // against the active model's whitelist. + const validSize = IMAGE_MODEL_SIZES[model]?.some((s) => s.size === imageSize) + ? imageSize + : defaultSizeFor(model); + body = JSON.stringify({ model, prompt, size: validSize, n: 1 }); + } } else if (kind === "video") { - const useSeed = !!reference; + // Video models still take a single seed image — extra attachments are + // dropped here rather than at the composer so the user can pre-stage + // images and switch modes without losing them. First image wins. + const useSeed = !!firstRef; const baseModel = modelOverride ?? videoModel; model = useSeed && !VIDEO_IMAGE_INPUT_MODELS.has(baseModel) ? DEFAULT_I2V_MODEL : baseModel; endpoint = VIDEO_ENDPOINT; - // aspect_ratio is token360 (Seedance) only; for other providers we - // omit it entirely so the upstream picks its own default. const ratios = VIDEO_MODEL_RATIOS[model]; const aspectRatio = ratios && ratios.includes(videoRatio) ? videoRatio : undefined; const resolutions = VIDEO_MODEL_RESOLUTIONS[model]; @@ -976,13 +1072,13 @@ export function useFranklinChat( body = JSON.stringify({ model, prompt, - ...(useSeed ? { image_url: reference } : {}), + ...(useSeed ? { image_url: firstRef } : {}), ...(aspectRatio ? { aspect_ratio: aspectRatio } : {}), ...(resolution ? { resolution } : {}), }); } else { // music — gateway picks the model via `model`; the rest of the params are - // optional (duration / lyrics / instrumental). Reference is ignored. + // optional (duration / lyrics / instrumental). References are ignored. model = modelOverride ?? musicModel; endpoint = MUSIC_ENDPOINT; body = JSON.stringify({ model, prompt }); From b3f7aec5b8894298890fa4c4b15394a346ea10a2 Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Wed, 3 Jun 2026 03:18:52 -0400 Subject: [PATCH 2/3] Clear staged attachments on mode change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Images staged in one mode (image fusion refs / chat vision) survived a mode switch and were sent on the next turn — leaking into chat vision or into a music request that drops them with an empty prompt. Reset attachments in the existing mode-change block, matching the flyout-reset pattern from #7. --- src/components/try/FranklinChat.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/try/FranklinChat.tsx b/src/components/try/FranklinChat.tsx index 40433d2..f44704f 100644 --- a/src/components/try/FranklinChat.tsx +++ b/src/components/try/FranklinChat.tsx @@ -121,6 +121,9 @@ export function FranklinChat() { setFlyoutMode(mode); setRatioOpen(false); setResOpen(false); + // Staged attachments are mode-specific (image fusion refs / chat vision); + // clearing them here keeps them from leaking into a different mode's send. + setAttachments([]); } const ratioOptions: { ratio: string; value: string }[] = mode === "image" From f87640c0ef21b5d80fa11f757cde7c254fe39629 Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Wed, 3 Jun 2026 03:22:22 -0400 Subject: [PATCH 3/3] Use attach row only for 2+ composer thumbnails The .try-attach-row comment states single-attachment turns reuse the .try-attach card directly, but the composer wrapped every count in the row, whose 'margin: 0' rule stripped the single thumbnail's spacing. Gate the row on 2+ items, matching the message-bubble render and the PR's zero-regression promise for 1-attachment turns. --- src/components/try/FranklinChat.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/try/FranklinChat.tsx b/src/components/try/FranklinChat.tsx index f44704f..f33bf7f 100644 --- a/src/components/try/FranklinChat.tsx +++ b/src/components/try/FranklinChat.tsx @@ -565,7 +565,7 @@ export function FranklinChat() { {/* Composer: textarea on top, tool row inside the same box */}
{attachments.length > 0 && ( -
+
1 ? "try-attach-row" : undefined}> {attachments.map((src, i) => (
{/* eslint-disable-next-line @next/next/no-img-element */}