From 2908bcb6d078890afed9263dc74cae3145d576dc Mon Sep 17 00:00:00 2001 From: luckyyzh Date: Mon, 3 Aug 2026 19:09:03 +0800 Subject: [PATCH] feat(vision): cache key include prompt, parallel multi-image, compressed hook prompt, PNG compression, retry logic --- pi-web/lib/image-attachments.ts | 157 ++++++++++++++++++-------------- resources/extensions/vision.ts | 103 ++++++++++++++------- 2 files changed, 161 insertions(+), 99 deletions(-) diff --git a/pi-web/lib/image-attachments.ts b/pi-web/lib/image-attachments.ts index 2826a07..c969728 100644 --- a/pi-web/lib/image-attachments.ts +++ b/pi-web/lib/image-attachments.ts @@ -7,90 +7,111 @@ export const MAX_IMAGE_EDGE_PX = 1600; export const IMAGE_JPEG_QUALITY = 0.85; export interface Base64ImageAttachment { - data: string; - mimeType: string; + data: string; + mimeType: string; } function isBase64DataChar(code: number): boolean { - return (code >= 0x41 && code <= 0x5a) - || (code >= 0x61 && code <= 0x7a) - || (code >= 0x30 && code <= 0x39) - || code === 0x2b - || code === 0x2f; + return ( + (code >= 0x41 && code <= 0x5a) || + (code >= 0x61 && code <= 0x7a) || + (code >= 0x30 && code <= 0x39) || + code === 0x2b || + code === 0x2f + ); } export function getBase64DecodedByteLength(data: string): number | null { - if (!data || data.length % 4 !== 0) return null; - const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; - const dataEnd = data.length - padding; - for (let index = 0; index < dataEnd; index += 1) { - if (!isBase64DataChar(data.charCodeAt(index))) return null; - } - for (let index = dataEnd; index < data.length; index += 1) { - if (data[index] !== "=") return null; - } - return (data.length / 4) * 3 - padding; + if (!data || data.length % 4 !== 0) return null; + const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; + const dataEnd = data.length - padding; + for (let index = 0; index < dataEnd; index += 1) { + if (!isBase64DataChar(data.charCodeAt(index))) return null; + } + for (let index = dataEnd; index < data.length; index += 1) { + if (data[index] !== "=") return null; + } + return (data.length / 4) * 3 - padding; } -export function isBase64ImageWithinLimits(value: unknown): value is Base64ImageAttachment { - if (!value || typeof value !== "object") return false; - const image = value as Partial; - if (typeof image.data !== "string" || typeof image.mimeType !== "string" || !image.mimeType.startsWith("image/")) { - return false; - } - const bytes = getBase64DecodedByteLength(image.data); - return bytes !== null && bytes <= MAX_ATTACHED_IMAGE_BYTES; +export function isBase64ImageWithinLimits( + value: unknown, +): value is Base64ImageAttachment { + if (!value || typeof value !== "object") return false; + const image = value as Partial; + if ( + typeof image.data !== "string" || + typeof image.mimeType !== "string" || + !image.mimeType.startsWith("image/") + ) { + return false; + } + const bytes = getBase64DecodedByteLength(image.data); + return bytes !== null && bytes <= MAX_ATTACHED_IMAGE_BYTES; } /** - * 上传前压缩：仅 JPEG 且长边超过 MAX_IMAGE_EDGE_PX 时缩放重编码（EXIF 方向由 - * createImageBitmap 默认 from-image 纠正）。PNG/WebP/GIF 原样返回（无损/动画场景）。 + * 上传前压缩：JPEG 和 PNG 且长边超过 MAX_IMAGE_EDGE_PX 时缩放并统一转 JPEG。 + * PNG 截图（代码编辑器/UI）通常无透明通道，转 JPEG 体积大幅减小。 + * WebP/GIF 原样返回（WebP 已经高效；GIF 可能是动画）。 * 重编码无收益（更小或失败）时退回原文件。 */ export async function compressImageFile(file: File): Promise { - if (file.type !== "image/jpeg") return file; - let bitmap: ImageBitmap; - try { - bitmap = await createImageBitmap(file); - } catch { - return file; // 解码失败退化为原文件 - } - try { - const edge = Math.max(bitmap.width, bitmap.height); - if (edge <= MAX_IMAGE_EDGE_PX) return file; // 本就不大，避免无谓重编码 - const scale = MAX_IMAGE_EDGE_PX / edge; - const width = Math.max(1, Math.round(bitmap.width * scale)); - const height = Math.max(1, Math.round(bitmap.height * scale)); - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext("2d"); - if (!ctx) return file; - ctx.drawImage(bitmap, 0, 0, width, height); - const blob = await new Promise((resolve) => - canvas.toBlob(resolve, "image/jpeg", IMAGE_JPEG_QUALITY) - ); - if (!blob || blob.size >= file.size) return file; // 重编码无收益则用原文件 - return new File([blob], file.name.replace(/\.\w+$/, ".jpg"), { type: "image/jpeg" }); - } finally { - bitmap.close(); - } + // 仅压缩 JPEG 和 PNG（最常见的截图/照片格式） + if ( + !file.type || + !(file.type.startsWith("image/jpeg") || file.type.startsWith("image/png")) + ) { + return file; + } + let bitmap: ImageBitmap; + try { + bitmap = await createImageBitmap(file); + } catch { + return file; // 解码失败退化为原文件 + } + try { + const edge = Math.max(bitmap.width, bitmap.height); + if (edge <= MAX_IMAGE_EDGE_PX) return file; // 本就不大，避免无谓重编码 + const scale = MAX_IMAGE_EDGE_PX / edge; + const width = Math.max(1, Math.round(bitmap.width * scale)); + const height = Math.max(1, Math.round(bitmap.height * scale)); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext("2d"); + if (!ctx) return file; + ctx.drawImage(bitmap, 0, 0, width, height); + const blob = await new Promise((resolve) => + canvas.toBlob(resolve, "image/jpeg", IMAGE_JPEG_QUALITY), + ); + if (!blob || blob.size >= file.size) return file; // 重编码无收益则用原文件 + return new File([blob], file.name.replace(/\.\w+$/, ".jpg"), { + type: "image/jpeg", + }); + } finally { + bitmap.close(); + } } /** Return an API-safe error for prompt, steering, and follow-up image arrays. */ export function validateAgentImages(value: unknown): string | null { - if (value === undefined) return null; - if (!Array.isArray(value)) return "images must be an array"; - if (value.length > MAX_ATTACHED_IMAGES) { - return `A message can include at most ${MAX_ATTACHED_IMAGES} images`; - } - for (const image of value) { - if (!image || typeof image !== "object" || (image as { type?: unknown }).type !== "image") { - return "Each attachment must be an image"; - } - if (!isBase64ImageWithinLimits(image)) { - return `Each image must be valid base64 image data of ${MAX_ATTACHED_IMAGE_BYTES / (1024 * 1024)}MB or smaller`; - } - } - return null; + if (value === undefined) return null; + if (!Array.isArray(value)) return "images must be an array"; + if (value.length > MAX_ATTACHED_IMAGES) { + return `A message can include at most ${MAX_ATTACHED_IMAGES} images`; + } + for (const image of value) { + if ( + !image || + typeof image !== "object" || + (image as { type?: unknown }).type !== "image" + ) { + return "Each attachment must be an image"; + } + if (!isBase64ImageWithinLimits(image)) { + return `Each image must be valid base64 image data of ${MAX_ATTACHED_IMAGE_BYTES / (1024 * 1024)}MB or smaller`; + } + } + return null; } diff --git a/resources/extensions/vision.ts b/resources/extensions/vision.ts index f85e0e8..dcb11c9 100644 --- a/resources/extensions/vision.ts +++ b/resources/extensions/vision.ts @@ -43,10 +43,10 @@ const DEFAULT_PROMPT = [ * the main model's context every turn, so verbosity costs both latency and * tokens. The `vision` tool keeps the detailed DEFAULT_PROMPT above. */ const HOOK_PROMPT = [ - "用中文简要描述这张图片（主模型依赖此转录理解图片，需准确但精简）：", - "1. 所有可见文字：按阅读顺序转录（含标签、数字、按钮、代码），无文字则写“无”。", - "2. 图片内容：主体、场景、布局，2-3 句。", - "总长约 100 字，不要分节模板，直接输出。", + `用中文简要描述此图（主模型完全依赖此转录）：`, + `1. 可见文字逐字转录（标签/代码/数字），无则写无文字。`, + `2. 画面内容 1-2 句。`, + `控制在 50 字内。`, ].join("\n"); const visionParams = Type.Object({ @@ -420,10 +420,10 @@ function dataUrlToLoadedImage(url: string): LoadedImage | null { return { base64: match[2], mime: match[1] }; } -function imageCacheKey(image: LoadedImage): string { - // Whole-image hash: PNG/JPG headers repeat for same dimensions, so a short - // prefix would collide across different images of the same size. - return createHash("md5").update(image.base64).digest("hex"); +function imageCacheKey(image: LoadedImage, prompt: string): string { + // Hash image + prompt so different prompts (hook vs tool) don't share cache. + // PNG/JPG headers repeat for same dimensions, so full base64 is needed. + return createHash("md5").update(image.base64).update(prompt).digest("hex"); } /** @@ -438,7 +438,7 @@ async function getImageDescription( prompt: string, signal: AbortSignal | undefined, ): Promise { - const key = imageCacheKey(image); + const key = imageCacheKey(image, prompt); await loadDescriptionCache(); const cached = IMAGE_DESCRIPTION_CACHE.get(key); if (cached) return cached; @@ -517,16 +517,26 @@ export default function visionExtension(pi: ExtensionAPI) { if (!Array.isArray(messages) || messages.length === 0) return; if (!isTextOnlyModel(ctx.model)) return; // vision-capable models pass through untouched + // Phase 1: collect image parts, replace with placeholders // Replace each image in place with its stable transcription. Do not move // historical descriptions to a later message: DeepSeek caches exact prompt // prefixes, so rewriting an old image message invalidates everything after it. - let dirty = false; + interface PendingImage { + img: LoadedImage; + num: number; + msgIndex: number; + partIndex: number; + } + const pendingImages: PendingImage[] = []; let imageNumber = 0; - for (const msg of messages) { + let dirty = false; + + for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) { + const msg = messages[msgIdx]; if (msg?.role !== "user" || !Array.isArray(msg.content)) continue; const newContent: unknown[] = []; - let messageDirty = false; - for (const part of msg.content) { + for (let partIdx = 0; partIdx < msg.content.length; partIdx++) { + const part = msg.content[partIdx]; const p = part as { type?: string; image_url?: { url?: string } }; if (p?.type !== "image_url" || typeof p.image_url?.url !== "string") { newContent.push(part); @@ -540,29 +550,60 @@ export default function visionExtension(pi: ExtensionAPI) { } imageNumber += 1; - let description: string; - try { - description = await getImageDescription( - img, - HOOK_PROMPT, - ctx.signal, - ); - } catch { - // Keep failures deterministic. Variable error details in the prompt - // would themselves invalidate the cache on every retry. - description = "[图片转录失败]"; - } - - newContent.push({ - type: "text", - text: `\n\n【图片${imageNumber}】\n${description}\n\n`, + pendingImages.push({ + img, + num: imageNumber, + msgIndex: msgIdx, + partIndex: partIdx, }); - messageDirty = true; + newContent.push(part); // placeholder, replaced after transcription dirty = true; } - if (messageDirty) msg.content = newContent; + if (newContent !== msg.content) { + msg.content = newContent; + } } + if (!dirty) return; + + // Phase 2: transcribe all images in parallel + const results = await Promise.all( + pendingImages.map(async ({ img, num, msgIndex, partIndex }) => { + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const desc = await getImageDescription( + img, + HOOK_PROMPT, + ctx.signal, + ); + return { msgIndex, partIndex, num, desc }; + } catch (err) { + console.warn( + `vision: transcription attempt ${attempt} failed:`, + err instanceof Error ? err.message : String(err), + ); + if (attempt === 2) { + // Deterministic fallback — variable errors would invalidate cache + return { msgIndex, partIndex, num, desc: "[图片转录失败]" }; + } + } + } + }), + ); + + // Phase 3: replace placeholders with descriptions + for (const r of results) { + if (!r) continue; + const { msgIndex, partIndex, num, desc } = r; + const msg = messages[msgIndex]; + if (msg && Array.isArray(msg.content)) { + msg.content[partIndex] = { + type: "text", + text: `\n\n【图片${num}】\n${desc}\n\n`, + }; + } + } + return payload; }); }