gitadmin/pi-agent-integrated
1
0
Fork 0
mirror of https://github.com/luckyyzh/pi-agent-integrated.git synced 2026-10-03 02:59:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(vision): cache key include prompt, parallel multi-image, compressed hook prompt, PNG compression, retry logic

Browse Source
This commit is contained in:
luckyyzh
2026-08-03 19:09:18 +08:00
parent dcce2468a0
commit 2908bcb6d0
2 changed files with 161 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pi-web/lib/image-attachments.ts
+89 -68
View File
@@ -7,90 +7,111 @@ export const MAX_IMAGE_EDGE_PX = 1600;
export const IMAGE_JPEG_QUALITY = 0.85; export const IMAGE_JPEG_QUALITY = 0.85;
export interface Base64ImageAttachment { export interface Base64ImageAttachment {
data: string; data: string;
mimeType: string; mimeType: string;
} }
function isBase64DataChar(code: number): boolean { function isBase64DataChar(code: number): boolean {
return (code >= 0x41 && code <= 0x5a) return (
|| (code >= 0x61 && code <= 0x7a) (code >= 0x41 && code <= 0x5a) ||
|| (code >= 0x30 && code <= 0x39) (code >= 0x61 && code <= 0x7a) ||
|| code === 0x2b (code >= 0x30 && code <= 0x39) ||
|| code === 0x2f; code === 0x2b ||
code === 0x2f
);
} }
export function getBase64DecodedByteLength(data: string): number | null { export function getBase64DecodedByteLength(data: string): number | null {
if (!data || data.length % 4 !== 0) return null; if (!data || data.length % 4 !== 0) return null;
const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0;
const dataEnd = data.length - padding; const dataEnd = data.length - padding;
for (let index = 0; index < dataEnd; index += 1) { for (let index = 0; index < dataEnd; index += 1) {
if (!isBase64DataChar(data.charCodeAt(index))) return null; if (!isBase64DataChar(data.charCodeAt(index))) return null;
} }
for (let index = dataEnd; index < data.length; index += 1) { for (let index = dataEnd; index < data.length; index += 1) {
if (data[index] !== "=") return null; if (data[index] !== "=") return null;
} }
return (data.length / 4) * 3 - padding; return (data.length / 4) * 3 - padding;
} }
export function isBase64ImageWithinLimits(value: unknown): value is Base64ImageAttachment { export function isBase64ImageWithinLimits(
if (!value || typeof value !== "object") return false; value: unknown,
const image = value as Partial<Base64ImageAttachment>; ): value is Base64ImageAttachment {
if (typeof image.data !== "string" || typeof image.mimeType !== "string" || !image.mimeType.startsWith("image/")) { if (!value || typeof value !== "object") return false;
return false; const image = value as Partial<Base64ImageAttachment>;
} if (
const bytes = getBase64DecodedByteLength(image.data); typeof image.data !== "string" ||
return bytes !== null && bytes <= MAX_ATTACHED_IMAGE_BYTES; typeof image.mimeType !== "string" ||
!image.mimeType.startsWith("image/")
) {
return false;
}
const bytes = getBase64DecodedByteLength(image.data);
return bytes !== null && bytes <= MAX_ATTACHED_IMAGE_BYTES;
} }
/** /**
* 上传前压缩：仅 JPEG 且长边超过 MAX_IMAGE_EDGE_PX 时缩放重编码（EXIF 方向由 * 上传前压缩：JPEG 和 PNG 且长边超过 MAX_IMAGE_EDGE_PX 时缩放并统一转 JPEG。
* createImageBitmap 默认 from-image 纠正）。PNG/WebP/GIF 原样返回（无损/动画场景）。 * PNG 截图（代码编辑器/UI）通常无透明通道，转 JPEG 体积大幅减小。
* WebP/GIF 原样返回（WebP 已经高效；GIF 可能是动画）。
* 重编码无收益（更小或失败）时退回原文件。 * 重编码无收益（更小或失败）时退回原文件。
*/ */
export async function compressImageFile(file: File): Promise<File> { export async function compressImageFile(file: File): Promise<File> {
if (file.type !== "image/jpeg") return file; // 仅压缩 JPEG 和 PNG（最常见的截图/照片格式）
let bitmap: ImageBitmap; if (
try { !file.type ||
bitmap = await createImageBitmap(file); !(file.type.startsWith("image/jpeg") || file.type.startsWith("image/png"))
} catch { ) {
return file; // 解码失败退化为原文件 return file;
} }
try { let bitmap: ImageBitmap;
const edge = Math.max(bitmap.width, bitmap.height); try {
if (edge <= MAX_IMAGE_EDGE_PX) return file; // 本就不大，避免无谓重编码 bitmap = await createImageBitmap(file);
const scale = MAX_IMAGE_EDGE_PX / edge; } catch {
const width = Math.max(1, Math.round(bitmap.width * scale)); return file; // 解码失败退化为原文件
const height = Math.max(1, Math.round(bitmap.height * scale)); }
const canvas = document.createElement("canvas"); try {
canvas.width = width; const edge = Math.max(bitmap.width, bitmap.height);
canvas.height = height; if (edge <= MAX_IMAGE_EDGE_PX) return file; // 本就不大，避免无谓重编码
const ctx = canvas.getContext("2d"); const scale = MAX_IMAGE_EDGE_PX / edge;
if (!ctx) return file; const width = Math.max(1, Math.round(bitmap.width * scale));
ctx.drawImage(bitmap, 0, 0, width, height); const height = Math.max(1, Math.round(bitmap.height * scale));
const blob = await new Promise<Blob | null>((resolve) => const canvas = document.createElement("canvas");
canvas.toBlob(resolve, "image/jpeg", IMAGE_JPEG_QUALITY) canvas.width = width;
); canvas.height = height;
if (!blob || blob.size >= file.size) return file; // 重编码无收益则用原文件 const ctx = canvas.getContext("2d");
return new File([blob], file.name.replace(/\.\w+$/, ".jpg"), { type: "image/jpeg" }); if (!ctx) return file;
} finally { ctx.drawImage(bitmap, 0, 0, width, height);
bitmap.close(); const blob = await new Promise<Blob | null>((resolve) =>
} canvas.toBlob(resolve, "image/jpeg", IMAGE_JPEG_QUALITY),
);
if (!blob || blob.size >= file.size) return file; // 重编码无收益则用原文件
return new File([blob], file.name.replace(/\.\w+$/, ".jpg"), {
type: "image/jpeg",
});
} finally {
bitmap.close();
}
} }
/** Return an API-safe error for prompt, steering, and follow-up image arrays. */ /** Return an API-safe error for prompt, steering, and follow-up image arrays. */
export function validateAgentImages(value: unknown): string | null { export function validateAgentImages(value: unknown): string | null {
if (value === undefined) return null; if (value === undefined) return null;
if (!Array.isArray(value)) return "images must be an array"; if (!Array.isArray(value)) return "images must be an array";
if (value.length > MAX_ATTACHED_IMAGES) { if (value.length > MAX_ATTACHED_IMAGES) {
return `A message can include at most ${MAX_ATTACHED_IMAGES} images`; return `A message can include at most ${MAX_ATTACHED_IMAGES} images`;
} }
for (const image of value) { for (const image of value) {
if (!image || typeof image !== "object" || (image as { type?: unknown }).type !== "image") { if (
return "Each attachment must be an image"; !image ||
} typeof image !== "object" ||
if (!isBase64ImageWithinLimits(image)) { (image as { type?: unknown }).type !== "image"
return `Each image must be valid base64 image data of ${MAX_ATTACHED_IMAGE_BYTES / (1024 * 1024)}MB or smaller`; ) {
} return "Each attachment must be an image";
} }
return null; if (!isBase64ImageWithinLimits(image)) {
return `Each image must be valid base64 image data of ${MAX_ATTACHED_IMAGE_BYTES / (1024 * 1024)}MB or smaller`;
}
}
return null;
} }
resources/extensions/vision.ts
+72 -31
View File
@@ -43,10 +43,10 @@ const DEFAULT_PROMPT = [
* the main model's context every turn, so verbosity costs both latency and * the main model's context every turn, so verbosity costs both latency and
* tokens. The `vision` tool keeps the detailed DEFAULT_PROMPT above. */ * tokens. The `vision` tool keeps the detailed DEFAULT_PROMPT above. */
const HOOK_PROMPT = [ const HOOK_PROMPT = [
"用中文简要描述这张图片（主模型依赖此转录理解图片，需准确但精简）：", `用中文简要描述此图（主模型完全依赖此转录）：`,
"1. 所有可见文字：按阅读顺序转录（含标签、数字、按钮、代码），无文字则写“无”。", `1. 可见文字逐字转录（标签/代码/数字），无则写无文字。`,
"2. 图片内容：主体、场景、布局，2-3 句。", `2. 画面内容 1-2 句。`,
"总长约 100 字，不要分节模板，直接输出。", `控制在 50 字内。`,
].join("\n"); ].join("\n");
const visionParams = Type.Object({ const visionParams = Type.Object({
@@ -420,10 +420,10 @@ function dataUrlToLoadedImage(url: string): LoadedImage | null {
return { base64: match[2], mime: match[1] }; return { base64: match[2], mime: match[1] };
} }
function imageCacheKey(image: LoadedImage): string { function imageCacheKey(image: LoadedImage, prompt: string): string {
// Whole-image hash: PNG/JPG headers repeat for same dimensions, so a short // Hash image + prompt so different prompts (hook vs tool) don't share cache.
// prefix would collide across different images of the same size. // PNG/JPG headers repeat for same dimensions, so full base64 is needed.
return createHash("md5").update(image.base64).digest("hex"); return createHash("md5").update(image.base64).update(prompt).digest("hex");
} }
/** /**
@@ -438,7 +438,7 @@ async function getImageDescription(
prompt: string, prompt: string,
signal: AbortSignal | undefined, signal: AbortSignal | undefined,
): Promise<string> { ): Promise<string> {
const key = imageCacheKey(image); const key = imageCacheKey(image, prompt);
await loadDescriptionCache(); await loadDescriptionCache();
const cached = IMAGE_DESCRIPTION_CACHE.get(key); const cached = IMAGE_DESCRIPTION_CACHE.get(key);
if (cached) return cached; if (cached) return cached;
@@ -517,16 +517,26 @@ export default function visionExtension(pi: ExtensionAPI) {
if (!Array.isArray(messages) || messages.length === 0) return; if (!Array.isArray(messages) || messages.length === 0) return;
if (!isTextOnlyModel(ctx.model)) return; // vision-capable models pass through untouched if (!isTextOnlyModel(ctx.model)) return; // vision-capable models pass through untouched
// Phase 1: collect image parts, replace with placeholders
// Replace each image in place with its stable transcription. Do not move // Replace each image in place with its stable transcription. Do not move
// historical descriptions to a later message: DeepSeek caches exact prompt // historical descriptions to a later message: DeepSeek caches exact prompt
// prefixes, so rewriting an old image message invalidates everything after it. // prefixes, so rewriting an old image message invalidates everything after it.
let dirty = false; interface PendingImage {
img: LoadedImage;
num: number;
msgIndex: number;
partIndex: number;
}
const pendingImages: PendingImage[] = [];
let imageNumber = 0; let imageNumber = 0;
for (const msg of messages) { let dirty = false;
for (let msgIdx = 0; msgIdx < messages.length; msgIdx++) {
const msg = messages[msgIdx];
if (msg?.role !== "user" || !Array.isArray(msg.content)) continue; if (msg?.role !== "user" || !Array.isArray(msg.content)) continue;
const newContent: unknown[] = []; const newContent: unknown[] = [];
let messageDirty = false; for (let partIdx = 0; partIdx < msg.content.length; partIdx++) {
for (const part of msg.content) { const part = msg.content[partIdx];
const p = part as { type?: string; image_url?: { url?: string } }; const p = part as { type?: string; image_url?: { url?: string } };
if (p?.type !== "image_url" || typeof p.image_url?.url !== "string") { if (p?.type !== "image_url" || typeof p.image_url?.url !== "string") {
newContent.push(part); newContent.push(part);
@@ -540,29 +550,60 @@ export default function visionExtension(pi: ExtensionAPI) {
} }
imageNumber += 1; imageNumber += 1;
let description: string; pendingImages.push({
try { img,
description = await getImageDescription( num: imageNumber,
img, msgIndex: msgIdx,
HOOK_PROMPT, partIndex: partIdx,
ctx.signal,
);
} catch {
// Keep failures deterministic. Variable error details in the prompt
// would themselves invalidate the cache on every retry.
description = "[图片转录失败]";
}
newContent.push({
type: "text",
text: `\n\n【图片${imageNumber}】\n${description}\n\n`,
}); });
messageDirty = true; newContent.push(part); // placeholder, replaced after transcription
dirty = true; dirty = true;
} }
if (messageDirty) msg.content = newContent; if (newContent !== msg.content) {
msg.content = newContent;
}
} }
if (!dirty) return; if (!dirty) return;
// Phase 2: transcribe all images in parallel
const results = await Promise.all(
pendingImages.map(async ({ img, num, msgIndex, partIndex }) => {
for (let attempt = 1; attempt <= 2; attempt++) {
try {
const desc = await getImageDescription(
img,
HOOK_PROMPT,
ctx.signal,
);
return { msgIndex, partIndex, num, desc };
} catch (err) {
console.warn(
`vision: transcription attempt ${attempt} failed:`,
err instanceof Error ? err.message : String(err),
);
if (attempt === 2) {
// Deterministic fallback — variable errors would invalidate cache
return { msgIndex, partIndex, num, desc: "[图片转录失败]" };
}
}
}
}),
);
// Phase 3: replace placeholders with descriptions
for (const r of results) {
if (!r) continue;
const { msgIndex, partIndex, num, desc } = r;
const msg = messages[msgIndex];
if (msg && Array.isArray(msg.content)) {
msg.content[partIndex] = {
type: "text",
text: `\n\n【图片${num}】\n${desc}\n\n`,
};
}
}
return payload; return payload;
}); });
} }