mirror of
https://github.com/luckyyzh/pi-agent-integrated.git
synced 2026-10-03 11:09:34 +00:00
feat: integrate Pi backend and Pi Web
This commit is contained in:
luckyyzh
@@ -0,0 +1,140 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect } from "vitest";
|
||||
import { createJudge, describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts";
|
||||
import { recordEvalSourceArtifact } from "./vitest-evals/artifacts.ts";
|
||||
import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
|
||||
|
||||
type ExtensionAuthoringOutput = {
|
||||
response: string;
|
||||
systemPromptHasGuidelines: boolean;
|
||||
systemPromptHasPiDocs: boolean;
|
||||
extensionErrors: Array<{ path: string; error: string }>;
|
||||
loadedExtensions: Array<{ path: string; tools: string[] }>;
|
||||
extensionSource: string | null;
|
||||
};
|
||||
|
||||
function createExtensionAuthoringHarness(name: string, transformSystemPrompt?: (defaultPrompt: string) => string) {
|
||||
return createPiCodingAgentHarness({
|
||||
name,
|
||||
...(transformSystemPrompt ? { transformSystemPrompt } : {}),
|
||||
output: ({ response, session }) => {
|
||||
const extensions = session.resourceLoader.getExtensions();
|
||||
const extensionPath = join(session.sessionManager.getCwd(), ".pi", "extensions", "hello.ts");
|
||||
const extensionSource = existsSync(extensionPath) ? readFileSync(extensionPath, "utf8") : null;
|
||||
return {
|
||||
response,
|
||||
systemPromptHasGuidelines: session.systemPrompt.includes("\nGuidelines:\n"),
|
||||
systemPromptHasPiDocs: session.systemPrompt.includes("\nPi documentation (read only"),
|
||||
extensionErrors: extensions.errors,
|
||||
loadedExtensions: extensions.extensions.map(({ path, tools }) => ({
|
||||
path,
|
||||
tools: [...tools.keys()],
|
||||
})),
|
||||
extensionSource,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function excludeGuidelinesAndDocumentation(defaultPrompt: string): string {
|
||||
const guidelinesStart = defaultPrompt.indexOf("\nGuidelines:\n");
|
||||
if (guidelinesStart === -1) throw new Error("Default Pi system prompt has no Guidelines section.");
|
||||
return defaultPrompt.slice(0, guidelinesStart);
|
||||
}
|
||||
|
||||
function prepareDefaultPromptOverride(defaultPrompt: string): string {
|
||||
const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: ");
|
||||
if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section.");
|
||||
return defaultPrompt.slice(0, cwdStart);
|
||||
}
|
||||
|
||||
const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthoringOutput>(
|
||||
"ExtensionAuthoringJudge",
|
||||
({ output, toolCalls }) => {
|
||||
const failures: string[] = [];
|
||||
if (output.extensionSource === null) {
|
||||
failures.push("generated extension source is unavailable");
|
||||
} else {
|
||||
const imports = Array.from(
|
||||
output.extensionSource.matchAll(/\b(?:from|import)\s+["']([^"']+)["']/g),
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!imports.includes("@earendil-works/pi-coding-agent")) {
|
||||
failures.push("extension does not import the canonical @earendil-works/pi-coding-agent package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@mariozechner/"))) {
|
||||
failures.push("extension imports a legacy @mariozechner package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@sinclair/typebox"))) {
|
||||
failures.push('extension imports legacy "@sinclair/typebox" instead of "typebox"');
|
||||
}
|
||||
}
|
||||
if (output.extensionErrors.length > 0) failures.push("extension loader reported errors");
|
||||
if (!output.loadedExtensions.some(({ tools }) => tools.includes("hello"))) {
|
||||
failures.push('no loaded extension registered the "hello" tool');
|
||||
}
|
||||
if (
|
||||
!toolCalls.some(
|
||||
(call) =>
|
||||
call.name === "hello" &&
|
||||
call.status === "ok" &&
|
||||
call.arguments?.name === "Bob" &&
|
||||
call.result === "Hello, Bob!",
|
||||
)
|
||||
) {
|
||||
failures.push('no successful hello({ name: "Bob" }) call returned "Hello, Bob!"');
|
||||
}
|
||||
if (output.response !== "Hello, Bob!") failures.push('final response was not exactly "Hello, Bob!"');
|
||||
|
||||
return {
|
||||
score: failures.length === 0 ? 1 : 0,
|
||||
metadata: {
|
||||
rationale: failures.length === 0 ? "Extension authoring workflow completed." : failures.join("; "),
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const extensionHarnessTable = evalHarnessTable("Pi extension authoring system prompt", {
|
||||
baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
|
||||
candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
|
||||
});
|
||||
|
||||
describe.for(extensionHarnessTable)("$name", ({ harness }) => {
|
||||
describeEval(
|
||||
"Pi extension authoring system prompt",
|
||||
{ harness, judges: [ExtensionAuthoringJudge], judgeThreshold: null },
|
||||
(it) => {
|
||||
it("creates, reloads, and uses a hello extension", async ({ run, task }) => {
|
||||
const result = await run([
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.",
|
||||
},
|
||||
{ type: "reload" },
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.",
|
||||
},
|
||||
]);
|
||||
if (result.output.extensionSource !== null) {
|
||||
const runId = result.artifacts?.runId;
|
||||
if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID.");
|
||||
await recordEvalSourceArtifact(task, runId, {
|
||||
name: "hello.ts",
|
||||
contentType: "text/typescript",
|
||||
body: result.output.extensionSource,
|
||||
bodyEncoding: "utf-8",
|
||||
});
|
||||
}
|
||||
const expectsFullPrompt = harness.name === "default-system-prompt";
|
||||
expect(result.output.systemPromptHasGuidelines).toBe(expectsFullPrompt);
|
||||
expect(result.output.systemPromptHasPiDocs).toBe(expectsFullPrompt);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { contentText } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
type AgentSession,
|
||||
type CreateAgentSessionOptions,
|
||||
createAgentSessionFromServices,
|
||||
createAgentSessionServices,
|
||||
ModelRuntime,
|
||||
SessionManager,
|
||||
SettingsManager,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import {
|
||||
createHarness,
|
||||
type Harness,
|
||||
type HarnessContext,
|
||||
type JsonValue,
|
||||
normalizeRecord,
|
||||
type SimpleHarnessResult,
|
||||
type TranscriptEvent,
|
||||
toJsonValue,
|
||||
} from "vitest-evals/harness";
|
||||
import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./vitest-evals/artifacts.ts";
|
||||
|
||||
export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
|
||||
|
||||
type PiCodingAgentModelSelection = {
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessOptions = {
|
||||
name?: string;
|
||||
model?: PiCodingAgentModelSelection;
|
||||
noTools?: CreateAgentSessionOptions["noTools"];
|
||||
transformSystemPrompt?: (defaultPrompt: string) => string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessWithOutput<TOutput extends JsonValue> = PiCodingAgentHarnessOptions & {
|
||||
output: (args: { response: string; session: AgentSession }) => TOutput | Promise<TOutput>;
|
||||
};
|
||||
|
||||
export function resolveModelSelection(
|
||||
explicitModel: PiCodingAgentModelSelection | undefined,
|
||||
environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env,
|
||||
): PiCodingAgentModelSelection {
|
||||
const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim();
|
||||
const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim();
|
||||
if (!provider || !id) {
|
||||
throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.");
|
||||
}
|
||||
return { provider, id };
|
||||
}
|
||||
|
||||
function toTranscriptEvents(messages: AgentSession["messages"]): TranscriptEvent[] {
|
||||
const events: TranscriptEvent[] = [];
|
||||
for (const message of messages) {
|
||||
if (message.role === "user") {
|
||||
events.push({ type: "message", role: "user", content: contentText(message.content) });
|
||||
} else if (message.role === "assistant") {
|
||||
const text = contentText(message.content);
|
||||
if (text) events.push({ type: "message", role: "assistant", content: text });
|
||||
for (const part of message.content) {
|
||||
if (part.type === "toolCall") {
|
||||
events.push({
|
||||
type: "tool_call",
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: normalizeRecord(part.arguments),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (message.role === "toolResult") {
|
||||
const text = contentText(message.content);
|
||||
events.push({
|
||||
type: "tool_result",
|
||||
toolCallId: message.toolCallId,
|
||||
name: message.toolName,
|
||||
content: message.content.every((part) => part.type === "text") ? text : toJsonValue(message.content),
|
||||
...(message.isError ? { error: { message: text || "Tool failed" } } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
async function promptAgent(session: AgentSession, input: string, signal: AbortSignal | undefined): Promise<string> {
|
||||
signal?.throwIfAborted();
|
||||
const previousMessageCount = session.messages.length;
|
||||
await session.prompt(input);
|
||||
const assistant = session.messages
|
||||
.slice(previousMessageCount)
|
||||
.reverse()
|
||||
.find((message) => message.role === "assistant");
|
||||
if (!assistant) throw new Error("Agent run completed without an assistant message.");
|
||||
if (assistant.stopReason !== "stop") {
|
||||
throw new Error(
|
||||
assistant.errorMessage ?? `Agent run ended with unexpected stop reason: ${assistant.stopReason}.`,
|
||||
);
|
||||
}
|
||||
const output = session.getLastAssistantText();
|
||||
if (!output) throw new Error("Agent run produced no assistant text.");
|
||||
return output;
|
||||
}
|
||||
|
||||
async function runPiCodingAgent<TOutput extends JsonValue>(
|
||||
input: PiCodingAgentInput,
|
||||
signal: AbortSignal | undefined,
|
||||
setArtifact: HarnessContext["setArtifact"],
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Promise<SimpleHarnessResult<string | TOutput>> {
|
||||
const startedAt = performance.now();
|
||||
signal?.throwIfAborted();
|
||||
const selection = resolveModelSelection(options.model);
|
||||
const modelRuntime = await ModelRuntime.create();
|
||||
const model = modelRuntime.getModel(selection.provider, selection.id);
|
||||
if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.id}`);
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
|
||||
const cwd = join(root, "workspace");
|
||||
const agentDir = join(root, "agent");
|
||||
let transformedSystemPrompt: string | undefined;
|
||||
let sessionManager: SessionManager | undefined;
|
||||
let session: AgentSession | undefined;
|
||||
let outcome: { success: true; result: SimpleHarnessResult<string | TOutput> } | { success: false; error: unknown };
|
||||
try {
|
||||
await Promise.all([mkdir(cwd), mkdir(agentDir)]);
|
||||
const services = await createAgentSessionServices({
|
||||
cwd,
|
||||
agentDir,
|
||||
modelRuntime,
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
...(options.transformSystemPrompt
|
||||
? { resourceLoaderOptions: { systemPromptOverride: () => transformedSystemPrompt } }
|
||||
: {}),
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
sessionManager = SessionManager.create(cwd, join(root, "sessions"));
|
||||
setArtifact("runId", sessionManager.getSessionId());
|
||||
session = (
|
||||
await createAgentSessionFromServices({
|
||||
services,
|
||||
sessionManager,
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
noTools: options.noTools,
|
||||
})
|
||||
).session;
|
||||
|
||||
const evalSession = session;
|
||||
if (options.transformSystemPrompt) {
|
||||
transformedSystemPrompt = options.transformSystemPrompt(evalSession.systemPrompt);
|
||||
if (!transformedSystemPrompt.trim()) throw new Error("Transformed eval system prompt must not be empty.");
|
||||
await evalSession.reload();
|
||||
}
|
||||
let abortPromise: Promise<void> | undefined;
|
||||
const abort = () => {
|
||||
abortPromise ??= evalSession.abort();
|
||||
};
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
if (evalSession.extensionRunner.getExtensionPaths().length !== 0) {
|
||||
throw new Error("Expected an isolated eval session to start without extensions.");
|
||||
}
|
||||
const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input;
|
||||
let response: string | undefined;
|
||||
for (const step of steps) {
|
||||
if (step.type === "prompt") {
|
||||
response = await promptAgent(evalSession, step.content, signal);
|
||||
} else {
|
||||
await evalSession.reload();
|
||||
}
|
||||
}
|
||||
if (response === undefined) throw new Error("Pi eval input must include at least one prompt step.");
|
||||
const output = "output" in options ? await options.output({ response, session: evalSession }) : response;
|
||||
const stats = evalSession.getSessionStats();
|
||||
const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some(
|
||||
({ input, output, cacheRead, cacheWrite }) => input > 0 || output > 0 || cacheRead > 0 || cacheWrite > 0,
|
||||
);
|
||||
outcome = {
|
||||
success: true,
|
||||
result: {
|
||||
output,
|
||||
events: toTranscriptEvents(evalSession.messages),
|
||||
usage: {
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
inputTokens: stats.tokens.input,
|
||||
outputTokens: stats.tokens.output,
|
||||
totalTokens: stats.tokens.total,
|
||||
toolCalls: stats.toolCalls,
|
||||
metadata: {
|
||||
cacheReadTokens: stats.tokens.cacheRead,
|
||||
cacheWriteTokens: stats.tokens.cacheWrite,
|
||||
...(hasPricing ? { estimatedCostUsd: stats.cost } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (abortPromise) await abortPromise;
|
||||
}
|
||||
} catch (error) {
|
||||
outcome = { success: false, error };
|
||||
}
|
||||
|
||||
const cleanupErrors: unknown[] = [];
|
||||
if (sessionManager) {
|
||||
try {
|
||||
const sessionPath = sessionManager.getSessionFile();
|
||||
if (sessionPath && existsSync(sessionPath)) {
|
||||
setArtifact(PI_SESSION_SNAPSHOT_ARTIFACT, await readFile(sessionPath, "utf8"));
|
||||
}
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
session?.dispose();
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
try {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
|
||||
if (!outcome.success) {
|
||||
if (cleanupErrors.length === 0) throw outcome.error;
|
||||
throw new AggregateError([outcome.error, ...cleanupErrors], "Agent run failed and cleanup also failed.");
|
||||
}
|
||||
if (cleanupErrors.length === 1) throw cleanupErrors[0];
|
||||
if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed.");
|
||||
return {
|
||||
...outcome.result,
|
||||
timings: { totalMs: performance.now() - startedAt },
|
||||
};
|
||||
}
|
||||
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Harness<PiCodingAgentInput, TOutput>;
|
||||
export function createPiCodingAgentHarness(options?: PiCodingAgentHarnessOptions): Harness<PiCodingAgentInput, string>;
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput> = {},
|
||||
) {
|
||||
return createHarness<PiCodingAgentInput, string | TOutput>({
|
||||
name: options.name ?? "pi-coding-agent",
|
||||
run: ({ input, signal, setArtifact }) => runPiCodingAgent(input, signal, setArtifact, options),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { expect } from "vitest";
|
||||
import { describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness } from "./pi-harness.ts";
|
||||
|
||||
const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" });
|
||||
|
||||
describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => {
|
||||
it("runs a basic prompt end to end", async ({ run }) => {
|
||||
const result = await run("What's the capital of France? Respond with only the city name.");
|
||||
|
||||
expect(result.output.trim()).toBe("Paris");
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.usage.provider).toBe(process.env.PI_PROVIDER);
|
||||
expect(result.usage.model).toBe(process.env.PI_MODEL);
|
||||
expect(result.usage.totalTokens).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { basename, join, relative } from "node:path";
|
||||
import {
|
||||
type RunnerTestCase,
|
||||
recordArtifact,
|
||||
type TestArtifact,
|
||||
type TestArtifactBase,
|
||||
type TestAttachment,
|
||||
} from "vitest";
|
||||
import type { HarnessRun } from "vitest-evals/harness";
|
||||
|
||||
export const PI_SESSION_SNAPSHOT_ARTIFACT = "piSessionJsonl";
|
||||
|
||||
const evalSessionArtifactKey = Symbol("pi-evals-session-artifact");
|
||||
const evalSourceArtifactKey = Symbol("pi-evals-source-artifact");
|
||||
|
||||
interface PiSessionAttachment extends TestAttachment {
|
||||
name: "session.jsonl";
|
||||
contentType: "application/jsonl";
|
||||
body: string;
|
||||
bodyEncoding: "utf-8";
|
||||
}
|
||||
|
||||
export interface SourceAttachment extends TestAttachment {
|
||||
name: string;
|
||||
contentType: string;
|
||||
body: string;
|
||||
bodyEncoding: "utf-8";
|
||||
}
|
||||
|
||||
interface PiSessionArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:session";
|
||||
runId: string;
|
||||
attachments: [PiSessionAttachment] | [];
|
||||
}
|
||||
|
||||
interface SourceArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:source";
|
||||
runId: string;
|
||||
attachments: [SourceAttachment] | [];
|
||||
}
|
||||
|
||||
declare module "vitest" {
|
||||
interface TestArtifactRegistry {
|
||||
[evalSessionArtifactKey]: PiSessionArtifact;
|
||||
[evalSourceArtifactKey]: SourceArtifact;
|
||||
}
|
||||
}
|
||||
|
||||
export async function recordEvalSessionArtifact(
|
||||
task: Readonly<RunnerTestCase>,
|
||||
run: Pick<HarnessRun, "artifacts">,
|
||||
): Promise<void> {
|
||||
const runId = run.artifacts?.runId;
|
||||
const session = run.artifacts?.[PI_SESSION_SNAPSHOT_ARTIFACT];
|
||||
if (session === undefined) return;
|
||||
if (typeof runId !== "string" || typeof session !== "string") {
|
||||
throw new TypeError("Pi eval session artifact metadata is invalid.");
|
||||
}
|
||||
await recordArtifact(task, {
|
||||
type: "@earendil-works/pi-evals:session",
|
||||
runId,
|
||||
attachments: [
|
||||
{
|
||||
name: "session.jsonl",
|
||||
contentType: "application/jsonl",
|
||||
body: session,
|
||||
bodyEncoding: "utf-8",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
export async function recordEvalSourceArtifact(
|
||||
task: Readonly<RunnerTestCase>,
|
||||
runId: string,
|
||||
attachment: SourceAttachment,
|
||||
): Promise<void> {
|
||||
await recordArtifact(task, {
|
||||
type: "@earendil-works/pi-evals:source",
|
||||
runId,
|
||||
attachments: [attachment],
|
||||
});
|
||||
}
|
||||
|
||||
export async function persistEvalArtifactReferences(
|
||||
artifacts: ReadonlyArray<TestArtifact>,
|
||||
runId: string,
|
||||
artifactDirectory: string,
|
||||
): Promise<Array<{ name: string; path: string }>> {
|
||||
const references: Array<{ name: string; path: string }> = [];
|
||||
for (const artifact of artifacts) {
|
||||
if (
|
||||
(artifact.type !== "@earendil-works/pi-evals:session" &&
|
||||
artifact.type !== "@earendil-works/pi-evals:source") ||
|
||||
artifact.runId !== runId
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const category = artifact.type === "@earendil-works/pi-evals:session" ? "sessions" : "sources";
|
||||
for (const attachment of artifact.attachments) {
|
||||
const name = basename(attachment.name);
|
||||
if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`);
|
||||
const directory = join(artifactDirectory, category, createHash("sha256").update(runId).digest("hex"));
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 });
|
||||
references.push({ name, path: relative(artifactDirectory, path) });
|
||||
}
|
||||
}
|
||||
return references;
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
attachHarnessRunToError,
|
||||
getHarnessRunFromError,
|
||||
type Harness,
|
||||
type HarnessRun,
|
||||
type JsonValue,
|
||||
} from "vitest-evals/harness";
|
||||
|
||||
export const EVAL_HARNESS_ITERATION_ARTIFACT = "vitestEvalsHarnessIteration";
|
||||
|
||||
export type EvalHarnessIterationArtifact = {
|
||||
schemaVersion: 1;
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
harness: string;
|
||||
baseline: string;
|
||||
candidates: string[];
|
||||
repetition: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableRow<TInput, TOutput extends JsonValue | undefined> = {
|
||||
harness: Harness<TInput, TOutput>;
|
||||
name: string;
|
||||
repetition: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTablePairOptions<TInput, TOutput extends JsonValue | undefined> = {
|
||||
baseline: Harness<TInput, TOutput>;
|
||||
candidate: Harness<TInput, TOutput>;
|
||||
repetitions?: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableCandidatesOptions<TInput, TOutput extends JsonValue | undefined> = {
|
||||
baseline: Harness<TInput, TOutput>;
|
||||
candidates: readonly Harness<TInput, TOutput>[];
|
||||
repetitions?: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableOptions<TInput, TOutput extends JsonValue | undefined> =
|
||||
| EvalHarnessTablePairOptions<TInput, TOutput>
|
||||
| EvalHarnessTableCandidatesOptions<TInput, TOutput>;
|
||||
|
||||
type EvalHarnessIterationPlan = Omit<EvalHarnessIterationArtifact, "groupKey">;
|
||||
|
||||
export function parseEvalHarnessIterationArtifact(
|
||||
value: JsonValue | undefined,
|
||||
): EvalHarnessIterationArtifact | undefined {
|
||||
if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition } = value;
|
||||
if (
|
||||
schemaVersion !== 1 ||
|
||||
typeof evalSet !== "string" ||
|
||||
typeof groupKey !== "string" ||
|
||||
typeof harness !== "string" ||
|
||||
typeof baseline !== "string" ||
|
||||
!Array.isArray(candidates) ||
|
||||
!candidates.every((name): name is string => typeof name === "string") ||
|
||||
typeof repetition !== "number"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition };
|
||||
}
|
||||
|
||||
function canonicalizeJson(value: unknown, ancestors: WeakSet<object>): JsonValue {
|
||||
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isFinite(value)) throw new TypeError("Eval input must contain only finite numbers.");
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "object") throw new TypeError("Eval input must be JSON-serializable.");
|
||||
if (ancestors.has(value)) throw new TypeError("Eval input must not contain circular references.");
|
||||
|
||||
ancestors.add(value);
|
||||
try {
|
||||
if (Array.isArray(value)) {
|
||||
const result: JsonValue[] = [];
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
if (!Object.hasOwn(value, index)) throw new TypeError("Eval input arrays must not be sparse.");
|
||||
result.push(canonicalizeJson(value[index], ancestors));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
if (prototype !== Object.prototype && prototype !== null) {
|
||||
throw new TypeError("Eval input must contain only plain objects and arrays.");
|
||||
}
|
||||
const entries: Array<[string, unknown]> = Object.entries(value);
|
||||
return Object.fromEntries(
|
||||
entries
|
||||
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
||||
.map(([key, item]): [string, JsonValue] => [key, canonicalizeJson(item, ancestors)]),
|
||||
);
|
||||
} finally {
|
||||
ancestors.delete(value);
|
||||
}
|
||||
}
|
||||
|
||||
function deriveInputKey(input: unknown): string {
|
||||
if (typeof input === "object" && input !== null && !Array.isArray(input) && "id" in input) {
|
||||
const id = input.id;
|
||||
if (typeof id === "string" && id.trim()) return id.trim();
|
||||
}
|
||||
const canonicalInput = JSON.stringify(canonicalizeJson(input, new WeakSet()));
|
||||
if (canonicalInput === undefined) throw new TypeError("Eval input must be JSON-serializable.");
|
||||
return createHash("sha256").update(canonicalInput).digest("hex");
|
||||
}
|
||||
|
||||
export function deriveEvalGroupKey(input: unknown, repetition: number): string {
|
||||
return JSON.stringify([deriveInputKey(input), repetition]);
|
||||
}
|
||||
|
||||
function validateOptions<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
baseline: Harness<TInput, TOutput>,
|
||||
candidates: readonly Harness<TInput, TOutput>[],
|
||||
repetitions: number,
|
||||
): void {
|
||||
if (!evalSet.trim()) throw new TypeError("evalSet must not be empty.");
|
||||
if (candidates.length === 0) throw new TypeError("At least one candidate harness is required.");
|
||||
const harnesses = [baseline, ...candidates];
|
||||
const names = new Set(harnesses.map((harness) => harness.name));
|
||||
if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set.");
|
||||
if (!Number.isSafeInteger(repetitions) || repetitions < 1) {
|
||||
throw new TypeError("repetitions must be a positive integer.");
|
||||
}
|
||||
}
|
||||
|
||||
function withIterationArtifact<TInput, TOutput extends JsonValue | undefined>(
|
||||
harness: Harness<TInput, TOutput>,
|
||||
plan: EvalHarnessIterationPlan,
|
||||
): Harness<TInput, TOutput> {
|
||||
return {
|
||||
name: harness.name,
|
||||
async run(input, context) {
|
||||
const groupKey = deriveEvalGroupKey(input, plan.repetition);
|
||||
const artifact: EvalHarnessIterationArtifact = { ...plan, groupKey };
|
||||
context.setArtifact(EVAL_HARNESS_ITERATION_ARTIFACT, artifact);
|
||||
const attachIterationArtifact = <TRun extends HarnessRun>(run: TRun): TRun => {
|
||||
run.artifacts = { ...context.artifacts, ...run.artifacts, [EVAL_HARNESS_ITERATION_ARTIFACT]: artifact };
|
||||
return run;
|
||||
};
|
||||
try {
|
||||
return attachIterationArtifact(await harness.run(input, context));
|
||||
} catch (error) {
|
||||
const partialRun = getHarnessRunFromError(error);
|
||||
if (partialRun) {
|
||||
throw attachHarnessRunToError(error, attachIterationArtifact(partialRun));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTablePairOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[];
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTableCandidatesOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[];
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTableOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[] {
|
||||
const repetitions = options.repetitions ?? 1;
|
||||
const candidates = "candidate" in options ? [options.candidate] : options.candidates;
|
||||
validateOptions(evalSet, options.baseline, candidates, repetitions);
|
||||
|
||||
const rows: EvalHarnessTableRow<TInput, TOutput>[] = [];
|
||||
const harnesses = [options.baseline, ...candidates];
|
||||
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
|
||||
for (const harness of harnesses) {
|
||||
const plan: EvalHarnessIterationPlan = {
|
||||
schemaVersion: 1,
|
||||
evalSet,
|
||||
harness: harness.name,
|
||||
baseline: options.baseline.name,
|
||||
candidates: candidates.map(({ name }) => name),
|
||||
repetition,
|
||||
};
|
||||
rows.push({
|
||||
harness: withIterationArtifact(harness, plan),
|
||||
name: harness.name,
|
||||
repetition,
|
||||
});
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { appendFile, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node";
|
||||
import { isHarnessRun } from "vitest-evals/harness";
|
||||
import { PI_SESSION_SNAPSHOT_ARTIFACT, persistEvalArtifactReferences } from "./artifacts.ts";
|
||||
import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts";
|
||||
import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts";
|
||||
|
||||
function readFiniteNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
async function appendHarnessRunReport(test: TestCase): Promise<void> {
|
||||
const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim();
|
||||
if (!artifactDirectory) return;
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) return;
|
||||
|
||||
const run = harness.run;
|
||||
const artifactRunId = run.artifacts?.runId;
|
||||
const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID();
|
||||
const metadata = Object.fromEntries(
|
||||
Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId" && name !== PI_SESSION_SNAPSHOT_ARTIFACT),
|
||||
);
|
||||
const record = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
test: {
|
||||
id: test.id,
|
||||
file: test.module.relativeModuleId,
|
||||
name: test.name,
|
||||
fullName: test.fullName,
|
||||
status: test.result().state,
|
||||
},
|
||||
harness: harness.name,
|
||||
usage: run.usage,
|
||||
...(run.timings ? { timings: run.timings } : {}),
|
||||
...(run.errors.length > 0 ? { errors: run.errors } : {}),
|
||||
artifacts: await persistEvalArtifactReferences(test.artifacts(), runId, artifactDirectory),
|
||||
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
||||
};
|
||||
await mkdir(artifactDirectory, { recursive: true, mode: 0o700 });
|
||||
await appendFile(join(artifactDirectory, "runs.jsonl"), `${JSON.stringify(record)}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "a",
|
||||
mode: 0o600,
|
||||
});
|
||||
}
|
||||
|
||||
function collectHarnessObservations(modules: ReadonlyArray<TestModule>): HarnessObservation[] {
|
||||
const observations: HarnessObservation[] = [];
|
||||
for (const module of modules) {
|
||||
for (const test of module.children.allTests()) {
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) continue;
|
||||
const run = harness.run;
|
||||
const iteration = parseEvalHarnessIterationArtifact(run.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT]);
|
||||
if (!iteration) continue;
|
||||
const score = readFiniteNumber(test.meta().eval?.avgScore);
|
||||
const estimatedCostUsd = readFiniteNumber(run.usage.metadata?.estimatedCostUsd);
|
||||
const observation = {
|
||||
evalSet: iteration.evalSet,
|
||||
groupKey: iteration.groupKey,
|
||||
testName: test.name,
|
||||
file: module.relativeModuleId,
|
||||
harness: iteration.harness,
|
||||
baseline: iteration.baseline,
|
||||
candidates: iteration.candidates,
|
||||
repetition: iteration.repetition,
|
||||
...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }),
|
||||
...(run.timings?.totalMs === undefined ? {} : { totalMs: run.timings.totalMs }),
|
||||
...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }),
|
||||
};
|
||||
if (run.errors.length > 0) observations.push({ ...observation, outcome: "errored" });
|
||||
else if (score !== undefined) observations.push({ ...observation, outcome: "scored", score });
|
||||
else {
|
||||
const state = test.result().state;
|
||||
const outcome = state === "passed" ? "unscored" : state === "failed" ? "errored" : state;
|
||||
observations.push({ ...observation, outcome });
|
||||
}
|
||||
}
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
export default class EvalHarnessReporter implements Reporter {
|
||||
private vitest: Vitest | undefined;
|
||||
|
||||
onInit(vitest: Vitest): void {
|
||||
this.vitest = vitest;
|
||||
}
|
||||
|
||||
async onTestCaseResult(test: TestCase): Promise<void> {
|
||||
await appendHarnessRunReport(test);
|
||||
}
|
||||
|
||||
onTestRunEnd(
|
||||
modules: ReadonlyArray<TestModule>,
|
||||
_errors: ReadonlyArray<SerializedError>,
|
||||
reason: TestRunEndReason,
|
||||
): void {
|
||||
if (reason === "interrupted") {
|
||||
this.vitest?.logger.log("\nEval comparisons unavailable: test run interrupted.");
|
||||
return;
|
||||
}
|
||||
const report = summarizeHarnessComparisons(collectHarnessObservations(modules));
|
||||
const formatted = formatHarnessComparisonReport(report);
|
||||
if (formatted) this.vitest?.logger.log(`\n${formatted}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { afterEach } from "vitest";
|
||||
import type {} from "vitest-evals";
|
||||
import { recordEvalSessionArtifact } from "./artifacts.ts";
|
||||
|
||||
afterEach(async ({ task }) => {
|
||||
const run = task.meta.harness?.run;
|
||||
if (run) await recordEvalSessionArtifact(task, run);
|
||||
});
|
||||
@@ -0,0 +1,438 @@
|
||||
import { styleText } from "node:util";
|
||||
|
||||
type HarnessObservationOutcome = "scored" | "unscored" | "skipped" | "pending" | "errored";
|
||||
|
||||
type HarnessObservationBase = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
harness: string;
|
||||
baseline: string;
|
||||
candidates: string[];
|
||||
repetition: number;
|
||||
totalTokens?: number;
|
||||
totalMs?: number;
|
||||
estimatedCostUsd?: number;
|
||||
};
|
||||
|
||||
export type HarnessObservation = HarnessObservationBase &
|
||||
({ outcome: "scored"; score: number } | { outcome: Exclude<HarnessObservationOutcome, "scored">; score?: never });
|
||||
|
||||
export type PairedMetricSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
baselineMean: number | null;
|
||||
candidateMean: number | null;
|
||||
meanDelta: number | null;
|
||||
};
|
||||
|
||||
export type CorrectnessLiftSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
baselinePassRate: number | null;
|
||||
candidatePassRate: number | null;
|
||||
lift: number | null;
|
||||
baselineWins: number;
|
||||
candidateWins: number;
|
||||
ties: number;
|
||||
};
|
||||
|
||||
export type HarnessPairComparison = {
|
||||
baseline: string;
|
||||
candidate: string;
|
||||
correctness: CorrectnessLiftSummary;
|
||||
totalTokens: PairedMetricSummary;
|
||||
totalMs: PairedMetricSummary;
|
||||
estimatedCostUsd: PairedMetricSummary;
|
||||
};
|
||||
|
||||
export type HarnessComparisonDiagnostic = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
harness: string;
|
||||
reason: "missing-observation" | "duplicate-observation" | "harness-error" | "missing-score" | "unscorable-outcome";
|
||||
};
|
||||
|
||||
export type HarnessEvalSetReport = {
|
||||
evalSet: string;
|
||||
comparisons: HarnessPairComparison[];
|
||||
};
|
||||
|
||||
export type HarnessComparisonReport = {
|
||||
schemaVersion: 1;
|
||||
evalSets: HarnessEvalSetReport[];
|
||||
diagnostics: HarnessComparisonDiagnostic[];
|
||||
};
|
||||
|
||||
type HarnessDescriptor = {
|
||||
name: string;
|
||||
index: number;
|
||||
};
|
||||
|
||||
type ObservationGroup = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
observationsByHarness: Map<string, HarnessObservation[]>;
|
||||
};
|
||||
|
||||
type EvalSetData = {
|
||||
baseline: HarnessDescriptor;
|
||||
candidatesByName: Map<string, HarnessDescriptor>;
|
||||
groupsByKey: Map<string, ObservationGroup>;
|
||||
};
|
||||
|
||||
type ObservationPair = {
|
||||
baseline: HarnessObservation;
|
||||
candidate: HarnessObservation;
|
||||
};
|
||||
|
||||
function getOrCreate<K, V extends object>(map: Map<K, V>, key: K, create: () => V): V {
|
||||
const existing = map.get(key);
|
||||
if (existing !== undefined) return existing;
|
||||
const value = create();
|
||||
map.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
function mean(values: readonly number[]): number | null {
|
||||
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function preciseDifference(left: number, right: number): number {
|
||||
return Number((left - right).toPrecision(15));
|
||||
}
|
||||
|
||||
function groupObservations(observations: readonly HarnessObservation[]): Map<string, EvalSetData> {
|
||||
const evalSets = new Map<string, EvalSetData>();
|
||||
for (const observation of observations) {
|
||||
const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({
|
||||
baseline: { name: observation.baseline, index: 0 },
|
||||
candidatesByName: new Map(),
|
||||
groupsByKey: new Map(),
|
||||
}));
|
||||
|
||||
for (const [index, name] of observation.candidates.entries()) {
|
||||
const existing = evalSet.candidatesByName.get(name);
|
||||
if (!existing || index < existing.index) evalSet.candidatesByName.set(name, { name, index });
|
||||
}
|
||||
|
||||
const group = getOrCreate(
|
||||
evalSet.groupsByKey,
|
||||
JSON.stringify([observation.file, observation.testName, observation.groupKey]),
|
||||
() => ({
|
||||
evalSet: observation.evalSet,
|
||||
groupKey: observation.groupKey,
|
||||
testName: observation.testName,
|
||||
file: observation.file,
|
||||
repetition: observation.repetition,
|
||||
observationsByHarness: new Map(),
|
||||
}),
|
||||
);
|
||||
getOrCreate(group.observationsByHarness, observation.harness, (): HarnessObservation[] => []).push(observation);
|
||||
}
|
||||
return evalSets;
|
||||
}
|
||||
|
||||
function orderedHarnesses(evalSet: EvalSetData): HarnessDescriptor[] {
|
||||
return [
|
||||
evalSet.baseline,
|
||||
...[...evalSet.candidatesByName.values()].sort(
|
||||
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function orderedCandidates(evalSet: EvalSetData): HarnessDescriptor[] {
|
||||
return [...evalSet.candidatesByName.values()].sort(
|
||||
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
|
||||
);
|
||||
}
|
||||
|
||||
function orderedGroups(evalSet: EvalSetData): ObservationGroup[] {
|
||||
return [...evalSet.groupsByKey.values()].sort(
|
||||
(left, right) => left.groupKey.localeCompare(right.groupKey) || left.repetition - right.repetition,
|
||||
);
|
||||
}
|
||||
|
||||
function collectDiagnostics(
|
||||
harnesses: readonly HarnessDescriptor[],
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessComparisonDiagnostic[] {
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const group of groups) {
|
||||
for (const { name: harness } of harnesses) {
|
||||
const observations = group.observationsByHarness.get(harness) ?? [];
|
||||
let reason: HarnessComparisonDiagnostic["reason"] | undefined;
|
||||
if (observations.length === 0) reason = "missing-observation";
|
||||
else if (observations.length > 1) reason = "duplicate-observation";
|
||||
else if (observations[0].outcome === "errored") reason = "harness-error";
|
||||
else if (observations[0].outcome === "unscored") {
|
||||
reason = "missing-score";
|
||||
} else if (observations[0].outcome !== "scored") {
|
||||
reason = "unscorable-outcome";
|
||||
}
|
||||
if (!reason) continue;
|
||||
diagnostics.push({
|
||||
evalSet: group.evalSet,
|
||||
groupKey: group.groupKey,
|
||||
testName: group.testName,
|
||||
file: group.file,
|
||||
repetition: group.repetition,
|
||||
harness,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
function pairObservations(
|
||||
groups: readonly ObservationGroup[],
|
||||
baselineHarness: string,
|
||||
candidateHarness: string,
|
||||
): ObservationPair[] {
|
||||
const pairs: ObservationPair[] = [];
|
||||
for (const group of groups) {
|
||||
const baseline = group.observationsByHarness.get(baselineHarness) ?? [];
|
||||
const candidate = group.observationsByHarness.get(candidateHarness) ?? [];
|
||||
if (baseline.length === 1 && candidate.length === 1) {
|
||||
pairs.push({ baseline: baseline[0], candidate: candidate[0] });
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function summarizeMetric(
|
||||
pairs: readonly ObservationPair[],
|
||||
select: (observation: HarnessObservation) => number | undefined,
|
||||
totalPairs: number,
|
||||
): PairedMetricSummary {
|
||||
const baselineValues: number[] = [];
|
||||
const candidateValues: number[] = [];
|
||||
for (const { baseline, candidate } of pairs) {
|
||||
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
|
||||
const baselineValue = select(baseline);
|
||||
const candidateValue = select(candidate);
|
||||
if (
|
||||
baselineValue === undefined ||
|
||||
candidateValue === undefined ||
|
||||
!Number.isFinite(baselineValue) ||
|
||||
!Number.isFinite(candidateValue)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
baselineValues.push(baselineValue);
|
||||
candidateValues.push(candidateValue);
|
||||
}
|
||||
|
||||
const baselineMean = mean(baselineValues);
|
||||
const candidateMean = mean(candidateValues);
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs: baselineValues.length,
|
||||
baselineMean,
|
||||
candidateMean,
|
||||
meanDelta:
|
||||
baselineMean === null || candidateMean === null ? null : preciseDifference(candidateMean, baselineMean),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary {
|
||||
let eligiblePairs = 0;
|
||||
let baselinePasses = 0;
|
||||
let candidatePasses = 0;
|
||||
let baselineWins = 0;
|
||||
let candidateWins = 0;
|
||||
let ties = 0;
|
||||
|
||||
for (const { baseline, candidate } of pairs) {
|
||||
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
|
||||
eligiblePairs += 1;
|
||||
const baselinePassed = baseline.score >= 1;
|
||||
const candidatePassed = candidate.score >= 1;
|
||||
if (baselinePassed) baselinePasses += 1;
|
||||
if (candidatePassed) candidatePasses += 1;
|
||||
if (baselinePassed === candidatePassed) ties += 1;
|
||||
else if (baselinePassed) baselineWins += 1;
|
||||
else candidateWins += 1;
|
||||
}
|
||||
|
||||
const baselinePassRate = eligiblePairs === 0 ? null : baselinePasses / eligiblePairs;
|
||||
const candidatePassRate = eligiblePairs === 0 ? null : candidatePasses / eligiblePairs;
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs,
|
||||
baselinePassRate,
|
||||
candidatePassRate,
|
||||
lift:
|
||||
baselinePassRate === null || candidatePassRate === null
|
||||
? null
|
||||
: preciseDifference(candidatePassRate, baselinePassRate),
|
||||
baselineWins,
|
||||
candidateWins,
|
||||
ties,
|
||||
};
|
||||
}
|
||||
|
||||
function compareHarnesses(
|
||||
baseline: HarnessDescriptor,
|
||||
candidate: HarnessDescriptor,
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessPairComparison {
|
||||
const pairs = pairObservations(groups, baseline.name, candidate.name);
|
||||
return {
|
||||
baseline: baseline.name,
|
||||
candidate: candidate.name,
|
||||
correctness: summarizeCorrectness(pairs, groups.length),
|
||||
totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length),
|
||||
totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length),
|
||||
estimatedCostUsd: summarizeMetric(pairs, ({ estimatedCostUsd }) => estimatedCostUsd, groups.length),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeHarnessComparisons(observations: readonly HarnessObservation[]): HarnessComparisonReport {
|
||||
const evalSets: HarnessEvalSetReport[] = [];
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const [evalSet, data] of [...groupObservations(observations)].sort(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
)) {
|
||||
const harnesses = orderedHarnesses(data);
|
||||
const candidates = orderedCandidates(data);
|
||||
const groups = orderedGroups(data);
|
||||
evalSets.push({
|
||||
evalSet,
|
||||
comparisons: candidates.map((candidate) => compareHarnesses(data.baseline, candidate, groups)),
|
||||
});
|
||||
diagnostics.push(...collectDiagnostics(harnesses, groups));
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
evalSets,
|
||||
diagnostics: diagnostics.sort(
|
||||
(left, right) =>
|
||||
left.evalSet.localeCompare(right.evalSet) ||
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.groupKey.localeCompare(right.groupKey) ||
|
||||
left.repetition - right.repetition ||
|
||||
left.harness.localeCompare(right.harness),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function formatPercentage(value: number | null): string {
|
||||
return value === null ? "unavailable" : `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function formatSigned(value: number, fractionDigits: number): string {
|
||||
return `${value >= 0 ? "+" : ""}${value.toFixed(fractionDigits)}`;
|
||||
}
|
||||
|
||||
function formatCoverage(eligiblePairs: number, totalPairs: number): string {
|
||||
return styleText("gray", `(${eligiblePairs}/${totalPairs} pairs)`);
|
||||
}
|
||||
|
||||
function formatReportLine(label: string, value: string): string {
|
||||
return ` ${styleText("gray", label.padStart(9))} ${value}`;
|
||||
}
|
||||
|
||||
function colorDelta(value: number, formatted: string, positiveIsBetter: boolean): string {
|
||||
if (value === 0) return styleText("gray", formatted);
|
||||
const improved = positiveIsBetter ? value > 0 : value < 0;
|
||||
return styleText(improved ? "green" : "red", formatted);
|
||||
}
|
||||
|
||||
function formatMetric(
|
||||
label: string,
|
||||
metric: PairedMetricSummary,
|
||||
formatValue: (value: number) => string,
|
||||
formatDelta: (value: number) => string,
|
||||
comparisonPairs: number,
|
||||
): string {
|
||||
const coverage =
|
||||
metric.eligiblePairs === 0 || metric.eligiblePairs === comparisonPairs
|
||||
? ""
|
||||
: ` ${formatCoverage(metric.eligiblePairs, metric.totalPairs)}`;
|
||||
if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) {
|
||||
return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`);
|
||||
}
|
||||
const delta = colorDelta(metric.meanDelta, formatDelta(metric.meanDelta), false);
|
||||
const values = styleText(
|
||||
"gray",
|
||||
`(candidate ${formatValue(metric.candidateMean)}, baseline ${formatValue(metric.baselineMean)})`,
|
||||
);
|
||||
return formatReportLine(label, `${delta} ${values}${coverage}`);
|
||||
}
|
||||
|
||||
export function formatHarnessComparisonReport(report: HarnessComparisonReport): string {
|
||||
if (report.evalSets.every(({ comparisons }) => comparisons.length === 0)) return "";
|
||||
const lines = [styleText("bold", "Eval Comparisons")];
|
||||
for (const evalSet of report.evalSets) {
|
||||
lines.push(` ${evalSet.evalSet}`);
|
||||
for (const [index, comparison] of evalSet.comparisons.entries()) {
|
||||
if (index > 0) lines.push("");
|
||||
const { correctness } = comparison;
|
||||
lines.push(formatReportLine("Baseline", comparison.baseline));
|
||||
lines.push(
|
||||
formatReportLine(
|
||||
"Candidate",
|
||||
`${comparison.candidate} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`,
|
||||
),
|
||||
);
|
||||
if (correctness.lift === null) {
|
||||
lines.push(formatReportLine("Pass rate", styleText("yellow", "unavailable")));
|
||||
} else {
|
||||
const lift = correctness.lift * 100;
|
||||
const delta = colorDelta(lift, `${formatSigned(lift, 1)} pp`, true);
|
||||
const values = styleText(
|
||||
"gray",
|
||||
`(candidate ${formatPercentage(correctness.candidatePassRate)}, baseline ${formatPercentage(correctness.baselinePassRate)})`,
|
||||
);
|
||||
lines.push(formatReportLine("Pass rate", `${delta} ${values}`));
|
||||
}
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Tokens",
|
||||
comparison.totalTokens,
|
||||
(value) => value.toFixed(1),
|
||||
(value) => formatSigned(value, 1),
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Latency",
|
||||
comparison.totalMs,
|
||||
(value) => `${value.toFixed(1)}ms`,
|
||||
(value) => `${formatSigned(value, 1)}ms`,
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Est. cost",
|
||||
comparison.estimatedCostUsd,
|
||||
(value) => `$${value.toFixed(4)}`,
|
||||
(value) => `${value >= 0 ? "+" : "-"}$${Math.abs(value).toFixed(4)}`,
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (report.diagnostics.length > 0) {
|
||||
lines.push(` ${styleText("yellow", "Incomplete observations")}`);
|
||||
for (const diagnostic of report.diagnostics) {
|
||||
lines.push(
|
||||
` ${diagnostic.reason}: ${diagnostic.file}/${diagnostic.testName} repetition ${diagnostic.repetition}, harness ${diagnostic.harness}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
Reference in New Issue
Block a user