feat: integrate Pi backend and Pi Web

scripts/check-integration.mjs

@@ -0,0 +1,83 @@
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const rootDir = dirname(dirname(fileURLToPath(import.meta.url)));
+const webDir = join(rootDir, "pi-web");
+const requiredProfilePaths = [
+  join(rootDir, "config", "settings.default.json"),
+  join(rootDir, "config", "models.example.json"),
+  join(rootDir, "resources", "skills"),
+  join(rootDir, "resources", "extensions"),
+  join(rootDir, "resources", "prompts"),
+  join(rootDir, "resources", "themes"),
+];
+
+const packages = [
+  ["@earendil-works/pi-agent-core", "agent"],
+  ["@earendil-works/pi-ai", "ai"],
+  ["@earendil-works/pi-coding-agent", "coding-agent"],
+  ["@earendil-works/pi-tui", "tui"],
+];
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+const webPackage = readJson(join(webDir, "package.json"));
+const failures = [];
+const versions = new Set();
+
+for (const path of requiredProfilePaths) {
+  if (!existsSync(path)) failures.push(`Missing managed profile path: ${path}`);
+}
+
+for (const [packageName, packageDirName] of packages) {
+  const localDir = resolve(rootDir, "pi", "packages", packageDirName);
+  const installedDir = resolve(webDir, "node_modules", ...packageName.split("/"));
+  const localPackagePath = join(localDir, "package.json");
+  const installedPackagePath = join(installedDir, "package.json");
+  const expectedSpec = `file:../pi/packages/${packageDirName}`;
+
+  if (webPackage.dependencies?.[packageName] !== expectedSpec) {
+    failures.push(`${packageName} must use ${expectedSpec}`);
+    continue;
+  }
+  if (!existsSync(localPackagePath)) {
+    failures.push(`Missing local package: ${localPackagePath}`);
+    continue;
+  }
+  if (!existsSync(installedPackagePath)) {
+    failures.push(`Missing installed package link: ${installedDir}; run npm run setup`);
+    continue;
+  }
+
+  const localPackage = readJson(localPackagePath);
+  const installedPackage = readJson(installedPackagePath);
+  versions.add(localPackage.version);
+
+  if (installedPackage.version !== localPackage.version) {
+    failures.push(`${packageName} installed ${installedPackage.version}, local source is ${localPackage.version}`);
+  }
+
+  const installedRealPath = realpathSync(installedDir);
+
+  const mainPath = join(installedDir, installedPackage.main ?? "dist/index.js");
+  if (!existsSync(mainPath)) {
+    failures.push(`${packageName} is not built: ${mainPath}`);
+  }
+
+  console.log(
+    `${packageName}@${localPackage.version} <- ${relative(rootDir, localDir)} ` +
+      `(installed at ${relative(rootDir, installedRealPath)})`,
+  );
+}
+
+if (versions.size > 1) failures.push(`Pi packages are not lockstep-versioned: ${[...versions].join(", ")}`);
+
+if (failures.length > 0) {
+  for (const failure of failures) console.error(`ERROR: ${failure}`);
+  process.exit(1);
+}
+
+console.log("Local Pi backend packages are built and installed into Pi Web.");

scripts/init-profile.mjs

@@ -0,0 +1,3 @@
+import { ensureProfile } from "./profile.mjs";
+
+ensureProfile();

scripts/migrate-profile.mjs

@@ -0,0 +1,102 @@
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { agentDir, ensureProfile, rootDir, skillsHomeDir } from "./profile.mjs";
+
+const args = process.argv.slice(2);
+const force = args.includes("--force");
+const dryRun = args.includes("--dry-run");
+
+function valueAfter(flag, fallback) {
+  const index = args.indexOf(flag);
+  return index >= 0 && args[index + 1] ? resolve(args[index + 1]) : fallback;
+}
+
+const sourceAgentDir = valueAfter("--from", join(homedir(), ".pi", "agent"));
+const sourceAgentsDir = valueAfter("--skills-from", join(homedir(), ".agents"));
+
+ensureProfile({ quiet: true });
+
+function copyContents(source, destination, label) {
+  if (!existsSync(source)) {
+    console.log(`Skip ${label}: source does not exist (${source})`);
+    return { copied: 0, skipped: 0 };
+  }
+
+  mkdirSync(destination, { recursive: true });
+  let copied = 0;
+  let skipped = 0;
+  for (const entry of readdirSync(source, { withFileTypes: true })) {
+    const sourcePath = join(source, entry.name);
+    const destinationPath = join(destination, entry.name);
+    if (existsSync(destinationPath) && !force) {
+      skipped++;
+      console.log(`Skip existing: ${destinationPath}`);
+      continue;
+    }
+    console.log(`${dryRun ? "Would copy" : "Copy"}: ${sourcePath} -> ${destinationPath}`);
+    if (!dryRun) cpSync(sourcePath, destinationPath, { recursive: true, force });
+    copied++;
+  }
+  return { copied, skipped };
+}
+
+function isInsideApp(path) {
+  const rel = relative(rootDir, resolve(path));
+  return rel === "" || (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel));
+}
+
+function isRemotePackageSource(source) {
+  return source.startsWith("npm:") || source.startsWith("git:") || /^[a-z]+:\/\//i.test(source);
+}
+
+function sanitizeMigratedSettings() {
+  const settingsPath = join(agentDir, "settings.json");
+  if (!existsSync(settingsPath)) return { changed: false, removed: [] };
+
+  let settings;
+  try {
+    settings = JSON.parse(readFileSync(settingsPath, "utf8"));
+  } catch {
+    console.log(`Skip settings cleanup: invalid JSON (${settingsPath})`);
+    return { changed: false, removed: [] };
+  }
+  if (!settings || typeof settings !== "object" || Array.isArray(settings)) {
+    console.log(`Skip settings cleanup: expected a JSON object (${settingsPath})`);
+    return { changed: false, removed: [] };
+  }
+
+  const removed = [];
+  for (const key of ["extensions", "skills", "prompts", "themes"]) {
+    if (!Array.isArray(settings[key])) continue;
+    settings[key] = settings[key].filter((entry) => {
+      if (typeof entry !== "string" || !isAbsolute(entry) || isInsideApp(entry)) return true;
+      removed.push(`${key}:${entry}`);
+      return false;
+    });
+  }
+
+  if (Array.isArray(settings.packages)) {
+    settings.packages = settings.packages.filter((entry) => {
+      const source = typeof entry === "string" ? entry : entry?.source;
+      if (typeof source !== "string" || isRemotePackageSource(source)) return true;
+      const resolvedSource = resolve(agentDir, source);
+      if (isInsideApp(resolvedSource)) return true;
+      removed.push(`packages:${source}`);
+      return false;
+    });
+  }
+
+  if (removed.length > 0) {
+    console.log(`Remove ${removed.length} external resource reference(s) from migrated settings.`);
+    for (const entry of removed) console.log(`  ${entry}`);
+    if (!dryRun) writeFileSync(settingsPath, `${JSON.stringify(settings, null, 2)}\n`, "utf8");
+  }
+  return { changed: removed.length > 0, removed };
+}
+
+const agentResult = copyContents(sourceAgentDir, agentDir, "Pi agent profile");
+const skillsResult = copyContents(sourceAgentsDir, join(skillsHomeDir, ".agents"), "global skills profile");
+const settingsCleanup = sanitizeMigratedSettings();
+
+console.log(JSON.stringify({ dryRun, force, agent: agentResult, skills: skillsResult, settingsCleanup }, null, 2));

scripts/migrate-profile.test.mjs

@@ -0,0 +1,49 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+test("profile migration copies state and removes external resource references", (t) => {
+  const root = mkdtempSync(join(tmpdir(), "pi-profile-migration-"));
+  const sourceAgent = join(root, "source-agent");
+  const sourceSkills = join(root, "source-agents");
+  const dataDir = join(root, "managed-data");
+  const externalExtension = join(root, "external-extension.js");
+  mkdirSync(join(sourceAgent, "sessions"), { recursive: true });
+  mkdirSync(join(sourceSkills, "skills", "example"), { recursive: true });
+  writeFileSync(join(sourceAgent, "sessions", "session.jsonl"), "{}\n");
+  writeFileSync(join(sourceSkills, "skills", "example", "SKILL.md"), "---\nname: example\n---\n");
+  writeFileSync(sourceAgent + "-source-marker", "preserved");
+  writeFileSync(join(sourceAgent, "settings.json"), JSON.stringify({
+    extensions: [externalExtension, "extensions/local.js"],
+    packages: ["npm:example@1.0.0", externalExtension],
+  }));
+
+  t.after(() => rmSync(root, { recursive: true, force: true }));
+  const result = spawnSync(
+    process.execPath,
+    [
+      join(process.cwd(), "scripts", "migrate-profile.mjs"),
+      "--from",
+      sourceAgent,
+      "--skills-from",
+      sourceSkills,
+      "--force",
+    ],
+    {
+      cwd: process.cwd(),
+      env: { ...process.env, PI_AGENT_DATA_DIR: dataDir },
+      encoding: "utf8",
+    },
+  );
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+
+  const settings = JSON.parse(readFileSync(join(dataDir, "agent", "settings.json"), "utf8"));
+  assert.deepEqual(settings.extensions, ["extensions/local.js"]);
+  assert.deepEqual(settings.packages, ["npm:example@1.0.0"]);
+  assert.equal(existsSync(join(dataDir, "agent", "sessions", "session.jsonl")), true);
+  assert.equal(existsSync(join(dataDir, "skills-home", ".agents", "skills", "example", "SKILL.md")), true);
+  assert.equal(existsSync(sourceAgent + "-source-marker"), true);
+});

scripts/profile.mjs

@@ -0,0 +1,67 @@
+import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const rootDir = dirname(dirname(fileURLToPath(import.meta.url)));
+export const configDir = join(rootDir, "config");
+export const resourcesDir = join(rootDir, "resources");
+export const dataDir = resolve(process.env.PI_AGENT_DATA_DIR ?? join(rootDir, "data"));
+export const agentDir = join(dataDir, "agent");
+export const skillsHomeDir = join(dataDir, "skills-home");
+export const stateDir = join(dataDir, "state");
+
+const managedDirectories = [
+  agentDir,
+  join(agentDir, "sessions"),
+  join(agentDir, "npm"),
+  join(agentDir, "git"),
+  join(agentDir, "bin"),
+  join(agentDir, "tools"),
+  join(agentDir, "tmp"),
+  join(dataDir, "logs"),
+  join(dataDir, "cache"),
+  join(skillsHomeDir, ".agents", "skills"),
+  join(skillsHomeDir, ".pi", "agent", "skills"),
+  stateDir,
+  join(resourcesDir, "skills"),
+  join(resourcesDir, "extensions"),
+  join(resourcesDir, "prompts"),
+  join(resourcesDir, "themes"),
+];
+
+const seedFiles = [
+  [join(configDir, "settings.default.json"), join(agentDir, "settings.json")],
+  [join(configDir, "models.example.json"), join(agentDir, "models.json")],
+];
+
+export function ensureProfile({ quiet = false } = {}) {
+  for (const path of managedDirectories) mkdirSync(path, { recursive: true });
+
+  const created = [];
+  for (const [source, destination] of seedFiles) {
+    if (existsSync(destination)) continue;
+    copyFileSync(source, destination);
+    created.push(destination);
+  }
+
+  if (!quiet) {
+    console.log(`Managed Pi profile: ${agentDir}`);
+    if (created.length > 0) console.log(`Initialized ${created.length} profile file(s).`);
+  }
+
+  return { rootDir, configDir, resourcesDir, dataDir, agentDir, skillsHomeDir, stateDir, created };
+}
+
+export function managedEnvironment(baseEnv = process.env) {
+  ensureProfile({ quiet: true });
+  return {
+    ...baseEnv,
+    PI_AGENT_MANAGED_RUNTIME: "1",
+    PI_AGENT_APP_ROOT: rootDir,
+    PI_AGENT_DATA_DIR: dataDir,
+    PI_AGENT_RESOURCES_DIR: resourcesDir,
+    PI_AGENT_SKILLS_HOME: skillsHomeDir,
+    PI_CODING_AGENT_DIR: agentDir,
+    XDG_STATE_HOME: stateDir,
+  };
+}

scripts/run.mjs

@@ -0,0 +1,39 @@
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+import { managedEnvironment, rootDir } from "./profile.mjs";
+
+const target = process.argv[2] ?? "dev";
+const allowedTargets = new Set(["dev", "dev:lan", "start", "start:lan"]);
+if (!allowedTargets.has(target)) {
+  console.error(`Unsupported Pi Web script: ${target}`);
+  process.exit(2);
+}
+
+const npmCliPath = process.env.npm_execpath;
+if (!npmCliPath) {
+  console.error("Run the integrated application through npm.");
+  process.exit(1);
+}
+
+const child = spawn(
+  process.execPath,
+  [npmCliPath, "--prefix", join(rootDir, "pi-web"), "run", target],
+  {
+    cwd: rootDir,
+    env: managedEnvironment(),
+    stdio: "inherit",
+  },
+);
+
+for (const signal of ["SIGINT", "SIGTERM"]) {
+  process.on(signal, () => child.kill(signal));
+}
+
+child.on("error", (error) => {
+  console.error(error);
+  process.exit(1);
+});
+child.on("exit", (code, signal) => {
+  if (signal) process.kill(process.pid, signal);
+  process.exit(code ?? 1);
+});

scripts/setup.mjs

@@ -0,0 +1,111 @@
+import { spawnSync } from "node:child_process";
+import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { ensureProfile } from "./profile.mjs";
+
+const rootDir = dirname(dirname(fileURLToPath(import.meta.url)));
+const npmCliPath = process.env.npm_execpath;
+const setupEnv = { ...process.env };
+
+if (process.env.PI_SETUP_USE_PROXY !== "1") {
+  for (const key of ["HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"]) {
+    delete setupEnv[key];
+  }
+}
+
+if (!npmCliPath) {
+  console.error("Run this setup through npm: npm run setup");
+  process.exit(1);
+}
+
+function run(args, cwd) {
+  const result = spawnSync(process.execPath, [npmCliPath, ...args], {
+    cwd,
+    stdio: "inherit",
+    env: setupEnv,
+  });
+
+  if (result.error) throw result.error;
+  if (result.status !== 0) process.exit(result.status ?? 1);
+}
+
+function hydrateModelData() {
+  const aiDir = join(rootDir, "pi", "packages", "ai");
+  const modelDataDir = join(aiDir, "src", "providers", "data");
+  if (existsSync(join(modelDataDir, "amazon-bedrock.json"))) return;
+
+  const aiPackage = JSON.parse(readFileSync(join(aiDir, "package.json"), "utf8"));
+  const tempDir = mkdtempSync(join(tmpdir(), "pi-ai-model-data-"));
+
+  try {
+    console.log(`Model data is absent; extracting it from ${aiPackage.name}@${aiPackage.version}...`);
+    const packResult = spawnSync(
+      process.execPath,
+      [
+        npmCliPath,
+        "pack",
+        `${aiPackage.name}@${aiPackage.version}`,
+        "--pack-destination",
+        tempDir,
+        "--ignore-scripts",
+        "--json",
+      ],
+      { cwd: rootDir, encoding: "utf8", env: setupEnv },
+    );
+    if (packResult.error) throw packResult.error;
+    if (packResult.status !== 0) {
+      process.stderr.write(packResult.stderr);
+      process.exit(packResult.status ?? 1);
+    }
+
+    const packOutput = JSON.parse(packResult.stdout);
+    const archiveName = packOutput[0]?.filename;
+    if (!archiveName) throw new Error("npm pack did not return an archive filename");
+
+    const extractResult = spawnSync("tar", ["-xzf", join(tempDir, archiveName), "-C", tempDir], {
+      stdio: "inherit",
+    });
+    if (extractResult.error) throw extractResult.error;
+    if (extractResult.status !== 0) process.exit(extractResult.status ?? 1);
+
+    const publishedDataDir = join(tempDir, "package", "dist", "providers", "data");
+    if (!existsSync(join(publishedDataDir, "amazon-bedrock.json"))) {
+      throw new Error(`Published model data is missing from ${publishedDataDir}`);
+    }
+    cpSync(publishedDataDir, modelDataDir, { recursive: true });
+  } finally {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
+}
+
+function assertSupportedNodeVersion() {
+  const [major, minor] = process.versions.node.split(".").map(Number);
+  if (major < 22 || (major === 22 && minor < 19)) {
+    console.error(`Node.js >=22.19.0 is required; current version is ${process.versions.node}.`);
+    process.exit(1);
+  }
+}
+
+assertSupportedNodeVersion();
+ensureProfile();
+
+console.log("[1/4] Installing Pi dependencies...");
+run(["ci", "--ignore-scripts"], join(rootDir, "pi"));
+
+console.log("[2/4] Hydrating model data and building local Pi packages...");
+hydrateModelData();
+run(["run", "build:offline"], join(rootDir, "pi"));
+
+console.log("[3/4] Installing Pi Web with local Pi package links...");
+run(["ci", "--ignore-scripts", "--install-links"], join(rootDir, "pi-web"));
+
+console.log("[4/4] Verifying the integration...");
+const checkResult = spawnSync(process.execPath, [join(rootDir, "scripts", "check-integration.mjs")], {
+  cwd: rootDir,
+  stdio: "inherit",
+  env: setupEnv,
+});
+if (checkResult.error) throw checkResult.error;
+process.exit(checkResult.status ?? 0);

scripts/smoke-isolation.mjs

@@ -0,0 +1,63 @@
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { agentDir } from "./profile.mjs";
+
+const baseUrl = process.env.PI_WEB_URL ?? "http://127.0.0.1:30141";
+const suffix = randomUUID().replaceAll("-", "");
+const managedSkillName = `managed-probe-${suffix}`;
+const externalSkillName = `external-probe-${suffix}`;
+const managedSkillDir = join(agentDir, "skills", managedSkillName);
+const workspace = mkdtempSync(join(tmpdir(), "pi-managed-workspace-"));
+const externalSkillDir = join(workspace, ".agents", "skills", externalSkillName);
+const externalExtensionDir = join(workspace, ".pi", "extensions");
+const externalExtensionMarker = join(workspace, "external-extension-executed");
+
+async function readJson(path, init) {
+  const response = await fetch(`${baseUrl}${path}`, init);
+  const data = await response.json();
+  if (!response.ok) throw new Error(`${path} returned ${response.status}: ${data.error ?? "unknown error"}`);
+  return data;
+}
+
+try {
+  mkdirSync(managedSkillDir, { recursive: true });
+  mkdirSync(externalSkillDir, { recursive: true });
+  mkdirSync(externalExtensionDir, { recursive: true });
+  writeFileSync(
+    join(managedSkillDir, "SKILL.md"),
+    `---\nname: ${managedSkillName}\ndescription: managed isolation probe\n---\nManaged probe.\n`,
+  );
+  writeFileSync(
+    join(externalSkillDir, "SKILL.md"),
+    `---\nname: ${externalSkillName}\ndescription: external isolation probe\n---\nExternal probe.\n`,
+  );
+  writeFileSync(
+    join(externalExtensionDir, "probe.js"),
+    `import { writeFileSync } from "node:fs";\nexport default () => { writeFileSync(${JSON.stringify(externalExtensionMarker)}, "executed"); };\n`,
+  );
+
+  await readJson("/api/cwd/validate", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ cwd: workspace }),
+  });
+  const result = await readJson(`/api/skills?cwd=${encodeURIComponent(workspace)}`);
+  const names = new Set(result.skills.map((skill) => skill.name));
+  const managedSkillLoaded = names.has(managedSkillName);
+  const externalWorkspaceSkillLoaded = names.has(externalSkillName);
+  const externalWorkspaceExtensionExecuted = existsSync(externalExtensionMarker);
+  if (!managedSkillLoaded || externalWorkspaceSkillLoaded || externalWorkspaceExtensionExecuted) {
+    throw new Error("Managed resource isolation check failed");
+  }
+
+  console.log(JSON.stringify({
+    managedSkillLoaded,
+    externalWorkspaceSkillLoaded,
+    externalWorkspaceExtensionExecuted,
+  }, null, 2));
+} finally {
+  rmSync(managedSkillDir, { recursive: true, force: true });
+  rmSync(workspace, { recursive: true, force: true });
+}

scripts/smoke.mjs

@@ -0,0 +1,73 @@
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const rootDir = dirname(dirname(fileURLToPath(import.meta.url)));
+const baseUrl = process.env.PI_WEB_URL ?? "http://127.0.0.1:30141";
+const cwd = resolve(process.env.PI_SMOKE_CWD ?? rootDir);
+
+async function readJson(path, init) {
+  const response = await fetch(`${baseUrl}${path}`, init);
+  const text = await response.text();
+  let data;
+  try {
+    data = JSON.parse(text);
+  } catch {
+    data = { error: text };
+  }
+  if (!response.ok) {
+    throw new Error(`${init?.method ?? "GET"} ${path} returned ${response.status}: ${data.error ?? text}`);
+  }
+  return data;
+}
+
+const pageResponse = await fetch(baseUrl);
+const pageHtml = await pageResponse.text();
+if (!pageResponse.ok || !pageHtml.includes("Pi Web")) {
+  throw new Error(`Pi Web page check failed with status ${pageResponse.status}`);
+}
+
+const jsonHeaders = { "Content-Type": "application/json" };
+const validated = await readJson("/api/cwd/validate", {
+  method: "POST",
+  headers: jsonHeaders,
+  body: JSON.stringify({ cwd }),
+});
+const models = await readJson(`/api/models?cwd=${encodeURIComponent(cwd)}`);
+const runtime = await readJson("/api/runtime");
+const created = await readJson("/api/agent/new", {
+  method: "POST",
+  headers: jsonHeaders,
+  body: JSON.stringify({ cwd, type: "ensure_session", toolNames: [] }),
+});
+const state = await readJson(`/api/agent/${encodeURIComponent(created.sessionId)}`);
+
+if (
+  !validated.success ||
+  !runtime.managed ||
+  resolve(runtime.appRoot) !== rootDir ||
+  resolve(runtime.agentDir) !== resolve(rootDir, "data", "agent") ||
+  !created.success ||
+  !created.sessionId ||
+  !state.running ||
+  !state.state
+) {
+  throw new Error("Pi AgentSession smoke check returned an incomplete state");
+}
+
+console.log(
+  JSON.stringify(
+    {
+      pageStatus: pageResponse.status,
+      cwdAccepted: validated.success,
+      managedRuntime: runtime.managed,
+      agentDir: runtime.agentDir,
+      modelCount: Array.isArray(models.modelList) ? models.modelList.length : 0,
+      defaultModelConfigured: Boolean(models.defaultModel),
+      sessionCreated: created.success,
+      runtimeRunning: state.running,
+      llmRequestSent: false,
+    },
+    null,
+    2,
+  ),
+);