From 8e5f180b2ac7f72ca2d84d8d28790da5e5fd4b29 Mon Sep 17 00:00:00 2001 From: luckyyzh Date: Fri, 31 Jul 2026 18:15:25 +0800 Subject: [PATCH] add automatic storage maintenance --- .env.example | 10 ++ README.md | 34 ++++ package.json | 5 +- scripts/run.mjs | 17 +- scripts/storage-maintenance.mjs | 255 +++++++++++++++++++++++++++ scripts/storage-maintenance.test.mjs | 92 ++++++++++ 6 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 scripts/storage-maintenance.mjs create mode 100644 scripts/storage-maintenance.test.mjs diff --git a/.env.example b/.env.example index 17a08b5..f59f3d4 100644 --- a/.env.example +++ b/.env.example @@ -15,3 +15,13 @@ GEMINI_API_KEY= # Optional: move mutable runtime data while retaining the same isolation model. # PI_AGENT_DATA_DIR=D:\path\to\pi-agent-data + +# Optional storage ceilings. Automatic maintenance runs before Pi Web starts, +# removes only rebuildable caches that exceed these limits, compacts Rewind Git +# data, and removes orphan checkpoints after the grace period. +# PI_STORAGE_AUTO_MAINTENANCE=1 +# PI_STORAGE_NEXT_DEV_MAX_MB=640 +# PI_STORAGE_NEXT_CACHE_MAX_MB=256 +# PI_STORAGE_NPM_CACHE_MAX_MB=256 +# PI_STORAGE_ORPHAN_GRACE_DAYS=7 +# PI_STORAGE_REWIND_GC_OBJECTS=1000 diff --git a/README.md b/README.md index e82fc3c..2ba157e 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,23 @@ npm run smoke:search -- "关键词" # 使用真实 SearXNG；需要配置 除 `smoke:search` 会访问用户配置的 SearXNG 外，以上验证不会发起付费模型请求。 +### 自动磁盘维护 + +每次 `dev`、`restart`、`build` 或 `start` 在启动 Pi Web 前都会检查项目内的可变数据。维护只在 30141 端口没有运行中的 Pi Web 时执行，并采用以下无损策略： + +- `.next/dev`、`.next/cache` 或项目 npm 缓存超过默认上限后才删除；下一次使用时会自动重建； +- Rewind 检查点包含大量松散 Git 对象时自动执行压缩，但保留所有有效检查点； +- 已找不到对应会话的孤儿 Rewind 检查点保留 7 天后自动删除； +- 会话、记忆、凭据、模型配置、已安装插件和仍有关联的检查点不会被自动删除。 + +```powershell +npm run storage:status # 只查看受管缓存大小 +npm run storage:maintain # 立即执行默认阈值维护 +npm run storage:clean # 清空可重建缓存并删除全部孤儿检查点 +``` + +运行中的服务不会被维护命令修改；如需在重启时自动回收，直接使用 `npm run restart`。可在 `.env` 中通过 `PI_STORAGE_*` 变量调整上限、宽限期或关闭自动维护，缺省值见 `.env.example`。 + ### 上游、更新和许可证 - Pi 基线：`0.83.0`，提交 `bb226f9c1f38d3c029156a690e97bbfc602336b9` @@ -323,6 +340,23 @@ npm run smoke:search -- "query" # Real configured SearXNG request Except for `smoke:search`, validation does not make paid model requests. +### Automatic storage maintenance + +Before `dev`, `restart`, `build`, or `start` launches Pi Web, the integrated launcher checks repository-local mutable storage. Maintenance runs only when no Pi Web process is listening on port 30141 and follows lossless defaults: + +- `.next/dev`, `.next/cache`, and the managed npm cache are removed only after exceeding their size ceilings and are rebuilt on demand; +- Rewind repositories with many loose Git objects are compacted without dropping valid checkpoints; +- orphan Rewind checkpoints whose session no longer exists are removed after a seven-day grace period; +- sessions, memory, credentials, model configuration, installed plugins, and linked checkpoints are never automatically deleted. + +```powershell +npm run storage:status # Report managed cache sizes without changing data +npm run storage:maintain # Apply the normal thresholds immediately +npm run storage:clean # Remove rebuildable caches and all orphan checkpoints +``` + +Maintenance refuses to modify a running service. Use `npm run restart` to reclaim space safely during a restart. `PI_STORAGE_*` variables in `.env` can tune the ceilings and grace period or disable automatic maintenance; `.env.example` documents the defaults. + ### Upstream, updates, and licensing - Pi baseline: `0.83.0`, commit `bb226f9c1f38d3c029156a690e97bbfc602336b9` diff --git a/package.json b/package.json index 980b84b..54e664b 100644 --- a/package.json +++ b/package.json @@ -26,9 +26,12 @@ "build": "node scripts/run.mjs build", "start": "node scripts/run.mjs start", "start:lan": "node scripts/run.mjs start:lan", + "storage:status": "node scripts/storage-maintenance.mjs status", + "storage:maintain": "node scripts/storage-maintenance.mjs auto", + "storage:clean": "node scripts/storage-maintenance.mjs clean", "typecheck": "node pi-web/node_modules/typescript/bin/tsc --noEmit -p pi-web/tsconfig.json", "lint": "npm --prefix pi-web run lint", - "test:managed": "node --test scripts/migrate-profile.test.mjs pi-web/lib/app-runtime.test.mjs pi-web/lib/project-trust.test.mjs pi-web/lib/searxng-extension.test.mjs pi-web/lib/skill-lock.test.mjs pi-web/lib/skill-updates.test.mjs" + "test:managed": "node --test scripts/migrate-profile.test.mjs scripts/storage-maintenance.test.mjs pi-web/lib/app-runtime.test.mjs pi-web/lib/project-trust.test.mjs pi-web/lib/searxng-extension.test.mjs pi-web/lib/skill-lock.test.mjs pi-web/lib/skill-updates.test.mjs" }, "engines": { "node": ">=22.19.0" diff --git a/scripts/run.mjs b/scripts/run.mjs index b8b51f6..7f82865 100644 --- a/scripts/run.mjs +++ b/scripts/run.mjs @@ -1,6 +1,11 @@ import { spawn } from "node:child_process"; import { join } from "node:path"; import { managedEnvironment, rootDir } from "./profile.mjs"; +import { + isLocalPortListening, + maintainStorage, + printMaintenanceResult, +} from "./storage-maintenance.mjs"; const target = process.argv[2] ?? "dev"; const allowedTargets = new Set(["build", "dev", "dev:lan", "start", "start:lan"]); @@ -15,12 +20,19 @@ if (!npmCliPath) { process.exit(1); } +const childEnvironment = managedEnvironment(); +if (await isLocalPortListening(30141)) { + console.log("[storage] skipped automatic maintenance because Pi Web is already running"); +} else { + printMaintenanceResult(maintainStorage({ mode: "auto", env: childEnvironment })); +} + const child = spawn( process.execPath, [npmCliPath, "--prefix", join(rootDir, "pi-web"), "run", target], { cwd: rootDir, - env: managedEnvironment(), + env: childEnvironment, stdio: "inherit", }, ); @@ -34,6 +46,9 @@ child.on("error", (error) => { process.exit(1); }); child.on("exit", (code, signal) => { + if (!signal && code === 0 && target === "build") { + printMaintenanceResult(maintainStorage({ mode: "auto", env: childEnvironment })); + } if (signal) process.kill(process.pid, signal); process.exit(code ?? 1); }); diff --git a/scripts/storage-maintenance.mjs b/scripts/storage-maintenance.mjs new file mode 100644 index 0000000..3705351 --- /dev/null +++ b/scripts/storage-maintenance.mjs @@ -0,0 +1,255 @@ +import { spawnSync } from "node:child_process"; +import { + existsSync, + lstatSync, + readdirSync, + rmSync, +} from "node:fs"; +import { createConnection } from "node:net"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { dataDir as managedDataDir, rootDir as projectRootDir } from "./profile.mjs"; + +const MIB = 1024 * 1024; +const DAY_MS = 24 * 60 * 60 * 1000; + +function numericSetting(env, name, fallback) { + const value = Number.parseFloat(env[name] ?? ""); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +function enabledSetting(env, name, fallback = true) { + const value = env[name]?.trim().toLowerCase(); + if (!value) return fallback; + return !["0", "false", "no", "off"].includes(value); +} + +export function storagePolicy(env = process.env) { + return { + enabled: enabledSetting(env, "PI_STORAGE_AUTO_MAINTENANCE", true), + nextDevMaxBytes: numericSetting(env, "PI_STORAGE_NEXT_DEV_MAX_MB", 640) * MIB, + nextBuildCacheMaxBytes: numericSetting(env, "PI_STORAGE_NEXT_CACHE_MAX_MB", 256) * MIB, + npmCacheMaxBytes: numericSetting(env, "PI_STORAGE_NPM_CACHE_MAX_MB", 256) * MIB, + orphanGraceMs: numericSetting(env, "PI_STORAGE_ORPHAN_GRACE_DAYS", 7) * DAY_MS, + rewindGcLooseObjects: numericSetting(env, "PI_STORAGE_REWIND_GC_OBJECTS", 1000), + }; +} + +export function directoryStats(path) { + if (!existsSync(path)) return { bytes: 0, files: 0, newestMtimeMs: 0 }; + + let bytes = 0; + let files = 0; + let newestMtimeMs = 0; + const pending = [path]; + + while (pending.length > 0) { + const current = pending.pop(); + let entries; + try { + entries = readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + const child = join(current, entry.name); + let stats; + try { + stats = lstatSync(child); + } catch { + continue; + } + newestMtimeMs = Math.max(newestMtimeMs, stats.mtimeMs); + if (stats.isSymbolicLink()) continue; + if (stats.isDirectory()) pending.push(child); + else if (stats.isFile()) { + bytes += stats.size; + files += 1; + } + } + } + + return { bytes, files, newestMtimeMs }; +} + +function collectSessionIds(path) { + const ids = new Set(); + if (!existsSync(path)) return ids; + const pending = [path]; + while (pending.length > 0) { + const current = pending.pop(); + let entries; + try { + entries = readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const child = join(current, entry.name); + if (entry.isDirectory()) pending.push(child); + else if (entry.isFile() && entry.name.endsWith(".jsonl")) ids.add(basename(entry.name, ".jsonl")); + } + } + return ids; +} + +function countLooseGitObjects(checkpointPath) { + const objectsPath = join(checkpointPath, ".git", "objects"); + if (!existsSync(objectsPath)) return 0; + let count = 0; + for (const entry of readdirSync(objectsPath, { withFileTypes: true })) { + if (!entry.isDirectory() || !/^[0-9a-f]{2}$/i.test(entry.name)) continue; + count += readdirSync(join(objectsPath, entry.name), { withFileTypes: true }) + .filter((child) => child.isFile()).length; + } + return count; +} + +function defaultCheckpointCompactor(path) { + const result = spawnSync("git", ["-C", path, "gc", "--quiet"], { + encoding: "utf8", + windowsHide: true, + }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(result.stderr.trim() || `git gc exited with ${result.status}`); +} + +function removeDirectory(path, label, result) { + const before = directoryStats(path); + if (!existsSync(path)) return; + rmSync(path, { recursive: true, force: true }); + result.reclaimedBytes += before.bytes; + result.actions.push({ type: "removed", label, path, bytes: before.bytes }); +} + +export function maintainStorage({ + mode = "auto", + rootDir = projectRootDir, + dataDir = managedDataDir, + env = process.env, + nowMs = Date.now(), + compactCheckpoint = defaultCheckpointCompactor, +} = {}) { + const policy = storagePolicy(env); + const result = { mode, actions: [], warnings: [], reclaimedBytes: 0, status: {} }; + if (mode === "auto" && !policy.enabled) return { ...result, disabled: true }; + + const cacheTargets = [ + { + label: "Next.js development output", + path: join(rootDir, "pi-web", ".next", "dev"), + limit: policy.nextDevMaxBytes, + }, + { + label: "Next.js build cache", + path: join(rootDir, "pi-web", ".next", "cache"), + limit: policy.nextBuildCacheMaxBytes, + }, + { + label: "managed npm cache", + path: join(dataDir, "cache", "npm"), + limit: policy.npmCacheMaxBytes, + }, + ]; + + for (const target of cacheTargets) { + const stats = directoryStats(target.path); + result.status[target.label] = stats; + if (mode === "clean" || (mode === "auto" && stats.bytes > target.limit)) { + removeDirectory(target.path, target.label, result); + } + } + + const sessionIds = collectSessionIds(join(dataDir, "agent", "sessions")); + const checkpointsPath = join(dataDir, "home", ".pi", "agent", "ayu", "checkpoints", "sessions"); + if (existsSync(checkpointsPath)) { + for (const entry of readdirSync(checkpointsPath, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const checkpointPath = join(checkpointsPath, entry.name); + const stats = directoryStats(checkpointPath); + const orphan = !sessionIds.has(entry.name); + const ageMs = Math.max(0, nowMs - stats.newestMtimeMs); + + if (orphan && (mode === "clean" || ageMs >= policy.orphanGraceMs)) { + removeDirectory(checkpointPath, `orphan Rewind checkpoint ${entry.name}`, result); + continue; + } + + if (mode !== "status") { + const looseObjects = countLooseGitObjects(checkpointPath); + if (looseObjects >= policy.rewindGcLooseObjects) { + const before = directoryStats(checkpointPath).bytes; + try { + compactCheckpoint(checkpointPath); + const after = directoryStats(checkpointPath).bytes; + result.reclaimedBytes += Math.max(0, before - after); + result.actions.push({ + type: "compacted", + label: `Rewind checkpoint ${entry.name}`, + path: checkpointPath, + bytes: Math.max(0, before - after), + }); + } catch (error) { + result.warnings.push(`Unable to compact ${checkpointPath}: ${error.message}`); + } + } + } + } + } + + return result; +} + +export function formatBytes(bytes) { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(2)} GiB`; + if (bytes >= MIB) return `${(bytes / MIB).toFixed(1)} MiB`; + return `${(bytes / 1024).toFixed(1)} KiB`; +} + +export function printMaintenanceResult(result) { + if (result.disabled) { + console.log("[storage] automatic maintenance is disabled"); + return; + } + if (result.mode === "status") { + for (const [label, stats] of Object.entries(result.status)) { + console.log(`[storage] ${label}: ${formatBytes(stats.bytes)} (${stats.files} files)`); + } + } + for (const action of result.actions) { + const verb = action.type === "compacted" ? "compacted" : "removed"; + console.log(`[storage] ${verb} ${action.label}; reclaimed ${formatBytes(action.bytes)}`); + } + for (const warning of result.warnings) console.warn(`[storage] ${warning}`); + if (result.actions.length > 0) console.log(`[storage] total reclaimed: ${formatBytes(result.reclaimedBytes)}`); + else if (result.mode !== "auto") console.log("[storage] nothing to clean"); +} + +export function isLocalPortListening(port, host = "127.0.0.1", timeoutMs = 250) { + return new Promise((resolvePromise) => { + const socket = createConnection({ port, host }); + const finish = (listening) => { + socket.destroy(); + resolvePromise(listening); + }; + socket.setTimeout(timeoutMs); + socket.once("connect", () => finish(true)); + socket.once("timeout", () => finish(false)); + socket.once("error", () => finish(false)); + }); +} + +const entryPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (entryPath === resolve(fileURLToPath(import.meta.url))) { + const mode = process.argv[2] ?? "status"; + if (!new Set(["status", "auto", "clean"]).has(mode)) { + console.error("Usage: node scripts/storage-maintenance.mjs [status|auto|clean]"); + process.exit(2); + } + if (mode !== "status" && await isLocalPortListening(30141)) { + console.error("[storage] Pi Web is running on port 30141. Stop it first or use npm run restart."); + process.exit(1); + } + printMaintenanceResult(maintainStorage({ mode })); +} diff --git a/scripts/storage-maintenance.test.mjs b/scripts/storage-maintenance.test.mjs new file mode 100644 index 0000000..13926e8 --- /dev/null +++ b/scripts/storage-maintenance.test.mjs @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { maintainStorage } from "./storage-maintenance.mjs"; + +function createFile(path, bytes = 2048) { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, Buffer.alloc(bytes)); +} + +function createCheckpoint(root, id, modifiedAt) { + const checkpoint = join(root, "home", ".pi", "agent", "ayu", "checkpoints", "sessions", id); + const objectDir = join(checkpoint, ".git", "objects", "ab"); + mkdirSync(objectDir, { recursive: true }); + createFile(join(objectDir, "object")); + createFile(join(checkpoint, "manifest.json"), 64); + for (const path of [join(objectDir, "object"), objectDir, join(checkpoint, ".git", "objects"), join(checkpoint, ".git"), join(checkpoint, "manifest.json"), checkpoint]) { + utimesSync(path, modifiedAt, modifiedAt); + } + return checkpoint; +} + +test("automatic maintenance bounds caches and removes only expired orphan checkpoints", (t) => { + const root = mkdtempSync(join(tmpdir(), "pi-storage-maintenance-")); + const dataDir = join(root, "data"); + const now = new Date("2026-07-31T00:00:00Z"); + const old = new Date(now.getTime() - 8 * 24 * 60 * 60 * 1000); + const recent = new Date(now.getTime() - 24 * 60 * 60 * 1000); + t.after(() => rmSync(root, { recursive: true, force: true })); + + const nextDev = join(root, "pi-web", ".next", "dev"); + const nextCache = join(root, "pi-web", ".next", "cache"); + const npmCache = join(dataDir, "cache", "npm"); + createFile(join(nextDev, "large.bin")); + createFile(join(nextCache, "large.bin")); + createFile(join(npmCache, "large.bin")); + + const sessions = join(dataDir, "agent", "sessions"); + createFile(join(sessions, "live-session.jsonl"), 16); + const liveCheckpoint = createCheckpoint(dataDir, "live-session", old); + const expiredOrphan = createCheckpoint(dataDir, "expired-orphan", old); + const recentOrphan = createCheckpoint(dataDir, "recent-orphan", recent); + const compacted = []; + + const result = maintainStorage({ + mode: "auto", + rootDir: root, + dataDir, + nowMs: now.getTime(), + env: { + PI_STORAGE_NEXT_DEV_MAX_MB: "0.001", + PI_STORAGE_NEXT_CACHE_MAX_MB: "0.001", + PI_STORAGE_NPM_CACHE_MAX_MB: "0.001", + PI_STORAGE_ORPHAN_GRACE_DAYS: "7", + PI_STORAGE_REWIND_GC_OBJECTS: "1", + }, + compactCheckpoint(path) { + compacted.push(path); + }, + }); + + assert.equal(existsSync(nextDev), false); + assert.equal(existsSync(nextCache), false); + assert.equal(existsSync(npmCache), false); + assert.equal(existsSync(expiredOrphan), false); + assert.equal(existsSync(recentOrphan), true); + assert.equal(existsSync(liveCheckpoint), true); + assert.deepEqual(new Set(compacted), new Set([recentOrphan, liveCheckpoint])); + assert.ok(result.reclaimedBytes > 0); +}); + +test("clean mode removes every orphan while preserving live checkpoints", (t) => { + const root = mkdtempSync(join(tmpdir(), "pi-storage-clean-")); + const dataDir = join(root, "data"); + t.after(() => rmSync(root, { recursive: true, force: true })); + + createFile(join(dataDir, "agent", "sessions", "live-session.jsonl"), 16); + const liveCheckpoint = createCheckpoint(dataDir, "live-session", new Date()); + const orphanCheckpoint = createCheckpoint(dataDir, "orphan-session", new Date()); + + maintainStorage({ + mode: "clean", + rootDir: root, + dataDir, + env: { PI_STORAGE_REWIND_GC_OBJECTS: "999999" }, + }); + + assert.equal(existsSync(liveCheckpoint), true); + assert.equal(existsSync(orphanCheckpoint), false); +});