[\s\S]*$/i, "").trimEnd();
}
return content.replace(/\n---\n+## Comments\n[\s\S]*$/i, "").trimEnd();
}
// ../core/src/extract.ts
var HTML_CONTENT_TYPES = [
"text/html",
"application/xhtml+xml",
"text/plain",
"text/markdown"
];
var MAX_CLIENT_SIDE_REDIRECTS = 5;
var MAX_ALTERNATE_LINK_FALLBACKS = 3;
var MIN_EXTRACTED_WORDS_BEFORE_ALTERNATE_FALLBACK = 30;
function normalizeContentType(contentType) {
return contentType.split(";")[0]?.trim().toLowerCase() ?? "";
}
function isAttachmentDisposition(contentDisposition) {
return /^attachment(?:\s*;|\s*$)/i.test(contentDisposition.trim());
}
function isTextualContentType(contentType) {
const normalized = normalizeContentType(contentType);
return normalized.startsWith("text/") || normalized === "application/json" || normalized === "text/json" || normalized.endsWith("+json") || normalized === "application/xml" || normalized === "text/xml" || normalized.endsWith("+xml") || normalized === "application/javascript" || normalized === "application/x-javascript" || normalized === "application/ecmascript" || normalized === "image/svg+xml";
}
function sanitizeBaseName(value) {
const sanitized = (0, import_deburr.default)(value).replace(/[\\/]+/g, "-").replace(/[^A-Za-z0-9._ -]+/g, "").trim().replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^\.+/, "").replace(/[. -]+$/g, "");
return sanitized;
}
function sanitizeExtension(value) {
const raw = (0, import_deburr.default)(value).replace(/^[.\s]+/, "").replace(/[\\/]+/g, "").replace(/[^A-Za-z0-9_-]+/g, "").toLowerCase();
return raw ? `.${raw}` : "";
}
function decodeContentDispositionFilename(value) {
try {
return decodeURIComponent(value);
} catch {
return value;
}
}
function extractContentDispositionFilename(contentDisposition) {
const filenameStarMatch = contentDisposition.match(/filename\*=([^;]+)/i);
const filenameMatch = contentDisposition.match(
/filename=(?:"([^"]+)"|([^;]+))/i
);
const rawFilename = filenameStarMatch?.[1] ? (() => {
const value = filenameStarMatch[1].trim();
const encoded = value.includes("''") ? value.split("''").slice(1).join("''") : value;
return decodeContentDispositionFilename(encoded.replace(/^"|"$/g, ""));
})() : (filenameMatch?.[1] ?? filenameMatch?.[2] ?? "").trim();
if (!rawFilename) {
return {};
}
const sanitizedFilename = rawFilename.replace(/^"|"$/g, "").replace(/[\\/]+/g, "-");
const parsed = parse(sanitizedFilename);
return {
baseName: sanitizeBaseName(parsed.name || sanitizedFilename),
extension: sanitizeExtension(parsed.ext)
};
}
function deriveUrlPathName(url) {
try {
const parsedUrl = new URL(url);
const lastSegment = parsedUrl.pathname.split("/").filter(Boolean).at(-1);
if (!lastSegment) {
return {};
}
const decodedSegment = decodeContentDispositionFilename(lastSegment);
const parsedSegment = parse(decodedSegment);
return {
baseName: sanitizeBaseName(parsedSegment.name || decodedSegment),
extension: sanitizeExtension(parsedSegment.ext)
};
} catch {
return {};
}
}
function resolveExtensionFromMimeType(contentType) {
const extension = (0, import_mime_types.extension)(normalizeContentType(contentType));
return sanitizeExtension(typeof extension === "string" ? extension : "") || ".dat";
}
function resolveDownloadTarget(finalUrl, contentDisposition, contentType) {
const fromDisposition = extractContentDispositionFilename(contentDisposition);
const fromUrl = deriveUrlPathName(finalUrl);
const baseName = fromDisposition.baseName || fromUrl.baseName || sanitizeBaseName(randomUUID());
const extension = fromDisposition.extension || resolveExtensionFromMimeType(contentType);
return {
fileName: `${baseName}${extension}`,
extension: extension || ".dat"
};
}
async function cleanupPartialFile(filePath) {
try {
await unlink(filePath);
} catch (error) {
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") {
throw error;
}
}
}
async function streamResponseToFile(response, filePath) {
await mkdir(parse(filePath).dir, { recursive: true });
let fileSize = 0;
if (response.body) {
const output = createWriteStream(filePath, { flags: "wx", mode: 384 });
const reader = response.body.getReader();
let opened = false;
try {
await new Promise((resolve, reject) => {
output.once("open", () => {
opened = true;
resolve();
});
output.once("error", reject);
});
const finished = new Promise((resolve, reject) => {
output.once("finish", () => resolve());
output.once("error", reject);
});
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
if (value) {
fileSize += value.byteLength;
if (!output.write(Buffer.from(value))) {
await once(output, "drain");
}
}
}
output.end();
await finished;
await chmod(filePath, 384);
return fileSize;
} catch (error) {
output.destroy();
try {
await reader.cancel(
error instanceof Error ? error.message : String(error)
);
} catch {
}
if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") {
throw error;
}
if (opened) {
await cleanupPartialFile(filePath);
}
throw error;
} finally {
reader.releaseLock();
}
}
if (response.readable) {
const source = response.readable();
source.on("data", (chunk) => {
fileSize += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.byteLength;
});
try {
await pipeline(
source,
createWriteStream(filePath, { flags: "wx", mode: 384 })
);
await chmod(filePath, 384);
return fileSize;
} catch (error) {
if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") {
throw error;
}
await cleanupPartialFile(filePath);
throw error;
}
}
const body = response.arrayBuffer ? new Uint8Array(await response.arrayBuffer()) : new TextEncoder().encode(await response.text());
fileSize = body.byteLength;
try {
await writeFile(filePath, body, { mode: 384, flag: "wx" });
await chmod(filePath, 384);
return fileSize;
} catch (error) {
if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") {
throw error;
}
await cleanupPartialFile(filePath);
throw error;
}
}
function isPlainTextContentType(contentType) {
const normalized = normalizeContentType(contentType);
return normalized === "text/plain" || normalized === "text/markdown";
}
function renderPlainTextContent(body, format) {
if (format === "html") {
return `
${body.replace(/&/g, "&").replace(//g, ">")}`;
}
return body;
}
function buildPlainTextResult(opts, finalUrl, rawBody, format, maxChars, browser, os) {
const normalizedBody = rawBody.replace(/\r\n/g, "\n").trim();
return {
kind: "content",
url: opts.url,
finalUrl,
title: "",
author: "",
published: "",
site: new URL(finalUrl).hostname,
language: "",
wordCount: estimateWordCount(normalizedBody),
content: truncateContent(
renderPlainTextContent(normalizedBody, format),
maxChars
),
browser,
os
};
}
function isTwitterJsDisabledPage(document, url) {
if (!/^(https?:\/\/)?(www\.)?(x\.com|twitter\.com)\//i.test(url))
return false;
const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
return text.includes("JavaScript is disabled") && text.includes("supported browser");
}
function extractDomTextFallback(document) {
const bodyText = document.body?.textContent ?? document.documentElement?.textContent ?? "";
return bodyText.replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").split("\n").map((line) => line.trim()).join("\n").replace(/[ \t]{2,}/g, " ").trim();
}
function escapeMarkdownText(value) {
return value.replace(/([\\`*_{}[\]()+#.!|>-])/g, "\\$1");
}
function normalizeInlineWhitespace(value) {
return value.replace(/\s+/g, " ").trim();
}
function renderInlineMarkdown(node) {
if (node.nodeType === 3) {
return normalizeInlineWhitespace(node.textContent ?? "");
}
if (node.nodeType !== 1) {
return "";
}
const element = node;
const tag = element.tagName.toLowerCase();
if (["script", "style", "meta", "link"].includes(tag)) {
return "";
}
if (tag === "br") {
return " \n";
}
if (tag === "code") {
const content = normalizeInlineWhitespace(element.textContent ?? "");
return content ? `\`${content}\`` : "";
}
if (tag === "img") {
const alt = element.getAttribute("alt") ?? "";
const src = element.getAttribute("src") ?? "";
return src ? `` : "";
}
const childContent = Array.from(element.childNodes).map(renderInlineMarkdown).join(" ").replace(/\s+/g, " ").trim();
if (tag === "a") {
const href = element.getAttribute("href") ?? "";
if (!href) return childContent;
return `[${childContent || href}](${href})`;
}
if (["strong", "b"].includes(tag)) {
return childContent ? `**${childContent}**` : "";
}
if (["em", "i"].includes(tag)) {
return childContent ? `*${childContent}*` : "";
}
return childContent;
}
function renderBlockMarkdown(node, depth = 0) {
if (node.nodeType === 3) {
const text = normalizeInlineWhitespace(node.textContent ?? "");
return text ? `${text}
` : "";
}
if (node.nodeType !== 1) {
return "";
}
const element = node;
const tag = element.tagName.toLowerCase();
if (["script", "style", "meta", "link"].includes(tag)) {
return "";
}
if (/^h[1-6]$/.test(tag)) {
const level = Number.parseInt(tag.slice(1), 10);
const content = Array.from(element.childNodes).map(renderInlineMarkdown).join(" ").replace(/\s+/g, " ").trim();
return content ? `${"#".repeat(level)} ${content}
` : "";
}
if (tag === "p") {
const content = Array.from(element.childNodes).map(renderInlineMarkdown).join(" ").replace(/\s+/g, " ").trim();
return content ? `${content}
` : "";
}
if (tag === "pre") {
const content = (element.textContent ?? "").trim();
return content ? `\`\`\`
${content}
\`\`\`
` : "";
}
if (tag === "blockquote") {
const content = Array.from(element.childNodes).map((child) => renderBlockMarkdown(child, depth)).join("").trim();
if (!content) return "";
return `${content.split("\n").map((line) => line ? `> ${line}` : ">").join("\n")}
`;
}
if (tag === "ul" || tag === "ol") {
const items = Array.from(element.children).filter((child) => child.tagName.toLowerCase() === "li").map((child, index) => {
const prefix = tag === "ol" ? `${index + 1}. ` : "- ";
const content = Array.from(child.childNodes).map((grandchild) => {
const childTag = grandchild.nodeType === 1 ? grandchild.tagName.toLowerCase() : "";
return childTag === "ul" || childTag === "ol" ? `
${renderBlockMarkdown(grandchild, depth + 1)}` : renderInlineMarkdown(grandchild);
}).join(" ").replace(/\s+\n/g, "\n").replace(/\n\s+/g, "\n").replace(/\s+/g, " ").trim();
if (!content) return "";
const indented = content.split("\n").map(
(line, lineIndex) => lineIndex === 0 ? `${" ".repeat(depth)}${prefix}${line}` : `${" ".repeat(depth + 1)}${line}`
).join("\n");
return indented;
}).filter(Boolean).join("\n");
return items ? `${items}
` : "";
}
if (tag === "hr") {
return "---\n\n";
}
const blockContent = Array.from(element.childNodes).map((child) => renderBlockMarkdown(child, depth)).join("");
if (blockContent.trim()) {
return blockContent;
}
const inlineContent = Array.from(element.childNodes).map(renderInlineMarkdown).join(" ").replace(/\s+/g, " ").trim();
return inlineContent ? `${inlineContent}
` : "";
}
function extractDomMarkdownFallback(document) {
const root = document.body ?? document.documentElement;
if (!root) return "";
return Array.from(root.childNodes).map((node) => renderBlockMarkdown(node)).join("").replace(/\r\n/g, "\n").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
}
function emitProgress(hooks, update) {
hooks.onProgressChange?.(update);
}
function emitStatus(hooks, status) {
hooks.onStatusChange?.(status);
}
function formatByteCount2(bytes) {
const units = ["B", "KB", "MB", "GB", "TB"];
let value = Math.max(0, bytes);
let unitIndex = 0;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex += 1;
}
const decimals = unitIndex === 0 ? 0 : value >= 10 ? 1 : 2;
return `${value.toFixed(decimals)} ${units[unitIndex]}`;
}
function parseContentLengthHeader(value) {
if (!value) return void 0;
const parsed = Number.parseInt(value, 10);
return Number.isFinite(parsed) && parsed >= 0 ? parsed : void 0;
}
function isTimeoutError(error) {
const message = error instanceof Error ? error.message : String(error);
return /timed out|timeout|deadline exceeded|abort(?:ed)?/i.test(message);
}
function isDnsError(error) {
const message = error instanceof Error ? error.message : String(error);
return /dns error|failed to lookup address|nodename nor servname provided|name resolution failed/i.test(
message
);
}
function isConnectError(error) {
const message = error instanceof Error ? error.message : String(error);
return /client error \(connect\)|connection refused|tcp connect error|connection reset|network unreachable|no route to host/i.test(
message
);
}
function isTlsError(error) {
const message = error instanceof Error ? error.message : String(error);
return /ssl.*error|tls.*error|bad certificate|certificate.*invalid|unknown.*issuer/i.test(
message
);
}
function extractHostname(url) {
try {
return new URL(url).hostname;
} catch {
return url;
}
}
function buildTimeoutError(context) {
const targetUrl = context.finalUrl ?? context.url;
const timeoutLabel = `${context.timeoutMs}ms`;
if (context.phase === "connecting") {
return {
error: `Timeout of ${timeoutLabel} exceeded while connecting to ${targetUrl}.`,
code: "timeout",
phase: "connecting",
retryable: true,
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl
};
}
if (context.phase === "waiting") {
return {
error: `Timeout of ${timeoutLabel} exceeded while waiting for ${targetUrl} to start responding.`,
code: "timeout",
phase: "waiting",
retryable: true,
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl
};
}
if (context.phase === "loading") {
const sizeHint = context.contentLength ? ` a ${formatByteCount2(context.contentLength)} ${context.mimeType && !isTextualContentType(context.mimeType) ? "file" : "response"}` : " the response body";
return {
error: `Timeout of ${timeoutLabel} exceeded while downloading${sizeHint} from ${targetUrl}.`,
code: "timeout",
phase: "loading",
retryable: true,
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl,
statusCode: context.statusCode,
statusText: context.statusText,
mimeType: context.mimeType,
contentLength: context.contentLength,
downloadedBytes: context.downloadedBytes
};
}
if (context.phase === "processing") {
return {
error: `Timeout of ${timeoutLabel} exceeded while processing the response from ${targetUrl}.`,
code: "timeout",
phase: "processing",
retryable: true,
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl,
statusCode: context.statusCode,
statusText: context.statusText,
mimeType: context.mimeType,
contentLength: context.contentLength,
downloadedBytes: context.downloadedBytes
};
}
return {
error: `Timeout of ${timeoutLabel} exceeded while fetching ${targetUrl}.`,
code: "timeout",
phase: context.phase,
retryable: true,
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl,
statusCode: context.statusCode,
statusText: context.statusText,
mimeType: context.mimeType,
contentLength: context.contentLength,
downloadedBytes: context.downloadedBytes
};
}
function buildThrownFetchError(error, context) {
if (isTimeoutError(error)) {
return buildTimeoutError(context);
}
const hostname = extractHostname(context.url);
if (isDnsError(error)) {
return {
error: `DNS error: failed to lookup address for ${hostname}. Check the URL and try again.`,
code: "network_error",
phase: "connecting",
retryable: false,
url: context.url,
finalUrl: context.finalUrl
};
}
if (isConnectError(error)) {
return {
error: `Connection failed to ${hostname}. The server may be unreachable or blocking requests.`,
code: "network_error",
phase: "connecting",
retryable: true,
url: context.url,
finalUrl: context.finalUrl
};
}
if (isTlsError(error)) {
return {
error: `TLS/SSL error connecting to ${hostname}. The server's certificate may be invalid.`,
code: "network_error",
phase: "connecting",
retryable: false,
url: context.url,
finalUrl: context.finalUrl
};
}
const isConnectLevel = isDnsError(error) || isConnectError(error) || isTlsError(error);
const effectivePhase = isConnectLevel ? "connecting" : context.phase;
const message = error instanceof Error ? error.message : String(error);
const targetUrl = context.finalUrl ?? context.url;
const phaseDescription = effectivePhase === "loading" ? "downloading the response" : effectivePhase === "waiting" ? "waiting for the server response" : effectivePhase === "connecting" ? "connecting" : "fetching";
return {
error: effectivePhase === "processing" ? `Failed while processing the response from ${targetUrl}: ${message}` : `Request failed while ${phaseDescription} for ${targetUrl}: ${message}`,
code: effectivePhase === "processing" ? "processing_error" : effectivePhase === "loading" && context.mimeType ? "download_error" : "network_error",
phase: effectivePhase,
retryable: effectivePhase !== "processing",
timeoutMs: context.timeoutMs,
url: context.url,
finalUrl: context.finalUrl,
statusCode: context.statusCode,
statusText: context.statusText,
mimeType: context.mimeType,
contentLength: context.contentLength,
downloadedBytes: context.downloadedBytes
};
}
function mapRequestEventToProgress(event) {
switch (event.type) {
case "request_start":
return { status: "connecting", progress: 0, phase: event.type };
case "request_sent":
return { status: "waiting", progress: 0.11, phase: event.type };
case "response_headers":
return { status: "loading", progress: 0.51, phase: event.type };
case "body_progress": {
const contentLength = event.contentLength;
const downloadedBytes = event.downloadedBytes ?? 0;
const bodyFraction = contentLength && contentLength > 0 ? Math.max(0, Math.min(1, downloadedBytes / contentLength)) : Math.max(0, Math.min(1, downloadedBytes / 65536));
return {
status: "loading",
progress: contentLength && contentLength > 0 ? 0.51 + bodyFraction * 0.44 : 0.51,
phase: event.type
};
}
case "body_complete":
return { status: "loading", progress: 0.95, phase: event.type };
case "done":
return { status: "done", progress: 1, phase: event.type };
case "error":
return { status: "error", progress: 1, phase: event.type };
default:
return null;
}
}
function resolveAcceptHeader(format) {
if (format === "json") return DEFAULT_JSON_ACCEPT_HEADER;
if (format === "raw") return DEFAULT_RAW_ACCEPT_HEADER;
return DEFAULT_ACCEPT_HEADER;
}
function isJsonContentType(contentType) {
const normalized = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
return normalized === "application/json" || normalized === "text/json" || normalized.endsWith("+json");
}
function isLikelyJsonBody(body) {
const trimmed = body.trim();
return trimmed.startsWith("{") || trimmed.startsWith("[");
}
function isJsonResponse(contentType, body) {
return isJsonContentType(contentType) || isLikelyJsonBody(body);
}
function decodeHtmlAttribute(value) {
return value.replace(/&/gi, "&").replace(/"/gi, '"').replace(/'|'/gi, "'").replace(/</gi, "<").replace(/>/gi, ">");
}
function extractQualifiedAlternateLinks(document, baseUrl, format) {
const acceptedTypes = {
markdown: ["text/markdown", "text/x-markdown"],
text: ["text/plain", "text/markdown", "text/x-markdown"],
html: ["text/html", "application/xhtml+xml"],
json: ["application/json", "text/json"],
raw: []
};
const accepted = acceptedTypes[format];
const head = document.head;
if (!head) return [];
const links = Array.from(head.querySelectorAll("link"));
const candidates = [];
for (const link of links) {
const rel = (link.getAttribute("rel") ?? "").toLowerCase().split(/\s+/);
if (!rel.includes("alternate")) continue;
const type = normalizeContentType(link.getAttribute("type") ?? "");
const isAccepted = accepted.some((value) => type === value) || format === "json" && type.endsWith("+json");
if (!isAccepted) continue;
const href = link.getAttribute("href");
if (!href) continue;
try {
const target = new URL(href, baseUrl).toString();
if (target !== baseUrl && !candidates.includes(target)) {
candidates.push(target);
}
} catch {
}
}
return candidates;
}
function extractClientSideRedirect(body, baseUrl) {
const snippet = body.slice(0, 4096);
const metaRefreshMatch = snippet.match(
/
]*http-equiv=["']?refresh["']?[^>]*content=["']?([^"'>]*)["']?[^>]*>/i
);
const refreshContent = metaRefreshMatch?.[1];
if (!refreshContent) {
return null;
}
const [delayPart = "", ...rest] = decodeHtmlAttribute(refreshContent).split(";");
const delaySeconds = Number.parseFloat(delayPart.trim());
const urlMatch = rest.join(";").match(/\burl\s*=\s*(.+)$/i);
const rawTarget = urlMatch?.[1]?.trim().replace(/^['"]|['"]$/g, "");
if (!rawTarget || !Number.isFinite(delaySeconds) || delaySeconds < 0 || delaySeconds >= 30) {
return null;
}
try {
const targetUrl = new URL(rawTarget, baseUrl).toString();
return targetUrl === baseUrl ? null : targetUrl;
} catch {
return null;
}
}
function buildJsonResult(opts, finalUrl, rawBody, format, maxChars, browser, os) {
const parsedJson = parseAndFormatJson(rawBody);
if ("error" in parsedJson) {
return parsedJson;
}
const content = truncateContent(
renderJsonContent(parsedJson.formatted, format),
maxChars
);
return {
kind: "content",
url: opts.url,
finalUrl,
title: "",
author: "",
published: "",
site: new URL(finalUrl).hostname,
language: "",
wordCount: estimateWordCount(parsedJson.formatted),
content,
browser,
os
};
}
async function buildFileResult(opts, response, finalUrl, contentType, contentDisposition, browser, os) {
const tempDir = opts.tempDir || join(tmpdir(), "smart-fetch");
await mkdir(tempDir, { recursive: true });
const { fileName, extension } = resolveDownloadTarget(
finalUrl,
contentDisposition,
contentType
);
let filePath = join(tempDir, fileName);
let attempt = 1;
while (attempt <= 100) {
try {
const fileSize = await streamResponseToFile(response, filePath);
return {
kind: "file",
url: opts.url,
finalUrl,
title: "",
author: "",
published: "",
site: new URL(finalUrl).hostname,
language: "",
wordCount: 0,
content: "",
browser,
os,
filePath,
fileSize,
mimeType: normalizeContentType(contentType) || void 0
};
} catch (error) {
if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") {
const nextBaseName = sanitizeBaseName(parse(fileName).name) || randomUUID();
filePath = join(tempDir, `${nextBaseName}-${attempt}${extension}`);
attempt += 1;
continue;
}
throw error;
}
}
return {
error: `Unable to create a unique temp file for ${finalUrl}`,
code: "download_error",
phase: "loading",
retryable: true,
timeoutMs: opts.timeoutMs,
url: opts.url,
finalUrl,
mimeType: normalizeContentType(contentType) || void 0
};
}
function shouldStripReplies(site) {
return site === "Hacker News" || site.startsWith("r/") || site.startsWith("GitHub - ");
}
function createDefuddleFetch(dependencies = runtimeDependencies) {
async function fetchWithClientRedirects(opts, hooks, clientSideRedirectCount, alternateLinkFallbackCount) {
const browser = opts.browser ?? DEFAULT_BROWSER;
const os = opts.os ?? DEFAULT_OS;
const format = opts.format ?? "markdown";
const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
const removeImages = opts.removeImages ?? false;
const includeReplies = opts.includeReplies ?? DEFAULT_INCLUDE_REPLIES;
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
let parsed;
try {
parsed = new URL(opts.url);
} catch {
return {
error: `Invalid URL: ${opts.url}`,
code: "invalid_url",
phase: "validation",
retryable: false,
url: opts.url
};
}
if (!["http:", "https:"].includes(parsed.protocol)) {
return {
error: `Only http/https URLs supported, got ${parsed.protocol}`,
code: "unsupported_protocol",
phase: "validation",
retryable: false,
url: opts.url
};
}
const fetchOptions = {
browser,
os,
headers: {
Accept: resolveAcceptHeader(format),
"Accept-Language": DEFAULT_ACCEPT_LANGUAGE_HEADER,
...opts.headers
},
redirect: "follow",
timeout: timeoutMs
};
if (opts.proxy) {
fetchOptions.proxy = opts.proxy;
}
const errorContext = {
url: opts.url,
phase: "connecting",
timeoutMs
};
try {
emitProgress(hooks, {
status: "connecting",
progress: 0,
phase: "fetch_start"
});
fetchOptions.onRequestEvent = (event) => {
if (event.url) {
errorContext.finalUrl = event.url;
}
if (event.status) {
errorContext.statusCode = event.status;
}
if (event.contentLength !== void 0 && event.contentLength !== null) {
errorContext.contentLength = event.contentLength;
}
if (event.downloadedBytes !== void 0) {
errorContext.downloadedBytes = event.downloadedBytes;
}
if (event.type === "request_start") {
errorContext.phase = "connecting";
} else if (event.type === "request_sent") {
errorContext.phase = "waiting";
} else if (event.type === "response_headers" || event.type === "body_progress" || event.type === "body_complete") {
errorContext.phase = "loading";
}
const mapped = mapRequestEventToProgress(event);
if (mapped) {
emitProgress(hooks, mapped);
}
};
fetchOptions.captureDiagnostics = true;
const response = await dependencies.fetch(opts.url, fetchOptions);
errorContext.finalUrl = response.url ?? opts.url;
errorContext.statusCode = response.status;
errorContext.statusText = response.statusText;
errorContext.mimeType = normalizeContentType(response.headers.get("content-type") ?? "") || void 0;
errorContext.contentLength = errorContext.contentLength ?? parseContentLengthHeader(response.headers.get("content-length"));
if (!response.ok) {
return {
error: `Server returned HTTP ${response.status} ${response.statusText} for ${opts.url}.`,
code: "http_error",
phase: errorContext.phase,
retryable: response.status >= 500 || response.status === 429,
url: opts.url,
finalUrl: errorContext.finalUrl,
statusCode: response.status,
statusText: response.statusText,
timeoutMs,
mimeType: errorContext.mimeType,
contentLength: errorContext.contentLength
};
}
const finalUrl = response.url ?? opts.url;
const contentType = response.headers.get("content-type") ?? "";
const contentDisposition = response.headers.get("content-disposition") ?? "";
const shouldDownloadToFile = isAttachmentDisposition(contentDisposition) || !isTextualContentType(contentType);
if (shouldDownloadToFile) {
errorContext.phase = "loading";
const fileResult = await buildFileResult(
opts,
response,
finalUrl,
contentType,
contentDisposition,
browser,
os
);
if (!isError(fileResult)) {
emitStatus(hooks, "done");
emitProgress(hooks, {
status: "done",
progress: 1,
phase: "file_done"
});
}
return fileResult;
}
errorContext.phase = "loading";
const rawBody = await response.text();
const clientSideRedirect = extractClientSideRedirect(rawBody, finalUrl);
if (clientSideRedirect) {
if (clientSideRedirectCount >= MAX_CLIENT_SIDE_REDIRECTS) {
return {
error: `Client-side redirect limit (${MAX_CLIENT_SIDE_REDIRECTS}) exceeded while fetching ${opts.url}.`,
code: "too_many_redirects",
phase: "loading",
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
return fetchWithClientRedirects(
{ ...opts, url: clientSideRedirect },
hooks,
clientSideRedirectCount + 1,
alternateLinkFallbackCount
);
}
const jsonResponse = isJsonResponse(contentType, rawBody);
if (format === "raw") {
const isXUrl2 = /^https?:\/\/(www\.)?(x\.com|twitter\.com)\//i.test(
opts.url
);
if (isXUrl2) {
let extractedContent2;
const suppressedErrors2 = [];
const origConsoleError = console.error;
console.error = (...args) => {
suppressedErrors2.push(args);
};
try {
const extractionDocument2 = parseLinkedomHTML(rawBody, finalUrl);
const extracted2 = await dependencies.defuddle(
extractionDocument2,
finalUrl,
{
markdown: true,
removeImages,
includeReplies
}
);
extractedContent2 = extracted2.content;
} finally {
console.error = origConsoleError;
}
const hasOembed404 = suppressedErrors2.some(
(args) => args.some(
(arg) => typeof arg === "string" && arg.includes("oEmbed request failed: 404")
)
);
const hasJsDisabledShell = isTwitterJsDisabledPage(
parseLinkedomHTML(rawBody, finalUrl),
opts.url
);
if ((hasOembed404 || hasJsDisabledShell) && !extractedContent2) {
return {
error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
code: "http_error",
phase: "loading",
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
statusCode: 404,
statusText: "Not Found",
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
}
const effectiveContent = opts.maxChars !== void 0 ? truncateContent(rawBody, maxChars) : rawBody;
const result2 = {
kind: "content",
url: opts.url,
finalUrl,
title: "",
author: "",
published: "",
site: new URL(finalUrl).hostname,
language: "",
wordCount: 0,
content: effectiveContent,
browser,
os,
contentType: normalizeContentType(contentType) || void 0
};
emitStatus(hooks, "done");
emitProgress(hooks, {
status: "done",
progress: 1,
phase: "raw_done"
});
return result2;
}
if (format === "json") {
if (!jsonResponse) {
if (HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {
const alternateLinks2 = extractQualifiedAlternateLinks(
parseLinkedomHTML(rawBody, finalUrl),
finalUrl,
format
);
if (alternateLinks2.length > 0 && alternateLinkFallbackCount < MAX_ALTERNATE_LINK_FALLBACKS) {
return fetchWithClientRedirects(
{ ...opts, url: alternateLinks2[0] },
hooks,
clientSideRedirectCount,
alternateLinkFallbackCount + 1
);
}
}
return {
error: `Not a JSON response (content-type: ${contentType})`,
code: "unexpected_response",
phase: errorContext.phase,
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
const result2 = buildJsonResult(
opts,
finalUrl,
rawBody,
format,
maxChars,
browser,
os
);
if (!isError(result2)) {
emitStatus(hooks, "done");
emitProgress(hooks, {
status: "done",
progress: 1,
phase: "json_done"
});
}
return result2;
}
if (jsonResponse) {
const result2 = buildJsonResult(
opts,
finalUrl,
rawBody,
format,
maxChars,
browser,
os
);
if (!isError(result2)) {
emitStatus(hooks, "done");
emitProgress(hooks, {
status: "done",
progress: 1,
phase: "json_done"
});
}
return result2;
}
if (isPlainTextContentType(contentType)) {
const result2 = buildPlainTextResult(
opts,
finalUrl,
rawBody,
format,
maxChars,
browser,
os
);
emitStatus(hooks, "done");
emitProgress(hooks, {
status: "done",
progress: 1,
phase: "plain_text_done"
});
return result2;
}
if (!HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {
return {
error: `Not an HTML page (content-type: ${contentType})`,
code: "unexpected_response",
phase: errorContext.phase,
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
errorContext.phase = "processing";
emitStatus(hooks, "processing");
emitProgress(hooks, {
status: "processing",
progress: 0.96,
phase: "extracting"
});
const fallbackDocument = parseLinkedomHTML(rawBody, finalUrl);
const extractionDocument = parseLinkedomHTML(rawBody, finalUrl);
const alternateLinks = extractQualifiedAlternateLinks(
fallbackDocument,
finalUrl,
format
);
const tryAlternateLinkFallback = async () => {
if (alternateLinks.length === 0 || alternateLinkFallbackCount >= MAX_ALTERNATE_LINK_FALLBACKS) {
return null;
}
return fetchWithClientRedirects(
{ ...opts, url: alternateLinks[0] },
hooks,
clientSideRedirectCount,
alternateLinkFallbackCount + 1
);
};
let extracted;
const suppressedErrors = [];
try {
const origConsoleError = console.error;
console.error = (...args) => {
suppressedErrors.push(args);
};
try {
extracted = await dependencies.defuddle(
extractionDocument,
finalUrl,
{
markdown: format !== "html",
removeImages,
includeReplies
}
);
} finally {
console.error = origConsoleError;
}
} catch (_error) {
extracted = {
content: void 0,
wordCount: 0
};
}
const isXUrl = /^https?:\/\/(www\.)?(x\.com|twitter\.com)\//i.test(
opts.url
);
if (isXUrl) {
const hasOembed404 = suppressedErrors.some(
(args) => args.some(
(arg) => typeof arg === "string" && arg.includes("oEmbed request failed: 404")
)
);
const hasJsDisabledShell = isTwitterJsDisabledPage(
fallbackDocument,
opts.url
);
if ((hasOembed404 || hasJsDisabledShell) && !extracted.content) {
return {
error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
code: "http_error",
phase: "loading",
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
statusCode: 404,
statusText: "Not Found",
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
}
let extractedContent = extracted.content;
let wordCount = extracted.wordCount;
if (!extractedContent || wordCount === 0) {
const fallbackText = extractDomTextFallback(fallbackDocument);
if (!fallbackText) {
const alternateResult = await tryAlternateLinkFallback();
if (alternateResult) return alternateResult;
return {
error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,
code: "no_content",
phase: "processing",
retryable: false,
timeoutMs,
url: opts.url,
finalUrl,
mimeType: normalizeContentType(contentType) || void 0,
contentLength: errorContext.contentLength
};
}
extractedContent = format === "html" ? rawBody : format === "markdown" ? extractDomMarkdownFallback(fallbackDocument) || fallbackText : fallbackText;
wordCount = estimateWordCount(fallbackText);
}
const extractedTextWordCount = estimateWordCount(
format === "text" ? extractedContent : markdownToText(extractedContent)
);
if (Math.min(wordCount, extractedTextWordCount) < MIN_EXTRACTED_WORDS_BEFORE_ALTERNATE_FALLBACK && alternateLinks.length > 0) {
const alternateResult = await tryAlternateLinkFallback();
if (alternateResult) return alternateResult;
}
if (includeReplies === false && shouldStripReplies(extracted.site ?? "")) {
const strippedContent = stripExtractorComments(
extractedContent,
format
);
if (strippedContent !== extractedContent) {
extractedContent = strippedContent;
wordCount = estimateWordCount(
format === "text" ? markdownToText(extractedContent) : extractedContent
);
}
}
const normalizedContent = format === "text" ? markdownToText(extractedContent) : extractedContent;
const result = {
kind: "content",
url: opts.url,
finalUrl,
title: extracted.title ?? "",
author: extracted.author ?? "",
published: extracted.published ?? "",
site: extracted.site ?? "",
language: extracted.language ?? "",
wordCount,
content: truncateContent(normalizedContent, maxChars),
browser,
os
};
emitStatus(hooks, "done");
emitProgress(hooks, { status: "done", progress: 1, phase: "done" });
return result;
} catch (error) {
const fetchError = buildThrownFetchError(error, errorContext);
emitStatus(hooks, "error");
emitProgress(hooks, { status: "error", progress: 1, phase: "error" });
return fetchError;
}
}
return function defuddleFetch2(opts, hooks = {}) {
return fetchWithClientRedirects(opts, hooks, 0, 0);
};
}
var defuddleFetch = createDefuddleFetch();
function isError(result) {
return "error" in result;
}
function resolveBatchConcurrency(value) {
if (!value || !Number.isFinite(value)) {
return DEFAULT_BATCH_CONCURRENCY;
}
return Math.max(1, Math.floor(value));
}
function resolveFetchToolDefaults(config = {}) {
return {
maxChars: config.maxChars ?? DEFAULT_MAX_CHARS,
timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,
browser: config.browser ?? DEFAULT_BROWSER,
os: config.os ?? DEFAULT_OS,
removeImages: config.removeImages ?? false,
includeReplies: config.includeReplies ?? DEFAULT_INCLUDE_REPLIES,
batchConcurrency: resolveBatchConcurrency(config.batchConcurrency),
tempDir: config.tempDir
};
}
function createBaseFetchToolParameterProperties(defaults) {
return {
url: Type.String({ description: "URL to fetch (http/https only)" }),
browser: Type.Optional(
Type.String({
description: `Browser profile for TLS fingerprinting. Default: "${defaults.browser}". Examples: chrome_145, firefox_147, safari_26, edge_145, opera_127`
})
),
os: Type.Optional(
Type.String({
description: `OS profile for fingerprinting. Default: "${defaults.os}". Options: windows, macos, linux, android, ios`
})
),
headers: Type.Optional(
Type.Record(Type.String(), Type.String(), {
description: "Custom HTTP headers to send. By default, Accept and Accept-Language are set automatically."
})
),
maxChars: Type.Optional(
Type.Number({
description: `Maximum characters to return. Default: ${defaults.maxChars}`
})
),
timeoutMs: Type.Optional(
Type.Number({
description: `Request timeout in milliseconds. Default: ${defaults.timeoutMs}`
})
),
format: Type.Optional(
Type.Union(
[
Type.Literal("markdown"),
Type.Literal("html"),
Type.Literal("text"),
Type.Literal("json"),
Type.Literal("raw")
],
{
description: 'Output format. "markdown" (default), "html" (cleaned HTML), "text" (plain text, no formatting), "json" (pretty-printed JSON), or "raw" (full raw server response without extraction or truncation, for further parsing)'
}
)
),
removeImages: Type.Optional(
Type.Boolean({
description: "Strip image references from output. Default: false"
})
),
includeReplies: Type.Optional(
Type.Union([Type.Boolean(), Type.Literal("extractors")], {
description: "Include replies/comments: 'extractors' for site-specific only (default), true for all, false for none"
})
),
proxy: Type.Optional(
Type.String({
description: "Proxy URL (http://user:pass@host:port or socks5://host:port)"
})
)
};
}
function createBatchFetchToolParameterProperties(defaults) {
return {
requests: Type.Array(
Type.Object(createBaseFetchToolParameterProperties(defaults), {
additionalProperties: false
}),
{
minItems: 1,
description: "Array of fetch requests. Each item accepts the same parameters as the single-item fetch tool."
}
)
};
}
function buildFetchOptionsFromParams(params, defaults) {
return {
url: params.url,
browser: params.browser ?? defaults.browser,
os: params.os ?? defaults.os,
headers: params.headers,
maxChars: params.maxChars ?? defaults.maxChars,
format: params.format ?? "markdown",
removeImages: params.removeImages ?? defaults.removeImages,
includeReplies: params.includeReplies ?? defaults.includeReplies,
proxy: params.proxy,
timeoutMs: params.timeoutMs ?? defaults.timeoutMs,
tempDir: defaults.tempDir
};
}
async function executeFetchToolCall(params, defaults, hooks = {}) {
return defuddleFetch(buildFetchOptionsFromParams(params, defaults), hooks);
}
var PROGRESS_BY_STATUS = {
queued: 0,
connecting: 0,
waiting: 0.11,
loading: 0.51,
processing: 0.96,
done: 1,
error: 1
};
function createInitialProgressItems(requests) {
return requests.map((request, index) => ({
index,
url: typeof request.url === "string" ? request.url : String(request.url ?? ""),
status: "queued",
progress: PROGRESS_BY_STATUS.queued,
statusStartedAt: Date.now()
}));
}
function buildProgressSnapshot(items, batchConcurrency) {
let completed = 0;
let succeeded = 0;
let failed = 0;
for (const item of items) {
if (item.status === "done" || item.status === "error") {
completed += 1;
}
if (item.status === "done") {
succeeded += 1;
}
if (item.status === "error") {
failed += 1;
}
}
return {
items: items.map((item) => ({ ...item })),
total: items.length,
completed,
succeeded,
failed,
batchConcurrency
};
}
async function executeBatchFetchToolCall(params, defaults, options = {}) {
const requests = (params.requests ?? []).map((request) => request ?? {});
const batchConcurrency = resolveBatchConcurrency(
options.batchConcurrency ?? defaults.batchConcurrency
);
const progressItems = createInitialProgressItems(requests);
const results = new Array(requests.length);
const emitProgress2 = () => {
options.onProgress?.(
buildProgressSnapshot(progressItems, batchConcurrency)
);
};
const updateProgress = (index, status, error, progress) => {
const previous = progressItems[index];
progressItems[index] = {
...previous,
status,
progress: progress === void 0 ? PROGRESS_BY_STATUS[status] : Math.max(0, Math.min(1, progress)),
statusStartedAt: previous?.status === status ? previous.statusStartedAt : Date.now(),
...error ? { error } : {}
};
emitProgress2();
};
emitProgress2();
let nextIndex = 0;
const worker = async () => {
while (true) {
const index = nextIndex;
nextIndex += 1;
if (index >= requests.length) {
return;
}
const request = requests[index] ?? {};
const normalizedRequest = buildFetchOptionsFromParams(request, defaults);
try {
const executeItem = options.executeItem ?? executeFetchToolCall;
const result = await executeItem(request, defaults, {
onStatusChange(status) {
if (status === "done") return;
updateProgress(index, status);
},
onProgressChange(update) {
if (update.status === "done") return;
updateProgress(index, update.status, void 0, update.progress);
}
});
if (isError(result)) {
const errorText = buildFetchErrorResponseText(result);
results[index] = {
index,
request: normalizedRequest,
status: "error",
progress: PROGRESS_BY_STATUS.error,
error: errorText
};
updateProgress(
index,
"error",
buildUserFacingFetchErrorSummary(result)
);
continue;
}
results[index] = {
index,
request: normalizedRequest,
status: "done",
progress: PROGRESS_BY_STATUS.done,
result
};
updateProgress(index, "done");
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
results[index] = {
index,
request: normalizedRequest,
status: "error",
progress: PROGRESS_BY_STATUS.error,
error: message
};
updateProgress(index, "error", message);
}
}
};
const workerCount = requests.length === 0 ? 0 : Math.min(batchConcurrency, requests.length);
await Promise.all(Array.from({ length: workerCount }, async () => worker()));
const finalSnapshot = buildProgressSnapshot(progressItems, batchConcurrency);
return {
items: results,
total: finalSnapshot.total,
succeeded: finalSnapshot.succeeded,
failed: finalSnapshot.failed,
batchConcurrency
};
}
var VALID_OS_VALUES = /* @__PURE__ */ new Set([
"windows",
"macos",
"linux",
"android",
"ios"
]);
function readBoolean(source, keys) {
for (const key of keys) {
if (typeof source[key] === "boolean") {
return source[key];
}
}
return void 0;
}
function readPositiveNumber(source, keys) {
for (const key of keys) {
const value = source[key];
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
return value;
}
}
return void 0;
}
function readNonEmptyString(source, keys) {
for (const key of keys) {
const value = source[key];
if (typeof value === "string" && value.trim() !== "") {
return value;
}
}
return void 0;
}
function readOs(source, keys) {
for (const key of keys) {
const value = source[key];
if (typeof value === "string" && VALID_OS_VALUES.has(value)) {
return value;
}
}
return void 0;
}
function readIncludeReplies(source, keys) {
for (const key of keys) {
const value = source[key];
if (typeof value === "boolean" || value === "extractors") {
return value;
}
}
return void 0;
}
function normalizePiSmartFetchSettings(input) {
if (!input || typeof input !== "object") return {};
const source = input;
return {
smartFetchVerboseByDefault: readBoolean(source, [
"smartFetchVerboseByDefault",
"webFetchVerboseByDefault"
]),
smartFetchDefaultMaxChars: readPositiveNumber(source, [
"smartFetchDefaultMaxChars",
"webFetchDefaultMaxChars"
]),
smartFetchDefaultTimeoutMs: readPositiveNumber(source, [
"smartFetchDefaultTimeoutMs"
]),
smartFetchDefaultBrowser: readNonEmptyString(source, [
"smartFetchDefaultBrowser"
]),
smartFetchDefaultOs: readOs(source, ["smartFetchDefaultOs"]),
smartFetchDefaultRemoveImages: readBoolean(source, [
"smartFetchDefaultRemoveImages"
]),
smartFetchDefaultIncludeReplies: readIncludeReplies(source, [
"smartFetchDefaultIncludeReplies"
]),
smartFetchDefaultBatchConcurrency: readPositiveNumber(source, [
"smartFetchDefaultBatchConcurrency",
"webFetchDefaultBatchConcurrency"
]),
smartFetchTempDir: readNonEmptyString(source, [
"smartFetchTempDir",
"webFetchTempDir"
])
};
}
function resolvePiSmartFetchSettings(globalSettings, projectSettings) {
const global2 = normalizePiSmartFetchSettings(globalSettings);
const project = normalizePiSmartFetchSettings(projectSettings);
return {
verboseByDefault: project.smartFetchVerboseByDefault ?? global2.smartFetchVerboseByDefault ?? false,
maxChars: project.smartFetchDefaultMaxChars ?? global2.smartFetchDefaultMaxChars,
timeoutMs: project.smartFetchDefaultTimeoutMs ?? global2.smartFetchDefaultTimeoutMs,
browser: project.smartFetchDefaultBrowser ?? global2.smartFetchDefaultBrowser,
os: project.smartFetchDefaultOs ?? global2.smartFetchDefaultOs,
removeImages: project.smartFetchDefaultRemoveImages ?? global2.smartFetchDefaultRemoveImages,
includeReplies: project.smartFetchDefaultIncludeReplies ?? global2.smartFetchDefaultIncludeReplies,
batchConcurrency: project.smartFetchDefaultBatchConcurrency ?? global2.smartFetchDefaultBatchConcurrency,
tempDir: project.smartFetchTempDir ?? global2.smartFetchTempDir ?? join(tmpdir(), "smart-fetch-pi")
};
}
async function readSettingsFile(path) {
try {
return JSON.parse(await readFile(path, "utf-8"));
} catch {
return {};
}
}
async function loadPiSmartFetchSettings(cwd, agentDir = getAgentDir()) {
const globalSettings = await readSettingsFile(
join(agentDir, "settings.json")
);
const projectSettings = await readSettingsFile(
join(cwd, ".pi", "settings.json")
);
return resolvePiSmartFetchSettings(globalSettings, projectSettings);
}
// src/index.ts
var toolDescription = [
"Fetch a URL with browser-grade TLS fingerprinting and extract clean, readable content.",
"Uses wreq-js for browser-like TLS/HTTP2 impersonation and Defuddle for article extraction.",
"Returns full metadata plus the extracted document to the agent while keeping the pi history preview brief.",
"Does NOT execute JavaScript \u2014 use a browser automation tool for JS-heavy pages."
].join(" ");
var batchToolDescription = [
"Fetch multiple URLs with browser-grade TLS fingerprinting and readable extraction.",
"Each request accepts the same parameters as web_fetch and fans out with bounded concurrency.",
"Returns full per-item metadata to the agent and streams compact per-item progress in the pi TUI.",
"Does NOT execute JavaScript \u2014 use a browser automation tool for JS-heavy pages."
].join(" ");
var SPINNER_FRAMES = ["\u280B", "\u2819", "\u2839", "\u2838", "\u283C", "\u2834", "\u2826", "\u2827", "\u2807", "\u280F"];
var SPINNER_INTERVAL_MS = 100;
function truncateMiddle(value, width) {
if (width <= 0) return "";
if (value.length <= width) return value.padEnd(width, " ");
if (width === 1) return "\u2026";
const left = Math.ceil((width - 1) / 2);
const right = Math.floor((width - 1) / 2);
return `${value.slice(0, left)}\u2026${value.slice(value.length - right)}`;
}
function getOptimisticProgress(item, now) {
const base = item.progress;
const startedAt = item.statusStartedAt ?? now;
const elapsedWholeSeconds = Math.max(0, Math.floor((now - startedAt) / 1e3));
switch (item.status) {
case "connecting":
return Math.min(0.1, Math.max(base, elapsedWholeSeconds * 0.01));
case "waiting":
return Math.min(0.5, Math.max(base, 0.11 + elapsedWholeSeconds * 0.01));
case "processing":
return Math.min(0.99, Math.max(base, 0.96 + elapsedWholeSeconds * 0.01));
default:
return base;
}
}
function renderProgressBar(item, width, theme, now) {
const innerWidth = Math.max(10, width - 2);
const progress = getOptimisticProgress(item, now);
const filled = Math.max(
0,
Math.min(innerWidth, Math.round(progress * innerWidth))
);
const barBgColor = item.status === "error" ? "toolErrorBg" : item.status === "done" ? "toolSuccessBg" : item.status === "queued" ? "toolPendingBg" : "selectedBg";
const centeredLabel = (() => {
const raw = item.status;
if (raw.length >= innerWidth) {
return raw.slice(0, innerWidth);
}
const totalPadding = innerWidth - raw.length;
const leftPadding = Math.floor(totalPadding / 2);
const rightPadding = totalPadding - leftPadding;
return `${" ".repeat(leftPadding)}${raw}${" ".repeat(rightPadding)}`;
})();
const filledLabel = centeredLabel.slice(0, filled);
const emptyLabel = centeredLabel.slice(filled);
return [
theme.fg("muted", "["),
theme.bg(barBgColor, theme.fg("text", filledLabel)),
theme.bg("toolPendingBg", theme.fg("muted", emptyLabel)),
theme.fg("muted", "]")
].join("");
}
function renderStatusGlyph(item, spinnerIndex, theme) {
switch (item.status) {
case "done":
return theme.fg("success", "\u2713");
case "error":
return theme.fg("error", "\u2717");
case "queued":
return theme.fg(
"muted",
SPINNER_FRAMES[spinnerIndex % SPINNER_FRAMES.length] ?? "\u280B"
);
default:
return theme.fg(
"accent",
SPINNER_FRAMES[spinnerIndex % SPINNER_FRAMES.length] ?? "\u280B"
);
}
}
function renderBatchProgressText(snapshot, width, expanded, theme, spinnerTick = 0) {
const summary = [
theme.fg("toolTitle", theme.bold("batch_web_fetch ")),
theme.fg(
"muted",
`${snapshot.completed}/${snapshot.total} done \xB7 ok ${snapshot.succeeded} \xB7 err ${snapshot.failed} \xB7 concurrency ${snapshot.batchConcurrency}`
)
].join("");
const availableRowWidth = Math.max(24, width);
const progressWidth = Math.max(
12,
Math.min(18, Math.floor(availableRowWidth * 0.2))
);
const glyphWidth = 2;
const urlWidth = Math.max(
12,
availableRowWidth - glyphWidth - progressWidth - 2
);
const now = Date.now();
const rows = snapshot.items.map((item, index) => {
const glyph = renderStatusGlyph(item, spinnerTick + index, theme);
const url = theme.fg("accent", truncateMiddle(item.url, urlWidth));
const bar = renderProgressBar(item, progressWidth, theme, now);
const baseRow = `${glyph} ${url} ${bar}`;
if (!expanded || !item.error) {
return baseRow;
}
return `${baseRow}
${theme.fg("error", `error: ${item.error}`)}`;
});
return [summary, ...rows].join("\n");
}
function renderUserMetadataLine(label, value, theme) {
return theme.fg("syntaxKeyword", `${label}: `) + theme.fg("syntaxString", String(value));
}
function buildWebFetchMetadataLines(details, theme) {
const fetchResult = details.fetchResult;
if (!fetchResult) {
return [];
}
const metadata = [
["Title", fetchResult.title],
["Published", fetchResult.published]
];
return metadata.flatMap(([label, value]) => {
if (value === void 0 || value === "") {
return [];
}
return [renderUserMetadataLine(label, value, theme)];
});
}
function shouldRenderHighlightedContent(format) {
return format === "markdown" || format === "json" || format === "html";
}
function buildHighlightedMarkdownContent(content, format) {
if (format === "json") {
return `\`\`\`json
${content}
\`\`\``;
}
if (format === "html") {
return `\`\`\`html
${content}
\`\`\``;
}
return content;
}
function buildWebFetchCollapsedPreview(details) {
const contentLines = (details.fetchResult?.content ?? "").split("\n").filter(
(line, index, lines) => line.length > 0 || index === 0 || index < lines.length - 1
);
const maxPreviewLines = 7;
const previewLines = contentLines.slice(0, maxPreviewLines);
return {
previewContent: previewLines.join("\n"),
remainingLines: Math.max(0, contentLines.length - previewLines.length)
};
}
function createWebFetchResultComponent(details, expanded, theme) {
const fetchResult = details.fetchResult;
if (!fetchResult) {
return new Text(theme.fg("muted", "No fetch result available."), 0, 0);
}
const metadataLines = buildWebFetchMetadataLines(details, theme);
const container = new Container();
if (metadataLines.length > 0) {
container.addChild(new Text(metadataLines.join("\n"), 0, 0));
}
if (isFileFetchResult(fetchResult)) {
const fileLines = [
theme.fg("muted", `File size: ${fetchResult.fileSize}`),
...fetchResult.mimeType ? [theme.fg("muted", `Mime type: ${fetchResult.mimeType}`)] : [],
theme.fg("muted", `File path: ${fetchResult.filePath}`)
];
if (metadataLines.length > 0 && fileLines.length > 0) {
container.addChild(new Spacer(1));
}
container.addChild(new Text(fileLines.join("\n"), 0, 0));
return container;
}
const { previewContent, remainingLines } = buildWebFetchCollapsedPreview(details);
const content = expanded ? fetchResult.content : previewContent;
const format = details.format ?? "markdown";
if (metadataLines.length > 0 && content) {
container.addChild(new Spacer(1));
}
if (content) {
if (shouldRenderHighlightedContent(format)) {
container.addChild(
new Markdown(
buildHighlightedMarkdownContent(content, format),
0,
0,
getMarkdownTheme()
)
);
} else {
container.addChild(new Text(content, 0, 0));
}
}
if (!expanded && remainingLines > 0) {
const expandKey = keyText("app.tools.expand") || "Ctrl+O";
if (content) {
container.addChild(new Spacer(1));
}
container.addChild(
new Text(
theme.fg("muted", `... (${remainingLines} more lines, `) + theme.fg("dim", expandKey) + theme.fg("muted", " to expand)"),
0,
0
)
);
}
return container;
}
function renderSingleFetchProgressText(details, width, theme) {
const status = details.status ?? "connecting";
const url = details.url ?? details.fetchResult?.finalUrl ?? details.fetchResult?.url ?? "";
const item = {
status,
progress: details.progress ?? 0,
statusStartedAt: Date.now()
};
const availableRowWidth = Math.max(24, width);
const progressWidth = Math.max(
12,
Math.min(18, Math.floor(availableRowWidth * 0.2))
);
const glyphWidth = 2;
const urlWidth = Math.max(
12,
availableRowWidth - glyphWidth - progressWidth - 2
);
const glyph = renderStatusGlyph(item, details.spinnerTick ?? 0, theme);
const renderedUrl = theme.fg("accent", truncateMiddle(url, urlWidth));
const bar = renderProgressBar(item, progressWidth, theme, Date.now());
return `${glyph} ${renderedUrl} ${bar}`;
}
function createResponsiveSingleFetchProgressComponent(details, theme) {
const text = new Text("", 0, 0);
return {
render(width) {
text.setText(renderSingleFetchProgressText(details, width, theme));
return text.render(width);
},
invalidate() {
text.invalidate();
}
};
}
function createResponsiveBatchComponent(details, expanded, theme) {
const text = new Text("", 0, 0);
return {
render(width) {
const snapshot = details.batchProgress;
if (!snapshot) {
text.setText(theme.fg("muted", "No batch progress available."));
return text.render(width);
}
const spinnerTick = details.spinnerTick ?? 0;
text.setText(
renderBatchProgressText(snapshot, width, expanded, theme, spinnerTick)
);
return text.render(width);
},
invalidate() {
text.invalidate();
}
};
}
function piSmartFetchExtension(pi) {
const defaults = resolveFetchToolDefaults();
pi.registerTool({
name: "web_fetch",
label: "web_fetch",
description: toolDescription,
promptSnippet: "web_fetch(url, browser?, os?, headers?, maxChars?, timeoutMs?, format?, removeImages?, includeReplies?, proxy?, verbose?): fetch browser-fingerprinted readable web content with full agent metadata and a compact pi preview",
parameters: Type.Object({
...createBaseFetchToolParameterProperties(defaults),
verbose: Type.Optional(
Type.Boolean({
description: "Compatibility flag. pi currently returns the full metadata header to the agent regardless, while keeping the history preview compact. Default: false, or smartFetchVerboseByDefault from pi settings."
})
)
}),
async execute(_toolCallId, params, _signal, onUpdate, ctx) {
const settings = await loadPiSmartFetchSettings(ctx.cwd, getAgentDir());
const runtimeDefaults = resolveFetchToolDefaults(settings);
const verbose = params.verbose ?? settings.verboseByDefault;
let spinnerTick = 0;
let spinnerTimer = null;
let latestDetails = {
verbose,
format: params.format ?? "markdown",
started: true,
status: "connecting",
progress: 0,
phase: "fetch_start",
url: typeof params.url === "string" ? params.url : void 0,
spinnerTick
};
const emitProgress2 = (details) => {
latestDetails = details;
onUpdate?.({
content: [
{
type: "text",
text: `Fetching ${details.url ?? params.url ?? "URL"}...`
}
],
details
});
};
try {
spinnerTimer = setInterval(() => {
if (latestDetails.status === "done") {
return;
}
spinnerTick += 1;
emitProgress2({
...latestDetails,
spinnerTick
});
}, SPINNER_INTERVAL_MS);
const result = await executeFetchToolCall(params, runtimeDefaults, {
onStatusChange(status) {
emitProgress2({
...latestDetails,
status,
progress: latestDetails.progress ?? (status === "processing" ? 0.96 : status === "loading" ? 0.51 : status === "waiting" ? 0.11 : 0),
spinnerTick
});
},
onProgressChange(update) {
emitProgress2({
...latestDetails,
status: update.status,
progress: update.progress,
phase: update.phase,
spinnerTick
});
}
});
if (isError(result)) {
const errorText = buildFetchErrorResponseText(result);
return {
content: [{ type: "text", text: errorText }],
details: {
error: true,
errorText,
userErrorSummary: buildUserFacingFetchErrorSummary(result),
verbose,
status: latestDetails.status,
phase: latestDetails.phase,
url: latestDetails.url,
spinnerTick
}
};
}
return {
content: [
{
type: "text",
text: buildFetchResponseText(result, { verbose: true })
}
],
details: {
verbose,
format: latestDetails.format,
maxChars: runtimeDefaults.maxChars,
fetchResult: result,
started: true,
status: "done",
progress: 1,
phase: "done",
url: result.finalUrl || result.url,
spinnerTick
}
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const errorText = `Error: Unexpected web_fetch failure for ${typeof params.url === "string" ? params.url : "URL"}.
${message}`;
return {
content: [{ type: "text", text: errorText }],
details: {
error: true,
errorText,
userErrorSummary: "The request failed before a usable response was returned.",
verbose,
status: latestDetails.status,
phase: latestDetails.phase,
url: latestDetails.url,
spinnerTick
}
};
} finally {
if (spinnerTimer) {
clearInterval(spinnerTimer);
}
}
},
renderCall(args, theme) {
const url = typeof args.url === "string" ? args.url : "...";
return new Text(
`${theme.fg("toolTitle", "web_fetch")} ${theme.fg("accent", url)}`,
0,
0
);
},
renderResult(result, { expanded, isPartial }, theme) {
if (isPartial) {
const details2 = result.details ?? {};
return createResponsiveSingleFetchProgressComponent(details2, theme);
}
const details = result.details ?? {};
const textContent = result.content.find((item) => item.type === "text");
const outputText = textContent?.type === "text" ? textContent.text : "";
if (details.error) {
return new Text(
theme.fg(
"error",
details.userErrorSummary || outputText || details.errorText || "Error"
),
0,
0
);
}
return createWebFetchResultComponent(details, expanded, theme);
}
});
pi.registerTool({
name: "batch_web_fetch",
label: "batch_web_fetch",
description: batchToolDescription,
promptSnippet: "batch_web_fetch(requests, verbose?): fetch multiple URLs concurrently with full agent metadata and per-item progress in the pi TUI",
parameters: Type.Object({
...createBatchFetchToolParameterProperties(defaults),
verbose: Type.Optional(
Type.Boolean({
description: "Compatibility flag. pi currently returns the full metadata header for successful results regardless, while keeping the history preview compact. Default: false, or smartFetchVerboseByDefault from pi settings."
})
)
}),
async execute(_toolCallId, params, _signal, onUpdate, ctx) {
const settings = await loadPiSmartFetchSettings(ctx.cwd, getAgentDir());
const runtimeDefaults = resolveFetchToolDefaults(settings);
const verbose = params.verbose ?? settings.verboseByDefault;
let latestSnapshot;
let spinnerTick = 0;
let spinnerTimer = null;
const emitProgress2 = (snapshot) => {
onUpdate?.({
content: [
{
type: "text",
text: `Started batch fetch for ${snapshot.total} URLs (${snapshot.completed}/${snapshot.total} complete).`
}
],
details: {
verbose,
started: true,
batchProgress: snapshot,
spinnerTick
}
});
};
try {
spinnerTimer = setInterval(() => {
if (!latestSnapshot || latestSnapshot.completed >= latestSnapshot.total) {
return;
}
spinnerTick += 1;
emitProgress2(latestSnapshot);
}, SPINNER_INTERVAL_MS);
const batchResult = await executeBatchFetchToolCall(
params,
runtimeDefaults,
{
batchConcurrency: runtimeDefaults.batchConcurrency,
onProgress(snapshot) {
latestSnapshot = snapshot;
emitProgress2(snapshot);
}
}
);
const finalProgress = {
items: batchResult.items.map((item) => ({
index: item.index,
url: item.request.url,
status: item.status,
progress: item.progress,
...item.error ? { error: item.error } : {}
})),
total: batchResult.total,
completed: batchResult.total,
succeeded: batchResult.succeeded,
failed: batchResult.failed,
batchConcurrency: batchResult.batchConcurrency
};
return {
content: [
{
type: "text",
text: buildBatchFetchResponseText(batchResult, {
verbose: true
})
}
],
details: {
verbose,
started: true,
batchProgress: finalProgress,
batchResult,
spinnerTick
}
};
} finally {
if (spinnerTimer) {
clearInterval(spinnerTimer);
}
}
},
renderCall(args, theme) {
const batchArgs = args;
const requestCount = Array.isArray(batchArgs.requests) ? batchArgs.requests.length : 0;
let text = theme.fg("toolTitle", theme.bold("batch_web_fetch "));
text += theme.fg("muted", `${requestCount} urls`);
return new Text(text, 0, 0);
},
renderResult(result, { expanded }, theme) {
return createResponsiveBatchComponent(
result.details ?? {},
expanded,
theme
);
}
});
}
/*! Bundled license information:
mime-db/index.js:
(*!
* mime-db
* Copyright(c) 2014 Jonathan Ong
* Copyright(c) 2015-2022 Douglas Christopher Wilson
* MIT Licensed
*)
mime-types/index.js:
(*!
* mime-types
* Copyright(c) 2014 Jonathan Ong
* Copyright(c) 2015 Douglas Christopher Wilson
* MIT Licensed
*)
*/
export { piSmartFetchExtension as default };
//# sourceMappingURL=index.js.map
//# sourceMappingURL=index.js.map