Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
- CLI cache: include local media `fileMtime` when writing transcript cache entries so repeated unchanged audio/video extraction can hit cache (#240, #241, thanks @alfozan).
- CLI: pass Codex image attachments to `codex exec` so local image summaries no longer fail before starting (#242, #243, thanks @alfozan).
- OpenAI-compatible gateways: honor `OPENAI_USE_CHAT_COMPLETIONS=false` and `openai.useChatCompletions=false` so custom base URLs can use the Responses API (#235, #236, thanks @mzbgf).
- RSS transcripts: block feed-controlled transcript URLs that target loopback, private, link-local, reserved, or redirected local-network addresses (#239, thanks @Hinotoi-agent).
- Chrome extension: abort stale side-panel summary streams on tab changes so delayed output from a closed or replaced tab cannot render under the new page title.
- Core: extract video IDs from YouTube `/live/` URLs so live and premiere links no longer abort summarization (#232, thanks @devYRPauli).
- Chrome extension: keep YouTube slide cards on the shared slide-summary path so local browser thumbnails receive the same summary text shape as CLI `--slides`.
Expand Down
3 changes: 2 additions & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@
"cheerio": "^1.2.0",
"es-toolkit": "^1.47.0",
"jsdom": "29.1.1",
"sanitize-html": "^2.17.4"
"sanitize-html": "^2.17.4",
"undici": "8.4.1"
},
"devDependencies": {
"@types/jsdom": "^28.0.3",
Expand Down
99 changes: 99 additions & 0 deletions packages/core/src/content/dns-pinned-fetch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import http from "node:http";
import https from "node:https";
import { Readable } from "node:stream";
import { readDnsPinnedAddresses, type DnsPinnedAddress } from "./fetch-capabilities.js";

type PinnedLookupAddress = { address: string; family: number };
type LookupCallback = (
error: Error | null,
address: string | PinnedLookupAddress[],
family?: number,
) => void;

function getInputUrl(input: RequestInfo | URL): string {
if (typeof input === "string") return input;
if (input instanceof URL) return input.href;
return input.url;
}

function createPinnedLookup(addresses: DnsPinnedAddress[]) {
const pinnedAddresses: PinnedLookupAddress[] = addresses.map((entry) => ({
address: entry.address,
family: entry.family ?? 4,
}));
return (_hostname: string, options: unknown, callback: LookupCallback): void => {
if ((options as { all?: boolean } | undefined)?.all) {
callback(null, pinnedAddresses);
return;
}
const first = pinnedAddresses[0];
callback(null, first?.address ?? "0.0.0.0", first?.family ?? 4);
};
}

function headersFrom(input: RequestInfo | URL, init?: RequestInit): Headers {
if (init?.headers) return new Headers(init.headers);
if (typeof input !== "string" && !(input instanceof URL)) return new Headers(input.headers);
return new Headers();
}

function methodFrom(input: RequestInfo | URL, init?: RequestInit): string {
if (init?.method) return init.method;
if (typeof input !== "string" && !(input instanceof URL)) return input.method;
return "GET";
}

function hasRequestBody(input: RequestInfo | URL, init?: RequestInit): boolean {
if (init && "body" in init && init.body != null) return true;
if (typeof input !== "string" && !(input instanceof URL)) return input.body != null;
return false;
}

export async function fetchWithDnsPinnedAddresses(
input: RequestInfo | URL,
init?: RequestInit,
): Promise<Response> {
const addresses = readDnsPinnedAddresses(init);
if (!addresses) throw new Error("Pinned DNS fetch missing validated addresses");
if (hasRequestBody(input, init)) {
throw new Error("Pinned DNS fetch does not support request bodies");
}

const url = new URL(getInputUrl(input));
const client = url.protocol === "https:" ? https : http;
const headers: Record<string, string> = {};
headersFrom(input, init).forEach((value, key) => {
headers[key] = value;
});

return await new Promise<Response>((resolve, reject) => {
const req = client.request(
url,
{
headers,
lookup: createPinnedLookup(addresses),
method: methodFrom(input, init),
...(init?.signal ? { signal: init.signal } : {}),
},
(res) => {
const responseHeaders = new Headers();
for (const [key, value] of Object.entries(res.headers)) {
if (Array.isArray(value)) {
for (const entry of value) responseHeaders.append(key, entry);
} else if (typeof value === "string") {
responseHeaders.set(key, value);
}
}
const response = new Response(Readable.toWeb(res) as ReadableStream<Uint8Array>, {
headers: responseHeaders,
status: res.statusCode ?? 200,
statusText: res.statusMessage,
});
Object.defineProperty(response, "url", { configurable: true, value: url.href });
resolve(response);
},
);
req.on("error", reject);
req.end();
});
}
54 changes: 54 additions & 0 deletions packages/core/src/content/fetch-capabilities.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
const DNS_PINNED_FETCH = Symbol.for("@steipete/summarize.dnsPinnedFetch");
const DNS_PINNED_ADDRESSES = Symbol.for("@steipete/summarize.dnsPinnedAddresses");

export type DnsPinnedAddress = { address: string; family?: number };

export function markFetchAsDnsPinned<T extends typeof fetch>(
fetchImpl: T,
pinnedFetchImpl: typeof fetch = fetchImpl,
): T {
Object.defineProperty(fetchImpl, DNS_PINNED_FETCH, {
configurable: false,
enumerable: false,
value: pinnedFetchImpl,
});
return fetchImpl;
}

export function resolveDnsPinnedFetch(fetchImpl: typeof fetch): typeof fetch | null {
const pinnedFetchImpl = (fetchImpl as { [DNS_PINNED_FETCH]?: typeof fetch })[DNS_PINNED_FETCH];
return pinnedFetchImpl ?? null;
}

export function supportsDnsPinnedFetch(fetchImpl: typeof fetch): boolean {
return resolveDnsPinnedFetch(fetchImpl) !== null;
}

export function isNativeOrBoundGlobalFetch(fetchImpl: typeof fetch): boolean {
if (fetchImpl === globalThis.fetch) return true;

const expectedBoundName = `bound ${globalThis.fetch.name || "fetch"}`;
return (
fetchImpl.name === expectedBoundName &&
Function.prototype.toString.call(fetchImpl).includes("[native code]")
);
}

export function attachDnsPinnedAddresses<T extends RequestInit>(
init: T,
addresses: DnsPinnedAddress[],
): T {
Object.defineProperty(init, DNS_PINNED_ADDRESSES, {
configurable: true,
enumerable: true,
value: addresses,
});
return init;
}

export function readDnsPinnedAddresses(init: RequestInit | undefined): DnsPinnedAddress[] | null {
const addresses = (init as { [DNS_PINNED_ADDRESSES]?: DnsPinnedAddress[] } | undefined)?.[
DNS_PINNED_ADDRESSES
];
return Array.isArray(addresses) && addresses.length > 0 ? addresses : null;
}
9 changes: 9 additions & 0 deletions packages/core/src/content/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ export {
type ExtractedLinkContent,
type FetchLinkContentOptions,
} from "./link-preview/content/types.js";
export {
attachDnsPinnedAddresses,
isNativeOrBoundGlobalFetch,
markFetchAsDnsPinned,
readDnsPinnedAddresses,
resolveDnsPinnedFetch,
supportsDnsPinnedFetch,
type DnsPinnedAddress,
} from "./fetch-capabilities.js";
export type {
ConvertHtmlToMarkdown,
FirecrawlScrapeResult,
Expand Down
3 changes: 1 addition & 2 deletions packages/core/src/content/link-preview/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ export interface LinkPreviewClientOptions {

/** Public factory for a link preview client with injectable dependencies. */
export function createLinkPreviewClient(options: LinkPreviewClientOptions = {}): LinkPreviewClient {
const fetchImpl: typeof fetch =
options.fetch ?? ((...args: Parameters<typeof fetch>) => globalThis.fetch(...args));
const fetchImpl: typeof fetch = options.fetch ?? globalThis.fetch;
const env = typeof options.env === "object" && options.env ? options.env : undefined;
const scrape: ScrapeWithFirecrawl | null = options.scrapeWithFirecrawl ?? null;
const apifyApiToken = typeof options.apifyApiToken === "string" ? options.apifyApiToken : null;
Expand Down
Loading