Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@
"cheerio": "^1.2.0",
"es-toolkit": "^1.46.1",
"jsdom": "29.1.1",
"sanitize-html": "^2.17.4"
"sanitize-html": "^2.17.4",
"undici": "8.3.0"
},
"devDependencies": {
"@types/jsdom": "^28.0.3",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import { lookup as dnsLookup } from "node:dns/promises";
import { createRequire } from "node:module";
import { isIP } from "node:net";
import type { TranscriptSegment } from "../../../link-preview/types.js";
import {
jsonTranscriptToPlainText,
Expand All @@ -14,17 +17,226 @@ import {
} from "./rss-feed.js";

type TranscriptCandidate = { url: string; type: string | null };
type LookupAddress = { address: string; family?: number };
type LookupFn = (hostname: string) => Promise<LookupAddress[]>;
type LookupCallback = (
error: Error | null,
address: string | LookupAddress[],
family?: number,
) => void;
type UndiciAgentConstructor = new (options: {
autoSelectFamily?: boolean;
autoSelectFamilyAttemptTimeout?: number;
connect: {
lookup: (hostname: string, options: unknown, callback: LookupCallback) => void;
};
}) => unknown;
type UndiciModule = { Agent: UndiciAgentConstructor; fetch: typeof fetch };

const MAX_TRANSCRIPT_REDIRECTS = 10;
const require = createRequire(import.meta.url);

function parseIpv4(address: string): number[] | null {
const parts = address.split(".");
if (parts.length !== 4) return null;
const octets = parts.map((part) => {
if (!/^\d{1,3}$/.test(part)) return null;
const value = Number(part);
return Number.isInteger(value) && value >= 0 && value <= 255 ? value : null;
});
return octets.every((value) => value != null) ? (octets as number[]) : null;
}

function isBlockedIpv4(address: string): boolean {
const octets = parseIpv4(address);
if (!octets) return true;
const [a, b] = octets;
return (
a === 0 ||
a === 10 ||
a === 127 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168) ||
(a === 192 && b === 0) ||
(a === 198 && (b === 18 || b === 19)) ||
a >= 224
);
}

function expandIpv6(address: string): number[] | null {
const normalized = address.split("%", 1)[0]?.toLowerCase() ?? "";
if (!normalized) return null;
const mapped = normalized.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/);
const ipv4 = mapped ? parseIpv4(mapped[2] ?? "") : null;
const head = mapped ? (mapped[1] ?? "") : normalized;
const partsAroundGap = head.split("::");
if (partsAroundGap.length > 2) return null;
const [leftRaw, rightRaw] = partsAroundGap;
const left = leftRaw ? leftRaw.split(":").filter(Boolean) : [];
const right = typeof rightRaw === "string" && rightRaw ? rightRaw.split(":").filter(Boolean) : [];
const ipv4Parts = ipv4
? [((ipv4[0] ?? 0) << 8) | (ipv4[1] ?? 0), ((ipv4[2] ?? 0) << 8) | (ipv4[3] ?? 0)]
: [];
const missing = 8 - left.length - right.length - ipv4Parts.length;
if (missing < 0 || (partsAroundGap.length === 1 && missing !== 0)) return null;
const parsePart = (part: string) => (/^[0-9a-f]{1,4}$/.test(part) ? parseInt(part, 16) : -1);
const parts = [
...left.map(parsePart),
...Array.from({ length: missing }, () => 0),
...right.map(parsePart),
...ipv4Parts,
];
return parts.length === 8 && parts.every((part) => part >= 0 && part <= 0xffff) ? parts : null;
}

function isBlockedIpv6(address: string): boolean {
const parts = expandIpv6(address);
if (!parts) return true;
const [first, second, , , , fifth, sixth, eighth] = parts;
const allZero = parts.every((part) => part === 0);
const loopback = parts.slice(0, 7).every((part) => part === 0) && eighth === 1;
const mappedIpv4 = parts.slice(0, 5).every((part) => part === 0) && fifth === 0xffff;
const compatibleIpv4 = parts.slice(0, 6).every((part) => part === 0) && !allZero && !loopback;
if (mappedIpv4 || compatibleIpv4) {
const ipv4 = `${((sixth ?? 0) >> 8) & 0xff}.${(sixth ?? 0) & 0xff}.${((eighth ?? 0) >> 8) & 0xff}.${(eighth ?? 0) & 0xff}`;
return isBlockedIpv4(ipv4);
}
return (
allZero ||
loopback ||
((first ?? 0) & 0xfe00) === 0xfc00 ||
((first ?? 0) & 0xffc0) === 0xfe80 ||
((first ?? 0) & 0xff00) === 0xff00 ||
(first === 0x2001 && second === 0xdb8)
);
}

function normalizeHostname(hostname: string): string {
return hostname
.trim()
.replace(/^\[|\]$/g, "")
.toLowerCase()
.replace(/\.$/, "");
}

export function isBlockedNetworkAddress(address: string): boolean {
const normalized = address.trim().replace(/^\[|\]$/g, "");
const family = isIP(normalized);
if (family === 4) return isBlockedIpv4(normalized);
if (family === 6) return isBlockedIpv6(normalized);
return true;
}

async function defaultLookup(hostname: string): Promise<LookupAddress[]> {
return await dnsLookup(hostname, { all: true, verbatim: true });
}

async function resolveSafeTranscriptUrl(
rawUrl: string,
{ lookup = defaultLookup }: { lookup?: LookupFn } = {},
): Promise<{ url: URL; addresses: LookupAddress[] }> {
let url: URL;
try {
url = new URL(rawUrl);
} catch {
throw new Error("RSS transcript URL is invalid");
}
if (url.protocol !== "http:" && url.protocol !== "https:") {
throw new Error("RSS transcript URL must use http or https");
}
const hostname = normalizeHostname(url.hostname);
if (hostname === "localhost" || hostname.endsWith(".localhost")) {
throw new Error("RSS transcript URL targets a blocked local network host");
}
if (isIP(hostname)) {
if (isBlockedNetworkAddress(hostname)) {
throw new Error("RSS transcript URL resolves to a blocked local network address");
}
return { url, addresses: [] };
}
const addresses = await lookup(hostname);
if (addresses.length === 0 || addresses.some((entry) => isBlockedNetworkAddress(entry.address))) {
throw new Error("RSS transcript URL resolves to a blocked local network address");
}
return { url, addresses };
}

function isRedirectStatus(status: number): boolean {
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
}

function isNativeFetchImpl(fetchImpl: typeof fetch): boolean {
return fetchImpl === globalThis.fetch || fetchImpl.name === "bound fetch";
}

function loadUndici(): UndiciModule {
return require("undici") as UndiciModule;
}

function createPinnedDispatcher(addresses: LookupAddress[]): unknown {
const { Agent } = loadUndici();
const pinnedAddresses = addresses.map((address) => ({
address: address.address,
family: address.family ?? (isIP(address.address) || 4),
}));
return new Agent({
autoSelectFamily: true,
autoSelectFamilyAttemptTimeout: 250,
connect: {
lookup: (_hostname, options, callback) => {
if ((options as { all?: boolean } | undefined)?.all) {
callback(null, pinnedAddresses);
return;
}
const first = pinnedAddresses[0];
callback(null, first?.address ?? "0.0.0.0", first?.family ?? 4);
},
},
});
}

async function fetchSafeTranscriptUrl(
fetchImpl: typeof fetch,
transcriptUrl: string,
{ lookup = defaultLookup }: { lookup?: LookupFn } = {},
redirectCount = 0,
): Promise<Response> {
const target = await resolveSafeTranscriptUrl(transcriptUrl, { lookup });
const pinnedInit = {
redirect: "manual" as const,
signal: AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS),
headers: { accept: "text/vtt,text/plain,application/json;q=0.9,*/*;q=0.8" },
...(target.addresses.length > 0
? { dispatcher: createPinnedDispatcher(target.addresses) }
: {}),
} as RequestInit & { dispatcher?: unknown };
const pinnedFetchImpl =
target.addresses.length > 0 && isNativeFetchImpl(fetchImpl) ? loadUndici().fetch : fetchImpl;
const res = await pinnedFetchImpl(target.url.href, pinnedInit);
if (!isRedirectStatus(res.status)) return res;
const location = res.headers.get("location");
if (!location) return res;
if (redirectCount >= MAX_TRANSCRIPT_REDIRECTS) {
throw new Error("RSS transcript URL redirected too many times");
}
const nextUrl = new URL(location, res.url || target.url.href).href;
return await fetchSafeTranscriptUrl(fetchImpl, nextUrl, { lookup }, redirectCount + 1);
}

export async function tryFetchTranscriptFromFeedXml({
fetchImpl,
feedXml,
episodeTitle,
notes,
lookup,
}: {
fetchImpl: typeof fetch;
feedXml: string;
episodeTitle: string | null;
notes: string[];
lookup?: LookupFn;
}): Promise<{
text: string;
transcriptUrl: string;
Expand All @@ -50,11 +262,7 @@ export async function tryFetchTranscriptFromFeedXml({

const transcriptUrl = decodeXmlEntities(preferred.url);
try {
const res = await fetchImpl(transcriptUrl, {
redirect: "follow",
signal: AbortSignal.timeout(TRANSCRIPTION_TIMEOUT_MS),
headers: { accept: "text/vtt,text/plain,application/json;q=0.9,*/*;q=0.8" },
});
const res = await fetchSafeTranscriptUrl(fetchImpl, transcriptUrl, { lookup });
if (!res.ok) throw new Error(`transcript fetch failed (${res.status})`);

const contentType =
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading