Llms txt

app/components/sidebar/tests/build-breadcrumbs.test.tsx

@@ -1,30 +1,31 @@
 import { describe, expect, it, vi } from "vitest"
-import { buildBreadcrumb } from "../build-breadcrumbs"
 import type { SidebarSection } from "../sidebar"
 
-vi.mock("~/utils/versions-utils", async () => {
-	const actual = await vi.importActual<typeof import("~/utils/versions-utils")>("~/utils/versions-utils")
-	return {
-		...actual,
-		useCurrentVersion: () => "v1.0.0",
-		getLatestVersion: () => "v1.0.0",
-		isKnownVersion: (v?: string) => v === "v1.0.0",
-	}
-})
+vi.mock("~/utils/split-slug-and-append-version", () => ({
+	splitSlugAndAppendVersion: (slug: string) => {
+		const parts = slug.split("/").filter(Boolean)
+		const version = "v1.0.0"
 
-vi.mock("~/utils/versions", async () => {
-	const actual = await vi.importActual<typeof import("~/utils/versions")>("~/utils/versions")
-	return {
-		...actual,
-		versions: ["v1.0.0"] as const,
-	}
-})
+		if (parts.length === 2) {
+			const [section, filename] = parts
+			return { version, section, filename }
+		}
+		if (parts.length === 3) {
+			const [section, subsection, filename] = parts
+			return { version, section, subsection, filename }
+		}
+
+		throw new Error(`Bad slug in test: ${slug}`)
+	},
+}))
+
+import { buildBreadcrumb } from "../build-breadcrumbs"
 
 type Doc = { slug: string; title: string }
 const makeDoc = (slug: string, title: string): Doc => ({ slug, title })
 
 type MinimalSection = Pick<SidebarSection, "title" | "slug" | "documentationPages" | "subsections">
-const makeSection = (overrides: Partial<MinimalSection> = {}): SidebarSection => ({
+const makeSection = (overrides: Partial<MinimalSection> = {}) => ({
 	title: "",
 	slug: "",
 	documentationPages: [],
@@ -41,9 +42,7 @@ describe("buildBreadcrumb (versioned paths via splitSlugAndAppendVersion)", () =
 				documentationPages: [makeDoc("getting-started/intro", "Intro")],
 			}),
 		]
-
-		const result = buildBreadcrumb(items, "/v1.0.0/getting-started/unknown")
-		expect(result).toEqual([])
+		expect(buildBreadcrumb(items, "/v1.0.0/getting-started/unknown")).toEqual([])
 	})
 
 	it("returns [section, doc] for a top-level doc", () => {
@@ -54,9 +53,7 @@ describe("buildBreadcrumb (versioned paths via splitSlugAndAppendVersion)", () =
 				documentationPages: [makeDoc("getting-started/intro", "Intro")],
 			}),
 		]
-
-		const result = buildBreadcrumb(items, "/v1.0.0/getting-started/intro")
-		expect(result).toEqual(["Getting Started", "Intro"])
+		expect(buildBreadcrumb(items, "/v1.0.0/getting-started/intro")).toEqual(["Getting Started", "Intro"])
 	})
 
 	it("returns full trail for a nested doc (root → sub → doc)", () => {
@@ -74,8 +71,10 @@ describe("buildBreadcrumb (versioned paths via splitSlugAndAppendVersion)", () =
 				documentationPages: [makeDoc("configuration/setup", "Setup")],
 			}),
 		]
-
-		const result = buildBreadcrumb(items, "/v1.0.0/configuration/advanced/tuning")
-		expect(result).toEqual(["Configuration", "Advanced", "Tuning"])
+		expect(buildBreadcrumb(items, "/v1.0.0/configuration/advanced/tuning")).toEqual([
+			"Configuration",
+			"Advanced",
+			"Tuning",
+		])
 	})
 })

app/components/versions-dropdown.tsx

@@ -1,21 +1,21 @@
 import { useState } from "react"
 import { useNavigate } from "react-router"
 import { Icon } from "~/ui/icon/icon"
+import { getCurrentVersion, homepageUrlWithVersion, isKnownVersion } from "~/utils/version-resolvers"
 import { versions } from "~/utils/versions"
-import { hrefForHomepage, isKnownVersion, useCurrentVersion } from "~/utils/versions-utils"
 
 export function VersionDropdown() {
 	const navigate = useNavigate()
-	const current = useCurrentVersion()
-	const [selectedVersion, setSelectedVersion] = useState(current)
+	const { version: currentVersion } = getCurrentVersion()
+	const [selectedVersion, setSelectedVersion] = useState(currentVersion)
 
 	function onChange(e: React.ChangeEvent<HTMLSelectElement>) {
 		const next = e.target.value
-		if (next === current) return
+		if (next === currentVersion) return
 
-		setSelectedVersion(isKnownVersion(next) ? next : current)
+		setSelectedVersion(isKnownVersion(next) ? next : currentVersion)
 
-		const to = hrefForHomepage(next)
+		const to = homepageUrlWithVersion(next)
 		const nav = () => {
 			navigate(to)
 			e.target.blur()

app/routes.ts

@@ -11,4 +11,5 @@ export default [
 	route("resource/*", "routes/resource.locales.ts"),
 	route("$", "routes/$.tsx"),
 	route("sitemap/:lang.xml", "routes/sitemap.$lang[.]xml.ts"),
+	route(":version?/llms.txt", "routes/llms[.]txt.ts"),
 ] satisfies RouteConfig

app/routes/documentation-homepage.tsx

@@ -1,12 +1,11 @@
 import GithubContributeLinks from "~/components/github-contribute-links"
 import PageMdxArticle from "~/components/page-mdx-article"
 import { loadContentCollections } from "~/utils/load-content-collections"
-import { resolveHomeVersionOrRedirect } from "~/utils/version-links"
+import { resolveVersionForHomepage } from "~/utils/version-resolvers"
 import type { Route } from "./+types/documentation-homepage"
 
 export async function loader({ params }: Route.LoaderArgs) {
-	const { version } = resolveHomeVersionOrRedirect(params.version)
-
+	const { version } = resolveVersionForHomepage(params.version)
 	const { allPages } = await loadContentCollections(version)
 	const page = allPages.find((p) => p._meta.path === "_index")
 	if (!page) throw new Response("Not Found", { status: 404 })

app/routes/documentation-layout.tsx

@@ -5,11 +5,11 @@ import { Sidebar } from "~/components/sidebar/sidebar"
 import { ThemeToggle } from "~/components/theme-toggle"
 import { VersionDropdown } from "~/components/versions-dropdown"
 import { createSidebarTree } from "~/utils/create-sidebar-tree"
-import { resolveLayoutVersion } from "~/utils/version-links"
+import { resolveVersionForLayout } from "~/utils/version-resolvers"
 import type { Route } from "./+types/documentation-layout"
 
 export async function loader({ params, request }: Route.LoaderArgs) {
-	const { version } = resolveLayoutVersion(params.version, request)
+	const { version } = resolveVersionForLayout(params.version, request)
 	const sidebarTree = await createSidebarTree(version)
 	return { sidebarTree, version }
 }

app/routes/documentation-page.tsx

@@ -6,14 +6,14 @@ import { useDocumentationLayoutLoaderData } from "~/hooks/use-documentation-layo
 import { usePreviousNextPages } from "~/hooks/use-previous-next-pages"
 import { extractHeadingTreeFromMarkdown } from "~/utils/extract-heading-tree-from-mdx"
 import { loadContentCollections } from "~/utils/load-content-collections"
-import { ensureVersion } from "~/utils/version-links"
+import { normalizeVersion } from "~/utils/version-resolvers"
 import type { Route } from "./+types/documentation-page"
 
 export async function loader({ params }: Route.LoaderArgs) {
 	const { version: v, section, subsection, filename } = params
 	if (!section || !filename) throw new Response("Not Found", { status: 404 })
 
-	const { version } = ensureVersion(v)
+	const { version } = normalizeVersion(v)
 
 	const slug = [section, subsection, filename].filter(Boolean).join("/")
 

app/routes/index.tsx

@@ -2,7 +2,8 @@ import { href, useNavigate } from "react-router"
 import { Header } from "~/components/header"
 import { Logo } from "~/components/logo"
 import { Icon } from "~/ui/icon/icon"
-import { getLatestVersion } from "~/utils/versions-utils"
+import { getLatestVersion } from "~/utils/version-resolvers"
+
 export default function Index() {
 	const navigate = useNavigate()
 	// Customize index page

app/routes/llms[.]txt.ts

@@ -0,0 +1,22 @@
+import { href, redirect } from "react-router"
+import { renderLlmsTxt } from "~/utils/llms-txt-builder"
+import { getLatestVersion, normalizeVersion } from "~/utils/version-resolvers"
+import type { Route } from "./+types/llms[.]txt"
+
+export async function loader({ request, params }: Route.LoaderArgs) {
+	const { version: paramVersion } = params
+	if (!paramVersion) {
+		const latest = getLatestVersion()
+		return redirect(href("/:version?/llms.txt", { version: latest }))
+	}
+
+	const { version } = normalizeVersion(paramVersion)
+	const body = await renderLlmsTxt({
+		request,
+		version,
+		title: "Documentation Template",
+		tagline: "Official documentation and guides.",
+	})
+
+	return new Response(body, { headers: { "Content-Type": "text/plain; charset=utf-8" } })
+}

app/utils/create-sidebar-tree.ts

@@ -1,6 +1,6 @@
 import type { SidebarSection } from "~/components/sidebar/sidebar"
 import { loadContentCollections } from "./load-content-collections"
-import type { Version } from "./versions-utils"
+import type { Version } from "./version-resolvers"
 
 const parentOf = (slug: string) => {
 	const i = slug.lastIndexOf("/")

app/utils/llms-txt-builder.ts

@@ -0,0 +1,59 @@
+import { createDomain } from "~/utils/http"
+import { loadContentCollections } from "~/utils/load-content-collections"
+import type { Page, Section } from "../../content-collections"
+import { type Version, pageUrlWithVersion } from "./version-resolvers"
+
+async function loadVersionData(version: Version) {
+	const { allPages, allSections } = await loadContentCollections(version)
+	return { version, pages: allPages, sections: allSections }
+}
+
+function buildSectionTitles(sections: Section[]) {
+	return new Map(sections.map((s) => [s.slug.split("/").pop() || "", s.title]))
+}
+
+function groupPagesByFolder(pages: Page[]) {
+	return pages.reduce((groups, p) => {
+		const id = p.section ?? p._meta?.path?.split("/")[0]
+		if (!id) return groups
+		const list = groups.get(id) ?? []
+		if (!groups.has(id)) groups.set(id, list)
+		list.push(p)
+		return groups
+	}, new Map<string, Page[]>())
+}
+
+function renderVersionBlock(domain: string, version: string, pages: Page[], sections: Section[]) {
+	if (!pages.length) return `## ${version}\n\n_No pages found._`
+
+	const sectionTitles = buildSectionTitles(sections)
+	const groups = groupPagesByFolder(pages)
+
+	const renderPageLink = (p: Page) => {
+		const url = pageUrlWithVersion(domain, version, p.slug)
+		const note = p.description
+		return `- [${p.title}](${url})${note ? `: ${note}` : ""}`
+	}
+
+	const renderSection = ([id, list]: [string, Page[]]) => {
+		const label = sectionTitles.get(id) ?? id
+		return `### ${label}\n\n${list.map(renderPageLink).join("\n")}`
+	}
+
+	return `\n## ${version}\n\n${Array.from(groups.entries()).map(renderSection).join("\n\n")}`
+}
+
+export async function renderLlmsTxt(opts: {
+	request: Request
+	version: Version
+	title: string
+	tagline: string
+}) {
+	const { request, version, title, tagline } = opts
+	const domain = createDomain(request)
+
+	const { pages, sections } = await loadVersionData(version)
+	const content = renderVersionBlock(domain, version, pages, sections)
+
+	return [`# ${title}`, `> ${tagline}`, content, ""].join("\n")
+}

app/utils/load-content-collections.ts

@@ -1,6 +1,6 @@
 import { dirname, resolve } from "node:path"
 import { fileURLToPath } from "node:url"
-import type { Version } from "~/utils/versions-utils"
+import type { Version } from "./version-resolvers"
 
 /**
  * Load content-collections outputs

app/utils/split-slug-and-append-version.ts

@@ -1,8 +1,8 @@
-import { useCurrentVersion } from "./versions-utils"
+import { getCurrentVersion } from "./version-resolvers"
 
 export function splitSlugAndAppendVersion(slug: string) {
 	const parts = slug.split("/").filter(Boolean)
-	const version = useCurrentVersion()
+	const { version } = getCurrentVersion()
 	if (parts.length === 2) {
 		const [section, filename] = parts
 		return { version, section, filename }

app/utils/tests/split-slug-and-append-version.test.ts

@@ -1,31 +1,28 @@
-const versionsMock = ["v1.0.0"] as const
+import { describe, expect, it, vi } from "vitest"
 
-vi.mock("~/utils/versions", () => ({
-	versions: versionsMock,
+vi.mock("../version-resolvers", () => ({
+	getLatestVersion: () => "v1.0.0",
+	isKnownVersion: (v?: string) => v === "v1.0.0",
+	getCurrentVersion: () => ({ version: "v1.0.0" }),
+}))
+
+vi.mock("../versions", () => ({
+	versions: ["v1.0.0"] as const,
 }))
 
-vi.mock("../versions-utils", () => {
-	return {
-		getLatestVersion: vi.fn(() => versionsMock[0]),
-		isKnownVersion: vi.fn(() => versionsMock[0]),
-		useCurrentVersion: vi.fn(() => versionsMock[0]),
-	}
-})
 import { splitSlugAndAppendVersion } from "../split-slug-and-append-version"
 
 describe("splitSlug test suite", () => {
 	it("splits a slug with 2 parts (no subsection)", () => {
-		const result = splitSlugAndAppendVersion("getting-started/intro")
-		expect(result).toEqual({
+		expect(splitSlugAndAppendVersion("getting-started/intro")).toEqual({
 			version: "v1.0.0",
 			section: "getting-started",
 			filename: "intro",
 		})
 	})
 
 	it("splits a slug with 3 parts (with subsection)", () => {
-		const result = splitSlugAndAppendVersion("getting-started/basics/intro")
-		expect(result).toEqual({
+		expect(splitSlugAndAppendVersion("getting-started/basics/intro")).toEqual({
 			version: "v1.0.0",
 			section: "getting-started",
 			subsection: "basics",
@@ -34,20 +31,19 @@ describe("splitSlug test suite", () => {
 	})
 
 	it("throws if slug has less than 2 parts", () => {
-		expect(() => splitSlugAndAppendVersion("getting-started")).toThrowError(
+		expect(() => splitSlugAndAppendVersion("getting-started")).toThrow(
 			/expected "section\/page" or "section\/subsection\/page"/i
 		)
 	})
 
 	it("throws if slug has more than 3 parts", () => {
-		expect(() => splitSlugAndAppendVersion("section/subsection/file/extra")).toThrowError(
+		expect(() => splitSlugAndAppendVersion("section/subsection/file/extra")).toThrow(
 			/expected "section\/page" or "section\/subsection\/page"/i
 		)
 	})
 
 	it("handles numeric or dashed parts", () => {
-		const result = splitSlugAndAppendVersion("01-intro/02-basics/file-name")
-		expect(result).toEqual({
+		expect(splitSlugAndAppendVersion("01-intro/02-basics/file-name")).toEqual({
 			version: "v1.0.0",
 			section: "01-intro",
 			subsection: "02-basics",
@@ -56,8 +52,7 @@ describe("splitSlug test suite", () => {
 	})
 
 	it("handles special characters in parts", () => {
-		const result = splitSlugAndAppendVersion("@special$/#weird$/file-name")
-		expect(result).toEqual({
+		expect(splitSlugAndAppendVersion("@special$/#weird$/file-name")).toEqual({
 			version: "v1.0.0",
 			section: "@special$",
 			subsection: "#weird$",
@@ -66,8 +61,7 @@ describe("splitSlug test suite", () => {
 	})
 
 	it("handles uppercase letters in slug", () => {
-		const result = splitSlugAndAppendVersion("GettingStarted/Intro")
-		expect(result).toEqual({
+		expect(splitSlugAndAppendVersion("GettingStarted/Intro")).toEqual({
 			version: "v1.0.0",
 			section: "GettingStarted",
 			filename: "Intro",