diff --git a/astro.config.mjs b/astro.config.mjs index 97d9800..1010dc9 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,7 +3,6 @@ import { defineConfig } from "astro/config"; import starlight from "@astrojs/starlight"; import ecTwoSlash from "expressive-code-twoslash"; import topics from "starlight-sidebar-topics"; -import starlightMarkdown from "starlight-markdown"; import mermaid from "astro-mermaid"; import { fileURLToPath } from "node:url"; @@ -104,7 +103,6 @@ export default defineConfig({ { icon: 'github', label: 'GitHub', href: 'https://bomb.sh/on/github' }, ], plugins: [ - starlightMarkdown(), topics([ { label: "Clack", diff --git a/package.json b/package.json index a075fc4..e15025d 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "build": "astro build && cp public/_headers dist/_headers", "preview": "astro preview", "astro": "astro", + "test": "node --test \"src/**/*.test.ts\"", "snapshot": "node --experimental-strip-types ./scripts/snapshot.ts", "generate:docs-index": "node --experimental-strip-types ./scripts/generate-docs-index.ts" }, @@ -31,11 +32,13 @@ "expressive-code-twoslash": "^0.5.3", "mermaid": "^11.16.0", "sharp": "^0.33.5", - "starlight-markdown": "^0.1.5", "starlight-sidebar-topics": "^0.6.2" }, "devDependencies": { "astro-vtbot": "^3.1.0", + "remark": "^15.0.1", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.1", "tinyexec": "^1.0.2", "wrangler": "^4.97.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9c08d42..5c814dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,9 +59,6 @@ importers: sharp: specifier: ^0.33.5 version: 0.33.5 - starlight-markdown: - specifier: ^0.1.5 - version: 0.1.5(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)) starlight-sidebar-topics: specifier: ^0.6.2 version: 0.6.2(@astrojs/starlight@0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2))) @@ -69,6 +66,15 @@ importers: astro-vtbot: specifier: ^3.1.0 version: 3.1.0 + remark: + specifier: ^15.0.1 + version: 15.0.1 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-mdx: + specifier: ^3.1.1 + version: 3.1.1 tinyexec: specifier: ^1.0.2 version: 1.0.2 @@ -77,6 +83,10 @@ importers: version: 4.97.0 router: + dependencies: + hono: + specifier: ^4.13.2 + version: 4.13.2 devDependencies: '@cloudflare/workers-types': specifier: ^4.20250327.0 @@ -1955,6 +1965,10 @@ packages: hastscript@9.0.1: resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==} + hono@4.13.2: + resolution: {integrity: sha512-JydRilDRkYBQMt9qR9U92mXxmbGqsqSn/IKOrh4e7/gEbn+0zSr8igTu0obwJoNGN4sez28DIql7FBHWydoJpA==} + engines: {node: '>=16.9.0'} + html-escaper@3.0.3: resolution: {integrity: sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==} @@ -2470,6 +2484,9 @@ packages: remark-stringify@11.0.0: resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + remark@15.0.1: + resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} + restructure@3.0.2: resolution: {integrity: sha512-gSfoiOEA0VPE6Tukkrr7I0RBdE0s7H1eFCDBk05l1KIQT1UIKNc5JZy6jdyW6eYH3aR3g5b3PuL77rq0hvwtAw==} @@ -2547,11 +2564,6 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} - starlight-markdown@0.1.5: - resolution: {integrity: sha512-23LXRaZp7pyE+r/HP6rxHfwic8HfvUBT4EImECA6encs/eTtrF0Z+7svANofdtfbiNt31D5q26i03B6FtcSmGg==} - peerDependencies: - astro: ^5.0.0 - starlight-sidebar-topics@0.6.2: resolution: {integrity: sha512-SNCTUZS/hcVor0ZcaXbaSVU37+V+qtvzNirkvnOg3Mqu/awuGpthkH5+uKpiZqWxLffp6TrOlsv5E5QsxrndNg==} engines: {node: '>=18'} @@ -4871,6 +4883,8 @@ snapshots: property-information: 7.1.0 space-separated-tokens: 2.0.2 + hono@4.13.2: {} + html-escaper@3.0.3: {} html-void-elements@3.0.0: {} @@ -5750,6 +5764,15 @@ snapshots: mdast-util-to-markdown: 2.1.2 unified: 11.0.5 + remark@15.0.1: + dependencies: + '@types/mdast': 4.0.4 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + restructure@3.0.2: {} retext-latin@4.0.0: @@ -5914,10 +5937,6 @@ snapshots: space-separated-tokens@2.0.2: {} - starlight-markdown@0.1.5(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)): - dependencies: - astro: 5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2) - starlight-sidebar-topics@0.6.2(@astrojs/starlight@0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2))): dependencies: '@astrojs/starlight': 0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)) diff --git a/router/package.json b/router/package.json index f1ce5d3..c198360 100644 --- a/router/package.json +++ b/router/package.json @@ -5,10 +5,14 @@ "type": "module", "scripts": { "dev": "wrangler dev", - "deploy": "wrangler deploy" + "deploy": "wrangler deploy", + "test": "node --test \"src/**/*.test.ts\"" }, "devDependencies": { "@cloudflare/workers-types": "^4.20250327.0", "wrangler": "^4.14.4" + }, + "dependencies": { + "hono": "^4.13.2" } } diff --git a/router/src/app.test.ts b/router/src/app.test.ts new file mode 100644 index 0000000..ecfe05f --- /dev/null +++ b/router/src/app.test.ts @@ -0,0 +1,135 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createApp } from "./app.ts"; + +/** App with a scripted upstream; unmatched paths 404 like the assets Worker. */ +function appWith(handler: (url: URL) => Response | undefined) { + const calls: URL[] = []; + const app = createApp({ + upstream: async (url) => { + calls.push(url); + return handler(url) ?? new Response("not found", { status: 404 }); + }, + passthrough: async (request) => + new Response(`passthrough:${request.url}`), + }); + return { app, calls }; +} + +const html = () => + new Response("page", { + status: 200, + headers: { "Content-Type": "text/html" }, + }); + +const markdown = () => + new Response("# Page\n", { + status: 200, + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); + +test("redirects bare project roots to their landing page", async () => { + const { app } = appWith(() => undefined); + const res = await app.request("https://bomb.sh/docs/clack", { + headers: { Accept: "text/markdown" }, + }); + assert.equal(res.status, 308); + assert.equal( + res.headers.get("Location"), + "https://bomb.sh/docs/clack/basics/getting-started/", + ); +}); + +test("serves the markdown twin on Accept: text/markdown", async () => { + const { app, calls } = appWith((url) => + url.pathname === "/docs/args/api/index.md" ? markdown() : undefined, + ); + const res = await app.request("https://bomb.sh/docs/args/api/", { + headers: { Accept: "text/markdown" }, + }); + assert.equal(res.status, 200); + assert.equal(calls[0].href, "https://docs.bomb.sh/docs/args/api/index.md"); + assert.match(res.headers.get("Content-Type") ?? "", /text\/markdown/); + assert.equal(res.headers.get("Vary"), "Accept"); + assert.equal( + res.headers.get("Link"), + '; rel="canonical"', + ); + assert.equal(await res.text(), "# Page\n"); +}); + +test("serves markdown for explicit .md paths without an Accept header", async () => { + const { app, calls } = appWith((url) => + url.pathname === "/docs/args/api/index.md" ? markdown() : undefined, + ); + const res = await app.request("https://bomb.sh/docs/args/api.md"); + assert.equal(res.status, 200); + assert.equal(calls[0].href, "https://docs.bomb.sh/docs/args/api/index.md"); +}); + +test("negotiates the docs root to the markdown index", async () => { + const { app, calls } = appWith((url) => + url.pathname === "/docs/index.md" ? markdown() : undefined, + ); + const res = await app.request("https://bomb.sh/docs", { + headers: { Accept: "text/markdown" }, + }); + assert.equal(res.status, 200); + assert.equal(calls[0].href, "https://docs.bomb.sh/docs/index.md"); +}); + +test("returns markdown 404 guidance when the twin is missing", async () => { + const { app } = appWith(() => undefined); + const res = await app.request("https://bomb.sh/docs/nope/", { + headers: { Accept: "text/markdown" }, + }); + assert.equal(res.status, 404); + assert.match(res.headers.get("Content-Type") ?? "", /text\/markdown/); + assert.match(await res.text(), /https:\/\/bomb\.sh\/docs\/index\.md/); +}); + +test("proxies HTML with security headers and a markdown alternate link", async () => { + const { app } = appWith((url) => + url.pathname === "/docs/args/api/" ? html() : undefined, + ); + const res = await app.request("https://bomb.sh/docs/args/api/"); + assert.equal(res.status, 200); + assert.equal( + res.headers.get("Cross-Origin-Embedder-Policy"), + "require-corp", + ); + assert.equal( + res.headers.get("Link"), + '; rel="alternate"; type="text/markdown"', + ); + assert.equal(await res.text(), "page"); +}); + +test("maps missing pages to the Starlight 404 page with status 404", async () => { + const { app } = appWith((url) => + url.pathname === "/docs/404.html" + ? new Response("404", { + status: 200, + headers: { "Content-Type": "text/html" }, + }) + : undefined, + ); + const res = await app.request("https://bomb.sh/docs/missing/"); + assert.equal(res.status, 404); + assert.equal(await res.text(), "404"); +}); + +test("does not negotiate asset paths", async () => { + const { app, calls } = appWith(() => html()); + await app.request("https://bomb.sh/docs/og-docs.png", { + headers: { Accept: "text/markdown" }, + }); + assert.equal(calls[0].href, "https://docs.bomb.sh/docs/og-docs.png"); +}); + +test("passes non-docs requests through untouched", async () => { + const { app, calls } = appWith(() => undefined); + const res = await app.request("https://bomb.sh/other"); + assert.equal(await res.text(), "passthrough:https://bomb.sh/other"); + assert.equal(calls.length, 0); +}); diff --git a/router/src/app.ts b/router/src/app.ts new file mode 100644 index 0000000..ea7d879 --- /dev/null +++ b/router/src/app.ts @@ -0,0 +1,27 @@ +import { Hono } from "hono/tiny"; +import { projectRedirects } from "./redirects.ts"; +import { markdownNegotiation } from "./markdown.ts"; +import { docsProxy } from "./proxy.ts"; +import { defaultUpstream, type Upstream } from "./upstream.ts"; + +export interface AppOptions { + /** Fetches from the docs assets Worker. */ + upstream?: Upstream; + /** Handles non-docs requests. */ + passthrough?: (request: Request) => Promise; +} + +export function createApp(options: AppOptions = {}) { + const upstream = options.upstream ?? defaultUpstream; + const passthrough = + options.passthrough ?? ((request: Request) => fetch(request)); + + const app = new Hono(); + for (const path of ["/docs", "/docs/*"]) { + app.use(path, projectRedirects()); + app.use(path, markdownNegotiation(upstream)); + app.all(path, docsProxy(upstream)); + } + app.all("*", (c) => passthrough(c.req.raw)); + return app; +} diff --git a/router/src/config.ts b/router/src/config.ts new file mode 100644 index 0000000..01aa10c --- /dev/null +++ b/router/src/config.ts @@ -0,0 +1,12 @@ +// Public origin used in Link headers (canonical/alternate) so agents always +// discover the proxied bomb.sh URLs, never the internal workers.dev ones. +export const SITE = "https://bomb.sh"; + +// Bare project roots don't have their own index page and should redirect to +// their actual landing page instead of 404ing. +export const PROJECT_LANDING_PAGES: Record = { + clack: "/docs/clack/basics/getting-started/", + tab: "/docs/tab/", + args: "/docs/args/getting-started/", + tty: "/docs/tty/basics/getting-started/", +}; diff --git a/router/src/index.ts b/router/src/index.ts index d7c6e6a..d66a9dc 100644 --- a/router/src/index.ts +++ b/router/src/index.ts @@ -1,70 +1,3 @@ -export interface Env { } +import { createApp } from "./app.ts"; -// Where to proxy docs requests. In production this is the live site. On -// Cloudflare branch previews both Workers share the same branch slug -// (e.g. `fix-404-bombsh-docs-router` ↔ `fix-404-bombshell-docs`), so we point -// the router at the matching docs preview by rewriting our own hostname. -// Per-version previews use an 8-char hex id that differs per Worker and can't -// be mapped, so those fall back to production. -function docsOrigin(host: string): string { - const match = host.match(/^(.+)-bombsh-docs-router\.(.+\.workers\.dev)$/); - if (match) { - const [, slug, zone] = match; - if (!/^[0-9a-f]{8}$/.test(slug)) { - return `https://${slug}-bombshell-docs.${zone}/`; - } - } - return "https://docs.bomb.sh/"; -} - -// Bare project roots don't have their own index page and should redirect to -// their actual landing page instead of 404ing. -const PROJECT_LANDING_PAGES: Record = { - clack: "/docs/clack/basics/getting-started/", - tab: "/docs/tab/", - args: "/docs/args/getting-started/", - tty: "/docs/tty/basics/getting-started/", -}; - -export default { - async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { - const url = new URL(request.url); - - const projectMatch = url.pathname.match(/^\/docs\/([^/]+)\/?$/); - if (projectMatch) { - const landingPage = PROJECT_LANDING_PAGES[projectMatch[1]]; - if (landingPage && url.pathname !== landingPage) { - return Response.redirect(new URL(landingPage, url).toString(), 308); - } - } - - if (url.pathname.startsWith("/docs")) { - const origin = docsOrigin(url.host); - let response = await fetch(new URL(url.pathname, docsOrigin(url.host))); - console.log({ from: url, to: new URL(url.pathname, docsOrigin(url.host)) }); - - - // Special case for Starlight's 404 page - let status = response.status; - if (status === 404) { - response = await fetch(new URL("/docs/404.html", origin)) - } - - const headers = new Headers(response.headers); - headers.set("Cross-Origin-Embedder-Policy", "require-corp"); - headers.set("Cross-Origin-Opener-Policy", "same-origin"); - headers.set("Cross-Origin-Resource-Policy", "cross-origin"); - headers.set("Referrer-Policy", "strict-origin-when-cross-origin"); - - // If we got 404, return the HTML, but set status to 404 manually, - // because the response status would be 200 - return new Response(response.body, { - status: status, - statusText: status === 404 ? "Not Found" : response.statusText, - headers, - }); - } - - return fetch(request); - }, -} satisfies ExportedHandler; +export default createApp(); diff --git a/router/src/markdown.ts b/router/src/markdown.ts new file mode 100644 index 0000000..15d3e1b --- /dev/null +++ b/router/src/markdown.ts @@ -0,0 +1,50 @@ +import type { MiddlewareHandler } from "hono"; +import { prefersMarkdown, toMarkdownPath } from "./negotiate.ts"; +import { docsOrigin } from "./origin.ts"; +import { SITE } from "./config.ts"; +import type { Upstream } from "./upstream.ts"; + +export const MARKDOWN_404 = `# 404: Not Found + +This page does not exist. An index of all Bombshell documentation is +available at ${SITE}/docs/index.md +`; + +// Agent-facing markdown: explicit `.md` paths always serve markdown; +// extensionless page routes negotiate on `Accept: text/markdown`. +// Negotiation rewrites to a distinct origin URL, so HTML and markdown +// variants get distinct cache keys — Cloudflare's cache ignores `Vary`. +export function markdownNegotiation(upstream: Upstream): MiddlewareHandler { + return async (c, next) => { + const url = new URL(c.req.url); + const markdownPath = toMarkdownPath(url.pathname); + const wantsMarkdown = + markdownPath !== null && + (url.pathname.endsWith(".md") || + prefersMarkdown(c.req.header("Accept") ?? null)); + if (!wantsMarkdown || !markdownPath) return next(); + + const response = await upstream( + new URL(markdownPath, docsOrigin(url.host)), + ); + if (response.status === 404) { + return new Response(MARKDOWN_404, { + status: 404, + headers: { + "Content-Type": "text/markdown; charset=utf-8", + Vary: "Accept", + }, + }); + } + + const headers = new Headers(response.headers); + headers.set("Content-Type", "text/markdown; charset=utf-8"); + headers.set("Vary", "Accept"); + const htmlPath = markdownPath.slice(0, -"index.md".length); + headers.set("Link", `<${SITE}${htmlPath}>; rel="canonical"`); + return new Response(response.body, { + status: response.status, + headers, + }); + }; +} diff --git a/router/src/negotiate.test.ts b/router/src/negotiate.test.ts new file mode 100644 index 0000000..6c5923d --- /dev/null +++ b/router/src/negotiate.test.ts @@ -0,0 +1,74 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { prefersMarkdown, toMarkdownPath } from "./negotiate.ts"; + +test("prefersMarkdown: no Accept header", () => { + assert.equal(prefersMarkdown(null), false); +}); + +test("prefersMarkdown: typical browser Accept header", () => { + assert.equal( + prefersMarkdown( + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + ), + false, + ); +}); + +test("prefersMarkdown: bare text/markdown", () => { + assert.equal(prefersMarkdown("text/markdown"), true); +}); + +test("prefersMarkdown: markdown preferred over html", () => { + assert.equal(prefersMarkdown("text/markdown,text/html;q=0.8"), true); +}); + +test("prefersMarkdown: markdown listed but html preferred", () => { + assert.equal(prefersMarkdown("text/html,text/markdown;q=0.5"), false); +}); + +test("prefersMarkdown: markdown explicitly refused", () => { + assert.equal(prefersMarkdown("text/markdown;q=0"), false); +}); + +test("prefersMarkdown: wildcard only is not markdown", () => { + assert.equal(prefersMarkdown("*/*"), false); +}); + +test("toMarkdownPath: docs root", () => { + assert.equal(toMarkdownPath("/docs"), "/docs/index.md"); + assert.equal(toMarkdownPath("/docs/"), "/docs/index.md"); +}); + +test("toMarkdownPath: page route with trailing slash", () => { + assert.equal( + toMarkdownPath("/docs/clack/basics/getting-started/"), + "/docs/clack/basics/getting-started/index.md", + ); +}); + +test("toMarkdownPath: page route without trailing slash", () => { + assert.equal( + toMarkdownPath("/docs/clack/basics/getting-started"), + "/docs/clack/basics/getting-started/index.md", + ); +}); + +test("toMarkdownPath: explicit .md request maps to index.md twin", () => { + assert.equal(toMarkdownPath("/docs/args/api.md"), "/docs/args/api/index.md"); +}); + +test("toMarkdownPath: already an index.md path is unchanged", () => { + assert.equal( + toMarkdownPath("/docs/tty/api/index.md"), + "/docs/tty/api/index.md", + ); +}); + +test("toMarkdownPath: assets and files with extensions are ignored", () => { + assert.equal(toMarkdownPath("/docs/_astro/hoisted.BQ1yu2o0.js"), null); + assert.equal(toMarkdownPath("/docs/favicon.svg"), null); + assert.equal(toMarkdownPath("/docs/docs-index.json"), null); + assert.equal(toMarkdownPath("/docs/og-docs.png"), null); + assert.equal(toMarkdownPath("/docs/404.html"), null); +}); diff --git a/router/src/negotiate.ts b/router/src/negotiate.ts new file mode 100644 index 0000000..ec2ca3c --- /dev/null +++ b/router/src/negotiate.ts @@ -0,0 +1,38 @@ +// Content negotiation for agent-facing markdown, per +// https://cra.mr/optimizing-content-for-agents: agents signal themselves with +// `Accept: text/markdown`; humans never do. + +/** Parse the quality value for a media type out of an Accept header. */ +function quality(accept: string, type: string): number { + for (const part of accept.split(",")) { + const [media, ...params] = part.trim().split(";"); + if (media.trim().toLowerCase() !== type) continue; + for (const param of params) { + const [key, value] = param.trim().split("="); + if (key === "q") return Number.parseFloat(value) || 0; + } + return 1; + } + return 0; +} + +export function prefersMarkdown(accept: string | null): boolean { + if (!accept) return false; + const markdown = quality(accept, "text/markdown"); + if (markdown === 0) return false; + return markdown >= quality(accept, "text/html"); +} + +/** + * Map a request path to its static markdown twin, or null if the path is not + * a documentation page (assets, feeds, and other files keep their extension). + */ +export function toMarkdownPath(pathname: string): string | null { + if (pathname.endsWith("/index.md")) return pathname; + if (pathname.endsWith(".md")) { + return `${pathname.slice(0, -".md".length)}/index.md`; + } + const lastSegment = pathname.slice(pathname.lastIndexOf("/") + 1); + if (lastSegment.includes(".")) return null; + return pathname.endsWith("/") ? `${pathname}index.md` : `${pathname}/index.md`; +} diff --git a/router/src/origin.test.ts b/router/src/origin.test.ts new file mode 100644 index 0000000..e4c7fa9 --- /dev/null +++ b/router/src/origin.test.ts @@ -0,0 +1,22 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { docsOrigin } from "./origin.ts"; + +test("falls back to production for plain hosts", () => { + assert.equal(docsOrigin("bomb.sh"), "https://docs.bomb.sh/"); + assert.equal(docsOrigin("localhost:8787"), "https://docs.bomb.sh/"); +}); + +test("maps branch preview router hosts to matching docs previews", () => { + assert.equal( + docsOrigin("fix-404-bombsh-docs-router.foo.workers.dev"), + "https://fix-404-bombshell-docs.foo.workers.dev/", + ); +}); + +test("per-version preview ids fall back to production", () => { + assert.equal( + docsOrigin("abcd1234-bombsh-docs-router.foo.workers.dev"), + "https://docs.bomb.sh/", + ); +}); diff --git a/router/src/origin.ts b/router/src/origin.ts new file mode 100644 index 0000000..5c03a5a --- /dev/null +++ b/router/src/origin.ts @@ -0,0 +1,16 @@ +// Where to proxy docs requests. In production this is the live site. On +// Cloudflare branch previews both Workers share the same branch slug +// (e.g. `fix-404-bombsh-docs-router` ↔ `fix-404-bombshell-docs`), so we point +// the router at the matching docs preview by rewriting our own hostname. +// Per-version previews use an 8-char hex id that differs per Worker and can't +// be mapped, so those fall back to production. +export function docsOrigin(host: string): string { + const match = host.match(/^(.+)-bombsh-docs-router\.(.+\.workers\.dev)$/); + if (match) { + const [, slug, zone] = match; + if (!/^[0-9a-f]{8}$/.test(slug)) { + return `https://${slug}-bombshell-docs.${zone}/`; + } + } + return "https://docs.bomb.sh/"; +} diff --git a/router/src/proxy.ts b/router/src/proxy.ts new file mode 100644 index 0000000..850af3f --- /dev/null +++ b/router/src/proxy.ts @@ -0,0 +1,47 @@ +import type { Handler } from "hono"; +import { toMarkdownPath } from "./negotiate.ts"; +import { docsOrigin } from "./origin.ts"; +import { SITE } from "./config.ts"; +import type { Upstream } from "./upstream.ts"; + +const SECURITY_HEADERS = { + "Cross-Origin-Embedder-Policy": "require-corp", + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "cross-origin", + "Referrer-Policy": "strict-origin-when-cross-origin", +} as const; + +export function docsProxy(upstream: Upstream): Handler { + return async (c) => { + const url = new URL(c.req.url); + const origin = docsOrigin(url.host); + let response = await upstream(new URL(url.pathname, origin)); + + // Starlight serves its 404 page at a fixed path with status 200; + // re-serve it under the requested URL with the right status. + const status = response.status; + if (status === 404) { + response = await upstream(new URL("/docs/404.html", origin)); + } + + const headers = new Headers(response.headers); + for (const [name, value] of Object.entries(SECURITY_HEADERS)) { + headers.set(name, value); + } + + // Advertise the markdown twin to agents crawling the HTML variant. + const markdownPath = toMarkdownPath(url.pathname); + if (markdownPath && status === 200) { + headers.set( + "Link", + `<${SITE}${markdownPath}>; rel="alternate"; type="text/markdown"`, + ); + } + + return new Response(response.body, { + status, + statusText: status === 404 ? "Not Found" : response.statusText, + headers, + }); + }; +} diff --git a/router/src/redirects.ts b/router/src/redirects.ts new file mode 100644 index 0000000..7f47da4 --- /dev/null +++ b/router/src/redirects.ts @@ -0,0 +1,14 @@ +import type { MiddlewareHandler } from "hono"; +import { PROJECT_LANDING_PAGES } from "./config.ts"; + +export function projectRedirects(): MiddlewareHandler { + return async (c, next) => { + const url = new URL(c.req.url); + const match = url.pathname.match(/^\/docs\/([^/]+)\/?$/); + const landingPage = match && PROJECT_LANDING_PAGES[match[1]]; + if (landingPage && url.pathname !== landingPage) { + return c.redirect(new URL(landingPage, url).toString(), 308); + } + return next(); + }; +} diff --git a/router/src/upstream.ts b/router/src/upstream.ts new file mode 100644 index 0000000..81d6a7b --- /dev/null +++ b/router/src/upstream.ts @@ -0,0 +1,4 @@ +/** Fetches a URL from the docs assets Worker. Injectable so tests can script responses. */ +export type Upstream = (url: URL) => Promise; + +export const defaultUpstream: Upstream = (url) => fetch(url); diff --git a/router/tsconfig.json b/router/tsconfig.json index 911fc10..96d360d 100644 --- a/router/tsconfig.json +++ b/router/tsconfig.json @@ -3,6 +3,7 @@ "target": "ESNext", "module": "ESNext", "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, "lib": ["ESNext"], "types": ["@cloudflare/workers-types"], "strict": true, diff --git a/scripts/generate-docs-index.ts b/scripts/generate-docs-index.ts index a1608ab..9b62b07 100644 --- a/scripts/generate-docs-index.ts +++ b/scripts/generate-docs-index.ts @@ -1,6 +1,6 @@ /** - * Walks `src/content/docs` and emits `public/llms.txt` plus - * `public/docs-index.json` for agent discoverability and offline search. + * Walks `src/content/docs` and emits `public/docs-index.json`, a + * machine-readable page index for offline search and local agent tooling. */ import fs from 'node:fs/promises'; import path from 'node:path'; @@ -98,42 +98,6 @@ function isIndexed(page: DocPage): boolean { return true; } -function generateLlmsTxt(pages: DocPage[]): string { - const indexed = pages.filter(isIndexed); - const lines = [ - '# Bombshell Documentation', - '', - '> Effortlessly build beautiful command-line apps. Docs for Clack, Args, Tab, and TTY.', - '', - `Canonical docs: ${BASE_URL}/`, - '', - ]; - - const homepage = indexed.find((page) => page.slug === ''); - if (homepage) { - lines.push(`- [${homepage.title}](${homepage.url}): ${homepage.description}`, ''); - } - - const sections = new Map(); - for (const page of indexed) { - if (page.slug === '') continue; - const section = page.slug.split('/')[0]; - if (!sections.has(section)) sections.set(section, []); - sections.get(section)!.push(page); - } - - for (const [section, sectionPages] of [...sections.entries()].sort()) { - const label = section.charAt(0).toUpperCase() + section.slice(1); - lines.push(`## ${label}`, ''); - for (const page of sectionPages.sort((a, b) => a.slug.localeCompare(b.slug))) { - lines.push(`- [${page.title}](${page.url}): ${page.description}`); - } - lines.push(''); - } - - return `${lines.join('\n').trimEnd()}\n`; -} - async function main() { const pages = await walkDocs(docsDir); const indexed = pages.filter(isIndexed); @@ -150,8 +114,6 @@ async function main() { 2, )}\n`, ); - await fs.writeFile(path.join(rootDir, 'public/llms.txt'), generateLlmsTxt(pages)); - console.log(`Generated docs index with ${indexed.length} pages`); } diff --git a/src/lib/mdx-to-markdown.test.ts b/src/lib/mdx-to-markdown.test.ts new file mode 100644 index 0000000..0f8e193 --- /dev/null +++ b/src/lib/mdx-to-markdown.test.ts @@ -0,0 +1,81 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mdxToMarkdown } from "./mdx-to-markdown.ts"; + +test("strips import statements", () => { + const out = mdxToMarkdown( + "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\n# Hello\n\nSome text.\n", + ); + assert.ok(!out.includes("import"), `import leaked into output:\n${out}`); + assert.ok(out.includes("# Hello")); + assert.ok(out.includes("Some text.")); +}); + +test("unwraps Tabs/TabItem with bold labels, preserving code fences", () => { + const out = mdxToMarkdown( + [ + "", + '', + "```sh", + "npm install @clack/prompts", + "```", + "", + '', + "```sh", + "pnpm add @clack/prompts", + "```", + "", + "", + ].join("\n"), + ); + assert.ok(!out.includes(" { + const out = mdxToMarkdown( + '\n', + ); + assert.ok(!out.includes(" **Caution:**"), `missing label:\n${out}`); + assert.ok(out.includes("Do not do the thing.")); +}); + +test("Aside with custom title uses the title", () => { + const out = mdxToMarkdown( + '\n', + ); + assert.ok(out.includes("> **Pro tip:**"), `missing title:\n${out}`); +}); + +test("plain markdown passes through, including tables", () => { + const src = [ + "# Title", + "", + "| a | b |", + "| - | - |", + "| 1 | 2 |", + "", + "Text with `code`.", + "", + ].join("\n"); + const out = mdxToMarkdown(src); + assert.ok(out.includes("| a | b |"), `table mangled:\n${out}`); + assert.ok(out.includes("Text with `code`.")); +}); + +test("drops JSX expressions and unwraps unknown components", () => { + const out = mdxToMarkdown( + 'export const x = 1;\n\n\n\nInner *content* here.\n\n\n\nValue: {x}\n', + ); + assert.ok(!out.includes("export const"), `export leaked:\n${out}`); + assert.ok(!out.includes("; + +const ASIDE_LABELS: Record = { + note: "Note", + tip: "Tip", + caution: "Caution", + danger: "Danger", +}; + +function attribute(node: Node, name: string): string | undefined { + for (const attr of node.attributes ?? []) { + if (attr.type === "mdxJsxAttribute" && attr.name === name) { + if (typeof attr.value === "string") return attr.value; + } + } + return undefined; +} + +function bold(text: string): Node { + return { + type: "paragraph", + children: [{ type: "strong", children: [{ type: "text", value: text }] }], + }; +} + +/** Replace a JSX element with plain markdown nodes (or [] to drop it). */ +function replaceElement(node: Node): Node[] { + const children = transformChildren(node.children ?? []); + switch (node.name) { + case "Aside": { + const label = + attribute(node, "title") ?? + ASIDE_LABELS[attribute(node, "type") ?? "note"] ?? + "Note"; + return [ + { type: "blockquote", children: [bold(`${label}:`), ...children] }, + ]; + } + case "TabItem": { + const label = attribute(node, "label"); + return label ? [bold(label), ...children] : children; + } + case "Card": { + const title = attribute(node, "title"); + return title ? [bold(title), ...children] : children; + } + case "LinkCard": + case "LinkButton": { + const href = attribute(node, "href"); + const title = + attribute(node, "title") ?? + (node.children?.[0]?.value as string | undefined); + if (href) { + return [ + { + type: "paragraph", + children: [ + { + type: "link", + url: href, + children: [{ type: "text", value: title ?? href }], + }, + ], + }, + ]; + } + return children; + } + // Tabs, Steps, CardGrid, FileTree, and anything unrecognized: unwrap. + default: + return children; + } +} + +function transformChildren(children: Node[]): Node[] { + const result: Node[] = []; + for (const child of children) { + switch (child.type) { + case "mdxjsEsm": + case "mdxFlowExpression": + case "mdxTextExpression": + break; + case "mdxJsxFlowElement": + case "mdxJsxTextElement": + result.push(...replaceElement(child)); + break; + default: + if (Array.isArray(child.children)) { + child.children = transformChildren(child.children); + } + result.push(child); + } + } + return result; +} + +const parser = remark().use(remarkMdx).use(remarkGfm); +// Stringify without the MDX extensions so output uses plain markdown escaping. +const printer = remark().use(remarkGfm); + +export function mdxToMarkdown(source: string): string { + const tree = parser.parse(source) as Node; + tree.children = transformChildren(tree.children); + return printer.stringify(tree as any); +} diff --git a/src/pages/[...path]/index.md.ts b/src/pages/[...path]/index.md.ts new file mode 100644 index 0000000..53cac1f --- /dev/null +++ b/src/pages/[...path]/index.md.ts @@ -0,0 +1,36 @@ +/** + * Markdown twin for every doc page at `/{slug}/index.md`, for agents. + * The root index and 404 are handled separately (`/index.md` is a + * sitemap-style index; 404 has no markdown twin). + */ +import type { APIRoute } from "astro"; +import { getCollection, type CollectionEntry } from "astro:content"; +import { mdxToMarkdown } from "../../lib/mdx-to-markdown"; + +const BASE_URL = "https://bomb.sh/docs"; + +export async function getStaticPaths() { + const docs = await getCollection("docs"); + return docs + .filter((doc) => doc.id !== "index" && doc.id !== "404") + .map((doc) => ({ params: { path: doc.id }, props: { doc } })); +} + +export const GET: APIRoute<{ doc: CollectionEntry<"docs"> }> = ({ props }) => { + const { doc } = props; + const frontmatter = [ + "---", + `title: ${JSON.stringify(doc.data.title)}`, + doc.data.description && + `description: ${JSON.stringify(doc.data.description)}`, + `canonical: ${BASE_URL}/${doc.id}/`, + "---", + ] + .filter(Boolean) + .join("\n"); + + const body = mdxToMarkdown(doc.body ?? ""); + return new Response(`${frontmatter}\n\n# ${doc.data.title}\n\n${body}`, { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); +}; diff --git a/src/pages/index.md.ts b/src/pages/index.md.ts new file mode 100644 index 0000000..8e32e71 --- /dev/null +++ b/src/pages/index.md.ts @@ -0,0 +1,62 @@ +/** + * Sitemap-style markdown index at `/docs/index.md` — the entry point agents + * land on via content negotiation. Purely descriptive: titles, descriptions, + * and markdown URLs, grouped by topic. + */ +import type { APIRoute } from "astro"; +import { getCollection } from "astro:content"; + +const BASE_URL = "https://bomb.sh/docs"; + +const SECTION_LABELS: Record = { + clack: "Clack — prompts", + args: "Args — argument parsing", + tab: "Tab — autocomplete", + tty: "TTY — layout & rendering", +}; + +export const GET: APIRoute = async () => { + const docs = await getCollection("docs"); + + const sections = new Map(); + for (const doc of docs) { + if (doc.id === "index" || doc.id === "404") continue; + const section = doc.id.split("/")[0]; + if (!sections.has(section)) sections.set(section, []); + sections.get(section)!.push({ + title: doc.data.title, + description: doc.data.description ?? "", + url: `${BASE_URL}/${doc.id}/index.md`, + }); + } + + const lines = [ + "---", + 'title: "Bombshell Documentation"', + `canonical: ${BASE_URL}/`, + "---", + "", + "# Bombshell Documentation", + "", + "> Effortlessly build beautiful command-line apps. Documentation for", + "> Clack, Args, Tab, and TTY — an ecosystem of terminal primitives for", + "> Node.js CLIs and TUIs.", + "", + `Every page is available as markdown at \`{page}/index.md\`, or by`, + `requesting any page URL with \`Accept: text/markdown\`.`, + "", + ]; + + for (const [section, pages] of [...sections.entries()].sort()) { + lines.push(`## ${SECTION_LABELS[section] ?? section}`, ""); + for (const page of pages.sort((a, b) => a.url.localeCompare(b.url))) { + const suffix = page.description ? `: ${page.description}` : ""; + lines.push(`- [${page.title}](${page.url})${suffix}`); + } + lines.push(""); + } + + return new Response(`${lines.join("\n").trimEnd()}\n`, { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); +}; diff --git a/src/starlightOverrides/Head.astro b/src/starlightOverrides/Head.astro index 127bb5b..44631a3 100644 --- a/src/starlightOverrides/Head.astro +++ b/src/starlightOverrides/Head.astro @@ -11,8 +11,19 @@ const ACCENTS = { tty: { dark: "300 100% 55%", light: "300 100% 38%" }, }; const accent = ACCENTS[Astro.locals.starlightRoute.id.split("/")[0]]; + +// Advertise the markdown twin of this page to agents (404 has none). +const routeId = Astro.locals.starlightRoute.id; +const markdownHref = + routeId === "404" + ? null + : `/docs/${routeId && routeId !== "index" ? `${routeId}/` : ""}index.md`; --- +{markdownHref && ( + +)} + {/* replaceSidebarContent: sidebar content differs per topic (starlight-sidebar-topics) */}