From 5c63bd3674b64a2d4a85cfc3a4236a7a85042e45 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 06:47:56 +0000 Subject: [PATCH 1/8] Add llms.txt, robots.txt, sitemap, structured data and a real 404 An is-agentic scan of magpie.apache.org found that unknown paths answer 200 with the app shell, that /llms.txt, /robots.txt and a sitemap do not exist, and that the site's developer resources cannot be found by name. It also inferred an HTTP API and marked it blocked; there is none. - astro.config.mjs, package.json: @astrojs/sitemap, so every route the build emits is enumerated at /sitemap-index.xml - public/.htaccess: ErrorDocument 404 /404.html so the ASF web server serves the site's own 404 page with the 404 status intact. Astro copies public/ into dist/, which CI pushes to the publish branch, so no workflow change - public/robots.txt: allow all, pointing at the sitemap - src/pages/llms.txt.ts: an llmstxt.org index built from the docs collection, so it tracks the synced docs rather than duplicating them. It states plainly that Magpie is a set of agent skills with no API on this domain, which is what tools currently get wrong about us - src/components/JsonLd.astro: Organization / WebSite / SoftwareSourceCode JSON-LD and a rel="llms-txt" link, with "<" escaped in the payload - src/pages/404.astro: recovery links, so a 404 is not a dead end - src/lib/site.ts: site constants, the hand-written route list, and HIDDEN_DOCS, which src/pages/docs/[...slug].astro now imports so the index and the site hide the same pages Titles in llms.txt come from frontmatter, else the page's opening H1, else the same dash-to-space fallback the docs page uses. Left for the checkout: mount in src/layouts/BaseLayout.astro. Generated-by: Claude (Fable 5.1) --- astro.config.mjs | 5 +- package-lock.json | 150 +++++++++++++++------------------ package.json | 1 + public/.htaccess | 29 +++++++ public/robots.txt | 4 + src/components/JsonLd.astro | 70 +++++++++++++++ src/lib/site.ts | 96 +++++++++++++++++++++ src/pages/404.astro | 49 +++++++++++ src/pages/docs/[...slug].astro | 4 +- src/pages/llms.txt.ts | 87 +++++++++++++++++++ 10 files changed, 408 insertions(+), 87 deletions(-) create mode 100644 public/.htaccess create mode 100644 public/robots.txt create mode 100644 src/components/JsonLd.astro create mode 100644 src/lib/site.ts create mode 100644 src/pages/404.astro create mode 100644 src/pages/llms.txt.ts diff --git a/astro.config.mjs b/astro.config.mjs index 92cc9a65..26470290 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,6 +1,7 @@ // @ts-check import { defineConfig } from 'astro/config'; import react from '@astrojs/react'; +import sitemap from '@astrojs/sitemap'; import tailwindcss from '@tailwindcss/vite'; // Production is served directly at the apex https://magpie.apache.org/ (root path). @@ -30,7 +31,9 @@ export default defineConfig({ site, base, redirects, - integrations: [react()], + // Markdown twins of docs pages (/docs/.md) are alternates of the HTML + // page, not pages, so they stay out of the sitemap. + integrations: [react(), sitemap({ filter: (page) => !page.endsWith('.md') })], vite: { plugins: [tailwindcss()], }, diff --git a/package-lock.json b/package-lock.json index 729bad0e..4a7b2548 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "devDependencies": { "@astrojs/check": "^0.9.10", "@astrojs/react": "^6.0.4", + "@astrojs/sitemap": "^3.7.4", "@tailwindcss/vite": "^4.3.3", "@types/node": "^26.2.0", "@types/react": "^19.2.18", @@ -152,9 +153,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -171,9 +169,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -190,9 +185,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -209,9 +201,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -386,6 +375,17 @@ "react-dom": "^17.0.2 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@astrojs/sitemap": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.7.4.tgz", + "integrity": "sha512-LbKNC24bdUWcQf/pThB6qLlSqHojxGjZDURIzFocY8rlWnAn2t74nnhnK6S5x0NHriHoAduLEpVjRykmeGiVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "sitemap": "^9.0.0", + "zod": "^4.3.6" + } + }, "node_modules/@astrojs/telemetry": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.3.3.tgz", @@ -881,9 +881,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -897,9 +894,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -913,9 +907,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -929,9 +920,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1901,9 +1889,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1920,9 +1905,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1939,9 +1921,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1958,9 +1937,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1977,9 +1953,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1996,9 +1969,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2015,9 +1985,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2034,9 +2001,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2053,9 +2017,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2078,9 +2039,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2103,9 +2061,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2128,9 +2083,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2153,9 +2105,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2178,9 +2127,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2203,9 +2149,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2228,9 +2171,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -4490,9 +4430,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4510,9 +4447,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4530,9 +4464,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -4550,9 +4481,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4845,6 +4773,16 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -5128,6 +5066,13 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -9795,6 +9740,43 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, + "node_modules/sitemap": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-9.0.1.tgz", + "integrity": "sha512-S6hzjGJSG3d6if0YoF5kTyeRJvia6FSTBroE5fQ0bu1QNxyJqhhinfUsXi9fH3MgtXODWvwo2BDyQSnhPQ88uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^24.9.2", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.4.1" + }, + "bin": { + "sitemap": "dist/esm/cli.js" + }, + "engines": { + "node": ">=20.19.5", + "npm": ">=10.8.2" + } + }, + "node_modules/sitemap/node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/sitemap/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", diff --git a/package.json b/package.json index 5d6d2566..60318acf 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "devDependencies": { "@astrojs/check": "^0.9.10", "@astrojs/react": "^6.0.4", + "@astrojs/sitemap": "^3.7.4", "@tailwindcss/vite": "^4.3.3", "@types/node": "^26.2.0", "@types/react": "^19.2.18", diff --git a/public/.htaccess b/public/.htaccess new file mode 100644 index 00000000..6d394886 --- /dev/null +++ b/public/.htaccess @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Astro copies public/ into dist/ verbatim, and CI force-pushes dist/ to the +# `publish` branch that ASF infrastructure serves at magpie.apache.org, so +# this file lands at the site root. + +# Unknown paths must answer 404, never a 200 with the app shell. Astro writes +# 404.html at build time; without ErrorDocument the ASF web server answers +# with its own bare "Not Found" body and the page's recovery links are lost. +ErrorDocument 404 /404.html + + + Header set Content-Type "text/markdown; charset=utf-8" + diff --git a/public/robots.txt b/public/robots.txt new file mode 100644 index 00000000..0b2441a4 --- /dev/null +++ b/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://magpie.apache.org/sitemap-index.xml diff --git a/src/components/JsonLd.astro b/src/components/JsonLd.astro new file mode 100644 index 00000000..4b00a775 --- /dev/null +++ b/src/components/JsonLd.astro @@ -0,0 +1,70 @@ +--- +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Organization / WebSite / SoftwareSourceCode structured data for the . +// Drop into BaseLayout.astro's once. + +import { SITE_URL, SITE_NAME, SITE_DESCRIPTION, SOURCE_REPO, DEV_LIST } from "../lib/site"; + + +const org = { + "@type": "Organization", + "@id": "https://www.apache.org/#organization", + name: "The Apache Software Foundation", + url: "https://www.apache.org/", +}; + +const payload = { + "@context": "https://schema.org", + "@graph": [ + org, + { + "@type": "WebSite", + "@id": `${SITE_URL}/#website`, + url: `${SITE_URL}/`, + name: SITE_NAME, + description: SITE_DESCRIPTION, + publisher: { "@id": org["@id"] }, + inLanguage: "en", + }, + { + "@type": "SoftwareSourceCode", + "@id": `${SITE_URL}/#software`, + name: SITE_NAME, + description: SITE_DESCRIPTION, + url: `${SITE_URL}/`, + codeRepository: SOURCE_REPO, + license: "https://www.apache.org/licenses/LICENSE-2.0", + programmingLanguage: "Markdown", + applicationCategory: "DeveloperApplication", + author: { "@id": org["@id"] }, + discussionUrl: `https://lists.apache.org/list.html?${DEV_LIST}`, + }, + ], +}; + +// JSON.stringify does not escape "<", so a value containing "" would +// close the tag early. Every value is a constant today; the escape keeps that +// independent of what the object later carries. +const json = JSON.stringify(payload).replace(/ + diff --git a/src/lib/site.ts b/src/lib/site.ts new file mode 100644 index 00000000..fba9ebf5 --- /dev/null +++ b/src/lib/site.ts @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getCollection, type CollectionEntry } from "astro:content"; + +export const SITE_URL = "https://magpie.apache.org"; +export const SITE_NAME = "Apache Magpie"; +export const SITE_DESCRIPTION = + "AI-powered assistant for open-source maintainers. Triage, mentor, draft, pair, and auto-merge under Apache Software Foundation governance."; +export const SOURCE_REPO = "https://github.com/apache/magpie"; +export const SITE_REPO = "https://github.com/apache/magpie-site"; +export const DEV_LIST = "dev@magpie.apache.org"; + +/** + * Hand-written routes that no collection knows about. The sitemap integration + * discovers pages from src/pages, so these only need listing for llms.txt. + * Keep the descriptions to one line: agents read them verbatim. + */ +export const STATIC_ROUTES: { path: string; title: string; description: string }[] = [ + { path: "/", title: "Home", description: "What Magpie is, the five agentic modes, and the ten skill families." }, + { path: "/docs/", title: "Documentation", description: "Setup, skill families, education stream, principles and RFCs." }, + { path: "/architecture/", title: "Architecture", description: "Organizations, tools and capability contracts behind vendor neutrality." }, + { path: "/tools/", title: "Tools", description: "The adapters that fulfil each capability, and which vendors each supports." }, + { path: "/downloads/", title: "Downloads", description: "Signed release artifacts and how to verify them." }, + { path: "/brand/", title: "Brand assets", description: "Logos, wordmark and usage rules for the Apache Magpie trademark." }, +]; + +/** + * Internal governance drafts that are in the synced collection but must not + * appear on the public site. src/pages/docs/[...slug].astro, llms.txt and the + * Markdown twins all filter on this one set. + */ +export const HIDDEN_DOCS = new Set(["board-resolution-draft"]); + +export type DocEntry = CollectionEntry<"docs">; + +/** Entry id with any trailing .md removed, as the docs page uses for its slug. */ +export function docSlug(entry: DocEntry): string { + return entry.id.replace(/\.md$/, ""); +} + +/** Path of the rendered HTML page, exactly as src/pages/docs/[...slug].astro emits it. */ +export function docHtmlPath(entry: DocEntry): string { + return `/docs/${docSlug(entry)}/`; +} + +/** Path of the synced file relative to apache/magpie/docs, original case preserved. */ +export function docRelPath(entry: DocEntry): string { + return entry.filePath ? entry.filePath.replace(/^.*\/content\/docs\//, "") : `${docSlug(entry)}.md`; +} + +/** Upstream file the page was synced from, so agents can read or edit the source. */ +export function docSourceUrl(entry: DocEntry): string { + return `${SOURCE_REPO}/blob/main/docs/${docRelPath(entry)}`; +} + +/** The body's opening "# heading", if it starts with one. */ +function leadingHeading(body: string): string | undefined { + const m = body.trimStart().match(/^#\s+(.+?)\s*(?:\n|$)/); + return m ? m[1].trim() : undefined; +} + +/** + * Display title: frontmatter, else the body's opening H1, else the same + * dash-to-space fallback the docs page uses. + */ +export function docTitle(entry: DocEntry): string { + return ( + entry.data.title ?? + leadingHeading(entry.body ?? "") ?? + docSlug(entry).split("/").pop()!.replace(/-/g, " ") + ); +} + +/** Public docs entries in stable order. */ +export async function allDocs(): Promise { + const entries = await getCollection("docs", (e) => !HIDDEN_DOCS.has(docSlug(e))); + return entries.sort((a, b) => a.id.localeCompare(b.id)); +} + diff --git a/src/pages/404.astro b/src/pages/404.astro new file mode 100644 index 00000000..3f89095a --- /dev/null +++ b/src/pages/404.astro @@ -0,0 +1,49 @@ +--- +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Recovery page for unknown paths. Astro emits this as /404.html; the +// .htaccess ErrorDocument points the ASF web server at it so the status stays +// 404 and the body gives both people and agents somewhere to go. Replace the +// bare shell with the site's base layout if one exports the . + +import { SITE_URL, SITE_NAME, SOURCE_REPO } from "../lib/site"; +--- + + + + + + + Page not found — {SITE_NAME} + + +
+

Page not found

+

There is no page at this address on {SITE_NAME}. It may have moved when the documentation was reorganised.

+ +
+ + diff --git a/src/pages/docs/[...slug].astro b/src/pages/docs/[...slug].astro index 4552d00b..7a9c4ba1 100644 --- a/src/pages/docs/[...slug].astro +++ b/src/pages/docs/[...slug].astro @@ -1,10 +1,10 @@ --- import { getCollection, render } from "astro:content"; import DocsLayout from "../../layouts/DocsLayout.astro"; +import { HIDDEN_DOCS } from "../../lib/site"; export async function getStaticPaths() { - // Internal governance drafts that should not appear on the public site. - const HIDDEN_DOCS = new Set(["board-resolution-draft"]); + // HIDDEN_DOCS lives in src/lib/site.ts so llms.txt hides the same pages. const docs = await getCollection("docs"); return docs .filter((entry) => !HIDDEN_DOCS.has(entry.id.replace(/\.md$/, ""))) diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts new file mode 100644 index 00000000..734b8d85 --- /dev/null +++ b/src/pages/llms.txt.ts @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// An llmstxt.org index, generated from the same docs collection the site +// renders, so it cannot drift from the sidebar. Links are absolute and the +// generated section sits under an H2: an llms.txt has exactly one H1. + +import type { APIRoute } from "astro"; +import { + SITE_URL, + SITE_NAME, + SITE_DESCRIPTION, + SOURCE_REPO, + SITE_REPO, + DEV_LIST, + STATIC_ROUTES, + allDocs, + docHtmlPath, + docTitle, +} from "../lib/site"; + +const HEADER = `# ${SITE_NAME} + +> ${SITE_DESCRIPTION} + +Apache Magpie is a set of agent skills (recipes) that maintainers run inside +their own agent harness: Claude Code, Codex, Copilot, Gemini, Cursor and +others. It is not a hosted service. There is no HTTP API, no OpenAPI spec and +nothing to call on this domain; integrate by installing the skills from the +source repository, or through an agent marketplace. + +- Source repository: ${SOURCE_REPO} +- Agent contract for the source tree: ${SOURCE_REPO}/blob/main/AGENTS.md +- Contributing: ${SOURCE_REPO}/blob/main/CONTRIBUTING.md +- Website source: ${SITE_REPO} +- Developer mailing list: ${DEV_LIST} (archive: https://lists.apache.org/list.html?${DEV_LIST}) + +`; + +export const GET: APIRoute = async () => { + const docs = await allDocs(); + + const pages = STATIC_ROUTES.map( + (r) => `- [${r.title}](${SITE_URL}${r.path}): ${r.description}`, + ); + + const docLines = docs.map((entry) => { + const desc = entry.data.description ? `: ${entry.data.description}` : ""; + return `- [${docTitle(entry)}](${SITE_URL}${docHtmlPath(entry)})${desc}`; + }); + + const body = [ + HEADER, + "## Site", + "", + ...pages, + "", + "## Documentation", + "", + ...docLines, + "", + "## Optional", + "", + `- [Sitemap](${SITE_URL}/sitemap-index.xml)`, + "", + ].join("\n"); + + return new Response(body, { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); +}; From 0d2f8b53bf5ca3a89a9bc2d34b5a52d8d7043480 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 07:35:51 +0000 Subject: [PATCH 2/8] Take the llms.txt title from the first H1, not the first line Synced docs open with an SPDX comment and a doctoc block, so the H1 is never on the first line and every readme was titled "readme". Generated-by: Claude (Fable 5.1) --- src/lib/site.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/site.ts b/src/lib/site.ts index fba9ebf5..a71e8ae1 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -70,9 +70,9 @@ export function docSourceUrl(entry: DocEntry): string { return `${SOURCE_REPO}/blob/main/docs/${docRelPath(entry)}`; } -/** The body's opening "# heading", if it starts with one. */ +/** The body's first "# heading", wherever it sits: synced files open with a licence comment and a TOC. */ function leadingHeading(body: string): string | undefined { - const m = body.trimStart().match(/^#\s+(.+?)\s*(?:\n|$)/); + const m = body.match(/^#\s+(.+?)\s*$/m); return m ? m[1].trim() : undefined; } From 76bb34c898a3c18adeefaec950c229c0d5724470 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 17:56:01 +1000 Subject: [PATCH 3/8] Add missing newline at end of site.ts --- src/lib/site.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/site.ts b/src/lib/site.ts index a71e8ae1..d4743f26 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -94,3 +94,4 @@ export async function allDocs(): Promise { return entries.sort((a, b) => a.id.localeCompare(b.id)); } + From 8d3f9f311d3c22e5b4a8c3e39204975be6f37ff8 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 18:21:27 +1000 Subject: [PATCH 4/8] Take the llms.txt title from the first H1; fix end of file --- src/lib/site.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/site.ts b/src/lib/site.ts index d4743f26..ef8a256d 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -93,5 +93,3 @@ export async function allDocs(): Promise { const entries = await getCollection("docs", (e) => !HIDDEN_DOCS.has(docSlug(e))); return entries.sort((a, b) => a.id.localeCompare(b.id)); } - - From 88148814981753945da3ef6d093e00c1cb977c70 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 17:26:31 +1000 Subject: [PATCH 5/8] Take the llms.txt title from the first H1, not the first line --- src/data/skill-counts.json | 4 ++-- src/data/tools.json | 31 ++++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/data/skill-counts.json b/src/data/skill-counts.json index 1ca738f3..eab7bfee 100644 --- a/src/data/skill-counts.json +++ b/src/data/skill-counts.json @@ -1,6 +1,6 @@ { "_comment": "GENERATED by scripts/gen-skill-counts.mjs — do not edit by hand. Counts derive from the `family:` frontmatter of apache/magpie skills/.", - "total": 70, + "total": 71, "uncategorized": [], "counts": { "contributor-growth": 6, @@ -12,6 +12,6 @@ "repo-health": 7, "security": 12, "setup": 9, - "utilities": 4 + "utilities": 5 } } diff --git a/src/data/tools.json b/src/data/tools.json index a34de4e4..092f027e 100644 --- a/src/data/tools.json +++ b/src/data/tools.json @@ -127,6 +127,10 @@ { "kind": "contract", "name": "change-request" + }, + { + "kind": "contract", + "name": "tracker" } ], "kind": "contract", @@ -881,7 +885,7 @@ { "name": "spec-inventory", "title": "spec-inventory", - "description": "A deterministic uv tool that emits a compact routing inventory for the spec-loop prompts. It summarizes spec frontmatter, where-it-lives hints, validation commands, known gaps, skill frontmatter, and tool/test presence so agents can…", + "description": "Two deterministic uv tools that help the spec-loop navigate the repository without scanning it from scratch each iteration.", "labels": [ { "kind": "substrate", @@ -1100,6 +1104,10 @@ ], "interfaces": [], "implementations": [ + { + "tool": "bitbucket", + "vendor": "Atlassian" + }, { "tool": "jira", "vendor": "Atlassian" @@ -1609,15 +1617,15 @@ "optIn": "adopter-declared in /privacy-llm.md (no fixed list)" }, "skills": { - "total": 70, - "neutral": 70, + "total": 71, + "neutral": 71, "byVerdict": { "capability-pure": 11, - "portable": 59, + "portable": 60, "vendor-coupled": 0 }, "byOrg": { - "agnostic": 56, + "agnostic": 57, "ASF": 14 }, "coupled": [], @@ -2022,6 +2030,15 @@ ], "coupled": [] }, + { + "skill": "report-framework-issue", + "verdict": "portable", + "organization": "agnostic", + "contractsUsed": [ + "contract:tracker" + ], + "coupled": [] + }, { "skill": "reviewer-routing", "verdict": "portable", @@ -2269,8 +2286,8 @@ } }, "skills": { - "total": 70, - "apacheLicensed": 70, + "total": 71, + "apacheLicensed": 71, "allApache": true, "license": "Apache-2.0", "byOrg": { From df60c22bc75612c74612ee1cc2564e68e1c9a192 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 06:47:56 +0000 Subject: [PATCH 6/8] Publish a Markdown copy of every docs page An agent that wants a docs page currently has to parse the rendered HTML. This publishes a Markdown twin next to each page, so /docs/modes/ is also available as /docs/modes.md. The docs are already Markdown synced from apache/magpie, and Astro emits endpoints as static files, so this is a route beside the existing src/pages/docs/[...slug].astro, using the same slug and the same HIDDEN_DOCS filter, not a prebuild script. - src/pages/docs/[...slug].md.ts: one twin per public collection entry. The body carries the title, the description as a blockquote, the rendered and source URLs, then the page as synced. A body that already opens with an H1 supplies the title rather than printing it twice - src/pages/llms-full.txt.ts: every twin concatenated, for one-fetch agents - src/pages/llms.txt.ts: links go to the twins and state the convention - src/lib/site.ts: docMarkdownPath, docSourceUrl and docMarkdown helpers The sitemap filter from the previous change already excludes the twins: they are alternates, not pages. How they are served and advertised is the next change. Generated-by: Claude (Fable 5.1) --- src/lib/site.ts | 25 +++++++++++++++++++ src/pages/docs/[...slug].md.ts | 44 ++++++++++++++++++++++++++++++++++ src/pages/llms-full.txt.ts | 39 ++++++++++++++++++++++++++++++ src/pages/llms.txt.ts | 9 ++++++- 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 src/pages/docs/[...slug].md.ts create mode 100644 src/pages/llms-full.txt.ts diff --git a/src/lib/site.ts b/src/lib/site.ts index ef8a256d..c5000eab 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -60,6 +60,11 @@ export function docHtmlPath(entry: DocEntry): string { return `/docs/${docSlug(entry)}/`; } +/** Path of the Markdown twin: same as the HTML path, .md in place of the slash. */ +export function docMarkdownPath(entry: DocEntry): string { + return `/docs/${docSlug(entry)}.md`; +} + /** Path of the synced file relative to apache/magpie/docs, original case preserved. */ export function docRelPath(entry: DocEntry): string { return entry.filePath ? entry.filePath.replace(/^.*\/content\/docs\//, "") : `${docSlug(entry)}.md`; @@ -93,3 +98,23 @@ export async function allDocs(): Promise { const entries = await getCollection("docs", (e) => !HIDDEN_DOCS.has(docSlug(e))); return entries.sort((a, b) => a.id.localeCompare(b.id)); } +<<<<<<< HEAD +======= + +/** + * The Markdown twin body. Mirrors the convention on iggy.apache.org: title, + * one-line description as a blockquote, the rendered and source URLs, then + * the body exactly as synced. If the body already opens with an H1 it becomes + * the title rather than being printed twice. + */ +export function docMarkdown(entry: DocEntry): string { + let body = (entry.body ?? "").trim(); + if (!entry.data.title && leadingHeading(body)) { + body = body.replace(/^#\s+.+?\s*(?:\n|$)/, "").trimStart(); + } + const lines = [`# ${docTitle(entry)}`, ""]; + if (entry.data.description) lines.push(`> ${entry.data.description}`, ""); + lines.push(`Rendered page: ${SITE_URL}${docHtmlPath(entry)}`, "", `Source: ${docSourceUrl(entry)}`, "", body, ""); + return lines.join("\n"); +} +>>>>>>> f03b2fc (Publish a Markdown copy of every docs page) diff --git a/src/pages/docs/[...slug].md.ts b/src/pages/docs/[...slug].md.ts new file mode 100644 index 00000000..251911b4 --- /dev/null +++ b/src/pages/docs/[...slug].md.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Publishes a Markdown copy of every docs page next to the rendered one, so +// /docs/modes/ is also available as /docs/modes.md. An agent that wants the +// page then never has to parse the HTML. Astro emits these as static files, so +// no prebuild script or server is involved. + +import type { APIRoute, GetStaticPaths } from "astro"; +import { allDocs, docMarkdown, docSlug } from "../../lib/site"; + +export const getStaticPaths: GetStaticPaths = async () => { + const docs = await allDocs(); + return docs.map((entry) => ({ + params: { slug: docSlug(entry) }, + props: { entry }, + })); +}; + +export const GET: APIRoute = ({ props }) => { + return new Response(docMarkdown(props.entry), { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + // Twins are alternates of the HTML page, not pages: keep them out of search results. + "X-Robots-Tag": "noindex", + }, + }); +}; diff --git a/src/pages/llms-full.txt.ts b/src/pages/llms-full.txt.ts new file mode 100644 index 00000000..7d8c8cd7 --- /dev/null +++ b/src/pages/llms-full.txt.ts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Every docs page concatenated, in sidebar order, separated by a rule. This is +// the same text the .md twins serve; agents that want one fetch use this one. + +import type { APIRoute } from "astro"; +import { SITE_NAME, SITE_URL, allDocs, docMarkdown } from "../lib/site"; + +export const GET: APIRoute = async () => { + const docs = await allDocs(); + const body = [ + `# ${SITE_NAME} — full documentation`, + "", + `Index: ${SITE_URL}/llms.txt`, + "", + ...docs.map((entry) => `---\n\n${docMarkdown(entry)}`), + ].join("\n"); + + return new Response(body, { + headers: { "Content-Type": "text/markdown; charset=utf-8" }, + }); +}; diff --git a/src/pages/llms.txt.ts b/src/pages/llms.txt.ts index 734b8d85..96f11aec 100644 --- a/src/pages/llms.txt.ts +++ b/src/pages/llms.txt.ts @@ -32,6 +32,7 @@ import { STATIC_ROUTES, allDocs, docHtmlPath, + docMarkdownPath, docTitle, } from "../lib/site"; @@ -51,6 +52,11 @@ source repository, or through an agent marketplace. - Website source: ${SITE_REPO} - Developer mailing list: ${DEV_LIST} (archive: https://lists.apache.org/list.html?${DEV_LIST}) +Every documentation page has a Markdown twin at the same path with a \`.md\` +suffix in place of the trailing slash, for example +${SITE_URL}/docs/modes/ is also ${SITE_URL}/docs/modes.md. HTML pages advertise +their twin with \`\`. Prefer the +twin: it is the synced source with no navigation. `; export const GET: APIRoute = async () => { @@ -62,7 +68,7 @@ export const GET: APIRoute = async () => { const docLines = docs.map((entry) => { const desc = entry.data.description ? `: ${entry.data.description}` : ""; - return `- [${docTitle(entry)}](${SITE_URL}${docHtmlPath(entry)})${desc}`; + return `- [${docTitle(entry)}](${SITE_URL}${docMarkdownPath(entry)})${desc} (rendered: ${SITE_URL}${docHtmlPath(entry)})`; }); const body = [ @@ -77,6 +83,7 @@ export const GET: APIRoute = async () => { "", "## Optional", "", + `- [Full documentation in one file](${SITE_URL}/llms-full.txt): every docs page concatenated, for agents that want one fetch.`, `- [Sitemap](${SITE_URL}/sitemap-index.xml)`, "", ].join("\n"); From ef74191e91270da461bdd73cbfe39e8a161ad560 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 07:35:51 +0000 Subject: [PATCH 7/8] Drop the body's H1 from the twin wherever it sits Generated-by: Claude (Fable 5.1) --- src/lib/site.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/site.ts b/src/lib/site.ts index c5000eab..2e2edecf 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -110,7 +110,7 @@ export async function allDocs(): Promise { export function docMarkdown(entry: DocEntry): string { let body = (entry.body ?? "").trim(); if (!entry.data.title && leadingHeading(body)) { - body = body.replace(/^#\s+.+?\s*(?:\n|$)/, "").trimStart(); + body = body.replace(/^#\s+.+?\s*$\n?/m, "").trimEnd(); } const lines = [`# ${docTitle(entry)}`, ""]; if (entry.data.description) lines.push(`> ${entry.data.description}`, ""); From b19e5441be8909395bcead06380a6e414c952eee Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 5 Sep 2026 18:29:41 +1000 Subject: [PATCH 8/8] Remove conflict markers from site.ts --- src/lib/site.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/site.ts b/src/lib/site.ts index 2e2edecf..bf0204c7 100644 --- a/src/lib/site.ts +++ b/src/lib/site.ts @@ -98,8 +98,6 @@ export async function allDocs(): Promise { const entries = await getCollection("docs", (e) => !HIDDEN_DOCS.has(docSlug(e))); return entries.sort((a, b) => a.id.localeCompare(b.id)); } -<<<<<<< HEAD -======= /** * The Markdown twin body. Mirrors the convention on iggy.apache.org: title, @@ -117,4 +115,3 @@ export function docMarkdown(entry: DocEntry): string { lines.push(`Rendered page: ${SITE_URL}${docHtmlPath(entry)}`, "", `Source: ${docSourceUrl(entry)}`, "", body, ""); return lines.join("\n"); } ->>>>>>> f03b2fc (Publish a Markdown copy of every docs page)