Remove clean markdown generation and artifacts.

Stop generating and committing clean markdown files in CI, simplify markdown serving/indexing to use source content directly, and remove .md URL suffixes from llms output and markdown alternates for the current main behavior.

Made-with: Cursor

Author: jottakka

.github/workflows/generate-markdown.yml

@@ -2,22 +2,15 @@ name: Generate LLMs.txt
 
 on:
   workflow_dispatch:
-  pull_request:
-    types: [opened, synchronize, reopened]
 
 permissions:
-  contents: write
-  pull-requests: write
+  contents: read
 
 jobs:
   llmstxt:
     name: Generate LLMSTXT
     runs-on: ubuntu-latest
 
-    permissions:
-      contents: write
-      pull-requests: write
-
     steps:
       - name: Use Node.js
         uses: actions/setup-node@v4
@@ -27,7 +20,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+          ref: ${{ github.ref }}
           token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
 
       - name: Install dependencies
@@ -40,41 +33,3 @@ jobs:
         run: pnpm llmstxt
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
-      - name: Check for changes
-        id: check-changes
-        run: |
-          if [ -n "$(git status --porcelain)" ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
-          else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Commit changes to PR
-        if: steps.check-changes.outputs.has_changes == 'true' && github.event_name == 'pull_request'
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add public/llms.txt
-          git commit -m "🤖 Regenerate LLMs.txt"
-          git push
-
-      - name: Create Pull Request (for scheduled/manual runs)
-        if: steps.check-changes.outputs.has_changes == 'true' && github.event_name != 'pull_request'
-        id: cpr
-        uses: peter-evans/create-pull-request@v7
-        with:
-          token: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}
-          commit-message: Regenerate LLMs.txt and related files
-          branch: auto-update-llms-txt
-          delete-branch: true
-          title: "🤖 Regenerate LLMs.txt"
-          reviewers: >
-            evantahler
-            torresmateo
-
-      - name: Enable Pull Request Automerge
-        if: steps.check-changes.outputs.has_changes == 'true' && github.event_name != 'pull_request'
-        run: gh pr merge --squash --auto ${{ steps.cpr.outputs.pull-request-number }}
-        env:
-          GH_TOKEN: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }}

.github/workflows/llmstxt.yml

@@ -15,6 +15,7 @@ pnpm vale:check       # Check docs against style rules
 ```
 
 Run a single test:
+
 ```bash
 pnpm vitest run tests/broken-link-check.test.ts
 ```
@@ -26,7 +27,7 @@ pnpm vitest run tests/broken-link-check.test.ts
 - **`app/_lib/`** — Data-fetching utilities (toolkit catalog, slug generation, static params).
 - **`app/api/`** — API routes (markdown export, toolkit-data, glossary).
 - **`toolkit-docs-generator/`** — Generates MCP toolkit documentation from server metadata JSON files in `toolkit-docs-generator/data/toolkits/`.
-- **`scripts/`** — Build/CI scripts (clean markdown export, Vale style fixes, redirect checking, pagefind indexing, i18n sync).
+- **`scripts/`** — Build/CI scripts (Vale style fixes, redirect checking, pagefind indexing, i18n sync).
 - **`tests/`** — Vitest tests (broken links, internal link validation, sitemap, smoke tests).
 - **`lib/`** — Next.js utilities (glossary remark plugin, llmstxt plugin).
 - **`next.config.ts`** — Contains ~138 redirect rules.

CLAUDE.md

@@ -10,14 +10,14 @@ const DROPDOWN_IDENTIFIER = "Markdown for LLMs";
 
 /**
  * This component overrides the default nextra-theme-docs "Copy page" button behavior
- * to fetch clean markdown from our API instead of copying raw MDX source.
+ * to fetch markdown from our API instead of copying raw MDX source.
  */
 export function CopyPageOverride() {
   const pathname = usePathname();
 
   const fetchAndCopyMarkdown = useCallback(async (): Promise<boolean> => {
     try {
-      const markdownUrl = `/api/markdown${pathname}.md`;
+      const markdownUrl = `/api/markdown${pathname}`;
       const response = await fetch(markdownUrl);
 
       if (!response.ok) {

app/_components/copy-page-override.tsx

@@ -60,7 +60,7 @@ function CopyPageButton() {
 
     setLoading(true);
     try {
-      const response = await fetch(`/api/markdown${pathname}.md`);
+      const response = await fetch(`/api/markdown${pathname}`);
       if (!response.ok) {
         throw new Error("Failed to fetch markdown");
       }

app/_components/toolkit-docs/components/page-actions.tsx

@@ -10,9 +10,6 @@ const MD_EXTENSION_REGEX = /\.md$/;
 const TOOLKIT_MARKDOWN_ROOT = join(process.cwd(), "public", "toolkit-markdown");
 const APP_ROOT = join(process.cwd(), "app");
 
-// Directory containing pre-generated clean markdown files
-const CLEAN_MARKDOWN_DIR = join(process.cwd(), "public", "_markdown");
-
 /**
  * Validates that a resolved path is within the allowed directory.
  * Prevents path traversal attacks (e.g., ../../../etc/passwd).
@@ -108,39 +105,6 @@ type ToolkitMarkdownTarget = {
   toolkitId: string;
 };
 
-/**
- * Try to serve clean pre-generated markdown.
- * Returns NextResponse if found, null otherwise.
- */
-async function tryServeCleanMarkdown(
-  request: NextRequest,
-  sanitizedPath: string
-): Promise<NextResponse | null> {
-  const cleanMarkdownPath = join(CLEAN_MARKDOWN_DIR, `${sanitizedPath}.md`);
-
-  try {
-    await access(cleanMarkdownPath);
-    if (!isPathWithinDirectory(cleanMarkdownPath, CLEAN_MARKDOWN_DIR)) {
-      return null;
-    }
-
-    const content = await readFile(cleanMarkdownPath, "utf-8");
-    await trackMarkdownRequest(request, sanitizedPath);
-
-    return new NextResponse(content, {
-      status: 200,
-      headers: {
-        "Content-Type": "text/markdown; charset=utf-8",
-        "Content-Disposition": "inline",
-        "Cache-Control": "public, max-age=3600",
-        Vary: "Accept, User-Agent",
-      },
-    });
-  } catch {
-    return null;
-  }
-}
-
 /**
  * Check if a path matches the toolkit documentation pattern.
  * Handles both actual toolkit IDs and the [toolkitId] dynamic route pattern.
@@ -248,13 +212,7 @@ export async function GET(
         filePath = join(APP_ROOT, `${sanitizedPath}/page.mdx`);
       }
     } else {
-      // Try clean markdown first (preferred)
-      const cleanResponse = await tryServeCleanMarkdown(request, sanitizedPath);
-      if (cleanResponse) {
-        return cleanResponse;
-      }
-
-      // Fallback: raw MDX file
+      // Serve raw MDX file.
       filePath = join(APP_ROOT, `${sanitizedPath}/page.mdx`);
     }
 

app/api/markdown/[[...slug]]/route.ts

@@ -5,7 +5,7 @@ description: "Learn how to speed up your development with agents in your IDEs"
 
 # Agentic Development
 
-Every page on the Arcade docs site renders as clean markdown. When an AI agent or coding assistant visits any docs URL, the site automatically returns `Content-Type: text/markdown` instead of HTML if:
+Every page on the Arcade docs site can be served as markdown. When an AI agent or coding assistant visits any docs URL, the site automatically returns `Content-Type: text/markdown` instead of HTML if:
 
 - The request `User-Agent` header matches a known AI agent (Claude, ChatGPT, Cursor, etc.)
 - The request includes the `Accept: text/markdown` header

app/en/get-started/setup/connect-arcade-docs/page.mdx

@@ -22,15 +22,16 @@ import {
 } from "nextra-theme-docs";
 
 const REGEX_LOCALE = /^\/([a-z]{2}(?:-[A-Z]{2})?)(?:\/|$)/;
+const ENABLE_MARKDOWN_ALTERNATE = false;
 
 function getMarkdownAlternatePath(pathname: string): string {
   // Handle root paths
   if (pathname === "/" || pathname === "") {
-    return "/index.md";
+    return "/";
   }
-  // Remove trailing slash if present, then add .md extension
+  // Remove trailing slash if present.
   const cleanPath = pathname.endsWith("/") ? pathname.slice(0, -1) : pathname;
-  return `${cleanPath}.md`;
+  return cleanPath;
 }
 
 export async function generateMetadata() {
@@ -77,11 +78,13 @@ export async function generateMetadata() {
     appleWebApp: {
       title: "Arcade Documentation",
     },
-    alternates: {
-      types: {
-        "text/markdown": getMarkdownAlternatePath(pathname),
-      },
-    },
+    alternates: ENABLE_MARKDOWN_ALTERNATE
+      ? {
+          types: {
+            "text/markdown": getMarkdownAlternatePath(pathname),
+          },
+        }
+      : undefined,
     other: {
       "apple-mobile-web-app-title": "Arcade Documentation",
       "twitter:url": "https://docs.arcade.dev",

app/layout.tsx

@@ -101,15 +101,15 @@ function getLocaleFromPathname(pathname: string, request: NextRequest): string {
 
 function buildMarkdownPath(pathname: string, locale: string): string {
   if (pathname === "/" || pathname === "") {
-    return `/${locale}/home.md`;
+    return `/${locale}/home`;
   }
   if (pathnameIsMissingLocale(pathname)) {
-    return `/${locale}${pathname}.md`;
+    return `/${locale}${pathname}`;
   }
   if (SUPPORTED_LOCALES.some((loc) => pathname === `/${loc}`)) {
-    return `${pathname}/home.md`;
+    return `${pathname}/home`;
   }
-  return `${pathname}.md`;
+  return pathname;
 }
 
 function isToolkitDetailPath(pathname: string): boolean {

middleware.ts

@@ -11,8 +11,7 @@
     "format": "pnpm exec ultracite fix",
     "prepare": "husky install",
     "toolkit-markdown": "pnpm dlx tsx toolkit-docs-generator/scripts/generate-toolkit-markdown.ts",
-    "postbuild": "if [ \"$SKIP_POSTBUILD\" != \"true\" ]; then pnpm run generate:markdown && pnpm run custompagefind; fi",
-    "generate:markdown": "pnpm dlx tsx scripts/generate-clean-markdown.ts",
+    "postbuild": "if [ \"$SKIP_POSTBUILD\" != \"true\" ]; then pnpm run custompagefind; fi",
     "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format",
     "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts",
     "custompagefind": "pnpm dlx tsx scripts/pagefind.ts",