Add sections to llms.txt and sitemap footer to *.md (#8270)

* Add sections to llms.txt

* Also add sitemap footer

Author: rickhanlonii

src/pages/api/md/[...path].ts

@@ -9,6 +9,14 @@ import type {NextApiRequest, NextApiResponse} from 'next';
 import fs from 'fs';
 import path from 'path';
 
+const FOOTER = `
+---
+
+## Sitemap
+
+[Overview of all docs pages](/llms.txt)
+`;
+
 export default function handler(req: NextApiRequest, res: NextApiResponse) {
   const pathSegments = req.query.path;
   if (!pathSegments) {
@@ -35,7 +43,7 @@ export default function handler(req: NextApiRequest, res: NextApiResponse) {
       const content = fs.readFileSync(fullPath, 'utf8');
       res.setHeader('Content-Type', 'text/plain; charset=utf-8');
       res.setHeader('Cache-Control', 'public, max-age=3600');
-      return res.status(200).send(content);
+      return res.status(200).send(content + FOOTER);
     } catch {
       // Try next candidate
     }

src/pages/llms.txt.tsx

@@ -9,45 +9,195 @@ import type {GetServerSideProps} from 'next';
 import {siteConfig} from '../siteConfig';
 import sidebarLearn from '../sidebarLearn.json';
 import sidebarReference from '../sidebarReference.json';
-import sidebarBlog from '../sidebarBlog.json';
 
 interface RouteItem {
   title?: string;
   path?: string;
   routes?: RouteItem[];
+  hasSectionHeader?: boolean;
+  sectionHeader?: string;
 }
 
 interface Sidebar {
   title: string;
   routes: RouteItem[];
 }
 
-function extractRoutes(
+interface Page {
+  title: string;
+  url: string;
+}
+
+interface SubGroup {
+  heading: string;
+  pages: Page[];
+}
+
+interface Section {
+  heading: string | null;
+  pages: Page[];
+  subGroups: SubGroup[];
+}
+
+// Clean up section header names (remove version placeholders)
+function cleanSectionHeader(header: string): string {
+  return header
+    .replace(/@\{\{version\}\}/g, '')
+    .replace(/-/g, ' ')
+    .split(' ')
+    .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+    .join(' ')
+    .trim();
+}
+
+// Extract routes for sidebars that use hasSectionHeader to define major sections
+// (like the API Reference sidebar)
+function extractSectionedRoutes(
   routes: RouteItem[],
   baseUrl: string
-): {title: string; url: string}[] {
-  const result: {title: string; url: string}[] = [];
+): Section[] {
+  const sections: Section[] = [];
+  let currentSection: Section | null = null;
 
   for (const route of routes) {
-    if (route.title && route.path) {
-      result.push({
+    // Skip external links
+    if (route.path?.startsWith('http')) {
+      continue;
+    }
+
+    // Start a new section when we hit a section header
+    if (route.hasSectionHeader && route.sectionHeader) {
+      if (currentSection) {
+        sections.push(currentSection);
+      }
+      currentSection = {
+        heading: cleanSectionHeader(route.sectionHeader),
+        pages: [],
+        subGroups: [],
+      };
+      continue;
+    }
+
+    // If no section started yet, skip
+    if (!currentSection) {
+      continue;
+    }
+
+    // Route with children - create a sub-group
+    if (route.title && route.routes && route.routes.length > 0) {
+      const subGroup: SubGroup = {
+        heading: route.title,
+        pages: [],
+      };
+
+      // Include parent page if it has a path
+      if (route.path) {
+        subGroup.pages.push({
+          title: route.title,
+          url: `${baseUrl}${route.path}.md`,
+        });
+      }
+
+      // Add child pages
+      for (const child of route.routes) {
+        if (child.title && child.path && !child.path.startsWith('http')) {
+          subGroup.pages.push({
+            title: child.title,
+            url: `${baseUrl}${child.path}.md`,
+          });
+        }
+      }
+
+      if (subGroup.pages.length > 0) {
+        currentSection.subGroups.push(subGroup);
+      }
+    }
+    // Single page without children
+    else if (route.title && route.path) {
+      currentSection.pages.push({
         title: route.title,
         url: `${baseUrl}${route.path}.md`,
       });
     }
-    if (route.routes) {
-      result.push(...extractRoutes(route.routes, baseUrl));
+  }
+
+  // Don't forget the last section
+  if (currentSection) {
+    sections.push(currentSection);
+  }
+
+  return sections;
+}
+
+// Extract routes for sidebars that use routes with children as the primary grouping
+// (like the Learn sidebar)
+function extractGroupedRoutes(
+  routes: RouteItem[],
+  baseUrl: string
+): SubGroup[] {
+  const groups: SubGroup[] = [];
+
+  for (const route of routes) {
+    // Skip section headers
+    if (route.hasSectionHeader) {
+      continue;
+    }
+
+    // Skip external links
+    if (route.path?.startsWith('http')) {
+      continue;
+    }
+
+    // Route with children - create a group
+    if (route.title && route.routes && route.routes.length > 0) {
+      const pages: Page[] = [];
+
+      // Include parent page if it has a path
+      if (route.path) {
+        pages.push({
+          title: route.title,
+          url: `${baseUrl}${route.path}.md`,
+        });
+      }
+
+      // Add child pages
+      for (const child of route.routes) {
+        if (child.title && child.path && !child.path.startsWith('http')) {
+          pages.push({
+            title: child.title,
+            url: `${baseUrl}${child.path}.md`,
+          });
+        }
+      }
+
+      if (pages.length > 0) {
+        groups.push({
+          heading: route.title,
+          pages,
+        });
+      }
+    }
+    // Single page without children - group under its own heading
+    else if (route.title && route.path) {
+      groups.push({
+        heading: route.title,
+        pages: [
+          {
+            title: route.title,
+            url: `${baseUrl}${route.path}.md`,
+          },
+        ],
+      });
     }
   }
 
-  return result;
+  return groups;
 }
 
-const sidebars: Sidebar[] = [
-  sidebarLearn as Sidebar,
-  sidebarReference as Sidebar,
-  sidebarBlog as Sidebar,
-];
+// Check if sidebar uses section headers as primary grouping
+function usesSectionHeaders(routes: RouteItem[]): boolean {
+  return routes.some((r) => r.hasSectionHeader && r.sectionHeader);
+}
 
 export const getServerSideProps: GetServerSideProps = async ({res}) => {
   const subdomain =
@@ -60,14 +210,48 @@ export const getServerSideProps: GetServerSideProps = async ({res}) => {
     '> The library for web and native user interfaces.',
   ];
 
+  const sidebars: Sidebar[] = [
+    sidebarLearn as Sidebar,
+    sidebarReference as Sidebar,
+  ];
+
   for (const sidebar of sidebars) {
     lines.push('');
     lines.push(`## ${sidebar.title}`);
-    lines.push('');
 
-    const routes = extractRoutes(sidebar.routes, baseUrl);
-    for (const route of routes) {
-      lines.push(`- [${route.title}](${route.url})`);
+    if (usesSectionHeaders(sidebar.routes)) {
+      // API Reference style: section headers define major groups
+      const sections = extractSectionedRoutes(sidebar.routes, baseUrl);
+      for (const section of sections) {
+        if (section.heading) {
+          lines.push('');
+          lines.push(`### ${section.heading}`);
+        }
+
+        // Output pages directly under section
+        for (const page of section.pages) {
+          lines.push(`- [${page.title}](${page.url})`);
+        }
+
+        // Output sub-groups with #### headings
+        for (const subGroup of section.subGroups) {
+          lines.push('');
+          lines.push(`#### ${subGroup.heading}`);
+          for (const page of subGroup.pages) {
+            lines.push(`- [${page.title}](${page.url})`);
+          }
+        }
+      }
+    } else {
+      // Learn style: routes with children define groups
+      const groups = extractGroupedRoutes(sidebar.routes, baseUrl);
+      for (const group of groups) {
+        lines.push('');
+        lines.push(`### ${group.heading}`);
+        for (const page of group.pages) {
+          lines.push(`- [${page.title}](${page.url})`);
+        }
+      }
     }
   }