fix: support Content-Signal

Fixes #487

Author: harlan-zw

docs/content/2.guides/1.robots-txt.md

@@ -60,13 +60,17 @@ The following rules are parsed from your `robots.txt` file:
 - `Disallow` - An array of paths to disallow for the user-agent.
 - `Allow` - An array of paths to allow for the user-agent.
 - `Sitemap` - An array of sitemap URLs to include in the generated sitemap.
-- `Content-Usage` - Content Signals for expressing AI usage preferences (see [Content Signals](#content-signals) below).
+- `Content-Usage` / `Content-Signal` - Directives for expressing AI usage preferences (see [Content Signals](#content-signals) below).
 
 This parsed data will be shown for environments that are `indexable`.
 
 ## Content Signals
 
-Content Signals allow you to express preferences about how AI systems should interact with your content using the `Content-Usage` directive.
+Content Signals allow you to express preferences about how AI systems should interact with your content. Both `Content-Usage` and `Content-Signal` directives are supported:
+
+### Content-Usage (IETF Standard)
+
+The `Content-Usage` directive follows the [IETF AI Preferences specification](https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/):
 
 ```txt [robots.txt]
 User-agent: *
@@ -76,7 +80,17 @@ Content-Usage: /public/ train-ai=y
 Content-Usage: /restricted/ ai=n train-ai=n
 ```
 
-See the emerging [IETF AI Preferences specification](https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/) for more details.
+### Content-Signal (Cloudflare Implementation)
+
+The `Content-Signal` directive is [Cloudflare's implementation](https://blog.cloudflare.com/content-signals-policy/), widely deployed across millions of domains:
+
+```txt [robots.txt]
+User-agent: *
+Allow: /
+Content-Signal: ai-train=no, search=yes, ai-input=yes
+```
+
+Both directives are parsed identically and output as `Content-Usage` in the generated robots.txt. Use whichever format matches your preferences or existing tooling.
 
 ## Conflicting `public/robots.txt`
 

src/util.ts

@@ -30,6 +30,7 @@ export { AiBots, NonHelpfulBots }
  * - disallow: a URL path that may not be crawled.
  * - sitemap: the complete URL of a sitemap.
  * - host: the host name of the site, this is optional non-standard directive.
+ * - content-usage / content-signal: AI content usage preferences (IETF spec / Cloudflare implementation).
  *
  * @see https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
  * @see https://github.com/google/robotstxt/blob/86d5836ba2d5a0b6b938ab49501be0e09d9c276c/robots.cc#L714C1-L720C2
@@ -110,6 +111,7 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
         }
         break
       case 'content-usage':
+      case 'content-signal':
         currentGroup.contentUsage = currentGroup.contentUsage || []
         currentGroup.contentUsage.push(val)
         break
@@ -315,7 +317,9 @@ export function generateRobotsTxt({ groups, sitemaps }: { groups: RobotsGroupRes
     for (const cleanParam of group.cleanParam || [])
       lines.push(`Clean-param: ${cleanParam}`)
 
-    // content signals / AI preferences (see https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/)
+    // content signals / AI preferences
+    // Both Content-Usage (IETF) and Content-Signal (Cloudflare) are accepted as input, output as Content-Usage
+    // See: https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/
     for (const contentUsage of group.contentUsage || [])
       lines.push(`Content-Usage: ${contentUsage}`)
 

test/unit/robotsTxtParser.test.ts

@@ -430,8 +430,37 @@ Content-Usage: /restricted/ ai=n train-ai=n
 User-Agent: *
 Content-Usage: invalid-preference
 Content-Usage: invalid-path ai=n
-Content-Usage: 
+Content-Usage:
     `
     expect(parseRobotsTxt(robotsTxt).errors).toEqual([])
   })
+
+  it('content-signal directive parsing', () => {
+    const robotsTxt = `
+User-Agent: *
+Allow: /
+Content-Signal: ai-train=no, search=yes, ai-input=yes
+    `
+    expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
+      {
+        "errors": [],
+        "groups": [
+          {
+            "allow": [
+              "/",
+            ],
+            "comment": [],
+            "contentUsage": [
+              "ai-train=no, search=yes, ai-input=yes",
+            ],
+            "disallow": [],
+            "userAgent": [
+              "*",
+            ],
+          },
+        ],
+        "sitemaps": [],
+      }
+    `)
+  })
 })