fix: comments being treated as errors (#152)

nuxt-modules · Oct 17, 2024 · ee973f1 · ee973f1
1 parent c895897
commit ee973f1
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 1 deletion.
diff --git a/src/runtime/util.ts b/src/runtime/util.ts
@@ -31,8 +31,9 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
   }
   let ln = -1
   // read the contents
-  for (const line of s.split('\n')) {
+  for (const _line of s.split('\n')) {
     ln++
+    const [line, comment] = _line.split('#').map(s => s.trim())
     const sepIndex = line.indexOf(':')
     // may not exist for comments
     if (sepIndex === -1)

diff --git a/test/fixtures/comments.txt b/test/fixtures/comments.txt
@@ -0,0 +1,4 @@
+# line comment
+
+User-agent: *   # 1st inline comment
+Allow: /        # 2nd inline comment
diff --git a/test/unit/robotsTxtParser.test.ts b/test/unit/robotsTxtParser.test.ts
@@ -355,4 +355,26 @@ Unknown: /bar
       ]
     `)
   })
+
+  it('comments do not error', async () => {
+    const robotsTxt = await fsp.readFile('./test/fixtures/comments.txt', { encoding: 'utf-8' })
+    expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
+      {
+        "errors": [],
+        "groups": [
+          {
+            "allow": [
+              "/",
+            ],
+            "comment": [],
+            "disallow": [],
+            "userAgent": [
+              "*",
+            ],
+          },
+        ],
+        "sitemaps": [],
+      }
+    `)
+  })
 })