feat(SEO): allow Browsershots, disallow /thanks in robots.txt

GaiAma · Oct 6, 2018 · 8db3280 · 8db3280
1 parent 962fcb8
commit 8db3280
Showing 1 changed file with 9 additions and 3 deletions.
diff --git a/gatsby-node.js b/gatsby-node.js
@@ -344,9 +344,15 @@ exports.onPostBuild = () => {
   }
 
   // add robots.txt to site root depending on $BRANCH env var
-  const robotsTxt = `User-agent: *\nDisallow:${isMaster ? `` : ` /`}`
-  console.log(`$BRANCH`, BRANCH, isMaster, robotsTxt)
-  writeFileSync(join(publicDir, `robots.txt`), robotsTxt)
+  const robotsTxt = []
+  if (isMaster) {
+    robotsTxt.push(`User-agent: *\nDisallow:`)
+  } else {
+    robotsTxt.push(`User-agent: *\nDisallow: /`)
+  }
+  robotsTxt.push(`User-agent: *\nDisallow: /de/danke\nDisallow: /en/thanks`)
+  robotsTxt.push(`User-agent: Browsershots\nDisallow:`)
+  writeFileSync(join(publicDir, `robots.txt`), robotsTxt.join(`\n\n`))
 }
 
 /**