chore(docs.angularjs.org): allow crawling of examples, don't deploy e2e test files

Author: Narretz

Gruntfile.js

@@ -341,7 +341,7 @@ module.exports = function(grunt) {
           },
           {
             cwd: 'build/docs',
-            src: '**',
+            src: ['**', '!ptore2e/**'],
             dest: 'deploy/docs/',
             expand: true
           },

docs/app/assets/robots.txt

@@ -1,13 +1,4 @@
 User-agent: *
 
-Disallow: /examples/
-Disallow: /ptore2e/
-Disallow: /Error404.html
-
-# The js / map files in the root are used by the embedded examples, not by the app itself
-Disallow: /*.js$
-Disallow: /*.map$
-
-# (Javascript) crawlers need to access JS files
-Allow: /components/*.js
-Allow: /js/*.js
+# The map files are not required by the app
+Disallow: /*.map$

scripts/docs.angularjs.org-firebase/firebase.json

@@ -29,7 +29,14 @@
     ],
     "headers": [
       {
-        "source": "/partials/**",
+        "source": "/Error404.html",
+        "headers" : [{
+          "key" : "X-Robots-Tag",
+          "value" : "noindex"
+        }]
+      },
+      {
+        "source": "/@(partials|examples)/**",
         "headers" : [{
           "key" : "X-Robots-Tag",
           "value" : "noindex"