Authenticate requests to the GitHub API (optionally)

This will alleviate somewhat the rate limits imposed on unauthenticated
requests.  As of this writing, authenticated requests get 5,000
requests/hour per authenticated user (_not_ per token).  Unauthenticated
requests get only 60 requests/hour per IP.¹

Resolves nextstrain#93.

Other future changes to address rate limiting might include switching
from simple fetch()es to an API client that does dynamic throttling to
respect the rate limits.  Octokit does a great job of making this easy,
for example.²  Complications with using Octokit include figuring out how
to incorporate our current caching, or an equivalent. Caching is an
important part of reducing these requests.  Another complication is how
long to retry before returning a 500 error to the nextstrain.org
request; the example above retries indefinitely, but that's likely not
what we want for nextstrain.org.

¹ https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
² https://github.com/nextstrain/community-search/blob/ce1a4146878e8336c442a175033ad242621f5a17/find#L2-L25

docs/infrastructure.md

@@ -33,6 +33,11 @@ Deploys of `master` happen automatically after Travis CI tests are successful.
 
   - `PAPERTRAIL_API_TOKEN` is used for logging through [papertrail](https://elements.heroku.com/addons/papertrail).
 
+  - `GITHUB_TOKEN` is used to make authenticated requests to the GitHub API so that we get increased rate limits.
+    The token should have public access only, i.e. no permission scopes granted (including `public_repo`, which grants write access to public repos).
+    If not provided, requests to the GitHub API are unauthenticated.
+    The token we use in production is associated with the `nextstrain-bot` GitHub user.
+
 ### Redis add-on
 
 The [Heroku Redis](https://elements.heroku.com/addons/heroku-redis) add-on is attached to our `nextstrain-server` and `nextstrain-dev` apps.

src/sources/community.js

@@ -5,6 +5,10 @@ const {NotFound} = require('http-errors');
 const utils = require("../utils");
 const {Source, Dataset, Narrative} = require("./models");
 
+const authorization = process.env.GITHUB_TOKEN
+  ? `token ${process.env.GITHUB_TOKEN}`
+  : "";
+
 class CommunitySource extends Source {
   constructor(owner, repoName) {
     super();
@@ -19,7 +23,7 @@ class CommunitySource extends Source {
 
     if (!this.repoName) throw new Error(`Cannot construct a ${this.constructor.name} without a repoName after splitting on /@/`);
 
-    this.defaultBranch = fetch(`https://api.github.com/repos/${this.owner}/${this.repoName}`)
+    this.defaultBranch = fetch(`https://api.github.com/repos/${this.owner}/${this.repoName}`, {headers: {authorization}})
       .then((res) => res.json())
       .then((data) => data.default_branch)
       .catch(() => {
@@ -55,7 +59,7 @@ class CommunitySource extends Source {
 
   async availableDatasets() {
     const qs = queryString.stringify({ref: await this.branch});
-    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents/auspice?${qs}`);
+    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents/auspice?${qs}`, {headers: {authorization}});
 
     if (response.status === 404) throw new NotFound();
     else if (response.status !== 200 && response.status !== 304) {
@@ -77,7 +81,7 @@ class CommunitySource extends Source {
 
   async availableNarratives() {
     const qs = queryString.stringify({ref: await this.branch});
-    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents/narratives?${qs}`);
+    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents/narratives?${qs}`, {headers: {authorization}});
 
     if (response.status !== 200 && response.status !== 304) {
       if (response.status !== 404) {

src/sources/core.js

@@ -4,6 +4,10 @@ const {NotFound} = require('http-errors');
 const utils = require("../utils");
 const {Source} = require("./models");
 
+const authorization = process.env.GITHUB_TOKEN
+  ? `token ${process.env.GITHUB_TOKEN}`
+  : "";
+
 class CoreSource extends Source {
   static get _name() { return "core"; }
   async baseUrl() { return "http://data.nextstrain.org/"; }
@@ -30,7 +34,7 @@ class CoreSource extends Source {
 
   async availableNarratives() {
     const qs = queryString.stringify({ref: this.branch});
-    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents?${qs}`);
+    const response = await fetch(`https://api.github.com/repos/${this.repo}/contents?${qs}`, {headers: {authorization}});
 
     if (response.status === 404) throw new NotFound();
     else if (response.status !== 200 && response.status !== 304) {