Skip indexed repositories, better error handling

indexRepoInfo.js

@@ -15,13 +15,14 @@ getUnindexedRepositories()
   .then(crawlNextChunk);
 
 function crawlNextChunk(projects) {
+  console.log((new Date().toString()) + ' remaining to index: '  + projects.length);
   if (projects.length === 0) {
     console.log('All done. No more projects to index.');
     client.unref();
     return;
   }
 
-  processNext(projects, 10);
+  processNext(projects, 5);
 }
 
 function processNext(array, count) {
@@ -52,14 +53,31 @@ function saveRepositories(repositories) {
 }
 
 function getUnindexedRepositories() {
+  var totalIndexed = 0;
   return client.keysAsync('repo:*').then(function(keys) {
     return keys.map(toName);
+  })
+  .map(toNameIfNotExist, {concurrency: 3})
+  .filter(function(x) {
+    if (!x) totalIndexed += 1;
+    return x;
+  })
+  .then(function(repositories) {
+    console.log('Total indexed: ' + totalIndexed);
+    return repositories;
   });
 
   function toName(repo) {
     // remove 'repo:' prefix from the key:
     return repo.substr(5);
   }
+
+  function toNameIfNotExist(repo) {
+    return client.existsAsync('rinfo:' + repo)
+      .then(function(exists) {
+        return exists ? '' : repo;
+      });
+  }
 }
 
 function printError(repo) {

lib/githubClient.js

@@ -29,7 +29,7 @@ function githubClient(token) {
 
   function getRepositoriesInfo(repositories) {
     return Promise.map(repositories, toRequest, {
-      concurrency: 3
+      concurrency: 2
     });
     function toRequest(repoName) {
       var detailsRequest = createRequestArgs(REPO_DETAILS + repoName + '?' + tokenPart);
@@ -38,9 +38,9 @@ function githubClient(token) {
       return githubRequest(detailsRequest).then(extractRepoInfo).catch(handleError);
 
       function handleError(err) {
-        if (err && err.statusCode === 404) {
-          console.log('repository not found: ' + repoName);
-          return { name: repoName, error: 404 };
+        if (err) {
+          console.log('!!Failed to fetch ' + repoName + ': ' + err);
+          return { name: repoName, error: err.statusCode };
         }
         throw err;
       }

lib/githubRequest.js

@@ -23,6 +23,12 @@ function githubRequest(options, followNext) {
     if (reason.statusCode === 403) {
       failedCount = 0;
       var headers = reason.response.headers;
+      var body = JSON.parse(reason.response.body);
+      if (body.block) {
+        // This means the repository is blocked.
+        throw reason;
+      }
+      // Otherwise, we just ran out of rate limit
       return getRateLimitPromiseFromHeaders(headers);
     } else if (reason.statusCode == 404) {
       failedCount = 0;