followersIndexer.js

/**
 * GitHub popuplar projects followers indexer. Based on list of popular repositories
 * finds all users who gave stars to these repositories.
 *
 * Usage:
 *   node followersIndexer.js ./repositories.json ./db/followers --tokens="A,B,C"
 *
 * Where:
 *   - repositories.json: file generated by repoIndexer.js
 *   - followers: leveldb name where to save found followers
 */

var tokens = require('./lib/tokens')();
var fs = require('fs');

if (tokens.enabled === 0) {
  printTokenHelp();
  return -1;
}

var githubClient = require('./lib/ghclient')(tokens);
var repositoriesFileName = process.argv[2];
var inputArgumentsValid = fs.existsSync(repositoriesFileName);

if (!inputArgumentsValid) {
  printInputArgumentsHelp();
  return -2;
}

var allRepositories = getIndexedRepositories(repositoriesFileName);
var processedRepositoriesName = process.argv[3];
var db = require('./lib/ldb')(processedRepositoriesName);

db.getAllKeys()
  .then(function (processedRepositoriesKeys) {
    var processedRepositories = {};
    processedRepositoriesKeys.forEach(function (key) { processedRepositories[key] = 1; });

    var remainingRepositories = getRemainingRepositories(allRepositories, processedRepositories);
    printStats(allRepositories, processedRepositories, remainingRepositories);

    var indexFollowers = require('./lib/indexFollowers');
    indexFollowers(remainingRepositories, db, githubClient);
  });

function printTokenHelp() {
  [
    'Github access token is not present in environment variables',
    'Go to https://github.com/settings/applications and click "Create new token"',
    'Pass tokens as a comma-separated argument --tokens="A,B,C"'
  ].forEach(function (line) { console.log(line); });
}

function printInputArgumentsHelp() {
  [
    'GitHub popuplar projects followers indexer. Based on list of popular repositories',
    'finds all users who gave stars to these repositories.',
    '',
    'Usage:',
    '  node followersIndexer.js ./repositories.json ./db/followers --tokens="A,B,C"',
    '',
    'Where:',
    '  - repositories.json: file generated by repoIndexer.js',
    '  - followers: leveldb name where to save found followers'
  ].forEach(function (line) { console.log(line); });
}

function printStats(allRepositories, processedRepositories, remainingRepositories) {
  console.log('Statistics:');
  console.log('  Total:', allRepositories.length);
  console.log('  Processed:', Object.keys(processedRepositories).length);
  console.log('  Remaining:', remainingRepositories.length);
}

function readJson(fileName) {
  return JSON.parse(fs.readFileSync(fileName, 'utf8'));
}

function getIndexedRepositories(repositoriesFileName) {
  console.log('Reading repositories file', repositoriesFileName);
  var repositories = readJson(repositoriesFileName);
  console.log('Read', repositories.length, 'repositories');
  return repositories;
}

function getRemainingRepositories(allRepositories, indexedRepositories) {
  return allRepositories.filter(function (x) { return !indexedRepositories[x.name]; });
}