Import projects descriptions into redis

05_export_description_stream.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# This script will export data saved by `create_description_stream.sh`
+# into gzipped CSV file in google storage
+
+source ./scripts_config
+
+echo "Exporting data from $DESCRIPTION_TABLE into $DESCRIPTION_BUCKET"
+
+bq --project_id $PROJECT_ID \
+  extract --compression=GZIP $DESCRIPTION_TABLE $DESCRIPTION_BUCKET

06_download_description_stream.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+source ./scripts_config
+
+echo "Downloading projects description from $DESCRIPTION_BUCKET"
+gsutil cp $DESCRIPTION_BUCKET ./
+
+echo "Extracting watchers information"
+gunzip $DESCRIPTION_FILE

07_import_description_stream_to_redis.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+source ./scripts_config
+
+echo "Importing projects information from $PLAIN_DESCRIPTION_FILE to redis"
+
+node ./import_description_to_redis.js $PLAIN_DESCRIPTION_FILE

import_description_to_redis.js

@@ -0,0 +1,47 @@
+/**
+ * This script imports CSV file with projects information into redis
+ *
+ * Projects information is stored as hashes, where keys is "desc:" + project name
+ * and value is { description: string, watchers: number }
+ */
+
+var fileName = process.argv[2];
+var fs = require('fs');
+
+if (!fs.existsSync(fileName)) {
+  throw new Error('Cannot find input file with csv data: ' + fileName);
+}
+
+var inputFile = require('fs').createReadStream('./description.csv'),
+  redis = require("redis"),
+  client = redis.createClient(),
+  csv = require('csv-parse'),
+  parser = csv();
+
+var processed = 0;
+parser.on('readable', saveLine);
+parser.on('end', function() {
+  client.unref();
+});
+
+inputFile.pipe(parser);
+
+function saveLine() {
+  var line = parser.read();
+  processed += 1;
+  if (processed % 10000 === 0) console.log('Saved: ', processed);
+  var repo = line[0];
+  var splitIndex = repo.indexOf('/');
+  if (splitIndex <= 0) return; // Ignore invalid repositories
+
+  client.hmset('desc:' + repo,
+    'description', line[1],
+    'watchers', line[2],
+    printError(repo));
+}
+
+function printError(name) {
+  return function(err, res) {
+    if (err) console.log('!! Failed to save ' + name, err);
+  };
+}