add remote running support for chrome distributed

Author: goelayu

system/node/chrome-distributed.js

@@ -61,6 +61,8 @@ program
   .option("--enableOPT", "Enables the entire system optimization pipeline")
   .option("--testing", "debug mode")
   .option("--azaddr <azaddr>", "addr for the az server")
+  .option("--id <id>", "id of the crawler")
+  .option("--remote", "remote mode")
   .parse(process.argv);
 
 var bashSanitize = (str) => {
@@ -101,17 +103,32 @@ var genBrowserArgs = (proxies) => {
         "--disable-setuid-sandbox",
         // "--blink-settings=scriptEnabled=false",
       ],
-    };
+    },
+    startHttpPort = 6080,
+    base = program.id * program.concurrency;
   program.testing && template.args.push("--auto-open-devtools-for-tabs");
   for (var i = 0; i < proxies.length; i++) {
     var proxy = proxies[i];
+    var httpAddr, httpsAddr;
+    if (program.remote) {
+      !program.id &&
+        console.log("FATAL: id is required in remote mode") &&
+        process.exit(1);
+
+      httpAddr = `lions.eecs.umich.edu:${startHttpPort + base + i}`;
+      httpsAddr = `lions.eecs.umich.edu:${startHttpPort + base + i + 1000}`;
+    } else {
+      httpAddr = `127.0.0.1:${proxy.http_port}`;
+      httpsAddr = `127.0.0.1:${proxy.https_port}`;
+    }
     var proxyFlags = [
-      `--host-resolver-rules=MAP *:80 127.0.0.1:${proxy.http_port},MAP *:443 127.0.0.1:${proxy.https_port},EXCLUDE localhost`,
+      `--host-resolver-rules=MAP *:80 ${httpAddr},MAP *:443 ${httpsAddr},EXCLUDE localhost`,
       // `--proxy-server=http=https://127.0.0.1:${proxy.https_port}`,
     ];
     var browserArgs = Object.assign({}, template);
     browserArgs.args = browserArgs.args.concat(proxyFlags);
     args.push(browserArgs);
+    console.log(proxyFlags);
   }
   // console.log(args)
   return args;
@@ -153,7 +170,8 @@ var genBrowserArgs = (proxies) => {
       `${program.output}/logs`,
       program.mode,
       program.enableOPT,
-      program.azaddr
+      program.azaddr,
+      program.remote
     );
     await proxyManager.createProxies();
     proxies = proxyManager.getAll();
@@ -196,13 +214,20 @@ var genBrowserArgs = (proxies) => {
       console.log("updating proxy path for ", pa, " to ", sanurl);
       process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
 
+      var host;
+      if (program.remote) {
+        host = "lions.eecs.umich.edu";
+      } else {
+        host = "127.0.0.1";
+      }
+
       var hr = await httpPromise(
-        `http://127.0.0.1:${pa - 1000}/update-archive-path?${
+        `http://${host}:${pa - 1000}/update-archive-path?${
           program.proxy
         }/${sanurl}.wprgo`
       );
       var hsr = await httpsPromise(
-        `https://127.0.0.1:${pa}/update-shared-object`
+        `https://${host}:${pa}/update-shared-object`
       );
 
       console.log(

system/node/chrome-server.js

@@ -161,7 +161,7 @@ async function main() {
     var url = req.url.slice(1);
     console.log(`Received request for ${url}`);
     console.log(`Pages loaded: ${++pagesload}`);
-    pagesload % 3 == 0 && cluster.queue(url);
+    cluster.queue(url);
     res.end("ok");
   });
   process.on("SIGINT", async () => {

system/node/lib/wpr-proxy.js

@@ -21,6 +21,7 @@ class Proxy {
     this.mode = options.mode;
     this.caching = options.caching;
     this.az_addr = options.az_addr;
+    this.remote = options.remote;
   }
 
   start() {
@@ -31,15 +32,21 @@ class Proxy {
     --az_addr ${this.az_addr}\
     ${this.caching ? "--caching" : ""}`;
     (this.stdout = ""), (this.stderr = "");
-    console.log(cmd);
-    //write dummy data to dataOutput before spawning command
-    this.process = child_process.spawn(cmd, { shell: true, cwd: WPRDIR });
-
-    var outStream = fs.createWriteStream(this.logOutput);
-    var errStream = fs.createWriteStream(this.logOutput);
-
-    this.process.stdout.pipe(outStream);
-    this.process.stderr.pipe(errStream);
+    if (!this.remote) {
+      console.log(cmd);
+      //write dummy data to dataOutput before spawning command
+      this.process = child_process.spawn(cmd, { shell: true, cwd: WPRDIR });
+
+      var outStream = fs.createWriteStream(this.logOutput);
+      var errStream = fs.createWriteStream(this.logOutput);
+
+      this.process.stdout.pipe(outStream);
+      this.process.stderr.pipe(errStream);
+    } else {
+      console.log(
+        `Running in remote mode, not starting proxy on ${this.http_port} and ${this.https_port}`
+      );
+    }
   }
 
   dump() {
@@ -59,7 +66,7 @@ class Proxy {
 }
 
 class ProxyManager {
-  constructor(nProxies, logDir, mode, caching, az_addr) {
+  constructor(nProxies, logDir, mode, caching, az_addr, remote) {
     this.nProxies = nProxies;
     this.proxies = [];
     this.startHttpsPort = 7080 + Math.floor(Math.random() * 1000);
@@ -68,6 +75,7 @@ class ProxyManager {
     this.mode = mode;
     this.caching = caching;
     this.az_addr = az_addr;
+    this.remote = remote;
   }
 
   async createProxies() {
@@ -78,13 +86,15 @@ class ProxyManager {
       var mode = this.mode;
       var caching = this.caching;
       var az_addr = this.az_addr;
+      var remote = this.remote;
       var p = new Proxy({
         http_port,
         https_port,
         logOutput,
         mode,
         caching,
         az_addr,
+        remote,
       });
       this.proxies.push(p);
     }