A dep change (#1520)

* Remove *fakes3* dep and all associated Ruby/Rails for pure JavaScript implementation
* New dev dep *S3rver* handles what we need with basic i/o for S3 on dev. Please read their homepage
* Leaving *fakeS3* `.gitignore`'s in for current data path... please delete your copies as this should eventually go away... or you can manually migrate it yourself perhaps. I'll just reimport from GH the sources on dev which is a better use of my time.
* Fix a potential `undefined` I put in trap a while back with a generic string.
* Change all docs to accommodate this
* Moved a native dependency up top (the TODO)... code style conformance
* DELETE OP RETESTED... PASS

NOTE:
* Contributors, and up, should no longer need to launch the server as it's launched in dev automatically and should terminate with a SIGINT (app exit). Remote local hosts still supported although I've never used that myself.

Closes #1519

Auto-merge

Author: Martii

.github/CONTRIBUTING.md

@@ -10,8 +10,6 @@ This project uses [editor config](http://editorconfig.org/), please make sure to
 * [Git](http://git-scm.com/)
 * [node.js](http://nodejs.org/) *(see [`./package.json`](https://github.com/OpenUserJs/OpenUserJS.org/blob/master/package.json) engines for specific requirements)*
 * [MongoDB](http://www.mongodb.org/) (Optional.  The project is preconfigured to use a dev DB on [MongoLab](https://mongolab.com/).)
-* [Ruby](https://www.ruby-lang.org/) (required to run [FakeS3](https://github.com/jubos/fake-s3/))
-* [FakeS3](https://github.com/jubos/fake-s3) (required to store libraries/scripts without [AWS S3](http://aws.amazon.com/s3/)) handled by [bundler](https://github.com/bundler/bundler)
 
 #### GitHub Fork Setup
 

.gitignore

@@ -20,6 +20,8 @@ data
 package-lock.json
 Gemfile.lock
 
+S3rver
+
 fakeS3
 dev/fakeS3
 

Gemfile

@@ -6,7 +6,6 @@ Repository | Reference | Recent Version
 :--- | :---: | :---
 [nodejs][nodeGHUrl] | [Documentation][nodejsDOCUrl] | <a href="https://raw.githubusercontent.com/nodejs/Release/master/schedule.png"><img src="https://raw.githubusercontent.com/nodejs/Release/master/schedule.png" alt="Current release schedule" title="Click to enlarge... Current release schedule" width="96" height="50"/></a> [CHANGELOG][nodejsReleasesUrl]
 [npm][npmGHUrl] | [Documentation][npmDOCUrl] | [![npm][npmNPMVersionImage]][npmNPMUrl] [CHANGELOG][npmGHReleasesUrl]
-[bundler][bundlerGHUrl] | [Documentation][bundlerDOCUrl] | [![GEM version][bundlerGEMVersionImage]][bundlerGEMUrl] [CHANGELOG][bundlerGHReleasesUrl]
 
 ### Contributing
 
@@ -38,7 +37,6 @@ Repository | Reference | Recent Version
 [express-brute-mongo][express-brute-mongoGHUrl]  <br />&#x22D4; [`MongoDBv3.x`][express-brute-mongoGHMongoDBv3.xUrl] | [Documentation][express-brute-mongoDOCUrl] | [![NPM version][express-brute-mongoNPMVersionImage]][express-brute-mongoNPMUrl]
 [express-minify][express-minifyGHUrl] | [Documentation][express-minifyDOCUrl] | [![NPM version][express-minifyNPMVersionImage]][express-minifyNPMUrl]
 [express-session][express-sessionGHUrl] | [Documentation][express-sessionDOCUrl] | [![NPM version][express-sessionNPMVersionImage]][express-sessionNPMUrl]
-[fakes3][fakes3GHUrl] | [Documentation][fakes3DOCUrl] | [![GEM version][fakes3GEMVersionImage]][fakes3GEMUrl]
 [font-awesome][font-awesomeGHUrl] | [Documentation][font-awesomeDOCUrl] | [![NPM version][font-awesomeNPMVersionImage]][font-awesomeNPMUrl]
 [formidable][formidableGHUrl] | [Documentation][formidableDOCUrl] | [![NPM version][formidableNPMVersionImage]][formidableNPMUrl]
 [git-rev][git-revGHUrl] | [Documentation][git-revDOCUrl] | [![NPM version][git-revNPMVersionImage]][git-revNPMUrl]
@@ -75,6 +73,7 @@ Repository | Reference | Recent Version
 [pegjs][pegjsGHUrl] | [Documentation][pegjsDOCUrl] | [![NPM version][pegjsNPMVersionImage]][pegjsNPMUrl]
 [request][requestGHUrl] | [Documentation][requestDOCUrl] | [![NPM version][requestNPMVersionImage]][requestNPMUrl]
 [rfc2047][rfc2047GHUrl] | [Documentation][rfc2047DOCUrl] | [![NPM version][rfc2047NPMVersionImage]][rfc2047NPMUrl]
+[S3rver][s3rverGHUrl] | [Documentation][s3rverDOCUrl] | [![NPM version][s3rverNPMVersionImage]][s3rverNPMUrl]
 [sanitize-html][sanitize-htmlGHUrl] | [Documentation][sanitize-htmlDOCUrl] | [![NPM version][sanitize-htmlNPMVersionImage]][sanitize-htmlNPMUrl]
 [select2][select2GHUrl] | [Documentation][select2DOCUrl] | [![NPM version][select2NPMVersionImage]][select2NPMUrl]
 [select2-bootstrap-css][select2-bootstrap-cssGHUrl] | [Documentation][select2-bootstrap-cssDOCUrl] | [![NPM version][select2-bootstrap-cssNPMVersionImage]][select2-bootstrap-cssNPMUrl]
@@ -166,12 +165,6 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated`
 [bootstrap-markdownNPMUrl]: https://www.npmjs.com/package/bootstrap-markdown
 [bootstrap-markdownNPMVersionImage]: https://img.shields.io/npm/v/bootstrap-markdown.svg?style=flat
 
-[bundlerGHUrl]: https://github.com/bundler/bundler
-[bundlerDOCUrl]: http://bundler.io/
-[bundlerGEMUrl]: http://rubygems.org/gems/bundler
-[bundlerGEMVersionImage]: http://img.shields.io/gem/v/bundler.svg?style=flat
-[bundlerGHReleasesUrl]: https://github.com/bundler/bundler/blob/master/CHANGELOG.md
-
 [clipboardGHUrl]: https://github.com/zenorocha/clipboard.js
 [clipboardDOCUrl]: https://github.com/zenorocha/clipboard.js/blob/master/readme.md
 [clipboardNPMUrl]: https://www.npmjs.com/package/clipboard
@@ -218,11 +211,6 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated`
 [express-sessionNPMUrl]: https://www.npmjs.com/package/express-session
 [express-sessionNPMVersionImage]: https://img.shields.io/npm/v/express-session.svg?style=flat
 
-[fakes3GHUrl]: https://github.com/jubos/fake-s3
-[fakes3DOCUrl]: http://www.rubydoc.info/gems/fakes3
-[fakes3GEMUrl]: http://rubygems.org/gems/fakes3
-[fakes3GEMVersionImage]: http://img.shields.io/gem/v/fakes3.svg?style=flat
-
 [font-awesomeGHUrl]: https://github.com/FortAwesome/Font-Awesome
 [font-awesomeDOCUrl]: http://fontawesome.io/
 [font-awesomeNPMUrl]: https://www.npmjs.com/package/font-awesome
@@ -417,6 +405,11 @@ Outdated dependencies list can also be achieved with `$ npm --depth 0 outdated`
 [rfc2047NPMUrl]: https://www.npmjs.com/package/rfc2047
 [rfc2047NPMVersionImage]: https://img.shields.io/npm/v/rfc2047.svg?style=flat
 
+[s3rverGHUrl]: https://github.com/jamhall/s3rver
+[s3rverDOCUrl]: https://github.com/jamhall/s3rver/blob/master/README.md
+[s3rverNPMUrl]: https://www.npmjs.com/package/s3rver
+[s3rverNPMVersionImage]: https://img.shields.io/npm/v/s3rver.svg?style=flat
+
 [sanitize-htmlGHUrl]: https://github.com/punkave/sanitize-html
 [sanitize-htmlDOCUrl]: https://github.com/punkave/sanitize-html/blob/master/README.md
 [sanitize-htmlNPMUrl]: https://www.npmjs.com/package/sanitize-html

README.md

@@ -11,12 +11,14 @@ var statusError = require('../libs/debug').statusError;
 //--- Dependency inclusions
 var fs = require('fs');
 var util = require('util');
+var http = require('http');
 var _ = require('underscore');
 var crypto = require('crypto');
 var request = require('request');
 var stream = require('stream');
 var peg = require('pegjs');
 var AWS = require('aws-sdk');
+var S3rver = require('s3rver');
 var UglifyJS = require("uglify-es");
 var rfc2047 = require('rfc2047');
 var mediaType = require('media-type');
@@ -114,24 +116,64 @@ var parsers = (function () {
 exports.parsers = parsers;
 
 var bucketName = 'OpenUserJS.org';
+if (isDev) {
+  bucketName = bucketName.toLowerCase(); // NOTE: *S3rver* requirement
+}
 
 var DEV_AWS_URL = null;
+var devAWSURL = null;
+var serverS3 = null;
 
 if (isPro) {
   AWS.config.update({
     region: 'us-east-1'
   });
 } else {
-  // You need to install (and ruby too): https://github.com/jubos/fake-s3
-  // Then run the fakes3.sh script or: fakes3 -r fakeS3 -p 10001
   DEV_AWS_URL = process.env.DEV_AWS_URL || 'http://localhost:10001';
   AWS.config.update({
     accessKeyId: 'fakeId',
     secretAccessKey: 'fakeKey',
     httpOptions: {
       proxy: DEV_AWS_URL,
-      agent: require('http').globalAgent // TODO: Move this up eventually
+      agent: http.globalAgent
+    }
+  });
+
+  devAWSURL = new URL(DEV_AWS_URL);
+
+  serverS3 = new S3rver({
+    hostname: devAWSURL.hostname,
+    port: devAWSURL.port,
+    directory: './S3rver' // WATCHPOINT: Technically this should be `..` Is probably upstream issue
+  }).run(function (aErr) {
+    if (aErr) {
+      console.error([
+        colors.red('ERROR: S3rver not initialized'),
+        aErr
+      ].join('\n'));
+      return;
     }
+    console.log(colors.green('S3rver initialized'));
+
+    var s3 = new AWS.S3();
+    s3.createBucket({ Bucket: bucketName }, function (aErr) {
+      if (aErr) {
+        switch (aErr.statusCode) {
+          case 409:
+            console.log(colors.green('Default dev S3 bucket already exists'));
+            break;
+          default:
+            console.error([
+              colors.red('Error creating default dev S3 bucket'),
+              aErr
+            ].join('\n'));
+            // fallthrough
+        }
+        return;
+      } else {
+        console.log(colors.green('Created default dev S3 bucket'));
+      }
+    });
   });
 }
 

controllers/scriptStorage.js

@@ -1952,7 +1952,7 @@ function getExistingScript(aReq, aOptions, aAuthedUser, aCallback) {
           'S3 GET (chunking indirect) ',
             aErr.code,
               'for', installNameBase + (isLib ? '.js' : '.user.js'),
-                'in the', bucketName, 'bucket\n' +
+                'in the S3 bucket\n' +
                   JSON.stringify(aErr, null, ' ') + '\n' +
                     aErr.stack
         );

controllers/user.js

@@ -26,123 +26,6 @@ var tasks = [
       aCallback(null, ['$ ' + cmd + '\n' + colors.gray(aStdout)]);
     });
   },
-  function (aStdouts, aCallback) {
-    var cmd = 'ruby -v';
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        aCallback(aErr);
-        return;
-      }
-
-      aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, aStdouts);
-    });
-  },
-  function (aStdouts, aCallback) {
-    var cmd = 'bundler -v';
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        if (aErr.code === 127) {
-//           aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-          aCallback(null, false, aStdouts);
-          return;
-        } else {
-          aCallback(aErr);
-          return;
-        }
-      }
-
-//       aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, true, aStdouts);
-    });
-  },
-  function (aSkip, aStdouts, aCallback) {
-    var cmd = 'sudo gem install bundler -v 1.16.4';
-
-    if (aSkip) {
-      aCallback(null, aStdouts);
-      return;
-    }
-
-    console.log(colors.cyan('Installing *bundler* gem as global...'));
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        aCallback(aErr);
-        return;
-      }
-
-//       aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, aStdouts);
-    });
-  },
-  function (aStdouts, aCallback) {
-    var cmd = 'bundler outdated';
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        if (aErr.code === 7) {
-//           aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-          aCallback(null, false, aStdouts);
-          return;
-        } else {
-          aCallback(aErr);
-          return;
-        }
-      }
-
-//       aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, true, aStdouts);
-    });
-  },
-  function (aSkip, aStdouts, aCallback) {
-    var cmd = 'bundler install';
-
-    if (aSkip) {
-      aCallback(null, aStdouts);
-      return;
-    }
-
-    console.log(colors.cyan('Installing bundled gem(s) as global...'));
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        aCallback(aErr);
-        return;
-      }
-
-//       aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, aStdouts);
-    });
-  },
-  function (aStdouts, aCallback) {
-    var cmd = 'gem list';
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        aCallback(aErr);
-        return;
-      }
-
-      aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, aStdouts);
-    });
-  },
-  function (aStdouts, aCallback) {
-    var cmd = 'gem outdated';
-
-    exec(cmd, function (aErr, aStdout, aStderr) {
-      if (aErr) {
-        aCallback(aErr);
-        return;
-      }
-
-      aStdouts.push('$ ' + cmd + '\n' + colors.gray(aStdout));
-      aCallback(null, aStdouts);
-    });
-  },
   function (aStdouts, aCallback) {
     var cmd = 'npm -v';
 

dev/postinstall.js

@@ -68,6 +68,9 @@
     "underscore": "1.9.1",
     "useragent": "2.3.0"
   },
+  "devDependencies": {
+    "s3rver": "2.2.6"
+  },
   "optionalDependencies": {
     "kerberos": "1.0.0"
   },