Make new atom.xml and data.min.js + build script fixes (microjs#958)

* Edits to config to make limits slightly more permissive

* Better error and edge case handling in sizer.js

* Fix some problematic source links in data.js

* More cleanup/fixes in data.js

* Use requests instead of hypervisor

- hypervisor is no longer being maintained
- My logs were friggin full of an annoying error even after updating to the latest version https://github.com/substack/hyperquest/issues/39

* Fix one last problem link in data.js

* Remove unneeded log line

(I put it there a bit ago for debugging purposes and forgot to remove it before committing)

* Update dependencies in package.json

- Update to latest version of after (fixes an error I was getting)
- Upgrade from uglify-js@2.x to uglify-es@3.x to support libraries using ES6+ syntax (old behavior was to always reject due to uglify errors)
- Use request package instead of the no-longer-maintained hyperquest; remove obviated hyperquest, hyperdirect, hyperzip, and bl libraries
- Errno also seems to be upgraded (this change wasn't deliberate?)

* make atom.xml and data-min.js

* Fix issue with github auth

(hopefully)

config.js

@@ -2,7 +2,7 @@ module.exports = {
     inFile                     : 'data.js'
   , outFile                    : 'data-min.js'
     // for now; need to tweak this later
-  , githubLastPushCutoffMonths : 48
+  , githubLastPushCutoffMonths : 60
   , sizeExceptions             : [
         'Backbone'
       , 'Zepto'
@@ -11,12 +11,13 @@ module.exports = {
         'cm.js'
     ]
   , errorCountThreshold        : 10
-  , maxGzipped                 : 5 * 1024
+    // normal limit is 5kb, but strictly enforce 6kb
+  , maxGzipped                 : 6 * 1024
     // these 3 are basic sanity cecks, minimums that we would expect the contents
     // to be, otherwise something might be wrong
-  , minRaw                     : 300
-  , minMinified                : 150
-  , minGzipped                 : 100
+  , minRaw                     : 100
+  , minMinified                : 50
+  , minGzipped                 : 25
     // this is to compare raw with minified, if it's too close then we may have
     // been given the minified source in the first place
   , minifierTollerance         : 50

data.js

@@ -28,7 +28,7 @@ module.exports = [{
     tags: ["modal", "javascript", "vanilla", "simple"],
     description: "Clean, Fast, Modular and customizable Modal Window controller",
     url: "http://juanbrujo.github.io/tinyModal/",
-    source: "https://github.com/juanbrujo/tinyModal/blob/master/src/tinyModal.js"
+    source: "https://raw.githubusercontent.com/juanbrujo/tinyModal/master/src/tinyModal.js"
   },
   {
     name: "NodeList.js",
@@ -44,15 +44,15 @@ module.exports = [{
     tags: ["Drag and Drop", "simple", "Library"],
     description: "Drag and drop so simple it hurts",
     url: "https://bevacqua.github.io/dragula/",
-    source: "https://github.com/bevacqua/dragula/blob/master/dragula.js"
+    source: "https://raw.githubusercontent.com/bevacqua/dragula/master/dragula.js"
   },
   {
     name: "fuzzysearch",
     github: "bevacqua/fuzzysearch",
     tags: ["Lightweight", "Tiny", "fast", "fuzzy", "fuzzy-search", "search"],
     description: "Tiny and blazing-fast fuzzy search in JavaScript",
     url: "https://github.com/bevacqua/fuzzysearch",
-    source: "https://github.com/bevacqua/fuzzysearch/blob/master/index.js"
+    source: "https://raw.githubusercontent.com/bevacqua/fuzzysearch/master/index.js"
   },
   {
     name: "Fuse.js",
@@ -76,7 +76,7 @@ module.exports = [{
     tags: ["pubsub", "component-interaction", "event", "jquery"],
     description: "A tiny pub/sub jQuery micro-framework for distant component interaction",
     url: "https://github.com/ahmed-musallam/syrupjs",
-    source: ["https://github.com/ahmed-musallam/syrupjs/blob/master/vendor/tiny-pub-sub.js","https://github.com/ahmed-musallam/syrupjs/blob/master/index.js"]
+    source: ["https://raw.githubusercontent.com/ahmed-musallam/syrupjs/master/vendor/tiny-pub-sub.js","https://raw.githubusercontent.com/ahmed-musallam/syrupjs/master/index.js"]
   },
   {
     name: "blobcounter.js",
@@ -138,7 +138,7 @@ module.exports = [{
     tags: ["jquery", "dom", "ajax"],
     description: "Lightweight modern jQuery-like DOM library",
     url: "http://www.bitstorm.org/javascript/miq/",
-    source: "http://www.bitstorm.org/javascript/miq/miq.js"
+    source: "https://raw.githubusercontent.com/edwinm/miq/master/miq.js"
   },
   {
     name: "RouterX",
@@ -276,7 +276,7 @@ module.exports = [{
     url: "https://github.com/Mapita/Canary",
     package: "canary-test",
     documentation: "http://canary.readthedocs.io/en/stable/",
-    source: "https://github.com/Mapita/Canary/blob/master/canary.js"
+    source: "https://raw.githubusercontent.com/Mapita/Canary/master/canary.js"
   },
   {
     name: "FrontExpress",
@@ -657,7 +657,7 @@ module.exports = [{
     tags: ["svg", "vml", "graphics", "drawing", "animation", "visualization", "charts", "data visualization", "api"],
     description: "A powerful lightweight JavaScript drawing library for graphics and animation, based on SVG/VML, with intuitive API",
     url: "http://www.graphicsjs.org",
-    source: "https://github.com/AnyChart/GraphicsJS/blob/master/dist/graphics.js"
+    source: "https://raw.githubusercontent.com/AnyChart/GraphicsJS/master/dist/graphics.js"
   },
   {
     name: "SaVaGe.js",
@@ -2181,7 +2181,7 @@ module.exports = [{
     tags: ["dom", "lightweight", "shorthand", "ajax"],
     description: "Small library for DOM and AJAX operations (jQuery function style).",
     url: "https://github.com/MrOnlineCoder/sul.js",
-    source: "https://github.com/MrOnlineCoder/sul.js/blob/master/src/sul.js"
+    source: "https://raw.githubusercontent.com/MrOnlineCoder/sul.js/master/src/sul.js"
   },
   {
     name: "Valentine",
@@ -3163,7 +3163,7 @@ module.exports = [{
     tags: ["ajax", "xhr", "rest", "fluent", "XMLHttpRequest"],
     description: "Fluent, extensible, easy-to-use ajax Framework.",
     url: "http://jccazeaux.github.io/jacks/",
-    source: "https://github.com/jccazeaux/jacks/blob/master/src/jacks.js"
+    source: "https://raw.githubusercontent.com/jccazeaux/jacks/master/src/jacks.js"
   },
   {
     name: "colorspaces.js",
@@ -5463,7 +5463,7 @@ module.exports = [{
     tags: ["DOM", "manipulation", "events", "css", "query selection", "data", "traversal", "es2015", "js:next", "functional"],
     description: "A lightweight DOM manipulation library",
     url: "https://github.com/enzyme/qoob",
-    source: "https://github.com/enzyme/qoob/blob/master/src/qoob.js"
+    source: "https://raw.githubusercontent.com/enzyme/qoob/master/src/qoob.js"
   },
   {
     name: "Prophet.js",
@@ -5566,6 +5566,6 @@ module.exports = [{
     tags: ["javascript", "Validation"],
     description: "1kb JavaScript Identify Number Validater.",
     url: "https://github.com/kucukharf/tiv",
-    source: "https://github.com/kucukharf/tiv/blob/master/tiv.min.js"
+    source: "https://raw.githubusercontent.com/kucukharf/tiv/master/tiv.min.js"
   }
 ];

lib/fetch-http.js

@@ -1,5 +1,4 @@
-const hyperquest    = require('hyperzip')(require('hyperdirect'))
-    , bl            = require('bl')
+const request = require('request')
 
 const maxConcurrent = 25
 
@@ -11,12 +10,16 @@ function go () {
     return
 
   var req = queue.shift()
+  req.options.url = req.url
   inFlight++
-  hyperquest(req.url, req.options).pipe(bl(function () {
+  request.get(req.options, function(err, response, body) {
     inFlight--
+    if(!err && response.statusCode !== 200){
+      err = "Error code " + response.statusCode + ": " + response.statusMessage;
+    }
     process.nextTick(go)
-    req.callback.apply(this, arguments)
-  }))
+    req.callback.call(this, err, body)
+  })
   process.nextTick(go)
 }
 

lib/ghauth.js

@@ -1,35 +1,35 @@
 const read       = require('read')
     , colors     = require('colors')
-    , hyperquest = require('hyperquest')
-    , bl         = require('bl')
     , path       = require('path')
+    , request    = require('request')
     , fs         = require('fs')
     , mkdirp     = require('mkdirp')
 
 function createAuth (config, callback) {
-  var req = hyperquest(
-      'https://api.github.com/authorizations'
-    , {
-          headers : {
-              'X-GitHub-OTP' : config.ghOtp || null
-            , 'User-Agent'   : 'microjs.com-build-script'
-          }
-        , method  : 'post'
-        , auth    : config.ghUser + ':' + config.ghPass
-      }
-    )
-  req.pipe(bl(function (err, data) {
-    if (err)
-      return callback(err)
-    data = JSON.parse(data.toString())
+  var req = request.post({
+    url: 'https://api.github.com/authorizations',
+    auth: {
+      user: config.ghUser,
+      pass: config.ghPass,
+    },
+    headers : {
+      'X-GitHub-OTP': config.ghOtp || null,
+      'User-Agent': 'microjs.com-build-script'
+    },
+  }, function(err, response, body) {
+    if (err || response.statusCode !== 201)
+      return callback(err || (
+        "Error code " + response.statusCode + ": " + response.statusMessage + " - " + body
+      ))
+    data = JSON.parse(body.toString())
     if (data.message)
       return callback(new Error(data.message))
     if (!data.token)
       return callback(new Error('No token from GitHub!'))
 
     config.ghToken = data.token
     callback()
-  }))
+  });
   req.end(JSON.stringify({ note: 'microjs.com build script'}))
 }
 

lib/process-github.js

@@ -6,9 +6,12 @@ const githubSpecRegex  = /^([^\/]+)\/(.*)$/
     , GIT_API_REPO_URL = 'https://api.github.com/repos/{user}/{project}'
 
 function processRepo (config, ee, lib, user, project, callback) {
-  var url     = GIT_API_REPO_URL.replace('{user}', user).replace('{project}', project)
-    , headers = { 'User-Agent' : 'microjs.com-build-script' }
-    , auth    = config.ghUser + ':' + config.ghToken
+  var url = GIT_API_REPO_URL.replace('{user}', user).replace('{project}', project)
+  var headers = { 'User-Agent' : 'microjs.com-build-script' }
+  var auth = {
+    user: config.ghUser,
+    pass: config.ghToken,
+  }
 
   function handle (err, content) {
     if (err)

lib/sizer.js

@@ -1,19 +1,30 @@
-const minify = require('uglify-js').minify
+const minify = require('uglify-es').minify
     , gzip   = require('zlib').gzip
 
 // calculate the 3 sizes, raw, minified & gzipped
 function sizer (source, callback) {
   var minified
-
+
+  if(!source || !source.length){
+    return callback('No source code found.')
+  }
+
+  if(source === '404: Not Found\n'){
+    return callback('404: Source code not found.')
+  }
+
   try {
-    minified = minify(source, { fromString: true }).code
+    minified = minify(source).code
   } catch (ex) {
     return callback('Uglify error: ' + (ex && ex.message ? ex.message : ex))
   }
+
+  if(!minified){
+    return callback('Uglify failed.')
+  }
 
   gzip(minified, function(err, data) {
     if (err) return callback(err)
-
     callback(null, {
         raw: source.length
       , minified: minified.length