Reproducible builds: reduce non-determinism in build process (WIP)

- Remove environment dependant fields from required()'d package.json files in browserify bundles
  using https://github.com/fanatid/browserify-package-json

- Lock-down npm dependencies using npm-shrinkwrap

- Lock-down bower dependencies using https://gist.github.com/shesek/029fbf279b826005095dad50697c0e27

- Monkey-patch archiver to produce deterministic zips (reset timestamps/modes and canonicalize file order),
  used for nwjs builds by grunt-nw-builder and grunt-contrib-compress

- Switch Chrome app build process to use the now determinstic archiver instead of the zip cli

- Canonincalize format for main package.json file (packed in nwjs builds)

Author: shesek

Gruntfile.js

@@ -1,5 +1,7 @@
 'use strict';
 
+require('./util/deterministic-zip')
+
 module.exports = function(grunt) {
 
   require('load-grunt-tasks')(grunt);
@@ -249,6 +251,15 @@ module.exports = function(grunt) {
         cwd: './webkitbuilds/<%= pkg.title %>/linux64/',
         src: ['**/*'],
         dest: '<%= pkg.title %>-linux/'
+      },
+      chrome: {
+        options: {
+          archive: './chrome-app/<%= pkg.name %>-chrome-extension.zip'
+        },
+        expand: true,
+        cwd: './chrome-app/build',
+        src: ['**/*'],
+        dest: '<%= pkg.name %>-chrome-extension'
       }
     },
     browserify: {
@@ -257,6 +268,11 @@ module.exports = function(grunt) {
           'angular-bitcore-wallet-client/angular-bitcore-wallet-client.js': ['angular-bitcore-wallet-client/index.js'],
           'angular-bitauth/angular-bitauth.js': ['angular-bitauth/index.js']
         },
+        options: {
+          // remove all package.json fields except for name and version
+          // (some fields are environment dependant, include full system paths, etc)
+          transform: [ [ 'browserify-package-json', { global: true, only: [ 'name', 'version' ] }  ] ]
+        }
       }
     }
   });

app-template/apply.js

@@ -62,6 +62,8 @@ Object.keys(templates).forEach(function(k) {
     k = 'config.xml';
   } else if (k === 'package-template.json') {
     k = 'package.json';
+    // canonical reproducible format for package.json (strip extra spaces, duplicate "//" field, etc)
+    content = JSON.stringify(JSON.parse(content), null, 2) + '\n'
   }
 
   if (!fs.existsSync('../' + targetDir)) {

app-template/package-template.json

@@ -77,7 +77,8 @@
     "grunt-nw-builder": "^2.0.3",
     "grunt-sass": "^1.2.0",
     "load-grunt-tasks": "^3.5.0",
-    "shelljs": "^0.3.0"
+    "shelljs": "^0.3.0",
+    "browserify-package-json": "^1.0.1"
   },
   "scripts": {
     "postinstall":            "bower install",

bower.json

@@ -7,23 +7,45 @@
     "bitcore"
   ],
   "dependencies": {
-    "angular-mocks": "1.4.10",
-    "angular-gettext": "2.2.1",
-    "angular-moment": "0.10.1",
-    "angular-qrcode": "bitpay/angular-qrcode#~6.3.0",
-    "ionic": "https://github.com/ionic-team/ionic-v1.git",
-    "moment": "2.10.3",
-    "ng-lodash": "0.2.3",
-    "qrcode-decoder-js": "*",
+    "angular-mocks": "https://github.com/angular/bower-angular-mocks.git#f5bcaf2b32c76f778704b7e92f31d661c24f3364",
+    "angular-gettext": "https://github.com/rubenv/angular-gettext.git#df72273b60377d6991911757cc0c5bbe1107eefd",
+    "angular-moment": "https://github.com/urish/angular-moment.git#8910240ee1872478a1b318d2d800c1c073526c37",
+    "angular-qrcode": "https://github.com/bitpay/angular-qrcode.git#290911217db50c3411821448401945d6c4180bbf",
+    "ionic": "https://github.com/ionic-team/ionic-v1.git#1b7414fabafc8a87252d9ec30d004bf6ac3fa5a7",
+    "moment": "https://github.com/moment/moment.git#6fa444238494318e3c488c81d8520ad2eba8bae7",
+    "ng-lodash": "https://github.com/rockabox/ng-lodash.git#0f144f0f9b1884a6d0ae9de5db91d8f7354e30ee",
+    "qrcode-decoder-js": "https://github.com/colkito/qrcode-decoder-js.git#b71009466546b975014d0b7e1cf7a158c8fdc7ef",
     "trezor-connect": "https://github.com/trezor/connect.git#c1a00f9f0f4a13239ba13ddd589c6ca61f2e7876",
-    "ng-csv": "~0.3.6",
-    "ionic-toast": "^0.4.1",
-    "angular-clipboard": "^1.4.2",
-    "angular-md5": "^0.1.10",
-    "ngtouch": "^1.0.1"
+    "ng-csv": "https://github.com/asafdav/ng-csv.git#6ad5e86785cbc585824268ba065b8a37e7526458",
+    "ionic-toast": "https://github.com/rajeshwarpatlolla/ionic-toast.git#d2e9fcab0bf0527ceecfe3fb78f8d1f47fecb436",
+    "angular-clipboard": "https://github.com/omichelsen/angular-clipboard.git#6861d5ead55956fb01d64520546c625605366f1a",
+    "angular-md5": "https://github.com/gdi2290/angular-md5.git#f0c5d61f92a807b4d3a0b2be98890d8948e03fe7",
+    "ngtouch": "https://github.com/nglar/ngTouch.git#1285101327290670a9b85d900254d46f771e6421",
+    "angular-animate": "https://github.com/angular/bower-angular-animate.git#671c738980fb0509b2b494716ccd8c004c39f368",
+    "angular": "https://github.com/angular/bower-angular.git#5a07c5107b4d24f41744a02b07717d55bad88e70",
+    "angular-sanitize": "https://github.com/angular/bower-angular-sanitize.git#d62a5eecedc71828f6f935fbde6c07217a95988a",
+    "angular-ui-router": "https://github.com/angular-ui/angular-ui-router-bower.git#2e580f271defdec34f464aab0cca519e41d1ee33",
+    "qrcode-generator": "git://github.com/monospaced/bower-qrcode-generator.git#64e898f180fbd76aa2740be41124d219aa735f0c"
   },
   "resolutions": {
-    "angular": "1.5.3",
-    "ionic": "1b7414faba"
+    "angular": "5a07c5107b4d24f41744a02b07717d55bad88e70",
+    "ionic": "1b7414fabafc8a87252d9ec30d004bf6ac3fa5a7",
+    "angular-animate": "671c738980fb0509b2b494716ccd8c004c39f368",
+    "angular-clipboard": "6861d5ead55956fb01d64520546c625605366f1a",
+    "angular-gettext": "df72273b60377d6991911757cc0c5bbe1107eefd",
+    "angular-md5": "f0c5d61f92a807b4d3a0b2be98890d8948e03fe7",
+    "angular-mocks": "f5bcaf2b32c76f778704b7e92f31d661c24f3364",
+    "angular-moment": "8910240ee1872478a1b318d2d800c1c073526c37",
+    "angular-qrcode": "290911217db50c3411821448401945d6c4180bbf",
+    "angular-sanitize": "d62a5eecedc71828f6f935fbde6c07217a95988a",
+    "angular-ui-router": "2e580f271defdec34f464aab0cca519e41d1ee33",
+    "ionic-toast": "d2e9fcab0bf0527ceecfe3fb78f8d1f47fecb436",
+    "qrcode-decoder-js": "b71009466546b975014d0b7e1cf7a158c8fdc7ef",
+    "moment": "6fa444238494318e3c488c81d8520ad2eba8bae7",
+    "ng-csv": "6ad5e86785cbc585824268ba065b8a37e7526458",
+    "ng-lodash": "0f144f0f9b1884a6d0ae9de5db91d8f7354e30ee",
+    "ngtouch": "1285101327290670a9b85d900254d46f771e6421",
+    "qrcode-generator": "64e898f180fbd76aa2740be41124d219aa735f0c",
+    "trezor-connect": "c1a00f9f0f4a13239ba13ddd589c6ca61f2e7876"
   }
 }

chrome-app/Makefile

@@ -23,5 +23,7 @@ files:
 $(ZIPFILE): dir files
 	cd $(BUILDDIR)
 	rm -f $(ZIPFILE)
-	zip -qr $(ZIPFILE) "`basename $(BUILDDIR)`"
+	# use grunt-contrib-compress, patched for reproducibility
+	../node_modules/.bin/grunt compress:chrome
+
 	@echo "** The Chrome Extension is ready at cobit-chrome-extension.zip"