use qiniu cdn for static files

cdn.js

@@ -0,0 +1,168 @@
+/* eslint no-console: 0 */
+require('dotenv').config();
+const qiniu = require('qiniu');
+const fs = require('fs');
+const path = require('path');
+const {
+    STATIC_DIR,
+    QINIU_ACCESS_KEY,
+    QINIU_SECRET_KEY,
+    QINIU_BUCKET,
+} = process.env;
+
+qiniu.conf.ACCESS_KEY = QINIU_ACCESS_KEY;
+qiniu.conf.SECRET_KEY = QINIU_SECRET_KEY;
+
+let totalCount = 0,
+    uploadedCount = 0;
+const retryCountMap = new Map(),
+    SINGLE_RETRY_COUNT_LIMIT = 10;
+
+function uptoken(bucket, key) {
+    const putPolicy = new qiniu.rs.PutPolicy(`${bucket}:${key}`);
+    return putPolicy.token();
+}
+
+function getKey(fullpath) {
+    return path.relative(STATIC_DIR, fullpath).replace(/^\w+\//, '');
+}
+
+function deduplicateFiles(files = [], bucket) {
+    const entries = files.map(getKey)
+        .map(key => new qiniu.rs.EntryPath(bucket, key));
+    console.info('[INFO] Qiniu upload file detect file duplication');
+    return new Promise((resolve, reject) => {
+        const client = new qiniu.rs.Client();
+        client.batchStat(entries, (err, res) => {
+            if (err) {
+                reject(err);
+                console.error(
+                    '[FAIL] Qiniu upload file detect file duplication fail' +
+                    ` (Response Code: ${err.code})`
+                );
+                const retryCount = retryCountMap.get('deduplicate') || 0;
+                if (retryCount < SINGLE_RETRY_COUNT_LIMIT) {
+                    const nextRetryCount = retryCount + 1;
+                    retryCountMap.set('deduplicate', nextRetryCount);
+                    console.info(
+                        '[INFO] Qiniu upload file detect retry' +
+                        `(${nextRetryCount}/${SINGLE_RETRY_COUNT_LIMIT})`
+                    );
+                    return deduplicateFiles(files, bucket);
+                }
+                process.exit(1);
+            }
+            const fileStats = res.map(r => r.code === 200);
+            const handled = files.reduce((all, file, index) => {
+                if (!fileStats[index]) {
+                    all.push(file);
+                } else {
+                    console.info(`[INFO] Qiniu upload file ignored - ${getKey(file)}`);
+                }
+                return all;
+            }, []);
+            return resolve(handled);
+        });
+    });
+}
+
+function uploadSingleFile(token, key, file) {
+    const size = fs.statSync(file).size;
+    return new Promise(resolve => {
+        const extra = new qiniu.io.PutExtra();
+        qiniu.io.putFile(token, key, file, extra, (err, ret) => {
+            if (err) {
+                // 用 Promise.all 来跟踪这个 promise 队列, reject 之后的所有 promise
+                // 其实依然会执行，只是 catch 会立马拿到这个 err
+                console.error(
+                    '[FAIL] Qiniu upload file fail ' +
+                    `(Response Code: ${err.code}, Msg: ${err.error}): ${key}`
+                );
+                const retryCount = retryCountMap.get(key) || 0;
+                if (retryCount < SINGLE_RETRY_COUNT_LIMIT) {
+                    const nextRetryCount = retryCount + 1;
+                    retryCountMap.set(key, nextRetryCount);
+                    console.info(
+                        '[INFO] Qiniu upload file retry' +
+                        `(${nextRetryCount}/${SINGLE_RETRY_COUNT_LIMIT}): ${key} - (${size})`
+                    );
+                    return uploadSingleFile(token, key, file)
+                        .then(resolve);
+                }
+                process.exit(1);
+                return Promise.reject();
+            }
+            const percentage = Math.ceil((++uploadedCount * 100) / totalCount);
+            console.info(`[${percentage}%] Qiniu upload file success: ${key} - (${size})`);
+            return resolve(ret);
+        });
+    });
+}
+
+function promiseUpload(localFile) {
+    const key = getKey(localFile);
+    const token = uptoken(QINIU_BUCKET, key);
+
+    return uploadSingleFile(token, key, localFile);
+}
+
+function walk(parent, pathname, fileHandler) {
+    const filepath = path.join(parent, pathname);
+    const stats = fs.statSync(filepath);
+
+    if (stats.isFile()) {
+        fileHandler(filepath);
+    } else if (stats.isDirectory()) {
+        fs.readdirSync(filepath).forEach(name => {
+            walk(filepath, name, fileHandler);
+        });
+    }
+}
+
+function uploadParallelCount(queue, parallelCount) {
+    totalCount = queue.length;
+    const slices = Math.floor(totalCount / parallelCount);
+    const remains = queue.length % parallelCount;
+    let promiseQueue = Promise.resolve();
+
+    for (let i = 0; i < slices; i++) {
+        promiseQueue = promiseQueue
+            .then(() => {
+                const start = parallelCount * i;
+
+                return Promise.all(
+                    queue.slice(start, start + parallelCount).map(promiseUpload)
+                );
+            });
+    }
+
+    return promiseQueue
+        .then(() => Promise.all(queue.slice(-remains).map(promiseUpload)))
+        .then(() => console.info('Qiniu upload completed'));
+}
+
+function upload(parallelCount = 10) {
+    const queue = [];
+    const ignorePattern = /\.gz$/;
+
+    try {
+        walk('', path.resolve(STATIC_DIR), filepath => {
+            if (ignorePattern.test(filepath)) return;
+            queue.push(filepath);
+        });
+
+        return deduplicateFiles(queue, QINIU_BUCKET)
+            .then((handled) => uploadParallelCount(handled, parallelCount));
+    } catch (err) {
+        console.error('Upload to qiniu unknown exception: %o', err);
+        return Promise.reject(err);
+    }
+}
+
+console.log('start upload built bundles to cdn')
+
+if (QINIU_BUCKET && STATIC_DIR && QINIU_ACCESS_KEY && QINIU_SECRET_KEY) {
+    upload()
+} else {
+     console.warn('require cdn config to upload static files');
+}

package.json

@@ -16,7 +16,8 @@
     "start-platform": "RUN_MODE=platform webpack-dev-server --port=8060 --progress --colors",
     "start": "webpack-dev-server --progress --colors",
     "clean": "rm -rf build",
-    "build": "npm run clean && cross-env NODE_ENV=production webpack --progress --colors",
+    "build": "npm run clean && cross-env NODE_ENV=production webpack --progress --colors && npm run after-build",
+    "after-build": "node cdn.js",
     "lint": "eslint app",
     "flow": "flow; test $? -eq 0 -o $? -eq 2"
   },
@@ -75,6 +76,7 @@
     "markdown": "^0.5.0",
     "moment": "^2.18.1",
     "octicons": "^4.3.0",
+    "qiniu": "^6.1.13",
     "qs": "^6.4.0",
     "react": "^15.3.0",
     "react-dom": "^15.3.1",

webpack_configs/common.config.js

@@ -14,14 +14,16 @@ module.exports = function (options={}) {
     staticDir = 'rs',
   } = options
 
+  const publicPath = process.env.QINIU_BUCKET ? // publicPath should end with '/'
+    `${process.env.QINIU_SERVER}/` : path.join('/', staticDir, '/');
 return {
   entry: {
     main: [path.join(rootDir, 'app')],
     workspaces: [path.join(rootDir, 'app/workspaces_standalone')],
     vendor: ['babel-polyfill', 'react', 'react-dom', 'redux', 'react-redux'],
   },
   output: {
-    publicPath: path.join('/', staticDir, '/'), // publicPath should end with '/'
+    publicPath,
     path: path.join(rootDir, 'build', staticDir),
     filename: '[name].[hash].js'
   },

yarn.lock

@@ -1093,6 +1093,10 @@ browserslist@^1.0.1, browserslist@^1.4.0, browserslist@^1.5.2, browserslist@^1.7
     caniuse-db "^1.0.30000631"
     electron-to-chromium "^1.2.5"
 
+buffer-concat@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.npmjs.org/buffer-concat/-/buffer-concat-0.0.1.tgz#43e1e3c77f3d22fb55bea6890a92d03778c4c10a"
+
 buffer-shims@^1.0.0:
   version "1.0.0"
   resolved "http://registry.npm.taobao.org/buffer-shims/download/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51"
@@ -1163,6 +1167,10 @@ caniuse-db@^1.0.30000346, caniuse-db@^1.0.30000631, caniuse-db@^1.0.30000634:
   version "1.0.30000636"
   resolved "http://registry.npm.taobao.org/caniuse-db/download/caniuse-db-1.0.30000636.tgz#08230e9dd26632193f29c9d935185c6f6f9fd9ef"
 
+caseless@~0.11.0:
+  version "0.11.0"
+  resolved "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz#715b96ea9841593cc33067923f5ec60ebda4f7d7"
+
 caseless@~0.12.0:
   version "0.12.0"
   resolved "http://registry.npm.taobao.org/caseless/download/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -1374,7 +1382,7 @@ concat-map@0.0.1:
   version "0.0.1"
   resolved "http://registry.npm.taobao.org/concat-map/download/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
 
-concat-stream@^1.4.6:
+concat-stream@^1.4.6, concat-stream@^1.4.7:
   version "1.6.0"
   resolved "http://registry.npm.taobao.org/concat-stream/download/concat-stream-1.6.0.tgz#0aac662fd52be78964d5532f694784e70110acf7"
   dependencies:
@@ -1449,6 +1457,10 @@ core-util-is@~1.0.0:
   version "1.0.2"
   resolved "http://registry.npm.taobao.org/core-util-is/download/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
 
+crc32@0.2.2:
+  version "0.2.2"
+  resolved "https://registry.npmjs.org/crc32/-/crc32-0.2.2.tgz#7ad220d6ffdcd119f9fc127a7772cacea390a4ba"
+
 create-ecdh@^4.0.0:
   version "4.0.0"
   resolved "http://registry.npm.taobao.org/create-ecdh/download/create-ecdh-4.0.0.tgz#888c723596cdf7612f6498233eebd7a35301737d"
@@ -1642,6 +1654,10 @@ debug@*, debug@2.3.3, debug@^2.1.1, debug@^2.2.0:
   dependencies:
     ms "0.7.2"
 
+debug@0.7.2:
+  version "0.7.2"
+  resolved "https://registry.npmjs.org/debug/-/debug-0.7.2.tgz#056692c86670977f115de82917918b8e8b9a10f0"
+
 debug@2.2.0, debug@~2.2.0:
   version "2.2.0"
   resolved "http://registry.npm.taobao.org/debug/download/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
@@ -2367,6 +2383,14 @@ form-data@~2.1.1:
     combined-stream "^1.0.5"
     mime-types "^2.1.12"
 
+formstream@0.0.7:
+  version "0.0.7"
+  resolved "https://registry.npmjs.org/formstream/-/formstream-0.0.7.tgz#981d0036743596c54820ee3dfb44baca18349788"
+  dependencies:
+    buffer-concat "0.0.1"
+    mime "1.2.9"
+    pause-stream ">=0.0.10"
+
 forwarded@~0.1.0:
   version "0.1.0"
   resolved "http://registry.npm.taobao.org/forwarded/download/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363"
@@ -2672,6 +2696,14 @@ htmlparser2@~3.3.0:
     domutils "1.1"
     readable-stream "1.0"
 
+http-basic@^2.5.1:
+  version "2.5.1"
+  resolved "https://registry.npmjs.org/http-basic/-/http-basic-2.5.1.tgz#8ce447bdb5b6c577f8a63e3fa78056ec4bb4dbfb"
+  dependencies:
+    caseless "~0.11.0"
+    concat-stream "^1.4.6"
+    http-response-object "^1.0.0"
+
 http-deceiver@^1.2.4:
   version "1.2.7"
   resolved "http://registry.npm.taobao.org/http-deceiver/download/http-deceiver-1.2.7.tgz#fa7168944ab9a519d337cb0bec7284dc3e723d87"
@@ -2709,6 +2741,10 @@ http-proxy@^1.16.2:
     eventemitter3 "1.x.x"
     requires-port "1.x.x"
 
+http-response-object@^1.0.0, http-response-object@^1.0.1, http-response-object@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.npmjs.org/http-response-object/-/http-response-object-1.1.0.tgz#a7c4e75aae82f3bb4904e4f43f615673b4d518c3"
+
 http-signature@~1.1.0:
   version "1.1.1"
   resolved "http://registry.npm.taobao.org/http-signature/download/http-signature-1.1.1.tgz#df72e267066cd0ac67fb76adf8e134a8fbcf91bf"
@@ -3332,6 +3368,10 @@ mime-types@^2.1.12, mime-types@~2.1.11, mime-types@~2.1.13, mime-types@~2.1.7:
   dependencies:
     mime-db "~1.26.0"
 
+mime@1.2.9:
+  version "1.2.9"
+  resolved "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz#009cd40867bd35de521b3b966f04e2f8d4d13d09"
+
 mime@1.3.4, mime@1.3.x, mime@^1.3.4:
   version "1.3.4"
   resolved "http://registry.npm.taobao.org/mime/download/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
@@ -3748,6 +3788,12 @@ path-type@^1.0.0:
     pify "^2.0.0"
     pinkie-promise "^2.0.0"
 
+pause-stream@>=0.0.10:
+  version "0.0.11"
+  resolved "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz#fe5a34b0cbce12b5aa6a2b403ee2e73b602f1445"
+  dependencies:
+    through "~2.3"
+
 pbkdf2@^3.0.3:
   version "3.0.9"
   resolved "http://registry.npm.taobao.org/pbkdf2/download/pbkdf2-3.0.9.tgz#f2c4b25a600058b3c3773c086c37dbbee1ffe693"
@@ -4116,7 +4162,17 @@ q@^1.1.2:
   version "1.4.1"
   resolved "http://registry.npm.taobao.org/q/download/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e"
 
-qs, qs@6.4.0, qs@~6.4.0:
+qiniu@^6.1.13:
+  version "6.1.13"
+  resolved "https://registry.npmjs.org/qiniu/-/qiniu-6.1.13.tgz#57f5d60974c64d8871d80f27df7d43211ae34f03"
+  dependencies:
+    crc32 "0.2.2"
+    formstream "0.0.7"
+    mime "1.2.9"
+    sync-request "3.0.1"
+    urllib "0.5.1"
+
+qs, qs@6.4.0, qs@^6.1.0, qs@~6.4.0:
   version "6.4.0"
   resolved "http://registry.npm.taobao.org/qs/download/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
 
@@ -4906,6 +4962,14 @@ symbol-observable@^1.0.2:
   version "1.0.4"
   resolved "http://registry.npm.taobao.org/symbol-observable/download/symbol-observable-1.0.4.tgz#29bf615d4aa7121bdd898b22d4b3f9bc4e2aa03d"
 
+sync-request@3.0.1:
+  version "3.0.1"
+  resolved "https://registry.npmjs.org/sync-request/-/sync-request-3.0.1.tgz#caa1235aaf889ba501076a1834c436830a82fb73"
+  dependencies:
+    concat-stream "^1.4.7"
+    http-response-object "^1.0.1"
+    then-request "^2.0.1"
+
 table@^3.7.8:
   version "3.8.3"
   resolved "http://registry.npm.taobao.org/table/download/table-3.8.3.tgz#2bbc542f0fda9861a755d3947fefd8b3f513855f"
@@ -4950,7 +5014,18 @@ text-table@~0.2.0:
   version "0.2.0"
   resolved "http://registry.npm.taobao.org/text-table/download/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
 
-through@^2.3.6:
+then-request@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.npmjs.org/then-request/-/then-request-2.2.0.tgz#6678b32fa0ca218fe569981bbd8871b594060d81"
+  dependencies:
+    caseless "~0.11.0"
+    concat-stream "^1.4.7"
+    http-basic "^2.5.1"
+    http-response-object "^1.1.0"
+    promise "^7.1.1"
+    qs "^6.1.0"
+
+through@^2.3.6, through@~2.3:
   version "2.3.8"
   resolved "http://registry.npm.taobao.org/through/download/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
@@ -5123,6 +5198,13 @@ url@^0.11.0:
     punycode "1.3.2"
     querystring "0.2.0"
 
+urllib@0.5.1:
+  version "0.5.1"
+  resolved "https://registry.npmjs.org/urllib/-/urllib-0.5.1.tgz#9e256b2b703dfe0a59637cb0dbb4308ed571971b"
+  dependencies:
+    buffer-concat "0.0.1"
+    debug "0.7.2"
+
 user-home@^2.0.0:
   version "2.0.0"
   resolved "http://registry.npm.taobao.org/user-home/download/user-home-2.0.0.tgz#9c70bfd8169bc1dcbf48604e0f04b8b49cde9e9f"