feat: Pull from GitHub instead of npm

srl295 · srl295 · commit 3a0a45a0bbf0 · 2021-09-24T17:34:45.000-05:00
- fetch from ICU’s GitHub release instead of npm (ICU v50+) - set env FULL_ICU_PREFER_NPM=1 to prefer npm instead - add .eslint - for ICU 67 and following, fetch from icu4c-___-data-_.zip otherwise fetch from icu4c-src.zip (will only work for little endian) Fixes: #36
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    commonjs: true,
+    es6: true,
+    node: true
+  },
+  extends: 'standard',
+  globals: {
+    Atomics: 'readonly',
+    SharedArrayBuffer: 'readonly'
+  },
+  parserOptions: {
+    ecmaVersion: 2018
+  },
+  rules: {
+  }
+}
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -3,7 +3,7 @@ name: Lint
 on: [push, pull_request]
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -12,3 +12,12 @@ jobs:
         node-version: 16
     - run: npm i
     - run: npm run lint
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 16
+    - run: npm i
+    - run: npm t
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,5 @@ node_modules
 *.dat
 npm-debug.log
 /yarn.lock
+package-lock.json
+/.nyc_output
diff --git a/.npmignore b/.npmignore
@@ -2,4 +2,5 @@
 node_modules
 .svn
 .git
-npm-debug.log
+npm-debug.log
+/test
diff --git a/.npmrc b/.npmrc
@@ -0,0 +1 @@
+package-lock=false
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,14 @@
+language: node_js
+node_js:
+- '8'
+- '10'
+- '11'
+- '12'
+script:
+- npm install
+- npm t
+cache:
+  directories:
+  - node_modules
+  - ".nvm"
+# this is a comment
diff --git a/install-gh-data.js b/install-gh-data.js
@@ -0,0 +1,66 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs');
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl');
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+//   const icupkg = fullIcu.icupkg
+  const { icudat, icuend } = fullIcu
+  if(fs.existsSync(icudat)) {
+	console.log(` √ ${icudat} (exists)`);
+	return;
+  }
+
+  // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+  // var npmPath = process.env.npm_execpath;
+
+  // var args;
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+  // now, unpack it
+
+  console.log(`Looking for ${icudat}`);
+  return new Promise((resolve, reject) =>
+	yauzl.open(srcZip, {lazyEntries: true}, (err, zipfile) => {
+		if (err) return reject(err);
+		zipfile.readEntry();
+		zipfile.on("end", () => reject(`Not found in zipfile: ${icudat}`));
+		zipfile.on("entry", (entry) => {
+			if (entry.fileName.endsWith('/')) {
+				zipfile.readEntry();
+			} else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) {
+				console.log('found ' + entry.fileName);
+				zipfile.openReadStream(entry, (err, readStream) => {
+					if (err) return reject(err);
+					// if entry.file
+					readStream.on("end", () => zipfile.readEntry());
+					const pipeOut = fs.createWriteStream(icudat);
+					readStream.pipe(pipeOut);
+					console.log(` √ ${icudat} (from ICU binary data tarball)`);
+					return resolve();
+				});
+			} else {
+				zipfile.readEntry(); // get next
+			}
+		});
+	}));
+}
diff --git a/install-gh.js b/install-gh.js
@@ -0,0 +1,71 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs');
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl');
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+  const {icupkg, icudat, icuend} = fullIcu
+
+  if(fs.existsSync(icudat)) {
+	console.log(` √ ${icudat} (exists)`);
+	return;
+  }
+
+  if (icuend != 'l') {
+	  // Should not hit this, as versions 67 and prior are already in NPM
+	  console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.');
+  }
+
+  // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+  // var npmPath = process.env.npm_execpath;
+
+  // var args;
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-src.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+  // now, unpack it
+
+  console.log(`Looking for ${icudat}`);
+  return new Promise((resolve, reject) =>
+	yauzl.open(srcZip, {lazyEntries: true}, (err, zipfile) => {
+		if (err) return reject(err);
+		zipfile.readEntry();
+		zipfile.on("end", () => reject(`Not found in zipfile: ${icudat}`));
+		zipfile.on("entry", (entry) => {
+			if (entry.fileName.endsWith('/')) {
+				zipfile.readEntry();
+			} else if(entry.fileName.endsWith('/'+icudat)) {
+				console.log('found ' + entry.fileName);
+				zipfile.openReadStream(entry, (err, readStream) => {
+					if (err) return reject(err);
+					// if entry.file
+					readStream.on("end", () => zipfile.readEntry());
+					const pipeOut = fs.createWriteStream(icudat);
+					readStream.pipe(pipeOut);
+					console.log(` √ ${icudat} (from ICU source tarball)`);
+					return resolve();
+				});
+			} else {
+				zipfile.readEntry(); // get next
+			}
+		});
+	}));
+}
diff --git a/myFetch.js b/myFetch.js
@@ -0,0 +1,58 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+
+function getFetcher (u) {
+  if (u.protocol === 'https:') return require('https')
+  if (u.protocol === 'http:') return require('http')
+  return null
+}
+
+/**
+ * @param {URL} fullUrl url to fetch
+ * @returns {Promse<String[]>} filename, tmpdir
+ */
+function myFetch (fullUrl) {
+  return new Promise((resolve, reject) => {
+    const fetcher = getFetcher(fullUrl)
+    console.log('Fetch:', fullUrl.toString())
+    if (!fetcher) {
+      return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`))
+    }
+
+    fetcher.get(fullUrl, res => {
+      const length = res.headers['content-length']
+      if (res.statusCode === 302 && res.headers.location) {
+        return resolve(myFetch(new URL(res.headers.location)))
+      } else if (res.statusCode !== 200) {
+        return reject(Error(`Bad status code ${res.statusCode}`))
+      }
+      const tmpd = fs.mkdtempSync(os.tmpdir())
+      const tmpf = path.join(tmpd, 'icu-download.zip')
+      let gotSoFar = 0
+      console.dir(tmpd)
+
+      res.on('data', data => {
+        gotSoFar += data.length
+        fs.appendFileSync(tmpf, data)
+        // console.dir(res.headers);
+        process.stdout.write(`${gotSoFar}/${length}\r`)
+        // console.log(`chunk: ${data.length}`);
+      })
+      res.on('end', () => {
+        resolve([tmpf, tmpd])
+        console.log(`${gotSoFar}/${length}\n`)
+      })
+      res.on('error', error => {
+        fs.unlinkSync(tmpf)
+        fs.rmdirSync(tmpd)
+        console.error(error)
+        return reject(error)
+      })
+    })
+  })
+}
+
+module.exports = myFetch
diff --git a/package.json b/package.json
@@ -3,8 +3,9 @@
   "version": "1.3.5-0",
   "description": "install 'full-icu' data for your current node",
   "scripts": {
-    "lint": "standard",
-    "postinstall": "node postinstall.js"
+    "lint": "standard && eslint *.js test/*.js",
+    "postinstall": "node postinstall.js",
+    "test": "tap test/*.js"
   },
   "keywords": [
     "icu4c"
@@ -22,7 +23,17 @@
   "bugs": {
     "url": "https://github.com/unicode-org/full-icu-npm/issues"
   },
+  "dependencies": {
+    "yauzl": "^2.10.0"
+  },
   "devDependencies": {
-    "standard": "^16.0.3"
+    "eslint": "^7.7.0",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-header": "^3.0.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "standard": "^16.0.3",
+    "tap": "^15.0.10"
   }
 }
diff --git a/postinstall.js b/postinstall.js
@@ -60,8 +60,20 @@ function advice () {
   console.log("... will show “enero”. If it shows “January” you don't have full data.")
 }
 
-// install by using spawn
-const npmInstall = require('./install-spawn')
+// Choose install method
+let npmInstall
+
+// GitHub has v50+ as releases
+// Experimentally, pull from GitHub for little endian
+if (fullIcu.icuend === 'l' && !process.env.FULL_ICU_PREFER_NPM) {
+  if (fullIcu.icuMaj < 67) { // https://unicode-org.atlassian.net/browse/ICU-20600
+    npmInstall = require('./install-gh')
+  } else {
+    npmInstall = require('./install-gh-data')
+  }
+} else {
+  npmInstall = require('./install-spawn')
+}
 
 if (fs.existsSync(fullIcu.icudat)) {
   console.log('√ ' + fullIcu.icudat + ' Already there (for Node ' + fullIcu.nodever + ' and small-icu ' + fullIcu.icuver + ')')
diff --git a/test/data/haystack.zip b/test/data/haystack.zip
diff --git a/test/test-unzipOne.js b/test/test-unzipOne.js
@@ -0,0 +1,32 @@
+const tap = require('tap')
+const fs = require('fs')
+const unzipOne = require('../unzipOne')
+
+tap.test('unzipOne', async t => {
+  t.test('setup', t => {
+    try {
+      fs.unlinkSync('test/tmp/needle.txt')
+    } catch (e) { /* ignore */ }
+    t.end()
+  })
+  t.test('no easteregg in haystack.zip', async t => {
+    const ee = await unzipOne('./test/data/haystack.zip', 'easteregg.txt', './test/tmp/')
+    t.notOk(ee, 'Did not expect to find easteregg in haystack: ' + ee)
+    t.end()
+  })
+  t.test('get needle.txt in haystack.zip', async t => {
+    const ee = await unzipOne('./test/data/haystack.zip', 'needle.txt', './test/tmp/')
+    t.ok(ee, 'Did expect to find needle.txt in haystack: ' + ee)
+    const truism = fs.readFileSync('./test/tmp/needle.txt', 'utf-8')
+    t.ok(truism)
+    t.equal(truism.trim(), 'true')
+    t.end()
+  })
+  t.test('cleanup', t => {
+    try {
+      fs.unlinkSync('test/tmp/needle.txt')
+    } catch (e) { /* ignore */ }
+    t.end()
+  })
+  t.end()
+})
diff --git a/test/tmp/.keep b/test/tmp/.keep
diff --git a/unzipOne.js b/unzipOne.js
@@ -0,0 +1,40 @@
+const yauzl = require('yauzl')
+const { basename, join } = require('path')
+const fs = require('fs')
+
+/**
+ * unzip and write file 'fn'  to 'dstDir'
+ * @param {String} srcZip source zipfile
+ * @param {String} fn to unzip
+ * @param {String} dstDir destination dir
+ * @returns {Promise<String>} to output filename if successful, or falsy if the file was not found.
+ */
+function unzipOne (srcZip, fn, dstDir) {
+  const outFile = join(dstDir, fn)
+  return new Promise((resolve, reject) => {
+    yauzl.open(srcZip, { lazyEntries: true },
+      (err, zipfile) => {
+        if (err) return reject(err)
+        zipfile.readEntry()
+        zipfile.on('entry', entry => {
+          if (basename(entry.fileName) === fn) {
+            zipfile.openReadStream(entry, (err, readStream) => {
+              if (err) return reject(err)
+              readStream.on('end', () => {
+                zipfile.close()
+                resolve(entry.fileName)
+              })
+              readStream.pipe(fs.createWriteStream(outFile))
+            })
+          } else {
+            zipfile.readEntry()
+          }
+        })
+        zipfile.on('end', () => {
+          resolve() // not found
+        })
+      })
+  })
+}
+
+module.exports = unzipOne
