Merge pull request #53 from nodejs/pull-l-from-gh

srl295 · web-flow · commit de367a840c44 · 2021-10-04T10:58:00.000-05:00
Pull data files from ICU GitHub instead of npm
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -0,0 +1,17 @@
+module.exports = {
+  env: {
+    commonjs: true,
+    es6: true,
+    node: true
+  },
+  extends: 'standard',
+  globals: {
+    Atomics: 'readonly',
+    SharedArrayBuffer: 'readonly'
+  },
+  parserOptions: {
+    ecmaVersion: 2018
+  },
+  rules: {
+  }
+}
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -3,7 +3,7 @@ name: Lint
 on: [push, pull_request]
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -12,3 +12,12 @@ jobs:
         node-version: 16
     - run: npm i
     - run: npm run lint
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 16
+    - run: npm i
+    - run: npm t
diff --git a/.github/workflows/test-gh.yml b/.github/workflows/test-gh.yml
@@ -1,4 +1,4 @@
-name: Integration Test
+name: Integration Test GitHub
 
 on: [push, pull_request]
 
@@ -28,4 +28,4 @@ jobs:
     - name: Install full-icu
       run: docker run --rm  -v $(pwd):/usr/src/app -w /usr/src/app  ${{ matrix.container }} npm i --no-package-lock --unsafe-perm
     - name: Test full-icu
-      run: docker run --rm  -v $(pwd):/usr/src/app -w /usr/src/app  ${{ matrix.container }} --icu-data-dir=. ./full-icu-test/test.js
+      run: docker run --rm -e NODE_ICU_DATA=. -v $(pwd):/usr/src/app -w /usr/src/app  ${{ matrix.container }} node ./full-icu-test/test.js
diff --git a/.github/workflows/test-npm.yml b/.github/workflows/test-npm.yml
@@ -0,0 +1,24 @@
+name: Integration Test npm
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        container:
+        - node:12
+        - node:12-slim
+        # Will fail on versions that aren't in icu4c-data
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/checkout@v2
+      with:
+        repository: nodejs/full-icu-test
+        path: full-icu-test
+    - name: Install full-icu
+      run: docker run --rm  -v $(pwd):/usr/src/app -w /usr/src/app  ${{ matrix.container }} env FULL_ICU_PREFER_NPM=1 npm i --no-package-lock --unsafe-perm
+    - name: Test full-icu
+      run: docker run --rm -e NODE_ICU_DATA=. -v $(pwd):/usr/src/app -w /usr/src/app  ${{ matrix.container }} node ./full-icu-test/test.js
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,5 @@ node_modules
 *.dat
 npm-debug.log
 /yarn.lock
+package-lock.json
+/.nyc_output
diff --git a/.npmignore b/.npmignore
@@ -2,4 +2,5 @@
 node_modules
 .svn
 .git
-npm-debug.log
+npm-debug.log
+/test
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,14 @@
+language: node_js
+node_js:
+- '8'
+- '10'
+- '11'
+- '12'
+script:
+- npm install
+- npm t
+cache:
+  directories:
+  - node_modules
+  - ".nvm"
+# this is a comment
diff --git a/README.md b/README.md
@@ -1,11 +1,14 @@
 # full-icu-npm
 
-Install full ICU data
+Install full ICU data from GitHub or npm
 
 ### To use
 
 `npm install full-icu`
 
+Note: Set env var `FULL_ICU_PREFER_NPM=true` to prefer using the `icu4c-data` npm module,
+otherwise the default is now to load from ICU4C’s GitHub release.
+
 ### To install globally
 
 `npm install -g full-icu`
diff --git a/install-gh-data.js b/install-gh-data.js
@@ -0,0 +1,61 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs')
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl')
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+  const { icudat, icuend } = fullIcu
+  if (fs.existsSync(icudat)) {
+    console.log(` √ ${icudat} (exists)`)
+    return
+  }
+
+  // Example URL:
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  // ICU v67/v68 use "68.1" and "67.1" in the filename instead of 68_1 and 69_1
+  // https://unicode-org.atlassian.net/browse/ICU-21764
+  // Can remove this conditional if the files are updated later.
+  const versionsAsUnderscore = (fullIcu.icumaj >= 69) ? fullIcu.icuver.replace(/\./g, '_') : fullIcu.icuver
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+
+  // now, unpack it
+  console.log(`Looking for ${icudat}`)
+  return new Promise((resolve, reject) =>
+    yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
+      if (err) return reject(err)
+      zipfile.readEntry()
+      zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
+      zipfile.on('entry', (entry) => {
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry()
+        } else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) {
+          console.log('found ' + entry.fileName)
+          zipfile.openReadStream(entry, (err, readStream) => {
+            if (err) return reject(err)
+            readStream.on('end', () => zipfile.readEntry())
+            const pipeOut = fs.createWriteStream(icudat)
+            readStream.pipe(pipeOut)
+            console.log(` √ ${icudat} (from ICU binary data tarball)`)
+            return resolve()
+          })
+        } else {
+          zipfile.readEntry()
+        }
+      })
+    }))
+}
diff --git a/install-gh.js b/install-gh.js
@@ -0,0 +1,64 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs')
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl')
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+  const { icudat, icuend } = fullIcu
+
+  if (fs.existsSync(icudat)) {
+    console.log(` √ ${icudat} (exists)`)
+    return
+  }
+
+  if (icuend !== 'l') {
+    // Should not hit this, as versions 67 and prior are already in NPM
+    console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.')
+  }
+
+  // var args;
+  // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+  const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+  const baseUrl = new URL(_baseUrl)
+  const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+  const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+  const tag = `release-${versionsAsHyphen}`
+  const file = `icu4c-${versionsAsUnderscore}-src.zip`
+  const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+  console.log(fullUrl.toString())
+  const [srcZip, tmpd] = await myFetch(fullUrl)
+
+  console.log(srcZip, tmpd)
+
+  // now, unpack it
+  console.log(`Looking for ${icudat}`)
+  return new Promise((resolve, reject) =>
+    yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
+      if (err) return reject(err)
+      zipfile.readEntry()
+      zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
+      zipfile.on('entry', (entry) => {
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry()
+        } else if (entry.fileName.endsWith('/' + icudat)) {
+          console.log('found ' + entry.fileName)
+          zipfile.openReadStream(entry, (err, readStream) => {
+            if (err) return reject(err)
+            readStream.on('end', () => zipfile.readEntry())
+            const pipeOut = fs.createWriteStream(icudat)
+            readStream.pipe(pipeOut)
+            console.log(` √ ${icudat} (from ICU source tarball)`)
+            return resolve()
+          })
+        } else {
+          zipfile.readEntry()
+        }
+      })
+    }))
+}
diff --git a/myFetch.js b/myFetch.js
@@ -0,0 +1,59 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+const { sep } = require('path')
+
+function getFetcher (u) {
+  if (u.protocol === 'https:') return require('https')
+  if (u.protocol === 'http:') return require('http')
+  return null
+}
+
+/**
+ * @param {URL} fullUrl url to fetch
+ * @returns {Promse<String[]>} filename, tmpdir
+ */
+function myFetch (fullUrl) {
+  return new Promise((resolve, reject) => {
+    const fetcher = getFetcher(fullUrl)
+    console.log('Fetch:', fullUrl.toString())
+    if (!fetcher) {
+      return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`))
+    }
+
+    fetcher.get(fullUrl, res => {
+      const length = res.headers['content-length']
+      if (res.statusCode === 302 && res.headers.location) {
+        return resolve(myFetch(new URL(res.headers.location)))
+      } else if (res.statusCode !== 200) {
+        return reject(Error(`Bad status code ${res.statusCode}`))
+      }
+      const tmpd = fs.mkdtempSync(`${os.tmpdir()}${sep}`)
+      const tmpf = path.join(tmpd, 'icu-download.zip')
+      let gotSoFar = 0
+      console.dir(tmpd)
+
+      res.on('data', data => {
+        gotSoFar += data.length
+        fs.appendFileSync(tmpf, data)
+        // console.dir(res.headers);
+        process.stdout.write(`${gotSoFar}/${length}\r`)
+        // console.log(`chunk: ${data.length}`);
+      })
+      res.on('end', () => {
+        resolve([tmpf, tmpd])
+        console.log(`${gotSoFar}/${length}\n`)
+      })
+      res.on('error', error => {
+        fs.unlinkSync(tmpf)
+        fs.rmdirSync(tmpd)
+        console.error(error)
+        return reject(error)
+      })
+    })
+  })
+}
+
+module.exports = myFetch
diff --git a/node-full-icu.js b/node-full-icu.js
@@ -1,9 +1,11 @@
 #!/usr/bin/env node
 const { spawn } = require('child_process')
 const data = require('./full-icu')
-const env = data.icu_small ? {
-  ...process.env,
-  NODE_ICU_DATA: data.datPath()
-} : process.env
+const env = data.icu_small
+  ? {
+      ...process.env,
+      NODE_ICU_DATA: data.datPath()
+    }
+  : process.env
 
 spawn('/usr/bin/env', ['node', ...process.argv.slice(2)], { env, stdio: 'inherit' })
diff --git a/package.json b/package.json
@@ -1,10 +1,11 @@
 {
   "name": "full-icu",
-  "version": "1.3.5-0",
+  "version": "1.4.0-0",
   "description": "install 'full-icu' data for your current node",
   "scripts": {
-    "lint": "standard",
-    "postinstall": "node postinstall.js"
+    "lint": "standard && eslint *.js test/*.js",
+    "postinstall": "node postinstall.js",
+    "test": "tap test/*.js"
   },
   "keywords": [
     "icu4c"
@@ -23,7 +24,17 @@
   "bugs": {
     "url": "https://github.com/unicode-org/full-icu-npm/issues"
   },
+  "dependencies": {
+    "yauzl": "^2.10.0"
+  },
   "devDependencies": {
-    "standard": "^16.0.3"
+    "eslint": "^7.7.0",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-header": "^3.0.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "standard": "^16.0.3",
+    "tap": "^15.0.10"
   }
 }
diff --git a/postinstall.js b/postinstall.js
@@ -60,8 +60,29 @@ function advice () {
   console.log("... will show “enero”. If it shows “January” you don't have full data.")
 }
 
-// install by using spawn
-const npmInstall = require('./install-spawn')
+// Choose install method
+let npmInstall
+
+// GitHub has v50+ as releases
+// Experimentally, pull from GitHub for little endian
+if (!process.env.FULL_ICU_PREFER_NPM) {
+  if (fullIcu.icumaj >= 67) {
+    // Pull from bin data zip, first arrived in ICU v67
+    // https://unicode-org.atlassian.net/browse/ICU-20600
+    npmInstall = require('./install-gh-data')
+  } else {
+    if (fullIcu.icuend === 'l') {
+      // Little Endian can pull from icu4c-src.zip which contains a prebuilt data file
+      npmInstall = require('./install-gh')
+    } else {
+      // Fall back to npm
+      console.log(`ICU data bin zip not available until ICU v${fullIcu.icumaj} for endianness ${fullIcu.icuend}: Falling back to npm`)
+      npmInstall = require('./install-spawn')
+    }
+  }
+} else {
+  npmInstall = require('./install-spawn')
+}
 
 if (fs.existsSync(fullIcu.icudat)) {
   console.log('√ ' + fullIcu.icudat + ' Already there (for Node ' + fullIcu.nodever + ' and small-icu ' + fullIcu.icuver + ')')
diff --git a/test/data/haystack.zip b/test/data/haystack.zip
diff --git a/test/test-unzipOne.js b/test/test-unzipOne.js
diff --git a/test/tmp/.keep b/test/tmp/.keep
diff --git a/unzipOne.js b/unzipOne.js
