Skip to content

Commit a57877d

Browse files
srl295srl295
committed
feat: Pull from GitHub instead of npm
Fixes: #36 - fetch from ICU’s GitHub release instead of npm (ICU v50+) - set env FULL_ICU_PREFER_NPM=1 to prefer npm instead - add .eslint (as well as standard) - for ICU 67 and following, fetch from icu4c-___-data-_.zip - otherwise fetch from icu4c-src.zip for little endian - otherwise, use npm as before - work around ICU issue with data file name https://unicode-org.atlassian.net/browse/ICU-21764 - document the FULL_ICU_PREFER_NPM - test with FULL_ICU_PREFER_NPM=1
1 parent 0a1b7a4 commit a57877d

File tree

18 files changed

+390
-14
lines changed

18 files changed

+390
-14
lines changed

‎.eslintrc.js‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
module.exports = {
2+
env: {
3+
commonjs: true,
4+
es6: true,
5+
node: true
6+
},
7+
extends: 'standard',
8+
globals: {
9+
Atomics: 'readonly',
10+
SharedArrayBuffer: 'readonly'
11+
},
12+
parserOptions: {
13+
ecmaVersion: 2018
14+
},
15+
rules: {
16+
}
17+
}

‎.github/workflows/lint.yml‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: Lint
33
on: [push, pull_request]
44

55
jobs:
6-
build:
6+
lint:
77
runs-on: ubuntu-latest
88
steps:
99
- uses: actions/checkout@v2
@@ -12,3 +12,12 @@ jobs:
1212
node-version: 16
1313
- run: npm i
1414
- run: npm run lint
15+
test:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v2
19+
- uses: actions/setup-node@v1
20+
with:
21+
node-version: 16
22+
- run: npm i
23+
- run: npm t

‎.github/workflows/test.yml‎ renamed to ‎.github/workflows/test-gh.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Integration Test
1+
name: Integration Test GitHub
22

33
on: [push, pull_request]
44

@@ -28,4 +28,4 @@ jobs:
2828
- name: Install full-icu
2929
run: docker run --rm -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} npm i --no-package-lock --unsafe-perm
3030
- name: Test full-icu
31-
run: docker run --rm -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} --icu-data-dir=. ./full-icu-test/test.js
31+
run: docker run --rm -e NODE_ICU_DATA=. -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} node ./full-icu-test/test.js

‎.github/workflows/test-npm.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: Integration Test npm
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
build:
7+
strategy:
8+
fail-fast: false
9+
matrix:
10+
container:
11+
- node:12
12+
- node:12-slim
13+
# Will fail on versions that aren't in icu4c-data
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v2
17+
- uses: actions/checkout@v2
18+
with:
19+
repository: nodejs/full-icu-test
20+
path: full-icu-test
21+
- name: Install full-icu
22+
run: docker run --rm -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} env FULL_ICU_PREFER_NPM=1 npm i --no-package-lock --unsafe-perm
23+
- name: Test full-icu
24+
run: docker run --rm -e NODE_ICU_DATA=. -v $(pwd):/usr/src/app -w /usr/src/app ${{ matrix.container }} node ./full-icu-test/test.js

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@ node_modules
22
*.dat
33
npm-debug.log
44
/yarn.lock
5+
package-lock.json
6+
/.nyc_output

‎.npmignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22
node_modules
33
.svn
44
.git
5-
npm-debug.log
5+
npm-debug.log
6+
/test

‎.travis.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
language: node_js
2+
node_js:
3+
- '8'
4+
- '10'
5+
- '11'
6+
- '12'
7+
script:
8+
- npm install
9+
- npm t
10+
cache:
11+
directories:
12+
- node_modules
13+
- ".nvm"
14+
# this is a comment

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
# full-icu-npm
22

3-
Install full ICU data
3+
Install full ICU data from GitHub or npm
44

55
### To use
66

77
`npm install full-icu`
88

9+
Note: Set env var `FULL_ICU_PREFER_NPM=true` to prefer using the `icu4c-data` npm module,
10+
otherwise the default is now to load from ICU4C’s GitHub release.
11+
912
### To install globally
1013

1114
`npm install -g full-icu`

‎install-gh-data.js‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
2+
3+
// Install by fetching ICU source tarball
4+
// This will only work for little endian systems, but will work for ancient ICU (back to v50)
5+
6+
const fs = require('fs')
7+
const { URL } = require('url')
8+
const process = require('process')
9+
const myFetch = require('./myFetch')
10+
const yauzl = require('yauzl')
11+
12+
// var isglobal = process.env.npm_config_global === 'true';
13+
14+
module.exports = async function installFromGithub (fullIcu, advice) {
15+
// const icupkg = fullIcu.icupkg
16+
const { icudat, icuend } = fullIcu
17+
if (fs.existsSync(icudat)) {
18+
console.log(` √ ${icudat} (exists)`)
19+
return
20+
}
21+
22+
// var cmdPath = nodePath = process.env.npm_node_execpath;
23+
24+
// var npmPath = process.env.npm_execpath;
25+
26+
// var args;
27+
// https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
28+
const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
29+
const baseUrl = new URL(_baseUrl)
30+
const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
31+
// ICU v67/v68 use "68.1" and "67.1" in the filename instead of 68_1 and 69_1
32+
// https://unicode-org.atlassian.net/browse/ICU-21764
33+
// Can remove this conditional if the files are updated later.
34+
const versionsAsUnderscore = (fullIcu.icumaj >= 69) ? fullIcu.icuver.replace(/\./g, '_') : fullIcu.icuver
35+
const tag = `release-${versionsAsHyphen}`
36+
const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip`
37+
const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
38+
console.log(fullUrl.toString())
39+
const [srcZip, tmpd] = await myFetch(fullUrl)
40+
41+
console.log(srcZip, tmpd)
42+
// now, unpack it
43+
44+
console.log(`Looking for ${icudat}`)
45+
return new Promise((resolve, reject) =>
46+
yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
47+
if (err) return reject(err)
48+
zipfile.readEntry()
49+
zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
50+
zipfile.on('entry', (entry) => {
51+
if (entry.fileName.endsWith('/')) {
52+
zipfile.readEntry()
53+
} else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) {
54+
console.log('found ' + entry.fileName)
55+
zipfile.openReadStream(entry, (err, readStream) => {
56+
if (err) return reject(err)
57+
// if entry.file
58+
readStream.on('end', () => zipfile.readEntry())
59+
const pipeOut = fs.createWriteStream(icudat)
60+
readStream.pipe(pipeOut)
61+
console.log(` √ ${icudat} (from ICU binary data tarball)`)
62+
return resolve()
63+
})
64+
} else {
65+
zipfile.readEntry() // get next
66+
}
67+
})
68+
}))
69+
}

‎install-gh.js‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
2+
3+
// Install by fetching ICU source tarball
4+
// This will only work for little endian systems, but will work for ancient ICU (back to v50)
5+
6+
const fs = require('fs')
7+
const { URL } = require('url')
8+
const process = require('process')
9+
const myFetch = require('./myFetch')
10+
const yauzl = require('yauzl')
11+
12+
// var isglobal = process.env.npm_config_global === 'true';
13+
14+
module.exports = async function installFromGithub (fullIcu, advice) {
15+
const { icudat, icuend } = fullIcu
16+
17+
if (fs.existsSync(icudat)) {
18+
console.log(` √ ${icudat} (exists)`)
19+
return
20+
}
21+
22+
if (icuend !== 'l') {
23+
// Should not hit this, as versions 67 and prior are already in NPM
24+
console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.')
25+
}
26+
27+
// var cmdPath = nodePath = process.env.npm_node_execpath;
28+
29+
// var npmPath = process.env.npm_execpath;
30+
31+
// var args;
32+
// https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
33+
const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
34+
const baseUrl = new URL(_baseUrl)
35+
const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
36+
const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
37+
const tag = `release-${versionsAsHyphen}`
38+
const file = `icu4c-${versionsAsUnderscore}-src.zip`
39+
const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
40+
console.log(fullUrl.toString())
41+
const [srcZip, tmpd] = await myFetch(fullUrl)
42+
43+
console.log(srcZip, tmpd)
44+
// now, unpack it
45+
46+
console.log(`Looking for ${icudat}`)
47+
return new Promise((resolve, reject) =>
48+
yauzl.open(srcZip, { lazyEntries: true }, (err, zipfile) => {
49+
if (err) return reject(err)
50+
zipfile.readEntry()
51+
zipfile.on('end', () => reject(Error(`Not found in zipfile: ${icudat}`)))
52+
zipfile.on('entry', (entry) => {
53+
if (entry.fileName.endsWith('/')) {
54+
zipfile.readEntry()
55+
} else if (entry.fileName.endsWith('/' + icudat)) {
56+
console.log('found ' + entry.fileName)
57+
zipfile.openReadStream(entry, (err, readStream) => {
58+
if (err) return reject(err)
59+
// if entry.file
60+
readStream.on('end', () => zipfile.readEntry())
61+
const pipeOut = fs.createWriteStream(icudat)
62+
readStream.pipe(pipeOut)
63+
console.log(` √ ${icudat} (from ICU source tarball)`)
64+
return resolve()
65+
})
66+
} else {
67+
zipfile.readEntry() // get next
68+
}
69+
})
70+
}))
71+
}

0 commit comments

Comments
 (0)