feat: Pull from GitHub instead of npm

- fetch from ICU’s GitHub release instead of npm (ICU v50+)
- set env FULL_ICU_PREFER_NPM=1 to prefer npm instead
- add .eslint
- for ICU 67 and following, fetch from icu4c-___-data-_.zip
- otherwise fetch from icu4c-src.zip for little endian
- otherwise, use npm as before

Fixes: #36

.eslintrc.js

@@ -0,0 +1,17 @@
+module.exports = {
+ env: {
+ commonjs: true,
+ es6: true,
+ node: true
+ },
+ extends: 'standard',
+ globals: {
+ Atomics: 'readonly',
+ SharedArrayBuffer: 'readonly'
+ },
+ parserOptions: {
+ ecmaVersion: 2018
+ },
+ rules: {
+ }
+}

.github/workflows/lint.yml

@@ -3,7 +3,7 @@ name: Lint
 on: [push, pull_request]
 
 jobs:
- build:
+ lint:
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v2
@@ -12,3 +12,12 @@ jobs:
  node-version: 16
  - run: npm i
  - run: npm run lint
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-node@v1
+ with:
+ node-version: 16
+ - run: npm i
+ - run: npm t

.gitignore

@@ -2,3 +2,5 @@ node_modules
 *.dat
 npm-debug.log
 /yarn.lock
+package-lock.json
+/.nyc_output

.npmignore

@@ -2,4 +2,5 @@
 node_modules
 .svn
 .git
-npm-debug.log
+npm-debug.log
+/test

.npmrc

@@ -0,0 +1 @@
+package-lock=false

.travis.yml

@@ -0,0 +1,14 @@
+language: node_js
+node_js:
+- '8'
+- '10'
+- '11'
+- '12'
+script:
+- npm install
+- npm t
+cache:
+ directories:
+ - node_modules
+ - ".nvm"
+# this is a comment

install-gh-data.js

@@ -0,0 +1,66 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs');
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl');
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+// const icupkg = fullIcu.icupkg
+ const { icudat, icuend } = fullIcu
+ if(fs.existsSync(icudat)) {
+ console.log(` √ ${icudat} (exists)`);
+ return;
+ }
+
+ // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+ // var npmPath = process.env.npm_execpath;
+
+ // var args;
+ // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+ const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+ const baseUrl = new URL(_baseUrl)
+ const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+ const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+ const tag = `release-${versionsAsHyphen}`
+ const file = `icu4c-${versionsAsUnderscore}-data-bin-${icuend}.zip`
+ const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+ console.log(fullUrl.toString())
+ const [srcZip, tmpd] = await myFetch(fullUrl)
+
+ console.log(srcZip, tmpd)
+ // now, unpack it
+
+ console.log(`Looking for ${icudat}`);
+ return new Promise((resolve, reject) =>
+ yauzl.open(srcZip, {lazyEntries: true}, (err, zipfile) => {
+ if (err) return reject(err);
+ zipfile.readEntry();
+ zipfile.on("end", () => reject(`Not found in zipfile: ${icudat}`));
+ zipfile.on("entry", (entry) => {
+ if (entry.fileName.endsWith('/')) {
+ zipfile.readEntry();
+ } else if (entry.fileName.endsWith(icudat) || entry.fileName.endsWith('/' + icudat)) {
+ console.log('found ' + entry.fileName);
+ zipfile.openReadStream(entry, (err, readStream) => {
+ if (err) return reject(err);
+ // if entry.file
+ readStream.on("end", () => zipfile.readEntry());
+ const pipeOut = fs.createWriteStream(icudat);
+ readStream.pipe(pipeOut);
+ console.log(` √ ${icudat} (from ICU binary data tarball)`);
+ return resolve();
+ });
+ } else {
+ zipfile.readEntry(); // get next
+ }
+ });
+ }));
+}

install-gh.js

@@ -0,0 +1,71 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+// Install by fetching ICU source tarball
+// This will only work for little endian systems, but will work for ancient ICU (back to v50)
+
+const fs = require('fs');
+const { URL } = require('url')
+const process = require('process')
+const myFetch = require('./myFetch')
+const yauzl = require('yauzl');
+
+// var isglobal = process.env.npm_config_global === 'true';
+
+module.exports = async function installFromGithub (fullIcu, advice) {
+ const {icupkg, icudat, icuend} = fullIcu
+
+ if(fs.existsSync(icudat)) {
+ console.log(` √ ${icudat} (exists)`);
+ return;
+ }
+
+ if (icuend != 'l') {
+ // Should not hit this, as versions 67 and prior are already in NPM
+ console.error('Warning: this method probably will fail, because the ICU source tarball only contains little endian data.');
+ }
+
+ // var cmdPath = nodePath = process.env.npm_node_execpath;
+
+ // var npmPath = process.env.npm_execpath;
+
+ // var args;
+ // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip
+ const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/'
+ const baseUrl = new URL(_baseUrl)
+ const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-')
+ const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_')
+ const tag = `release-${versionsAsHyphen}`
+ const file = `icu4c-${versionsAsUnderscore}-src.zip`
+ const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl)
+ console.log(fullUrl.toString())
+ const [srcZip, tmpd] = await myFetch(fullUrl)
+
+ console.log(srcZip, tmpd)
+ // now, unpack it
+
+ console.log(`Looking for ${icudat}`);
+ return new Promise((resolve, reject) =>
+ yauzl.open(srcZip, {lazyEntries: true}, (err, zipfile) => {
+ if (err) return reject(err);
+ zipfile.readEntry();
+ zipfile.on("end", () => reject(`Not found in zipfile: ${icudat}`));
+ zipfile.on("entry", (entry) => {
+ if (entry.fileName.endsWith('/')) {
+ zipfile.readEntry();
+ } else if(entry.fileName.endsWith('/'+icudat)) {
+ console.log('found ' + entry.fileName);
+ zipfile.openReadStream(entry, (err, readStream) => {
+ if (err) return reject(err);
+ // if entry.file
+ readStream.on("end", () => zipfile.readEntry());
+ const pipeOut = fs.createWriteStream(icudat);
+ readStream.pipe(pipeOut);
+ console.log(` √ ${icudat} (from ICU source tarball)`);
+ return resolve();
+ });
+ } else {
+ zipfile.readEntry(); // get next
+ }
+ });
+ }));
+}

myFetch.js

@@ -0,0 +1,58 @@
+// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved.
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+
+function getFetcher (u) {
+ if (u.protocol === 'https:') return require('https')
+ if (u.protocol === 'http:') return require('http')
+ return null
+}
+
+/**
+ * @param {URL} fullUrl url to fetch
+ * @returns {Promse<String[]>} filename, tmpdir
+ */
+function myFetch (fullUrl) {
+ return new Promise((resolve, reject) => {
+ const fetcher = getFetcher(fullUrl)
+ console.log('Fetch:', fullUrl.toString())
+ if (!fetcher) {
+ return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`))
+ }
+
+ fetcher.get(fullUrl, res => {
+ const length = res.headers['content-length']
+ if (res.statusCode === 302 && res.headers.location) {
+ return resolve(myFetch(new URL(res.headers.location)))
+ } else if (res.statusCode !== 200) {
+ return reject(Error(`Bad status code ${res.statusCode}`))
+ }
+ const tmpd = fs.mkdtempSync(os.tmpdir())
+ const tmpf = path.join(tmpd, 'icu-download.zip')
+ let gotSoFar = 0
+ console.dir(tmpd)
+
+ res.on('data', data => {
+ gotSoFar += data.length
+ fs.appendFileSync(tmpf, data)
+ // console.dir(res.headers);
+ process.stdout.write(`${gotSoFar}/${length}\r`)
+ // console.log(`chunk: ${data.length}`);
+ })
+ res.on('end', () => {
+ resolve([tmpf, tmpd])
+ console.log(`${gotSoFar}/${length}\n`)
+ })
+ res.on('error', error => {
+ fs.unlinkSync(tmpf)
+ fs.rmdirSync(tmpd)
+ console.error(error)
+ return reject(error)
+ })
+ })
+ })
+}
+
+module.exports = myFetch

package.json

@@ -3,8 +3,9 @@
  "version": "1.3.5-0",
  "description": "install 'full-icu' data for your current node",
  "scripts": {
- "lint": "standard",
- "postinstall": "node postinstall.js"
+ "lint": "standard && eslint *.js test/*.js",
+ "postinstall": "node postinstall.js",
+ "test": "tap test/*.js"
  },
  "keywords": [
  "icu4c"
@@ -22,7 +23,17 @@
  "bugs": {
  "url": "https://github.com/unicode-org/full-icu-npm/issues"
  },
+ "dependencies": {
+ "yauzl": "^2.10.0"
+ },
  "devDependencies": {
- "standard": "^16.0.3"
+ "eslint": "^7.7.0",
+ "eslint-config-standard": "^16.0.3",
+ "eslint-plugin-header": "^3.0.0",
+ "eslint-plugin-import": "^2.24.2",
+ "eslint-plugin-node": "^11.1.0",
+ "eslint-plugin-promise": "^4.2.1",
+ "standard": "^16.0.3",
+ "tap": "^15.0.10"
  }
 }

postinstall.js

@@ -60,8 +60,29 @@ function advice () {
  console.log("... will show “enero”. If it shows “January” you don't have full data.")
 }
 
-// install by using spawn
-const npmInstall = require('./install-spawn')
+// Choose install method
+let npmInstall
+
+// GitHub has v50+ as releases
+// Experimentally, pull from GitHub for little endian
+if (!process.env.FULL_ICU_PREFER_NPM) {
+ if (fullIcu.icumaj >= 67) {
+ // Pull from bin data zip, first arrived in ICU v67
+ // https://unicode-org.atlassian.net/browse/ICU-20600
+ npmInstall = require('./install-gh-data')
+ } else {
+ if (fullIcu.icuend === 'l') {
+ // Little Endian can pull from icu4c-src.zip which contains a prebuilt data file
+ npmInstall = require('./install-gh')
+ } else {
+ // Fall back to npm
+ console.log(`ICU data bin zip not available until ICU v${fullIcu.icumaj} for endianness ${fullIcu.icuend}: Falling back to npm`);
+ npmInstall = require('./install-spawn')
+ }
+ }
+} else {
+ npmInstall = require('./install-spawn')
+}
 
 if (fs.existsSync(fullIcu.icudat)) {
  console.log('√ ' + fullIcu.icudat + ' Already there (for Node ' + fullIcu.nodever + ' and small-icu ' + fullIcu.icuver + ')')

test/test-unzipOne.js

@@ -0,0 +1,32 @@
+const tap = require('tap')
+const fs = require('fs')
+const unzipOne = require('../unzipOne')
+
+tap.test('unzipOne', async t => {
+ t.test('setup', t => {
+ try {
+ fs.unlinkSync('test/tmp/needle.txt')
+ } catch (e) { /* ignore */ }
+ t.end()
+ })
+ t.test('no easteregg in haystack.zip', async t => {
+ const ee = await unzipOne('./test/data/haystack.zip', 'easteregg.txt', './test/tmp/')
+ t.notOk(ee, 'Did not expect to find easteregg in haystack: ' + ee)
+ t.end()
+ })
+ t.test('get needle.txt in haystack.zip', async t => {
+ const ee = await unzipOne('./test/data/haystack.zip', 'needle.txt', './test/tmp/')
+ t.ok(ee, 'Did expect to find needle.txt in haystack: ' + ee)
+ const truism = fs.readFileSync('./test/tmp/needle.txt', 'utf-8')
+ t.ok(truism)
+ t.equal(truism.trim(), 'true')
+ t.end()
+ })
+ t.test('cleanup', t => {
+ try {
+ fs.unlinkSync('test/tmp/needle.txt')
+ } catch (e) { /* ignore */ }
+ t.end()
+ })
+ t.end()
+})