build: i18n: Autodownload ICU, Intl small-icu by default!

This is to support joyent#7676, Intl by default.

* make `--with-intl=small-icu` the default
 * Download, verify (md5), unpack ICU's zip if not there
 * update docs

There's a "list" of URLs being used, but right now only the
first is picked up. The logic works something like this:

* if there is no directory `deps/icu`,
 * if no zip file (currently `icu4c-54_1-src.zip`),
  * download zip file (icu-project.org -> sf.net)
 * verify the MD5 sum of the zipfile
  * if bad, print error and exit
 * unpack the zipfile into `deps/icu`
* if `deps/icu` now exists, use it, else fail with help text

Also:
* refactor some code into tools/configure.d/nodedownload.py
* add `intl-none` option for `vcbuild.bat`

To rebuild `deps/icu-small` -
```
bash tools/icu/prepare-icu-source.sh
```

Also:
Reduce space by about 1MB with ICU 54 (over without this patch).
Also trims a few other source files, but only conditional on the exact ICU
version used. This is to future-proof - a file that is unneeded now may
be needed in future ICUs.

Author: srl295

.gitignore

@@ -46,6 +46,8 @@ ipch/
 email.md
 deps/v8-*
 deps/icu
+deps/icu*.zip
+deps/icu*.tgz
 ./node_modules
 .svn/
 

README.md

@@ -83,20 +83,73 @@ make doc
 man doc/node.1
 ```
 
-### To build `Intl` (ECMA-402) support:
+### `Intl` (ECMA-402) support:
 
-*Note:* more docs, including how to reduce disk footprint, are on
+#### "small" (English only) support
+
+The default is to build with "small" (English only) support, so the
+`small-icu` parameter isn't strictly necessary. It's included here
+for documentation.
+
+Unix/Macintosh:
+
+```sh
+./configure --with-intl=small-icu
+```
+
+Windows:
+
+```sh
+vcbuild small-icu
+```
+
+The `small-icu` mode builds
+with English-only data. You can add full data at runtime.
+
+*Note:* more docs are on
 [the wiki](https://github.com/joyent/node/wiki/Intl).
 
+#### Build with full ICU support (all locales supported by ICU):
+
+*Note*, this may download ICU if you don't have an ICU in `deps/icu`
+
+Unix/Macintosh:
+
+```sh
+./configure --with-intl=full-icu
+```
+
+Windows:
+
+```sh
+vcbuild full-icu
+```
+
+#### Build with no Intl support `:-(`
+
+The `Intl` object will not be available.
+
+Unix/Macintosh:
+
+```sh
+./configure --with-intl=none
+```
+
+Windows:
+
+```sh
+vcbuild intl-none
+```
+
 #### Use existing installed ICU (Unix/Macintosh only):
 
 ```sh
 pkg-config --modversion icu-i18n && ./configure --with-intl=system-icu
 ```
 
-#### Build ICU from source:
+#### Build with a specific ICU:
 
-First: Unpack latest ICU
+First: Unpack latest ICU to `deps/icu`
   [icu4c-**##.#**-src.tgz](http://icu-project.org/download) (or `.zip`)
   as `deps/icu` (You'll have: `deps/icu/source/...`)
 

configure

@@ -6,13 +6,21 @@ import re
 import shlex
 import subprocess
 import sys
+import urllib
+import zipfile
+import hashlib
+import shutil
 
 CC = os.environ.get('CC', 'cc')
 
 root_dir = os.path.dirname(__file__)
 sys.path.insert(0, os.path.join(root_dir, 'tools', 'gyp', 'pylib'))
 from gyp.common import GetFlavor
 
+# imports in tools/configure.d
+sys.path.insert(0, os.path.join(root_dir, 'tools', 'configure.d'))
+import nodedownload
+
 # parse our options
 parser = optparse.OptionParser()
 
@@ -244,7 +252,7 @@ parser.add_option('--with-icu-path',
 parser.add_option('--with-intl',
     action='store',
     dest='with_intl',
-    help='Intl mode: none, full-icu, small-icu (default is none)')
+    help='Intl mode: none, full-icu, small-icu (default is small-icu)')
 
 parser.add_option('--with-perfctr',
     action='store_true',
@@ -712,6 +720,58 @@ def glob_to_var(dir_base, dir_sub):
   return list
 
 def configure_intl(o):
+  icus = [
+    {
+      'url': 'http://download.icu-project.org/files/icu4c/54.1/icu4c-54_1-src.zip',
+      # from https://ssl.icu-project.org/files/icu4c/54.1/icu4c-src-54_1.md5:
+      'md5': '6b89d60e2f0e140898ae4d7f72323bca',
+    },
+  ]
+  class ConfigOpener(urllib.FancyURLopener):
+    # append to existing version (UA)
+    version = '%s (node.js/configure)' % urllib.URLopener.version
+  def icu_download(path):
+    # download ICU, if needed
+    def fmtMb(amt):
+      return "{:.1f}".format(amt / 1024000.)
+    def reporthook(count, size, total):
+      sys.stdout.write(' ICU: %c %sMB total, %sMB downloaded   \r' %
+                       (nodedownload.spin(count),
+                        nodedownload.formatSize(total),
+                        nodedownload.formatSize(count*size)))
+    for icu in icus:
+      url = icu['url']
+      md5 = icu['md5']
+      local = url.split('/')[-1]
+      targetfile = os.path.join(root_dir, 'deps', local)
+      if not os.path.isfile(targetfile):
+        try:
+          sys.stdout.write(' <%s>\nConnecting...\r' % url)
+          sys.stdout.flush()
+          msg = urllib.urlretrieve(url, targetfile, reporthook=reporthook)
+          print ''  # clear the line
+        except:
+          print ' ** Error occurred while downloading\n <%s>' % url
+          raise
+      else:
+        print ' Re-using existing %s' % targetfile
+      if os.path.isfile(targetfile):
+        digest = hashlib.md5()
+        count = 0
+        sys.stdout.write(' Checking file integrity with MD5:\r')
+        with open(targetfile, 'rb') as f:
+          chunk = f.read(1024)
+          while chunk !=  "":
+            digest.update(chunk)
+            chunk = f.read(1024)
+        gotmd5 = digest.hexdigest()
+        print ' MD5:      %s  %s' % (gotmd5, targetfile)
+        if (md5 == gotmd5):
+          return targetfile
+        else:
+          print ' Expected: %s      *MISMATCH*' % md5
+          print '\n ** Corrupted ZIP? Delete %s to retry download.\n' % targetfile
+    return None
   icu_config = {
     'variables': {}
   }
@@ -723,7 +783,6 @@ def configure_intl(o):
   write(icu_config_name, do_not_edit +
         pprint.pformat(icu_config, indent=2) + '\n')
 
-  # small ICU is off by default.
   # always set icu_small, node.gyp depends on it being defined.
   o['variables']['icu_small'] = b(False)
 
@@ -739,6 +798,8 @@ def configure_intl(o):
     o['variables']['icu_gyp_path'] = options.with_icu_path
     return
   # --with-intl=<with_intl>
+  if with_intl is None:
+    with_intl = 'small-icu'  # The default mode of Intl
   if with_intl == 'none' or with_intl is None:
     o['variables']['v8_enable_i18n_support'] = 0
     return  # no Intl
@@ -769,20 +830,47 @@ def configure_intl(o):
   # Note: non-ICU implementations could use other 'with_intl'
   # values.
 
+  icu_parent_path = os.path.join(root_dir, 'deps')
+  icu_full_path = os.path.join(icu_parent_path, 'icu')
+  icu_small_path = os.path.join(icu_parent_path, 'icu-small')
+  icu_small_tag = os.path.join(icu_full_path, 'is-small-icu.txt')
+
+  ## Use (or not) an embedded small-icu.
+  if with_intl == 'small-icu':
+    if not os.path.isdir(icu_full_path) and os.path.isdir(icu_small_path):
+      # deps/small-icu -> deps/icu
+      print 'Copying small ICU %s to %s' % (icu_small_path, icu_full_path)
+      shutil.copytree(icu_small_path, icu_full_path)
+    #else:
+    #  print 'Not copying %s to %s' % (icu_small_path, icu_full_path)
+  elif os.path.isfile(icu_small_tag):
+    print 'deleting small-icu %s for --with-intl=%s' % (icu_full_path, with_intl)
+    shutil.rmtree(icu_full_path)
+
   # ICU mode. (icu-generic.gyp)
   byteorder = sys.byteorder
   o['variables']['icu_gyp_path'] = 'tools/icu/icu-generic.gyp'
   # ICU source dir relative to root
-  icu_full_path = os.path.join(root_dir, 'deps/icu')
   o['variables']['icu_path'] = icu_full_path
   if not os.path.isdir(icu_full_path):
-    print 'Error: ICU path is not a directory: %s' % (icu_full_path)
+    print '* ECMA-402 (Intl) support didn\'t find ICU in %s..' % (icu_full_path)
+    # can we download (or find) a zipfile?
+    localzip = icu_download(icu_full_path)
+    if localzip:
+      with zipfile.ZipFile(localzip, 'r') as icuzip:
+        print ' Extracting ICU source zip: %s' % localzip
+        icuzip.extractall(icu_parent_path)
+  if not os.path.isdir(icu_full_path):
+    print ' Cannot build Intl without ICU in %s.' % (icu_full_path)
+    print ' (Fix, or disable with "--with-intl=none" )'
     sys.exit(1)
+  else:
+    print '* Using ICU in %s' % (icu_full_path)
   # Now, what version of ICU is it? We just need the "major", such as 54.
   # uvernum.h contains it as a #define.
   uvernum_h = os.path.join(icu_full_path, 'source/common/unicode/uvernum.h')
   if not os.path.isfile(uvernum_h):
-    print 'Error: could not load %s - is ICU installed?' % uvernum_h
+    print ' Error: could not load %s - is ICU installed?' % uvernum_h
     sys.exit(1)
   icu_ver_major = None
   matchVerExp = r'^\s*#define\s+U_ICU_VERSION_SHORT\s+"([^"]*)".*'
@@ -792,7 +880,7 @@ def configure_intl(o):
     if m:
       icu_ver_major = m.group(1)
   if not icu_ver_major:
-    print 'Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h
+    print ' Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h
     sys.exit(1)
   icu_endianness = sys.byteorder[0];  # TODO(srl295): EBCDIC should be 'e'
   o['variables']['icu_ver_major'] = icu_ver_major
@@ -819,8 +907,8 @@ def configure_intl(o):
   # this is the icudt*.dat file which node will be using (platform endianness)
   o['variables']['icu_data_file'] = icu_data_file
   if not os.path.isfile(icu_data_path):
-    print 'Error: ICU prebuilt data file %s does not exist.' % icu_data_path
-    print 'See the README.md.'
+    print ' Error: ICU prebuilt data file %s does not exist.' % icu_data_path
+    print ' See the README.md.'
     # .. and we're not about to build it from .gyp!
     sys.exit(1)
   # map from variable name to subdirs

tools/configure.d/nodedownload.py

@@ -0,0 +1,11 @@
+# Moved some utilities here from ../../configure
+
+def formatSize(amt):
+    """Format a size as a string"""
+    return "{:.1f}".format(amt / 1024000.)
+
+def spin(c):
+    """print out a spinner based on 'c'"""
+#    spin = "\\|/-"
+    spin = ".:|'"
+    return (spin[c % len(spin)])