add-remote-git: Add support for git submodules in git remotes

This is a fairly simple approach, which does not leverage the git caching
mechansim to cache submodules.  It also doesn't provide a means to disable
automatic initialization, e.g. via a setting in the .gitmodules file.

This also adds documentation for the git+file protocol, since we have
a test case ensuring that this works.  We have no test case for the
git+rsync protocol, even though npm-package-arg seems to have support
for that as well, so that wasn't added to the documentation.

PR-URL: npm/npm#11094
Credit: @gagern
Reviewed-By: @othiym23
Fixes: #1876

doc/cli/npm-install.md

@@ -169,9 +169,12 @@ after packing it up into a tarball (b).
 
           <protocol>://[<user>[:<password>]@]<hostname>[:<port>][:][/]<path>[#<commit-ish>]
 
-    `<protocol>` is one of `git`, `git+ssh`, `git+http`, or
-    `git+https`.  If no `<commit-ish>` is specified, then `master` is
-    used.
+    `<protocol>` is one of `git`, `git+ssh`, `git+http`, `git+https`,
+    or `git+file`.
+    If no `<commit-ish>` is specified, then `master` is used.
+
+    If the repository makes use of submodules, those submodules will
+    be cloned as well.
 
     The following git environment variables are recognized by npm and will be added
     to the environment when running git:

lib/cache/add-remote-git.js

@@ -346,6 +346,24 @@ function checkoutTreeish (from, resolvedURL, resolvedTreeish, tmpdir, cb) {
       }
       log.verbose('checkoutTreeish', from, 'checkout', stdout)
 
+      updateSubmodules(from, resolvedURL, tmpdir, cb)
+    }
+  )
+}
+
+function updateSubmodules (from, resolvedURL, tmpdir, cb) {
+  var args = ['submodule', '-q', 'update', '--init', '--recursive']
+  git.whichAndExec(
+    args,
+    { cwd: tmpdir, env: gitEnv() },
+    function (er, stdout, stderr) {
+      stdout = (stdout + '\n' + stderr).trim()
+      if (er) {
+        log.error('git ' + args.join(' ') + ':', stderr)
+        return cb(er)
+      }
+      log.verbose('updateSubmodules', from, 'submodule update', stdout)
+
       // convince addLocal that the checkout is a local dependency
       realizePackageSpecifier(tmpdir, function (er, spec) {
         if (er) {

test/tap/add-remote-git-submodule.js

@@ -0,0 +1,145 @@
+var fs = require('fs')
+var resolve = require('path').resolve
+
+var osenv = require('osenv')
+var mkdirp = require('mkdirp')
+var rimraf = require('rimraf')
+var test = require('tap').test
+
+var npm = require('../../lib/npm.js')
+var common = require('../common-tap.js')
+
+var pkg = resolve(__dirname, 'add-remote-git-submodule')
+var repos = resolve(__dirname, 'add-remote-git-submodule-repos')
+var subwt = resolve(repos, 'subwt')
+var topwt = resolve(repos, 'topwt')
+var suburl = 'git://localhost:1234/sub.git'
+var topurl = 'git://localhost:1234/top.git'
+
+var daemon
+var daemonPID
+var git
+
+var pjParent = JSON.stringify({
+  name: 'parent',
+  version: '1.2.3',
+  dependencies: {
+    child: topurl
+  }
+}, null, 2) + '\n'
+
+var pjChild = JSON.stringify({
+  name: 'child',
+  version: '1.0.3'
+}, null, 2) + '\n'
+
+test('setup', function (t) {
+  setup(function (er, r) {
+    t.ifError(er, 'git started up successfully')
+    t.end()
+  })
+})
+
+test('install from repo', function (t) {
+  bootstrap(t)
+  npm.commands.install('.', [], function (er) {
+    t.ifError(er, 'npm installed via git')
+    t.end()
+  })
+})
+
+test('has file in submodule', function (t) {
+  bootstrap(t)
+  npm.commands.install('.', [], function (er) {
+    t.ifError(er, 'npm installed via git')
+    var fooPath = resolve('node_modules', 'child', 'subpath', 'foo.txt')
+    fs.stat(fooPath, function (er) {
+      t.ifError(er, 'file in submodule exists')
+      t.end()
+    })
+  })
+})
+
+test('clean', function (t) {
+  daemon.on('close', function () {
+    cleanup()
+    t.end()
+  })
+  process.kill(daemonPID)
+})
+
+function bootstrap (t) {
+  mkdirp.sync(pkg)
+  process.chdir(pkg)
+  fs.writeFileSync('package.json', pjParent)
+  t.tearDown(function () {
+    process.chdir(osenv.tmpdir())
+    rimraf.sync(pkg)
+  })
+}
+
+function setup (cb) {
+  mkdirp.sync(topwt)
+  fs.writeFileSync(resolve(topwt, 'package.json'), pjChild)
+  mkdirp.sync(subwt)
+  fs.writeFileSync(resolve(subwt, 'foo.txt'), 'This is provided by submodule')
+  npm.load({ registry: common.registry, loglevel: 'silent' }, function () {
+    git = require('../../lib/utils/git.js')
+
+    function startDaemon (cb) {
+      // start git server
+      var d = git.spawn(
+        [
+          'daemon',
+          '--verbose',
+          '--listen=localhost',
+          '--export-all',
+          '--base-path=.',
+          '--reuseaddr',
+          '--port=1234'
+        ],
+        {
+          cwd: repos,
+          env: process.env,
+          stdio: ['pipe', 'pipe', 'pipe']
+        }
+      )
+      d.stderr.on('data', childFinder)
+
+      function childFinder (c) {
+        var cpid = c.toString().match(/^\[(\d+)\]/)
+        if (cpid[1]) {
+          this.removeListener('data', childFinder)
+          daemon = d
+          daemonPID = cpid[1]
+          cb(null)
+        }
+      }
+    }
+
+    var env = { PATH: process.env.PATH }
+    var topopt = { cwd: topwt, env: env }
+    var reposopt = { cwd: repos, env: env }
+    common.makeGitRepo({
+      path: subwt,
+      added: ['foo.txt'],
+      commands: [
+        git.chainableExec(['clone', '--bare', subwt, 'sub.git'], reposopt),
+        startDaemon,
+        [common.makeGitRepo, {
+          path: topwt,
+          commands: [
+            git.chainableExec(['submodule', 'add', suburl, 'subpath'], topopt),
+            git.chainableExec(['commit', '-m', 'added submodule'], topopt),
+            git.chainableExec(['clone', '--bare', topwt, 'top.git'], reposopt)
+          ]
+        }]
+      ]
+    }, cb)
+  })
+}
+
+function cleanup () {
+  process.chdir(osenv.tmpdir())
+  rimraf.sync(repos)
+}