vfs: keep default dlopen flags for real file paths

The dlopen hook installed while a VFS is mounted always forwarded its
flags parameter, so a two-argument process.dlopen() call for a real
file system path reached the original implementation with `undefined`
as the flags. That coerces to 0, which is not a valid dlopen(2) mode,
instead of applying the default flags, and loading any addon from the
real file system failed with EINVAL while a VFS was mounted.

Signed-off-by: Matteo Collina <hello@matteocollina.com>

Author: mcollina

lib/internal/vfs/setup.js

@@ -968,6 +968,10 @@ function installAddonLoader() {
       const { dlopenBinary } = internalBinding('process_methods');
       return dlopenBinary(module, filename, flags, readFileSync(filename));
     }
+    // Do not forward a missing flags argument as `undefined`:
+    // process.dlopen() coerces it to 0, which is not a valid dlopen(2)
+    // mode, instead of applying the default flags.
+    if (flags === undefined) return originalDlopen(module, filename);
     return originalDlopen(module, filename, flags);
   };
 }

test/parallel/test-vfs-addon.js

@@ -34,4 +34,12 @@ if (process.platform !== 'win32') {
   assert.deepStrictEqual(leaked, [], `addon temp not cleaned up: ${leaked}`);
 }
 
+// Regression check: while a VFS is mounted, process.dlopen() of a
+// real-file-system addon without a flags argument must keep the default
+// flags rather than forwarding `undefined`, which coerces to 0 - not a
+// valid dlopen(2) mode.
+const realMod = { exports: {} };
+process.dlopen(realMod, addonPath);
+assert.strictEqual(realMod.exports.hello(), 'world');
+
 myVfs.unmount();