Default metro.config.js should be configured to support symbol links to packages

[ryantrem]

Describe the Feature

npm install <path to local package> creates a symbolic link to the package directory. This is useful for monorepos as well as iterating on a package in the context of the consuming app. Unfortunately, the Metro bundler does not support this out of the box (see facebook/metro#1). It is possible though to configure the Metro bundler (via metro.config.js) to make it work with symbolically linked packages. It would be extremely helpful if react-native init MyApp provided a metro.config.js that made symbolically linked packages work by default.

Possible Implementations

There are many work arounds proposed in facebook/metro#1. All of them had problems for me. My solution (used in multiple projects) is to modify metro.config.js to manually follow symbolic links:

/**
 * Metro configuration for React Native
 * https://github.com/facebook/react-native
 *
 * @format
 */

const path = require('path');
const fs = require('fs');
const appendExclusions = require('metro-config/src/defaults/blacklist');

// Escape function taken from the MDN documentation: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#Escaping
function escapeRegExp(string) {
  return string.replace(/[.*+\-?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}

// NOTE: The Metro bundler does not support symlinks (see https://github.com/facebook/metro/issues/1), which NPM uses for local packages.
//       To work around this, we supplement the logic to follow symbolic links.

// Create a mapping of package ids to linked directories.
function processModuleSymLinks() {
  const nodeModulesPath = path.resolve(__dirname, 'node_modules');
  let moduleMappings = {};
  let moduleExclusions = [];

  function findPackageDirs(directory) {
    fs.readdirSync(directory).forEach(item => {
      const itemPath = path.resolve(directory, item);
      const itemStat = fs.lstatSync(itemPath);
      if (itemStat.isSymbolicLink()) {
        let linkPath = fs.readlinkSync(itemPath);
        // Sym links are relative in Unix, absolute in Windows.
        if (!path.isAbsolute(linkPath)) {
          linkPath = path.resolve(directory, linkPath);
        }
        const linkStat = fs.statSync(linkPath);
        if (linkStat.isDirectory()) {
          const packagePath = path.resolve(linkPath, "package.json");
          if (fs.existsSync(packagePath)) {
            const packageId = path.relative(nodeModulesPath, itemPath);
            moduleMappings[packageId] = linkPath;

            const packageInfoData = fs.readFileSync(packagePath);
            const packageInfo = JSON.parse(packageInfoData);

            const dependencies = packageInfo.dependencies ? Object.keys(packageInfo.dependencies) : [];
            const peerDependencies = packageInfo.peerDependencies ? Object.keys(packageInfo.peerDependencies) : [];
            const devDependencies = packageInfo.devDependencies ? Object.keys(packageInfo.devDependencies) : [];

            // Exclude dependencies that appear in devDependencies or peerDependencies but not in dependencies. Otherwise,
            // the metro bundler will package those devDependencies/peerDependencies as unintended copies.
            for (const devDependency of devDependencies.concat(peerDependencies).filter(dependency => !dependencies.includes(dependency))) {
              moduleExclusions.push(new RegExp(escapeRegExp(path.join(linkPath, "node_modules", devDependency)) + "\/.*"));
            }
          }
        }
      } else if (itemStat.isDirectory()) {
        findPackageDirs(itemPath);
      }
    });
  }

  findPackageDirs(nodeModulesPath);

  return [moduleMappings, moduleExclusions];
}

const [moduleMappings, moduleExclusions] = processModuleSymLinks();
console.log("Mapping the following sym linked packages:");
console.log(moduleMappings);

module.exports = {
  transformer: {
    getTransformOptions: async () => ({
      transform: {
        experimentalImportSupport: false,
        inlineRequires: false,
      },
    }),
  },

  resolver: {
    // Register an "extra modules proxy" for resolving modules outside of the normal resolution logic.
    extraNodeModules: new Proxy(
        // Provide the set of known local package mappings.
        moduleMappings,
        {
            // Provide a mapper function, which uses the above mappings for associated package ids,
            // otherwise fall back to the standard behavior and just look in the node_modules directory.
            get: (target, name) => name in target ? target[name] : path.join(__dirname, `node_modules/${name}`),
        },
    ),
    blacklistRE: appendExclusions(moduleExclusions),
  },

  projectRoot: path.resolve(__dirname),

  // Also additionally watch all the mapped local directories for changes to support live updates.
  watchFolders: Object.values(moduleMappings),
};

Related Issues

facebook/metro#1

[Braincompiler]

Thanks for this, it helps us to manage a bigger codebase where we use a lot of symlinks caused by multirepos.

But we had a problem the symlinks where not considered even with your metro.config.js. The problem is, you use fs.lstatSync(linkPath); for the already resolved symlink directory, this leads to the problem linkStat.isDirectory() returned false. I found out, you should use lstat only for the symbolic not for the resolved target anymore (nodejs/node#25342). After changing it to fs.statSync(linkPath); it was working as expected.

[ryantrem]

Thanks @Braincompiler for sharing back your findings! I read the link you shared and it certainly sounds like fs.lstatSync(linkPath) should not work, but it does work for me. Can you describe your specific scenario a bit more? Like what your directory structure looks like under node_modules and where in that structure the sym link is?

Also, I tested it out with fs.statSync(linkPath) and it also worked as expected for me. Since based on the links you shared this seems more likely correct, and since it also fixed the issue you were running into, I am updating my original post to reflect this change. I'd still love to better understand which specific scenarios are broken with the old way though!

[Braincompiler]

We debugged the metro bundler start and found out that linkStat.isDirectory() returned false which was not possible cause the already resolved symbolic link was a directory. Then we searched for that error in case of symbolic links and we found the linked issue. After using statSync rather than 'lstatSync' it worked as expcted.

Our directory structure looks like:

node_modules/
  @vendor-name
    module1 -> symbolic link
    module2 -> symbolic link
    module3 -> symbolic link

The only problem we now have: In case we change something in one of the linked modules, the app reloads but is not rebundled. Thus we have to restart the bundler yarn start to see our changes. Thats very annoying but we haven't found a working solution for that. The watchFolders: Object.values(moduleMappings), seems to work as the app reloads, but its not rebundled. For sake of completeness: We use Typescript, so the modules will also build after changing. We notice a double reload of the app, however it's not rebundled.

[ryantrem]

That's really interesting, I don't think we see any of those issues. We have exactly the same directory structure as you describe above and no issues with the link resolution even with lstatSync. When you say the app reloads, but it is not rebundled, does that effectively mean fast refresh does not work for you? This is also something that works as expected for us.

[Braincompiler]

Do you use Mac Books? We have currently Mid 2015 Mac Books, could this be a problem? I don't think so.

The refresh works (at least the app is reloading (fast)) but the changes we made in one of the dependent modules are not there. We have to exit the bundler and start it again. But this takes a lot of time, obviously.

[ryantrem]

Yes we use both Mac and Windows. We did have an issue at one point kind of like what you are describing, but for us if we saved the top level App.js it would refresh as expected. However this stopped reproing before we had a chance to investigate so we assumed it was fixed by a React Native update.

[Braincompiler]

Since you said you saved the "App.js", means you don't use Typescript? We use Typescript (formerly the cli generated typescript project with babel, now we use a custom build step using tsc so we can use tsconfig and better typechecking support). So maybe the problem is indicated due to the extra transpile step? But the changes were never built in even after manual refresh (⌘+R).

[ryantrem]

We use TypeScript + Babel, I just said App.js to keep things simple. In reality it is App.tsx for us. What version of React Native and the CLI tools are you using? I’m not sure what other tools come into play here... node, watchman, other?

[Braincompiler]

We use always the latest versions for all the tools and clis you named. Currently we don't use symlinks anymore because the problems with the bundler were too hard to work with. But hopefully there will be a solution for this provided by the metro bundler itself. I cannot imagine Facebook is not using symlinks or similar in there big dev environments.

[Merlier]

Describe the Feature

npm install <path to local package> creates a symbolic link to the package directory. This is useful for monorepos as well as iterating on a package in the context of the consuming app. Unfortunately, the Metro bundler does not support this out of the box (see facebook/metro#1). It is possible though to configure the Metro bundler (via metro.config.js) to make it work with symbolically linked packages. It would be extremely helpful if react-native init MyApp provided a metro.config.js that made symbolically linked packages work by default.

Possible Implementations

There are many work arounds proposed in facebook/metro#1. All of them had problems for me. My solution (used in multiple projects) is to modify metro.config.js to manually follow symbolic links:

/**
 * Metro configuration for React Native
 * https://github.com/facebook/react-native
 *
 * @format
 */

const path = require('path');
const fs = require('fs');

// NOTE: The Metro bundler does not support symlinks (see https://github.com/facebook/metro/issues/1), which NPM uses for local packages.
//       To work around this, we supplement the logic to follow symbolic links.

// Create a mapping of package ids to linked directories.
function getModuleMappings() {
  const nodeModulesPath = path.resolve(__dirname, 'node_modules');
  let moduleMappings = {};

  function findPackageDirs(directory) {
    fs.readdirSync(directory).forEach(item => {
      const itemPath = path.resolve(directory, item);
      const itemStat = fs.lstatSync(itemPath);
      if (itemStat.isSymbolicLink()) {
        let linkPath = fs.readlinkSync(itemPath);
        // Sym links are relative in Unix, absolute in Windows.
        if (!path.isAbsolute(linkPath)) {
          linkPath = path.resolve(directory, linkPath);
        }
        const linkStat = fs.statSync(linkPath);
        if (linkStat.isDirectory()) {
          const packagePath = path.resolve(linkPath, "package.json");
          if (fs.existsSync(packagePath)) {
            const packageId = path.relative(nodeModulesPath, itemPath);
            moduleMappings[packageId] = linkPath;
          }
        }
      } else if (itemStat.isDirectory()) {
        findPackageDirs(itemPath);
      }
    });
  }

  findPackageDirs(nodeModulesPath);

  return moduleMappings;
}

const moduleMappings = getModuleMappings();
console.log("Mapping the following sym linked packages:");
console.log(moduleMappings);

module.exports = {
  transformer: {
    getTransformOptions: async () => ({
      transform: {
        experimentalImportSupport: false,
        inlineRequires: false,
      },
    }),
  },

  resolver: {
    // Register an "extra modules proxy" for resolving modules outside of the normal resolution logic.
    extraNodeModules: new Proxy(
        // Provide the set of known local package mappings.
        moduleMappings,
        {
            // Provide a mapper function, which uses the above mappings for associated package ids,
            // otherwise fall back to the standard behavior and just look in the node_modules directory.
            get: (target, name) => name in target ? target[name] : path.join(__dirname, `node_modules/${name}`),
        },
    ),
  },

  projectRoot: path.resolve(__dirname),

  // Also additionally watch all the mapped local directories for changes to support live updates.
  watchFolders: Object.values(moduleMappings),
};

Related Issues

facebook/metro#1

Thanks for your solution 👍

[ryantrem]

We found that in cases where you sym link to a package that has devDependencies, those dependencies are installed under that package's node_modules, and those get picked up (as a copy) by the Metro bundler. In a non-symlinked package, this wouldn't be an issue because because devDependencies would not be installed with the package. To fix this, we added some additional logic that adds any devDependencies of sym linked packages to the path exclusion list. I've updated my original post to include this fix.

[carloseustaquio]

Thanks for this @ryantrem ! I've been searching for a way to do this for many weeks.
Unfortulately, I couldn't manage this to work yet. When I do the linking and run the metro I get the following log:

$ react-native start --config dev.metro.config.js
Mapping the following sym linked packages:
{ '@my-company/my-package':
   '/home/myuser/.config/yarn/link/@my-company/my-package' }

And in fact, it's looking for files in the linked path:

Looking for JS files in
   /home/carlos/code/vizir/my-app
   /home/carlos/.config/yarn/link/@my-company/my-package

It compiles ok, but when I try to open the app I get the following error:

  error: bundling failed: Error: Unable to resolve module `@my-company/my-package/dist/navigation/app_stack.native` from `src/index.native.tsx`: @my-company/my-package/dist/navigation/app_stack.native could not be found within the project or in these directories:
  /home/carlos/.config/yarn/link/@my-company/my-package/dist/navigation

When open the symlink path, the package is there.
I'm kind of a beginner in this metro configuration thing, and I'm not really shure about how I should try to debug this.
ps: @my-company/my-package has dependencies, devDependencies and peerDependencies