feat: auto external node native module when using farm.config.ts (#81)

* feat: auto external node native module when using farm.config.ts

* feat: update lockfile

README.md

@@ -21,7 +21,7 @@ Farm is a super fast, light-weight web building tool written in Rust. Benchmark
 
 ## Features
 
-- ⚡ **Super Fast**: Written in Rust, start a react / vue(incoming) project in milliseconds, perform a HMR update within 10ms for situations.
+- ⚡ **Super Fast**: Written in Rust, start a react / vue(incoming) project in milliseconds, perform a HMR update within 10ms for most situations.
 - 🧰 **Fully Pluggable**: Everything inside Farm is powered by plugins, achieve anything you want by creating a plugin. Support both Rust and Js plugins.
 - ⚙️ **Powerful**: Compiling JS/TS/JSX/TSX, css, html and static assets out of box.
 - ⏱️ **Lazy Compilation**: Dynamic imported resources are compiled only when they are requested.

crates/compiler/src/build/mod.rs

@@ -18,7 +18,7 @@ use farmfe_core::{
   rayon,
   rayon::ThreadPool,
 };
-use farmfe_toolkit::tracing::{self, debug, info};
+use farmfe_toolkit::tracing::{self, debug};
 
 use crate::{
   build::{
@@ -240,7 +240,13 @@ impl Compiler {
           resolve_module_id_result,
         }) => {
           if resolve_module_id_result.resolve_result.external {
-            // skip external modules
+            // insert external module to the graph
+            let module_id: ModuleId = resolve_param.source.as_str().into();
+            let mut module = Module::new(module_id.clone());
+            module.external = true;
+
+            Self::add_module(module, &resolve_param.kind, &context);
+            Self::add_edge(&resolve_param, module_id, order, &context);
             return;
           }
 

crates/compiler/src/generate/render_resource_pots.rs

@@ -6,9 +6,9 @@ use farmfe_core::{
   parking_lot::Mutex,
   plugin::PluginHookContext,
   rayon::prelude::{IntoParallelIterator, ParallelIterator},
-  resource::{resource_pot::ResourcePot, Resource},
+  resource::{resource_pot::ResourcePot, Resource, ResourceType},
 };
-use farmfe_toolkit::{fs::transform_output_filename, hash::sha256, tracing};
+use farmfe_toolkit::{fs::transform_output_filename, tracing};
 
 #[tracing::instrument(skip_all)]
 pub fn render_resource_pots_and_generate_resources(

crates/compiler/src/update/regenerate_resources.rs

@@ -132,7 +132,7 @@ fn generate_and_diff_resource_pots(
   context: &Arc<CompilationContext>,
 ) -> farmfe_core::error::Result<Vec<ResourcePotId>> {
   let mut module_group_graph = context.module_group_graph.write();
-
+  // TODO: Make swc helpers for commonjs module like default and wildcard exports embedded in the module system to optimize the HMR time, as these two modules may be imported by most modules
   let modules = module_groups
     .iter()
     .fold(HashSet::new(), |mut acc, module_group_id| {

crates/plugin_html/src/resources_injector.rs

@@ -175,6 +175,9 @@ impl VisitMut for ResourcesInjector {
         )));
       }
 
+      // inject global define
+      self.inject_global_define(element);
+
       // inject runtime <script>
       let script_element = create_element(
         "script",
@@ -183,8 +186,6 @@ impl VisitMut for ResourcesInjector {
       );
       element.children.push(Child::Element(script_element));
     } else if element.tag_name.to_string() == "body" {
-      self.inject_global_define(element);
-
       for script in &self.script_resources {
         element.children.push(Child::Element(create_element(
           "script",

crates/plugin_partial_bundling/src/lib.rs

@@ -54,6 +54,10 @@ impl Plugin for FarmPluginPartialBundling {
 
     for module_id in modules {
       let module = module_graph.module(module_id).unwrap();
+      // Skip the external modules
+      if module.external {
+        continue;
+      }
 
       if module.resource_pot.is_some() {
         panic!(

crates/plugin_runtime/src/lib.rs

@@ -408,8 +408,9 @@ impl Plugin for FarmPluginRuntime {
             .body
             .insert(0, runtime_ast.body.to_vec().remove(0));
 
+          // TODO move this logic to the entry module plugin, and should do this work in the finalize_resources hook
+          // TODO should collect the exports of the entry module, and only export the exports of the entry module
           // TODO support top level await, and only support reexport default export now, should support more export type in the future
-          // TODO inject global define
           // call the entry module
           let call_entry = parse_module(
             "farm-internal-call-entry-module",
@@ -428,7 +429,17 @@ impl Plugin for FarmPluginRuntime {
             context.config.script.target.clone(),
             context.meta.script.cm.clone(),
           )?;
-
+          // temporary solutions, move this logic to separate plugin when support configuring target env.
+          let global_var = parse_module(
+            "farm-global-var",
+            r#"import module from 'node:module';
+            global.__farmNodeRequire = module.createRequire(import.meta.url);
+            global.__farmNodeBuiltinModules = module.builtinModules;"#,
+            Syntax::Es(context.config.script.parser.es_config.clone()),
+            context.config.script.target.clone(),
+            context.meta.script.cm.clone(),
+          )?;
+          resource_pot_ast.body.splice(0..0, global_var.body);
           resource_pot_ast.body.extend(call_entry.body);
         }
         _ => { /* only inject entry execution for script, html entry will be injected after all resources generated */

crates/plugin_runtime/src/render_resource_pot/source_replacer.rs

@@ -88,8 +88,13 @@ impl SourceReplacer<'_> {
         // only execute script module
         let dep_module = self.module_graph.module(&id).unwrap();
 
+        if dep_module.external {
+          return SourceReplaceResult::NotReplaced;
+        }
+
         if dep_module.module_type.is_script() || dep_module.module_type == ModuleType::Runtime {
           *value = id.id(self.mode.clone()).into();
+
           call_expr.visit_mut_children_with(self);
           return SourceReplaceResult::Replaced;
         } else {

examples/script-entry/farm.config.js

@@ -0,0 +1,29 @@
+// change to @farmfe/core/config when resolve support conditional exports
+import { builtinModules } from 'module';
+
+/**
+ * @type {import('@farmfe/core').UserConfig}
+ */
+export default {
+  compilation: {
+    input: {
+      index: './index.ts'
+    },
+    output: {
+      path: 'dist',
+      filename: 'index.[ext]'
+    },
+    external: builtinModules,
+    partialBundling: {
+      moduleBuckets: [
+        {
+          name: 'node.bundle.js',
+          test: ['.+']
+        }
+      ]
+    }
+  },
+  server: {
+    hmr: false
+  }
+};

examples/script-entry/index.ts

@@ -1,9 +1,12 @@
+import module from 'module';
+
 import { UserConfig } from '@farmfe/core';
 
 export default {
   compilation: {
     input: {
-      main: './main.tsx',
+      main: './main.tsx'
     },
-  },
+    external: module.builtinModules
+  }
 } as UserConfig;

examples/script-entry/test.mjs

@@ -0,0 +1,4 @@
+// test that the script is working
+import config from './dist/index.js';
+
+console.log(config);

packages/core/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @farmfe/core
 
+## 0.4.5
+
+### Patch Changes
+
+- Auto external node native module when reading farm.config.ts
+- Updated dependencies
+  - @farmfe/runtime@0.3.4
+
 ## 0.4.4
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farmfe/core",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
@@ -62,7 +62,7 @@
     "type-check": "tsc -p tsconfig.build.json --noEmit"
   },
   "dependencies": {
-    "@farmfe/runtime": "workspace:^0.3.3",
+    "@farmfe/runtime": "workspace:^0.3.4",
     "@farmfe/runtime-plugin-hmr": "workspace:^3.0.5",
     "@swc/helpers": "^0.4.9",
     "boxen": "^7.0.1",

packages/core/src/config/index.ts

@@ -229,6 +229,10 @@ async function readConfigFile(
             filename: fileName,
             path: outputPath,
           },
+          external: [
+            ...module.builtinModules,
+            ...module.builtinModules.map((m) => `node:${m}`),
+          ],
           partialBundling: {
             moduleBuckets: [
               {
@@ -245,7 +249,7 @@ async function readConfigFile(
       });
       const compiler = new Compiler(normalizedConfig);
       await compiler.compile();
-      await compiler.writeResourcesToDisk();
+      compiler.writeResourcesToDisk();
 
       const filePath = path.join(outputPath, fileName);
       // Change to vm.module of node or loaders as far as it is stable
@@ -255,8 +259,12 @@ async function readConfigFile(
         return (await import(filePath)).default;
       }
     } else {
-      const config = (await import(resolvedPath)).default;
-      return config;
+      // Change to vm.module of node or loaders as far as it is stable
+      if (process.platform === 'win32') {
+        return (await import(pathToFileURL(resolvedPath).toString())).default;
+      } else {
+        return (await import(resolvedPath)).default;
+      }
     }
   }
 }

packages/core/tests/config.spec.ts

@@ -1,3 +1,4 @@
+import { builtinModules } from 'module';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { test, expect } from 'vitest';
@@ -22,6 +23,7 @@ test('resolveUserConfig', async () => {
       input: {
         main: './main.tsx',
       },
+      external: builtinModules,
     },
     root: path.join(filePath, 'fixtures', 'config'),
   });

packages/core/tests/fixtures/config/farm.config.ts

@@ -1,8 +1,10 @@
+import { builtinModules } from 'module';
 import { defineFarmConfig } from '../../../src/config';
 import input from './util';
 
 export default defineFarmConfig({
   compilation: {
     input,
+    external: builtinModules,
   },
 });

packages/runtime/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @farmfe/runtime
 
+## 0.3.4
+
+### Patch Changes
+
+- Auto external node native module when reading farm.config.ts
+
 ## 0.3.3
 
 ### Patch Changes

packages/runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@farmfe/runtime",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "Runtime of Farm",
   "author": {
     "name": "bright wu",

packages/runtime/src/module-system.ts

@@ -1,6 +1,6 @@
 import { Module } from './module';
 import { FarmRuntimePlugin, FarmRuntimePluginContainer } from './plugin';
-import { Resource, ResourceLoader } from './resource-loader';
+import { Resource, ResourceLoader, targetEnv } from './resource-loader';
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 type ModuleInitialization = (
@@ -49,6 +49,23 @@ export class ModuleSystem {
       }
     }
 
+    // if running on node, using native require to load node built-in modules
+    if (targetEnv === 'node') {
+      const { __farmNodeRequire, __farmNodeBuiltinModules } =
+        // TODO: polyfill globalThis
+        globalThis as unknown as {
+          __farmNodeRequire: (id: string) => any;
+          __farmNodeBuiltinModules: string[];
+        };
+
+      if (moduleId.startsWith('node:')) {
+        const nodeModuleId = moduleId.slice(5);
+        return __farmNodeRequire(nodeModuleId);
+      } else if (__farmNodeBuiltinModules.includes(moduleId)) {
+        return __farmNodeRequire(moduleId);
+      }
+    }
+
     const initializer = this.modules[moduleId];
 
     if (!initializer) {

packages/runtime/src/resource-loader.ts

@@ -5,9 +5,10 @@ export interface Resource {
   type: 'script' | 'link';
 }
 
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore do not check type here
-const targetEnv = (globalThis || global || window || self).__FARM_TARGET_ENV__;
+export const targetEnv =
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore do not check type here
+  (globalThis || global || window || self).__FARM_TARGET_ENV__ || 'node';
 
 /**
  * Loading resources according to their type and target env.