add new backend feature

Cargo.toml

@@ -45,6 +45,7 @@ swc-ast-explorer = { path = "turbopack/crates/turbopack-swc-ast-explorer" }
 turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
 turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
 turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
+turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
 turbo-tasks-build = { path = "turbopack/crates/turbo-tasks-build" }
 turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" }
 turbo-tasks-env = { path = "turbopack/crates/turbo-tasks-env" }

crates/napi/Cargo.toml

@@ -37,6 +37,8 @@ __internal_dhat-heap = ["dhat"]
 # effectively does nothing.
 __internal_dhat-ad-hoc = ["dhat"]
 
+new-backend = ["dep:turbo-tasks-backend"]
+
 # Enable specific tls features per-target.
 [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
 next-core = { workspace = true, features = ["native-tls"] }
@@ -105,6 +107,7 @@ lightningcss-napi = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 turbo-tasks = { workspace = true }
 turbo-tasks-memory = { workspace = true }
+turbo-tasks-backend = { workspace = true, optional = true }
 turbo-tasks-fs = { workspace = true }
 next-api = { workspace = true }
 next-build = { workspace = true }

crates/napi/src/next_api/project.rs

@@ -1,4 +1,4 @@
-use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
+use std::{path::PathBuf, sync::Arc, thread, time::Duration};
 
 use anyhow::{anyhow, bail, Context, Result};
 use napi::{
@@ -43,8 +43,8 @@ use url::Url;
 use super::{
     endpoint::ExternalEndpoint,
     utils::{
-        get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, NextBackend, RootTask,
-        TurbopackResult, VcArc,
+        create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue,
+        NextBackend, RootTask, TurbopackResult, VcArc,
     },
 };
 use crate::register;
@@ -88,7 +88,7 @@ pub struct NapiProjectOptions {
 
     /// next.config's distDir. Project initialization occurs eariler than
     /// deserializing next.config, so passing it as separate option.
-    pub dist_dir: Option<String>,
+    pub dist_dir: String,
 
     /// Whether to watch he filesystem for file changes.
     pub watch: bool,
@@ -279,10 +279,7 @@ pub async fn project_new(
         let subscriber = Registry::default();
 
         let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap());
-        let dist_dir = options
-            .dist_dir
-            .as_ref()
-            .map_or_else(|| ".next".to_string(), |d| d.to_string());
+        let dist_dir = options.dist_dir.clone();
 
         let internal_dir = PathBuf::from(&options.project_path).join(dist_dir);
         std::fs::create_dir_all(&internal_dir)
@@ -308,27 +305,29 @@ pub async fn project_new(
         subscriber.init();
     }
 
-    let turbo_tasks = TurboTasks::new(NextBackend::new(
-        turbo_engine_options
-            .memory_limit
-            .map(|m| m as usize)
-            .unwrap_or(usize::MAX),
-    ));
-    let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
-    if let Some(stats_path) = stats_path {
-        let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
-        exit.on_exit(async move {
-            tokio::task::spawn_blocking(move || {
-                let mut file = std::fs::File::create(&stats_path)
-                    .with_context(|| format!("failed to create or open {stats_path:?}"))?;
-                serde_json::to_writer(&file, &task_stats)
-                    .context("failed to serialize or write task statistics")?;
-                file.flush().context("failed to flush file")
-            })
-            .await
-            .unwrap()
-            .unwrap();
-        });
+    let memory_limit = turbo_engine_options
+        .memory_limit
+        .map(|m| m as usize)
+        .unwrap_or(usize::MAX);
+    let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?;
+    #[cfg(not(feature = "new-backend"))]
+    {
+        let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
+        if let Some(stats_path) = stats_path {
+            let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
+            exit.on_exit(async move {
+                tokio::task::spawn_blocking(move || {
+                    let mut file = std::fs::File::create(&stats_path)
+                        .with_context(|| format!("failed to create or open {stats_path:?}"))?;
+                    serde_json::to_writer(&file, &task_stats)
+                        .context("failed to serialize or write task statistics")?;
+                    file.flush().context("failed to flush file")
+                })
+                .await
+                .unwrap()
+                .unwrap();
+            });
+        }
     }
     let options = options.into();
     let container = turbo_tasks

crates/napi/src/next_api/utils.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc};
+use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
 use napi::{
@@ -9,15 +9,31 @@ use napi::{
 use serde::Serialize;
 use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
 use turbo_tasks_fs::FileContent;
-use turbo_tasks_memory::MemoryBackend;
 use turbopack_core::{
     diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
     error::PrettyPrintError,
     issue::{IssueDescriptionExt, PlainIssue, PlainIssueSource, PlainSource, StyledString},
     source_pos::SourcePos,
 };
 
-pub type NextBackend = MemoryBackend;
+#[cfg(not(feature = "new-backend"))]
+pub type NextBackend = turbo_tasks_memory::MemoryBackend;
+#[cfg(feature = "new-backend")]
+pub type NextBackend = turbo_tasks_backend::TurboTasksBackend;
+
+#[allow(unused_variables, reason = "feature-gated")]
+pub fn create_turbo_tasks(
+    output_path: PathBuf,
+    memory_limit: usize,
+) -> Result<Arc<TurboTasks<NextBackend>>> {
+    #[cfg(not(feature = "new-backend"))]
+    let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit));
+    #[cfg(feature = "new-backend")]
+    let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
+        turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
+    )));
+    Ok(backend)
+}
 
 /// A helper type to hold both a Vc operation and the TurboTasks root process.
 /// Without this, we'd need to pass both individually all over the place

crates/napi/src/turbotrace.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{path::PathBuf, sync::Arc};
 
 use napi::bindgen_prelude::*;
 use node_file_trace::{start, Args};
@@ -8,29 +8,28 @@ use turbopack::{
     resolve_options_context::ResolveOptionsContext,
 };
 
-use crate::next_api::utils::NextBackend;
+use crate::next_api::utils::{self, NextBackend};
 
 #[napi]
-pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<NextBackend>>> {
-    let turbo_tasks = TurboTasks::new(NextBackend::new(
-        memory_limit.map(|m| m as usize).unwrap_or(usize::MAX),
-    ));
-    External::new_with_size_hint(
-        turbo_tasks,
-        memory_limit.map(|u| u as usize).unwrap_or(usize::MAX),
-    )
+pub fn create_turbo_tasks(
+    output_path: String,
+    memory_limit: Option<i64>,
+) -> External<Arc<TurboTasks<NextBackend>>> {
+    let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX);
+    let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit)
+        .expect("Failed to create TurboTasks");
+    External::new_with_size_hint(turbo_tasks, limit)
 }
 
 #[napi]
 pub async fn run_turbo_tracing(
     options: Buffer,
-    turbo_tasks: Option<External<Arc<TurboTasks<NextBackend>>>>,
+    turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
 ) -> napi::Result<Vec<String>> {
     let args: Args = serde_json::from_slice(options.as_ref())?;
-    let turbo_tasks = turbo_tasks.map(|t| t.clone());
     let files = start(
         Arc::new(args),
-        turbo_tasks.as_ref(),
+        turbo_tasks.clone(),
         Some(ModuleOptionsContext {
             ecmascript: EcmascriptOptionsContext {
                 enable_types: true,

packages/next/src/build/collect-build-traces.ts

@@ -118,6 +118,7 @@ export async function collectBuildTraces({
       let turbotraceOutputPath: string | undefined
       let turbotraceFiles: string[] | undefined
       turboTasksForTrace = bindings.turbo.createTurboTasks(
+        distDir,
         (config.experimental.turbotrace?.memoryLimit ??
           TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
           1024 *

packages/next/src/build/index.ts

@@ -1266,6 +1266,7 @@ export default async function build(
           {
             projectPath: dir,
             rootPath: config.outputFileTracingRoot || dir,
+            distDir,
             nextConfig: config,
             jsConfig: await getTurbopackJsConfig(dir, config),
             watch: false,

packages/next/src/build/swc/index.ts

@@ -412,6 +412,11 @@ export interface ProjectOptions {
    */
   projectPath: string
 
+  /**
+   * The path to the .next directory.
+   */
+  distDir: string
+
   /**
    * The next.config.js contents.
    */
@@ -1547,15 +1552,18 @@ function loadNative(importPath?: string) {
       initHeapProfiler: bindings.initHeapProfiler,
       teardownHeapProfiler: bindings.teardownHeapProfiler,
       turbo: {
-        startTrace: (options = {}, turboTasks: unknown) => {
+        startTrace: (options = {}, turboTasks: { __napi: 'TurboTasks' }) => {
           initHeapProfiler()
           return (customBindings ?? bindings).runTurboTracing(
             toBuffer({ exact: true, ...options }),
             turboTasks
           )
         },
-        createTurboTasks: (memoryLimit?: number): unknown =>
-          bindings.createTurboTasks(memoryLimit),
+        createTurboTasks: (
+          outputPath: string,
+          memoryLimit?: number
+        ): { __napi: 'TurboTasks' } =>
+          bindings.createTurboTasks(outputPath, memoryLimit),
         entrypoints: {
           stream: (
             turboTasks: any,

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -130,6 +130,7 @@ export async function createHotReloaderTurbopack(
     {
       projectPath: dir,
       rootPath: opts.nextConfig.outputFileTracingRoot || dir,
+      distDir,
       nextConfig: opts.nextConfig,
       jsConfig: await getTurbopackJsConfig(dir, nextConfig),
       watch: true,

test/development/basic/next-rs-api.test.ts

@@ -191,13 +191,20 @@ describe('next.rs api', () => {
     console.log(next.testDir)
     const nextConfig = await loadConfig(PHASE_DEVELOPMENT_SERVER, next.testDir)
     const bindings = await loadBindings()
+    const distDir = path.join(
+      process.env.NEXT_SKIP_ISOLATE
+        ? path.resolve(__dirname, '../../..')
+        : next.testDir,
+      '.next'
+    )
     project = await bindings.turbo.createProject({
       env: {},
       jsConfig: {
         compilerOptions: {},
       },
       nextConfig: nextConfig,
       projectPath: next.testDir,
+      distDir,
       rootPath: process.env.NEXT_SKIP_ISOLATE
         ? path.resolve(__dirname, '../../..')
         : next.testDir,
@@ -208,12 +215,7 @@ describe('next.rs api', () => {
         clientRouterFilters: undefined,
         config: nextConfig,
         dev: true,
-        distDir: path.join(
-          process.env.NEXT_SKIP_ISOLATE
-            ? path.resolve(__dirname, '../../..')
-            : next.testDir,
-          '.next'
-        ),
+        distDir: distDir,
         fetchCacheKeyPrefix: undefined,
         hasRewrites: false,
         middlewareMatchers: undefined,