Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Turbopack] remove exceeding cells #69059

Closed
wants to merge 49 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
9d1592f
add new backend
sokra Jul 26, 2024
023f16e
new backend
sokra Jul 29, 2024
32b447a
add more backend method implementations
sokra Aug 2, 2024
fcea2fd
very basic operation working
sokra Aug 7, 2024
c6f4e70
add own cells reading
sokra Aug 7, 2024
db0bc43
call transient tasks
sokra Aug 7, 2024
ca6cbb4
call persistent tasks
sokra Aug 7, 2024
da0f04c
run tests multiple times to test caching
sokra Aug 7, 2024
41d716c
add cell/output reading and creating cached tasks
sokra Aug 7, 2024
f573143
serialization fixup
sokra Aug 9, 2024
5029e1c
remove unused stuff
sokra Aug 9, 2024
ed8b087
clippy
sokra Aug 9, 2024
2fb17f3
remove things not yet used (persistence)
sokra Aug 9, 2024
0d8a666
clippy
sokra Aug 9, 2024
37df4cc
add more tests for new backend
sokra Aug 9, 2024
eb0b6c9
add support for connect_child
sokra Aug 13, 2024
78af7a5
set root type on root tasks
sokra Aug 9, 2024
ffc9e21
remove old edges when task recomputation completes
sokra Aug 12, 2024
c448018
add try_get_function_id
sokra Aug 14, 2024
b0977cb
Revert "remove things not yet used (persistence)"
sokra Aug 9, 2024
dd40246
add lmdb persisting and restoring
sokra Aug 8, 2024
bdec3c3
pass test name to test_config to construct db name
sokra Aug 9, 2024
2cce8d9
continue uncompleted operations
sokra Aug 9, 2024
d3fcc67
create dir and logging
sokra Aug 12, 2024
024c2f1
improve error messages
sokra Aug 13, 2024
95e3f2c
handle keys larger than 511 bytes
sokra Aug 13, 2024
6dac853
set root type on root tasks
sokra Aug 9, 2024
4d5b42e
move backend impl into type alias
sokra Aug 12, 2024
733d42a
add new backend feature
sokra Aug 12, 2024
c736b0d
add missing import
sokra Aug 14, 2024
36b118c
initial aggregation update
sokra Aug 13, 2024
03abe58
enable new backend
sokra Aug 13, 2024
686c198
WIP: remove todos
sokra Aug 13, 2024
e126a75
more aggregation operations
sokra Aug 13, 2024
bc62ead
handle state serialization
sokra Aug 14, 2024
aa5ec04
validate serialization and improve errors
sokra Aug 14, 2024
c39ec7d
avoid some serde skip and untagged
sokra Aug 14, 2024
40aab63
use TransientState
sokra Aug 14, 2024
1aa5572
validate serialization and improve errors
sokra Aug 14, 2024
4835ae1
store project options in state
sokra Aug 14, 2024
44e902a
show lookup error
sokra Aug 14, 2024
d4f7561
log restored db entries
sokra Aug 14, 2024
26b7652
improve error
sokra Aug 14, 2024
2a0dd5a
improve benchmark_file_io for persistent cache
sokra Aug 14, 2024
4866761
WIP: print new tasks
sokra Aug 14, 2024
6db46f3
gracefully stop turbo-tasks to allow persisting to complete
sokra Aug 14, 2024
a96143d
avoid storing transient tasks
sokra Aug 14, 2024
92aff18
WIP: logging
sokra Aug 14, 2024
15950e6
remove exceeding cells
sokra Aug 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
move backend impl into type alias
  • Loading branch information
sokra committed Sep 4, 2024
commit 4d5b42ee2c1f03f2d00156fbde1bb013ee4befde
7 changes: 3 additions & 4 deletions crates/napi/src/app_structure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ use turbo_tasks::{
ValueToString, Vc,
};
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath};
use turbo_tasks_memory::MemoryBackend;
use turbopack_core::PROJECT_FILESYSTEM_NAME;

use crate::register;
use crate::{next_api::utils::NextBackend, register};

#[turbo_tasks::function]
async fn project_fs(project_dir: RcStr, watching: bool) -> Result<Vc<Box<dyn FileSystem>>> {
Expand Down Expand Up @@ -365,7 +364,7 @@ async fn get_value(

#[napi]
pub fn stream_entrypoints(
turbo_tasks: External<Arc<TurboTasks<MemoryBackend>>>,
turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
root_dir: String,
project_dir: String,
page_extensions: Vec<String>,
Expand Down Expand Up @@ -411,7 +410,7 @@ pub fn stream_entrypoints(

#[napi]
pub async fn get_entrypoints(
turbo_tasks: External<Arc<TurboTasks<MemoryBackend>>>,
turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
root_dir: String,
project_dir: String,
page_extensions: Vec<String>,
Expand Down
13 changes: 6 additions & 7 deletions crates/napi/src/next_api/project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ use tracing::Instrument;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc};
use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
use turbo_tasks_memory::MemoryBackend;
use turbopack_core::{
diagnostics::PlainDiagnostic,
error::PrettyPrintError,
Expand All @@ -44,7 +43,7 @@ use url::Url;
use super::{
endpoint::ExternalEndpoint,
utils::{
get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, RootTask,
get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, NextBackend, RootTask,
TurbopackResult, VcArc,
},
};
Expand Down Expand Up @@ -244,7 +243,7 @@ impl From<NapiDefineEnv> for DefineEnv {
}

pub struct ProjectInstance {
turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: Arc<TurboTasks<NextBackend>>,
container: Vc<ProjectContainer>,
exit_receiver: tokio::sync::Mutex<Option<ExitReceiver>>,
}
Expand Down Expand Up @@ -309,7 +308,7 @@ pub async fn project_new(
subscriber.init();
}

let turbo_tasks = TurboTasks::new(MemoryBackend::new(
let turbo_tasks = TurboTasks::new(NextBackend::new(
turbo_engine_options
.memory_limit
.map(|m| m as usize)
Expand Down Expand Up @@ -463,7 +462,7 @@ impl NapiRoute {
fn from_route(
pathname: String,
value: Route,
turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Self {
let convert_endpoint = |endpoint: Vc<Box<dyn Endpoint>>| {
Some(External::new(ExternalEndpoint(VcArc::new(
Expand Down Expand Up @@ -530,7 +529,7 @@ struct NapiMiddleware {
impl NapiMiddleware {
fn from_middleware(
value: &Middleware,
turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Result<Self> {
Ok(NapiMiddleware {
endpoint: External::new(ExternalEndpoint(VcArc::new(
Expand All @@ -550,7 +549,7 @@ struct NapiInstrumentation {
impl NapiInstrumentation {
fn from_instrumentation(
value: &Instrumentation,
turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: &Arc<TurboTasks<NextBackend>>,
) -> Result<Self> {
Ok(NapiInstrumentation {
node_js: External::new(ExternalEndpoint(VcArc::new(
Expand Down
12 changes: 7 additions & 5 deletions crates/napi/src/next_api/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,24 @@ use turbopack_core::{
source_pos::SourcePos,
};

pub type NextBackend = MemoryBackend;

/// A helper type to hold both a Vc operation and the TurboTasks root process.
/// Without this, we'd need to pass both individually all over the place
#[derive(Clone)]
pub struct VcArc<T> {
turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: Arc<TurboTasks<NextBackend>>,
/// The Vc. Must be resolved, otherwise you are referencing an inactive
/// operation.
vc: T,
}

impl<T> VcArc<T> {
pub fn new(turbo_tasks: Arc<TurboTasks<MemoryBackend>>, vc: T) -> Self {
pub fn new(turbo_tasks: Arc<TurboTasks<NextBackend>>, vc: T) -> Self {
Self { turbo_tasks, vc }
}

pub fn turbo_tasks(&self) -> &Arc<TurboTasks<MemoryBackend>> {
pub fn turbo_tasks(&self) -> &Arc<TurboTasks<NextBackend>> {
&self.turbo_tasks
}
}
Expand All @@ -55,7 +57,7 @@ pub fn serde_enum_to_string<T: Serialize>(value: &T) -> Result<String> {
/// The root of our turbopack computation.
pub struct RootTask {
#[allow(dead_code)]
turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: Arc<TurboTasks<NextBackend>>,
#[allow(dead_code)]
task_id: Option<TaskId>,
}
Expand Down Expand Up @@ -299,7 +301,7 @@ impl<T: ToNapiValue> ToNapiValue for TurbopackResult<T> {
}

pub fn subscribe<T: 'static + Send + Sync, F: Future<Output = Result<T>> + Send, V: ToNapiValue>(
turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
turbo_tasks: Arc<TurboTasks<NextBackend>>,
func: JsFunction,
handler: impl 'static + Sync + Send + Clone + Fn() -> F,
mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext<T>) -> napi::Result<Vec<V>>,
Expand Down
9 changes: 5 additions & 4 deletions crates/napi/src/turbotrace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,16 @@ use std::sync::Arc;
use napi::bindgen_prelude::*;
use node_file_trace::{start, Args};
use turbo_tasks::TurboTasks;
use turbo_tasks_memory::MemoryBackend;
use turbopack::{
module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
resolve_options_context::ResolveOptionsContext,
};

use crate::next_api::utils::NextBackend;

#[napi]
pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<MemoryBackend>>> {
let turbo_tasks = TurboTasks::new(MemoryBackend::new(
pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<NextBackend>>> {
let turbo_tasks = TurboTasks::new(NextBackend::new(
memory_limit.map(|m| m as usize).unwrap_or(usize::MAX),
));
External::new_with_size_hint(
Expand All @@ -23,7 +24,7 @@ pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<
#[napi]
pub async fn run_turbo_tracing(
options: Buffer,
turbo_tasks: Option<External<Arc<TurboTasks<MemoryBackend>>>>,
turbo_tasks: Option<External<Arc<TurboTasks<NextBackend>>>>,
) -> napi::Result<Vec<String>> {
let args: Args = serde_json::from_slice(options.as_ref())?;
let turbo_tasks = turbo_tasks.map(|t| t.clone());
Expand Down