From e3ff96a510074ba7a4600aa113e2f9536e208ba0 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Fri, 24 Jan 2025 17:15:02 -0800 Subject: [PATCH 1/2] refactor(turbo-tasks): Port task statistics to the new backend --- crates/napi/src/next_api/project.rs | 38 ++++++++----------- crates/napi/src/next_api/utils.rs | 17 +++++---- .../turbo-tasks-backend/src/backend/mod.rs | 25 ++++++++++-- .../crates/turbo-tasks-memory/src/lib.rs | 2 - .../turbo-tasks-memory/src/memory_backend.rs | 14 +++---- turbopack/crates/turbo-tasks/src/backend.rs | 3 ++ turbopack/crates/turbo-tasks/src/lib.rs | 1 + .../src/task_statistics.rs | 10 ++--- 8 files changed, 64 insertions(+), 46 deletions(-) rename turbopack/crates/{turbo-tasks-memory => turbo-tasks}/src/task_statistics.rs (90%) diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index 03b25122136c4..f2b486cb14233 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, sync::Arc, thread, time::Duration}; +use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration}; use anyhow::{anyhow, bail, Context, Result}; use napi::{ @@ -382,27 +382,21 @@ pub async fn project_new( persistent_caching, memory_limit, )?; - if !persistent_caching { - use std::io::Write; - let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); - if let Some(stats_path) = stats_path { - let Some(backend) = turbo_tasks.memory_backend() else { - return Err(anyhow!("task statistics require a memory backend").into()); - }; - let task_stats = backend.task_statistics().enable().clone(); - exit.on_exit(async move { - tokio::task::spawn_blocking(move || { - let mut file = std::fs::File::create(&stats_path) - .with_context(|| format!("failed to create or open {stats_path:?}"))?; - serde_json::to_writer(&file, &task_stats) - .context("failed to serialize or write task statistics")?; - file.flush().context("failed to flush file") - }) - .await - .unwrap() - .unwrap(); - }); - } + let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); + if let Some(stats_path) = stats_path { + let task_stats = turbo_tasks.task_statistics().enable().clone(); + exit.on_exit(async move { + tokio::task::spawn_blocking(move || { + let mut file = std::fs::File::create(&stats_path) + .with_context(|| format!("failed to create or open {stats_path:?}"))?; + serde_json::to_writer(&file, &task_stats) + .context("failed to serialize or write task statistics")?; + file.flush().context("failed to flush file") + }) + .await + .unwrap() + .unwrap(); + }); } let options: ProjectOptions = options.into(); let container = turbo_tasks diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index 0d19fcfda94d9..c745ce71ecfb4 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -10,7 +10,8 @@ use napi::{ }; use serde::Serialize; use turbo_tasks::{ - trace::TraceRawVcs, OperationVc, ReadRef, TaskId, TryJoinIterExt, TurboTasks, UpdateInfo, Vc, + backend::Backend, task_statistics::TaskStatisticsApi, trace::TraceRawVcs, OperationVc, ReadRef, + TaskId, TryJoinIterExt, TurboTasks, UpdateInfo, Vc, }; use turbo_tasks_backend::{ default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage, @@ -108,17 +109,19 @@ impl NextTurboTasks { } } - pub fn memory_backend(&self) -> Option<&turbo_tasks_memory::MemoryBackend> { + pub async fn stop_and_wait(&self) { match self { - NextTurboTasks::Memory(_) => None, - NextTurboTasks::PersistentCaching(_) => None, + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.stop_and_wait().await, + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.stop_and_wait().await, } } - pub async fn stop_and_wait(&self) { + pub fn task_statistics(&self) -> &TaskStatisticsApi { match self { - NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.stop_and_wait().await, - NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.stop_and_wait().await, + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.backend().task_statistics(), + NextTurboTasks::PersistentCaching(turbo_tasks) => { + turbo_tasks.backend().task_statistics() + } } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 5b79be373becc..5faf146e5b119 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -31,6 +31,7 @@ use turbo_tasks::{ }, event::{Event, EventListener}, registry, + task_statistics::TaskStatisticsApi, util::IdFactoryWithReuse, CellId, FunctionId, RawVc, ReadConsistency, SessionId, TaskId, TraitTypeId, TurboTasksBackendApi, ValueTypeId, TRANSIENT_TASK_BIT, @@ -178,6 +179,8 @@ struct TurboTasksBackendInner { idle_start_event: Event, idle_end_event: Event, + task_statistics: TaskStatisticsApi, + backing_storage: B, } @@ -222,6 +225,7 @@ impl TurboTasksBackendInner { stopping_event: Event::new(|| "TurboTasksBackend::stopping_event".to_string()), idle_start_event: Event::new(|| "TurboTasksBackend::idle_start_event".to_string()), idle_end_event: Event::new(|| "TurboTasksBackend::idle_end_event".to_string()), + task_statistics: TaskStatisticsApi::default(), backing_storage, } } @@ -339,6 +343,16 @@ impl TurboTasksBackendInner { fn should_track_children(&self) -> bool { self.options.children_tracking } + + fn track_cache_hit(&self, task_type: &CachedTaskType) { + self.task_statistics + .map(|stats| stats.increment_cache_hit(task_type.fn_type)); + } + + fn track_cache_miss(&self, task_type: &CachedTaskType) { + self.task_statistics + .map(|stats| stats.increment_cache_miss(task_type.fn_type)); + } } pub(crate) struct OperationGuard<'a, B: BackingStorage> { @@ -813,10 +827,12 @@ impl TurboTasksBackendInner { turbo_tasks: &dyn TurboTasksBackendApi>, ) -> TaskId { if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { + self.track_cache_hit(&task_type); self.connect_child(parent_task, task_id, turbo_tasks); return task_id; } + self.track_cache_miss(&task_type); let tx = self .should_restore() .then(|| self.backing_storage.start_read_transaction()) @@ -871,11 +887,12 @@ impl TurboTasksBackendInner { ); } if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { - // Safety: `tx` is a valid transaction from `self.backend.backing_storage`. + self.track_cache_hit(&task_type); self.connect_child(parent_task, task_id, turbo_tasks); return task_id; } + self.track_cache_miss(&task_type); let task_type = Arc::new(task_type); let task_id = self.transient_task_id_factory.get(); if let Err(existing_task_id) = self.task_cache.try_insert(task_type, task_id) { @@ -883,12 +900,10 @@ impl TurboTasksBackendInner { unsafe { self.transient_task_id_factory.reuse(task_id); } - // Safety: `tx` is a valid transaction from `self.backend.backing_storage`. self.connect_child(parent_task, existing_task_id, turbo_tasks); return existing_task_id; } - // Safety: `tx` is a valid transaction from `self.backend.backing_storage`. self.connect_child(parent_task, task_id, turbo_tasks); task_id @@ -1962,6 +1977,10 @@ impl Backend for TurboTasksBackend { fn dispose_root_task(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { self.0.dispose_root_task(task_id, turbo_tasks); } + + fn task_statistics(&self) -> &TaskStatisticsApi { + &self.0.task_statistics + } } // from https://github.com/tokio-rs/tokio/blob/29cd6ec1ec6f90a7ee1ad641c03e0e00badbcb0e/tokio/src/time/instant.rs#L57-L63 diff --git a/turbopack/crates/turbo-tasks-memory/src/lib.rs b/turbopack/crates/turbo-tasks-memory/src/lib.rs index 751727936e63e..9f1f3e3ba67e4 100644 --- a/turbopack/crates/turbo-tasks-memory/src/lib.rs +++ b/turbopack/crates/turbo-tasks-memory/src/lib.rs @@ -14,7 +14,5 @@ mod map_guard; mod memory_backend; mod output; mod task; -mod task_statistics; pub use memory_backend::MemoryBackend; -pub use task_statistics::{TaskStatistics, TaskStatisticsApi}; diff --git a/turbopack/crates/turbo-tasks-memory/src/memory_backend.rs b/turbopack/crates/turbo-tasks-memory/src/memory_backend.rs index a1c06a36802d9..113e04f685958 100644 --- a/turbopack/crates/turbo-tasks-memory/src/memory_backend.rs +++ b/turbopack/crates/turbo-tasks-memory/src/memory_backend.rs @@ -23,6 +23,7 @@ use turbo_tasks::{ TransientTaskType, TypedCellContent, }, event::EventListener, + task_statistics::TaskStatisticsApi, util::{IdFactoryWithReuse, NoMoveVec}, CellId, FunctionId, RawVc, ReadConsistency, TaskId, TaskIdSet, TraitTypeId, TurboTasksBackendApi, Unused, ValueTypeId, TRANSIENT_TASK_BIT, @@ -36,7 +37,6 @@ use crate::{ }, output::Output, task::{ReadCellError, Task, TaskType}, - task_statistics::TaskStatisticsApi, }; fn prehash_task_type(task_type: CachedTaskType) -> PreHashed { @@ -333,16 +333,12 @@ impl MemoryBackend { } } - pub fn task_statistics(&self) -> &TaskStatisticsApi { - &self.task_statistics - } - - fn track_cache_hit(&self, task_type: &PreHashed) { + fn track_cache_hit(&self, task_type: &CachedTaskType) { self.task_statistics() .map(|stats| stats.increment_cache_hit(task_type.fn_type)); } - fn track_cache_miss(&self, task_type: &PreHashed) { + fn track_cache_miss(&self, task_type: &CachedTaskType) { self.task_statistics() .map(|stats| stats.increment_cache_miss(task_type.fn_type)); } @@ -777,6 +773,10 @@ impl Backend for MemoryBackend { fn dispose_root_task(&self, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { Task::unset_root(task, self, turbo_tasks); } + + fn task_statistics(&self) -> &TaskStatisticsApi { + &self.task_statistics + } } pub(crate) enum Job { diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index 2e00328cd73b4..7fbb5c1c4aee7 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -20,6 +20,7 @@ use crate::{ raw_vc::CellId, registry, task::shared_reference::TypedSharedReference, + task_statistics::TaskStatisticsApi, triomphe_utils::unchecked_sidecast_triomphe_arc, FunctionId, RawVc, ReadRef, SharedReference, TaskId, TaskIdSet, TraitRef, TraitTypeId, ValueTypeId, VcRead, VcValueTrait, VcValueType, @@ -604,4 +605,6 @@ pub trait Backend: Sync + Send { ) -> TaskId; fn dispose_root_task(&self, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi); + + fn task_statistics(&self) -> &TaskStatisticsApi; } diff --git a/turbopack/crates/turbo-tasks/src/lib.rs b/turbopack/crates/turbo-tasks/src/lib.rs index 8bcc910a6544e..38a1c7766a377 100644 --- a/turbopack/crates/turbo-tasks/src/lib.rs +++ b/turbopack/crates/turbo-tasks/src/lib.rs @@ -73,6 +73,7 @@ mod shrink_to_fit; pub mod small_duration; mod state; pub mod task; +pub mod task_statistics; pub mod trace; mod trait_helpers; mod trait_ref; diff --git a/turbopack/crates/turbo-tasks-memory/src/task_statistics.rs b/turbopack/crates/turbo-tasks/src/task_statistics.rs similarity index 90% rename from turbopack/crates/turbo-tasks-memory/src/task_statistics.rs rename to turbopack/crates/turbo-tasks/src/task_statistics.rs index 6df3abc0ffc3a..481ec2975bbc1 100644 --- a/turbopack/crates/turbo-tasks-memory/src/task_statistics.rs +++ b/turbopack/crates/turbo-tasks/src/task_statistics.rs @@ -6,7 +6,8 @@ use std::{ use dashmap::DashMap; use rustc_hash::FxHasher; use serde::{ser::SerializeMap, Serialize, Serializer}; -use turbo_tasks::{registry, FunctionId}; + +use crate::{registry, FunctionId}; /// An API for optionally enabling, updating, and reading aggregated statistics. #[derive(Default)] @@ -40,18 +41,17 @@ impl TaskStatisticsApi { } } -/// A type representing the enabled state of [`TaskStatisticsApi`]. Implements -/// [`serde::Serialize`]. +/// A type representing the enabled state of [`TaskStatisticsApi`]. Implements [`serde::Serialize`]. pub struct TaskStatistics { inner: DashMap>, } impl TaskStatistics { - pub(crate) fn increment_cache_hit(&self, function_id: FunctionId) { + pub fn increment_cache_hit(&self, function_id: FunctionId) { self.with_task_type_statistics(function_id, |stats| stats.cache_hit += 1) } - pub(crate) fn increment_cache_miss(&self, function_id: FunctionId) { + pub fn increment_cache_miss(&self, function_id: FunctionId) { self.with_task_type_statistics(function_id, |stats| stats.cache_miss += 1) } From 0a7696fdac06bc7b81894706685bd3166d3c22e4 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Sun, 26 Jan 2025 19:44:55 -0800 Subject: [PATCH 2/2] port unit test to new backend --- Cargo.lock | 2 + crates/napi/src/next_api/utils.rs | 10 +- .../crates/turbo-tasks-backend/Cargo.toml | 2 + .../tests/task_statistics.rs | 1 + .../tests/task_statistics.rs | 275 +----------------- .../crates/turbo-tasks-testing/src/lib.rs | 4 + .../tests/task_statistics.rs | 272 +++++++++++++++++ turbopack/crates/turbo-tasks/src/manager.rs | 7 + 8 files changed, 293 insertions(+), 280 deletions(-) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/task_statistics.rs mode change 100644 => 120000 turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs create mode 100644 turbopack/crates/turbo-tasks-testing/tests/task_statistics.rs diff --git a/Cargo.lock b/Cargo.lock index fe16fd2708366..606790abf1732 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9153,8 +9153,10 @@ dependencies = [ "pot", "rand", "rayon", + "regex", "rustc-hash 1.1.0", "serde", + "serde_json", "serde_path_to_error", "smallvec", "thread_local", diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index c745ce71ecfb4..d74ea3700eee4 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -10,8 +10,8 @@ use napi::{ }; use serde::Serialize; use turbo_tasks::{ - backend::Backend, task_statistics::TaskStatisticsApi, trace::TraceRawVcs, OperationVc, ReadRef, - TaskId, TryJoinIterExt, TurboTasks, UpdateInfo, Vc, + task_statistics::TaskStatisticsApi, trace::TraceRawVcs, OperationVc, ReadRef, TaskId, + TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, Vc, }; use turbo_tasks_backend::{ default_backing_storage, noop_backing_storage, DefaultBackingStorage, NoopBackingStorage, @@ -118,10 +118,8 @@ impl NextTurboTasks { pub fn task_statistics(&self) -> &TaskStatisticsApi { match self { - NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.backend().task_statistics(), - NextTurboTasks::PersistentCaching(turbo_tasks) => { - turbo_tasks.backend().task_statistics() - } + NextTurboTasks::Memory(turbo_tasks) => turbo_tasks.task_statistics(), + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.task_statistics(), } } } diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 9974bacfbeac7..163f135d62396 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -54,6 +54,8 @@ turbo-tasks-testing = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } +regex = { workspace = true } +serde_json = { workspace = true } [build-dependencies] turbo-tasks-build = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/tests/task_statistics.rs b/turbopack/crates/turbo-tasks-backend/tests/task_statistics.rs new file mode 120000 index 0000000000000..bc0c0bbb90f5a --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/task_statistics.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/task_statistics.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs b/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs deleted file mode 100644 index d435060f5b9a2..0000000000000 --- a/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs +++ /dev/null @@ -1,274 +0,0 @@ -#![feature(arbitrary_self_types)] -#![feature(arbitrary_self_types_pointers)] -#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this - -use std::{ - future::{Future, IntoFuture}, - sync::Arc, -}; - -use anyhow::Result; -use once_cell::sync::Lazy; -use regex::Regex; -use serde_json::json; -use turbo_tasks::{TurboTasks, Vc}; -use turbo_tasks_memory::MemoryBackend; -use turbo_tasks_testing::{register, Registration}; - -static REGISTRATION: Registration = register!(); - -#[tokio::test] -async fn test_simple_task() { - run_with_tt(|tt| async move { - for i in 0..10 { - double(i).await.unwrap(); - // use cached results - double(i).await.unwrap(); - } - for i in 0..5 { - double(i).await.unwrap(); - } - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::double": { - "cache_miss": 10, - "cache_hit": 15, - }, - }) - ); - }) - .await; -} - -#[tokio::test] -async fn test_await_same_vc_multiple_times() { - run_with_tt(|tt| async move { - let dvc = double(0); - // this is awaited multiple times, but only resolved once - tokio::try_join!(dvc.into_future(), dvc.into_future()).unwrap(); - dvc.await.unwrap(); - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::double": { - "cache_miss": 1, - "cache_hit": 0, - }, - }) - ); - }) - .await; -} - -#[tokio::test] -async fn test_vc_receiving_task() { - run_with_tt(|tt| async move { - for i in 0..10 { - let dvc = double(i); - double_vc(dvc).await.unwrap(); - // use cached results - double_vc(dvc).await.unwrap(); - } - for i in 0..5 { - let dvc = double(i); - double_vc(dvc).await.unwrap(); - } - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::double": { - "cache_miss": 10, - "cache_hit": 5, - }, - "turbo-tasks-memory::::double_vc": { - "cache_miss": 10, - "cache_hit": 15, - }, - }) - ); - }) - .await; -} - -#[tokio::test] -async fn test_trait_methods() { - run_with_tt(|tt| async move { - for i in 0..10 { - let wvc = wrap(i); - tokio::try_join!(wvc.double().into_future(), wvc.double().into_future()).unwrap(); - tokio::try_join!(wvc.double_vc().into_future(), wvc.double_vc().into_future()).unwrap(); - } - // use cached results - for i in 0..5 { - let wvc = wrap(i); - wvc.double().await.unwrap(); - wvc.double_vc().await.unwrap(); - } - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::wrap": { - "cache_miss": 10, - "cache_hit": 5, - }, - "turbo-tasks-memory::::WrappedU64::Doublable::double": { - "cache_miss": 10, - "cache_hit": 15, - }, - "turbo-tasks-memory::::WrappedU64::Doublable::double_vc": { - "cache_miss": 10, - "cache_hit": 15, - }, - }) - ); - }) - .await; -} - -#[tokio::test] -async fn test_dyn_trait_methods() { - run_with_tt(|tt| async move { - for i in 0..10 { - let wvc: Vc> = Vc::upcast(wrap(i)); - let _ = tokio::try_join!(wvc.double().resolve(), wvc.double().resolve()).unwrap(); - let _ = tokio::try_join!(wvc.double_vc().resolve(), wvc.double_vc().resolve()).unwrap(); - } - // use cached results - for i in 0..5 { - let wvc: Vc> = Vc::upcast(wrap(i)); - let _ = wvc.double().resolve().await.unwrap(); - let _ = wvc.double_vc().resolve().await.unwrap(); - } - // use cached results without dynamic dispatch - for i in 0..2 { - let wvc = wrap(i); - let _ = wvc.double().await.unwrap(); - let _ = wvc.double_vc().await.unwrap(); - } - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::wrap": { - "cache_miss": 10, - "cache_hit": 7, - }, - "turbo-tasks-memory::::WrappedU64::Doublable::double": { - "cache_miss": 10, - "cache_hit": 17, - }, - "turbo-tasks-memory::::WrappedU64::Doublable::double_vc": { - "cache_miss": 10, - "cache_hit": 17, - }, - }) - ); - }) - .await; -} - -// creates Vcs, but doesn't ever execute them -#[tokio::test] -async fn test_no_execution() { - run_with_tt(|tt| async move { - // don't await this! - let _ = wrap_vc(double_vc(double(123))).double().double_vc(); - assert_eq!( - stats_json(&tt), - json!({ - "turbo-tasks-memory::::double": { - "cache_miss": 1, - "cache_hit": 0, - }, - }) - ); - }) - .await; -} - -// Internally, this function uses `PersistentTaskType`. -#[turbo_tasks::function] -fn double(val: u64) -> Vc { - Vc::cell(val * 2) -} - -// Internally, this function uses `LocalTaskType::ResolveNative`. -#[turbo_tasks::function] -async fn double_vc(val: Vc) -> Result> { - let val = *val.await?; - Ok(Vc::cell(val * 2)) -} - -#[turbo_tasks::value] -struct WrappedU64(u64); - -#[turbo_tasks::function] -fn wrap(val: u64) -> Vc { - WrappedU64(val).cell() -} - -#[turbo_tasks::function] -async fn wrap_vc(val: Vc) -> Result> { - Ok(WrappedU64(*val.await?).cell()) -} - -#[turbo_tasks::value_trait] -pub trait Doublable { - fn double(&self) -> Vc; - fn double_vc(self: Vc) -> Vc; -} - -#[turbo_tasks::value_impl] -impl Doublable for WrappedU64 { - #[turbo_tasks::function] - fn double(&self) -> Vc { - WrappedU64(self.0 * 2).cell() - } - - #[turbo_tasks::function] - async fn double_vc(&self) -> Result> { - let val = self.0; - Ok(WrappedU64(val * 2).cell()) - } -} - -#[turbo_tasks::function] -async fn fail(val: u64) -> Result> { - anyhow::bail!("failed using {val}"); -} - -async fn run_with_tt(func: impl FnOnce(Arc>) -> Fut) -where - Fut: Future + Send + 'static, -{ - REGISTRATION.ensure_registered(); - let tt = TurboTasks::new(MemoryBackend::default()); - tt.backend().task_statistics().enable(); - let fut = func(Arc::clone(&tt)); - tt.run_once(async move { - fut.await; - Ok(()) - }) - .await - .unwrap(); -} - -fn stats_json(tt: &TurboTasks) -> serde_json::Value { - remove_hashes(serde_json::to_value(tt.backend().task_statistics().get()).unwrap()) -} - -// Global task identifiers can contain a hash of the crate and dependencies. -// Remove that so that we can compare against a stable value in tests. -fn remove_hashes(mut json: serde_json::Value) -> serde_json::Value { - static HASH_RE: Lazy = Lazy::new(|| Regex::new("@[^:]+").unwrap()); - match &mut json { - serde_json::Value::Object(map) => { - let old_map = std::mem::take(map); - for (k, v) in old_map { - map.insert(HASH_RE.replace(&k, "").into_owned(), v); - } - } - _ => unreachable!("expected object"), - }; - json -} diff --git a/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs b/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs new file mode 120000 index 0000000000000..bc0c0bbb90f5a --- /dev/null +++ b/turbopack/crates/turbo-tasks-memory/tests/task_statistics.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/task_statistics.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-testing/src/lib.rs b/turbopack/crates/turbo-tasks-testing/src/lib.rs index 38368e2a5e54a..6a98bb9f81f66 100644 --- a/turbopack/crates/turbo-tasks-testing/src/lib.rs +++ b/turbopack/crates/turbo-tasks-testing/src/lib.rs @@ -295,6 +295,10 @@ impl TurboTasksApi for VcStorage { unimplemented!() } + fn task_statistics(&self) -> &turbo_tasks::task_statistics::TaskStatisticsApi { + unimplemented!() + } + fn stop_and_wait(&self) -> std::pin::Pin + Send + 'static>> { Box::pin(async {}) } diff --git a/turbopack/crates/turbo-tasks-testing/tests/task_statistics.rs b/turbopack/crates/turbo-tasks-testing/tests/task_statistics.rs new file mode 100644 index 0000000000000..50315593ea023 --- /dev/null +++ b/turbopack/crates/turbo-tasks-testing/tests/task_statistics.rs @@ -0,0 +1,272 @@ +#![feature(arbitrary_self_types)] +#![feature(arbitrary_self_types_pointers)] +#![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this + +use std::future::IntoFuture; + +use anyhow::Result; +use once_cell::sync::Lazy; +use regex::Regex; +use serde_json::json; +use turbo_tasks::Vc; +use turbo_tasks_testing::{register, run_without_cache_check, Registration}; + +static REGISTRATION: Registration = register!(); + +#[tokio::test] +async fn test_simple_task() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + for i in 0..10 { + double(i).await.unwrap(); + // use cached results + double(i).await.unwrap(); + } + for i in 0..5 { + double(i).await.unwrap(); + } + assert_eq!( + stats_json(), + json!({ + "double": { + "cache_miss": 10, + "cache_hit": 15, + }, + }) + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn test_await_same_vc_multiple_times() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + let dvc = double(0); + // this is awaited multiple times, but only resolved once + tokio::try_join!(dvc.into_future(), dvc.into_future()).unwrap(); + dvc.await.unwrap(); + assert_eq!( + stats_json(), + json!({ + "double": { + "cache_miss": 1, + "cache_hit": 0, + }, + }) + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn test_vc_receiving_task() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + for i in 0..10 { + let dvc = double(i); + double_vc(dvc).await.unwrap(); + // use cached results + double_vc(dvc).await.unwrap(); + } + for i in 0..5 { + let dvc = double(i); + double_vc(dvc).await.unwrap(); + } + assert_eq!( + stats_json(), + json!({ + "double": { + "cache_miss": 10, + "cache_hit": 5, + }, + "double_vc": { + "cache_miss": 10, + "cache_hit": 15, + }, + }) + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn test_trait_methods() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + for i in 0..10 { + let wvc = wrap(i); + tokio::try_join!(wvc.double().into_future(), wvc.double().into_future()).unwrap(); + tokio::try_join!(wvc.double_vc().into_future(), wvc.double_vc().into_future()).unwrap(); + } + // use cached results + for i in 0..5 { + let wvc = wrap(i); + wvc.double().await.unwrap(); + wvc.double_vc().await.unwrap(); + } + assert_eq!( + stats_json(), + json!({ + "wrap": { + "cache_miss": 10, + "cache_hit": 5, + }, + "WrappedU64::Doublable::double": { + "cache_miss": 10, + "cache_hit": 15, + }, + "WrappedU64::Doublable::double_vc": { + "cache_miss": 10, + "cache_hit": 15, + }, + }) + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn test_dyn_trait_methods() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + for i in 0..10 { + let wvc: Vc> = Vc::upcast(wrap(i)); + let _ = tokio::try_join!(wvc.double().resolve(), wvc.double().resolve()).unwrap(); + let _ = tokio::try_join!(wvc.double_vc().resolve(), wvc.double_vc().resolve()).unwrap(); + } + // use cached results + for i in 0..5 { + let wvc: Vc> = Vc::upcast(wrap(i)); + let _ = wvc.double().resolve().await.unwrap(); + let _ = wvc.double_vc().resolve().await.unwrap(); + } + // use cached results without dynamic dispatch + for i in 0..2 { + let wvc = wrap(i); + let _ = wvc.double().await.unwrap(); + let _ = wvc.double_vc().await.unwrap(); + } + assert_eq!( + stats_json(), + json!({ + "wrap": { + "cache_miss": 10, + "cache_hit": 7, + }, + "WrappedU64::Doublable::double": { + "cache_miss": 10, + "cache_hit": 17, + }, + "WrappedU64::Doublable::double_vc": { + "cache_miss": 10, + "cache_hit": 17, + }, + }) + ); + Ok(()) + }) + .await +} + +// creates Vcs, but doesn't ever execute them +#[tokio::test] +async fn test_no_execution() -> Result<()> { + run_without_cache_check(®ISTRATION, async move { + enable_stats(); + // don't await this! + let _ = wrap_vc(double_vc(double(123))).double().double_vc(); + assert_eq!( + stats_json(), + json!({ + "double": { + "cache_miss": 1, + "cache_hit": 0, + }, + }) + ); + Ok(()) + }) + .await +} + +// Internally, this function uses `PersistentTaskType`. +#[turbo_tasks::function] +fn double(val: u64) -> Vc { + Vc::cell(val * 2) +} + +// Internally, this function uses `LocalTaskType::ResolveNative`. +#[turbo_tasks::function] +async fn double_vc(val: Vc) -> Result> { + let val = *val.await?; + Ok(Vc::cell(val * 2)) +} + +#[turbo_tasks::value] +struct WrappedU64(u64); + +#[turbo_tasks::function] +fn wrap(val: u64) -> Vc { + WrappedU64(val).cell() +} + +#[turbo_tasks::function] +async fn wrap_vc(val: Vc) -> Result> { + Ok(WrappedU64(*val.await?).cell()) +} + +#[turbo_tasks::value_trait] +pub trait Doublable { + fn double(&self) -> Vc; + fn double_vc(self: Vc) -> Vc; +} + +#[turbo_tasks::value_impl] +impl Doublable for WrappedU64 { + #[turbo_tasks::function] + fn double(&self) -> Vc { + WrappedU64(self.0 * 2).cell() + } + + #[turbo_tasks::function] + async fn double_vc(&self) -> Result> { + let val = self.0; + Ok(WrappedU64(val * 2).cell()) + } +} + +#[turbo_tasks::function] +async fn fail(val: u64) -> Result> { + anyhow::bail!("failed using {val}"); +} + +fn enable_stats() { + let tt = turbo_tasks::turbo_tasks(); + tt.task_statistics().enable(); +} + +fn stats_json() -> serde_json::Value { + let tt = turbo_tasks::turbo_tasks(); + remove_crate_and_hashes(serde_json::to_value(tt.task_statistics().get()).unwrap()) +} + +// Global task identifiers can contain a hash of the crate and dependencies. +// Remove that so that we can compare against a stable value in tests. +fn remove_crate_and_hashes(mut json: serde_json::Value) -> serde_json::Value { + static HASH_RE: Lazy = Lazy::new(|| Regex::new("^[^:@]+@[^:]+:+").unwrap()); + match &mut json { + serde_json::Value::Object(map) => { + let old_map = std::mem::take(map); + for (k, v) in old_map { + map.insert(HASH_RE.replace(&k, "").into_owned(), v); + } + } + _ => unreachable!("expected object"), + }; + json +} diff --git a/turbopack/crates/turbo-tasks/src/manager.rs b/turbopack/crates/turbo-tasks/src/manager.rs index ab3da0ddffc62..e1c87d5ccc970 100644 --- a/turbopack/crates/turbo-tasks/src/manager.rs +++ b/turbopack/crates/turbo-tasks/src/manager.rs @@ -44,6 +44,7 @@ use crate::{ local_task::{LocalTask, LocalTaskType}, shared_reference::TypedSharedReference, }, + task_statistics::TaskStatisticsApi, trace::TraceRawVcs, trait_helpers::get_trait_method, util::StaticOrArc, @@ -176,6 +177,8 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { f: Pin> + Send + 'static>>, ) -> Pin> + Send + 'static>>; + fn task_statistics(&self) -> &TaskStatisticsApi; + fn stop_and_wait(&self) -> Pin + Send>>; } @@ -1418,6 +1421,10 @@ impl TurboTasksApi for TurboTasks { )) } + fn task_statistics(&self) -> &TaskStatisticsApi { + self.backend.task_statistics() + } + fn stop_and_wait(&self) -> Pin + Send + 'static>> { let this = self.pin(); Box::pin(async move {