move ProcessEnv into turbo-tasks-env (#462)

make ProcessEnv a trait to allow other implemenetations of it

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "crates/swc-ast-explorer",
   "crates/turbo-malloc",
   "crates/turbo-tasks-build",
+  "crates/turbo-tasks-env",
   "crates/turbo-tasks-fs",
   "crates/turbo-tasks-macros",
   "crates/turbo-tasks-memory",

crates/next-core/Cargo.toml

@@ -16,6 +16,7 @@ serde = "1.0.136"
 serde_json = "1.0.85"
 tokio = { version = "1.11.0", features = ["full"] }
 turbo-tasks = { path = "../turbo-tasks" }
+turbo-tasks-env = { path = "../turbo-tasks-env" }
 turbo-tasks-fs = { path = "../turbo-tasks-fs" }
 turbopack = { path = "../turbopack" }
 turbopack-core = { path = "../turbopack-core" }

crates/next-core/src/env.rs

@@ -1,12 +1,15 @@
 use anyhow::Result;
+use turbo_tasks_env::{CommandLineProcessEnvVc, FilterProcessEnvVc, ProcessEnvVc};
 use turbo_tasks_fs::FileSystemPathVc;
-use turbopack_env::ProcessEnvVc;
+use turbopack_env::TryDotenvProcessEnvVc;
 
 /// Loads a series of dotenv files according to the precedence rules set by
 /// https://nextjs.org/docs/basic-features/environment-variables#environment-variable-load-order
 #[turbo_tasks::function]
 pub async fn load_env(project_path: FileSystemPathVc) -> Result<ProcessEnvVc> {
-    let node_env = std::env::var("NODE_ENV").unwrap_or_else(|_| "development".into());
+    let env = CommandLineProcessEnvVc::new().as_process_env();
+    let node_env = env.read("NODE_ENV").await?;
+    let node_env = node_env.as_deref().unwrap_or("development");
 
     let files = [
         Some(format!(".env.{node_env}.local")),
@@ -21,14 +24,14 @@ pub async fn load_env(project_path: FileSystemPathVc) -> Result<ProcessEnvVc> {
     .into_iter()
     .flatten();
 
-    let env = files.fold(ProcessEnvVc::from_command_line(), |prior, f| {
+    let env = files.fold(env, |prior, f| {
         let path = project_path.join(&f);
-        ProcessEnvVc::from_dotenv_file(path, Some(prior))
+        TryDotenvProcessEnvVc::new(prior, path).as_process_env()
     });
 
     Ok(env)
 }
 
 pub fn filter_for_client(env: ProcessEnvVc) -> ProcessEnvVc {
-    ProcessEnvVc::filter(env, "NEXT_PUBLIC_".to_string())
+    FilterProcessEnvVc::new(env, "NEXT_PUBLIC_".to_string()).into()
 }

crates/next-core/src/next_client/context.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 
 use anyhow::Result;
 use turbo_tasks::Value;
+use turbo_tasks_env::ProcessEnvVc;
 use turbo_tasks_fs::FileSystemPathVc;
 use turbopack::{
     module_options::module_options_context::{ModuleOptionsContext, ModuleOptionsContextVc},
@@ -17,7 +18,7 @@ use turbopack_core::{
 };
 use turbopack_dev_server::html_runtime_asset::HtmlRuntimeAssetVc;
 use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc;
-use turbopack_env::{ProcessEnvAssetVc, ProcessEnvVc};
+use turbopack_env::ProcessEnvAssetVc;
 
 use crate::{
     env::filter_for_client,

crates/next-core/src/server_rendered_source.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use anyhow::Result;
 use serde_json::json;
 use turbo_tasks::{primitives::JsonValueVc, Value};
+use turbo_tasks_env::ProcessEnvVc;
 use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemEntryType, FileSystemPathVc};
 use turbopack::{
     module_options::ModuleOptionsContext, resolve_options_context::ResolveOptionsContext,
@@ -21,7 +22,7 @@ use turbopack_dev_server::source::{
     ContentSourceVc, NoContentSourceVc,
 };
 use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc;
-use turbopack_env::{ProcessEnvAssetVc, ProcessEnvVc};
+use turbopack_env::ProcessEnvAssetVc;
 
 use crate::{
     next_client::{

crates/next-core/src/web_entry_source.rs

@@ -2,6 +2,7 @@ use std::future::IntoFuture;
 
 use anyhow::{anyhow, Result};
 use futures::{prelude::*, stream};
+use turbo_tasks_env::ProcessEnvVc;
 use turbo_tasks_fs::FileSystemPathVc;
 use turbopack::ecmascript::EcmascriptModuleAssetVc;
 use turbopack_core::{
@@ -12,7 +13,6 @@ use turbopack_dev_server::{
     html::DevHtmlAsset,
     source::{asset_graph::AssetGraphContentSourceVc, ContentSourceVc},
 };
-use turbopack_env::ProcessEnvVc;
 
 use crate::next_client::context::{
     get_client_asset_context, get_client_chunking_context, get_resolved_client_runtime_entries,

crates/turbo-tasks-env/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "turbo-tasks-env"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+bench = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.47"
+dotenvy = "0.15.5"
+indexmap = "1.8.0"
+serde = "1.0.136"
+serde_json = "1.0.85"
+tokio = "1.11.0"
+turbo-tasks = { path = "../turbo-tasks" }
+turbo-tasks-fs = { path = "../turbo-tasks-fs" }
+
+[build-dependencies]
+turbo-tasks-build = { path = "../turbo-tasks-build" }

crates/turbo-tasks-env/build.rs

@@ -0,0 +1,5 @@
+use turbo_tasks_build::generate_register;
+
+fn main() {
+    generate_register();
+}

crates/turbo-tasks-env/src/command_line.rs

@@ -0,0 +1,31 @@
+use std::env;
+
+use indexmap::IndexMap;
+
+use crate::{EnvMapVc, ProcessEnv, ProcessEnvVc, GLOBAL_ENV_LOCK};
+
+/// Load the environment variables defined via command line.
+#[turbo_tasks::value]
+pub struct CommandLineProcessEnv;
+
+#[turbo_tasks::value_impl]
+impl CommandLineProcessEnvVc {
+    #[turbo_tasks::function]
+    pub fn new() -> Self {
+        CommandLineProcessEnv.cell()
+    }
+}
+
+/// Clones the current env vars into a IndexMap.
+fn env_snapshot() -> IndexMap<String, String> {
+    let _lock = GLOBAL_ENV_LOCK.lock().unwrap();
+    env::vars().collect::<IndexMap<_, _>>()
+}
+
+#[turbo_tasks::value_impl]
+impl ProcessEnv for CommandLineProcessEnv {
+    #[turbo_tasks::function]
+    fn read_all(&self) -> EnvMapVc {
+        EnvMapVc::cell(env_snapshot())
+    }
+}

crates/turbo-tasks-env/src/dotenv.rs

@@ -0,0 +1,89 @@
+use std::env;
+
+use anyhow::{anyhow, Context, Result};
+use indexmap::IndexMap;
+use turbo_tasks::ValueToString;
+use turbo_tasks_fs::{FileContent, FileSystemPathVc};
+
+use crate::{EnvMapVc, ProcessEnv, ProcessEnvVc, GLOBAL_ENV_LOCK};
+
+/// Load the environment variables defined via a dotenv file, with an
+/// optional prior state that we can lookup already defined variables
+/// from.
+#[turbo_tasks::value]
+pub struct DotenvProcessEnv {
+    prior: Option<ProcessEnvVc>,
+    path: FileSystemPathVc,
+}
+
+#[turbo_tasks::value_impl]
+impl DotenvProcessEnvVc {
+    #[turbo_tasks::function]
+    pub fn new(prior: Option<ProcessEnvVc>, path: FileSystemPathVc) -> Self {
+        DotenvProcessEnv { prior, path }.cell()
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl ProcessEnv for DotenvProcessEnv {
+    #[turbo_tasks::function]
+    async fn read_all(&self) -> Result<EnvMapVc> {
+        let prior = if let Some(p) = self.prior {
+            Some(p.read_all().await?)
+        } else {
+            None
+        };
+        let empty = IndexMap::new();
+        let prior = prior.as_deref().unwrap_or(&empty);
+
+        let file = self.path.read().await?;
+        if let FileContent::Content(f) = &*file {
+            let res;
+            let vars;
+            {
+                let _lock = GLOBAL_ENV_LOCK.lock().unwrap();
+
+                // Unfortunately, dotenvy only looks up variable references from the global env.
+                // So we must mutate while we process. Afterwards, we can restore the initial
+                // state.
+                let initial = env::vars().collect();
+
+                restore_env(&initial, prior);
+
+                // from_read will load parse and evalute the Read, and set variables
+                // into the global env. If a later dotenv defines an already defined
+                // var, it'll be ignored.
+                res = dotenvy::from_read(f.content());
+
+                vars = env::vars().collect();
+                restore_env(&vars, &initial);
+            }
+
+            if res.is_err() {
+                res.context(anyhow!(
+                    "unable to read {} for env vars",
+                    self.path.to_string().await?
+                ))?;
+            }
+
+            Ok(EnvMapVc::cell(vars))
+        } else {
+            Ok(EnvMapVc::cell(prior.clone()))
+        }
+    }
+}
+
+/// Restores the global env variables to mirror `to`.
+fn restore_env(from: &IndexMap<String, String>, to: &IndexMap<String, String>) {
+    for key in from.keys() {
+        if !to.contains_key(key) {
+            env::remove_var(key);
+        }
+    }
+    for (key, value) in to {
+        match from.get(key) {
+            Some(v) if v == value => {}
+            _ => env::set_var(key, value),
+        }
+    }
+}

crates/turbo-tasks-env/src/filter.rs

@@ -0,0 +1,43 @@
+use anyhow::Result;
+use indexmap::IndexMap;
+use turbo_tasks::primitives::OptionStringVc;
+
+use crate::{EnvMapVc, ProcessEnv, ProcessEnvVc};
+
+#[turbo_tasks::value]
+pub struct FilterProcessEnv {
+    prior: ProcessEnvVc,
+    filter: String,
+}
+
+#[turbo_tasks::value_impl]
+impl FilterProcessEnvVc {
+    #[turbo_tasks::function]
+    pub fn new(prior: ProcessEnvVc, filter: String) -> Self {
+        FilterProcessEnv { prior, filter }.cell()
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl ProcessEnv for FilterProcessEnv {
+    #[turbo_tasks::function]
+    async fn read_all(&self) -> Result<EnvMapVc> {
+        let prior = self.prior.read_all().await?;
+        let mut filtered = IndexMap::new();
+        for (key, value) in &*prior {
+            if key.starts_with(&self.filter) {
+                filtered.insert(key.clone(), value.clone());
+            }
+        }
+        Ok(EnvMapVc::cell(filtered))
+    }
+
+    #[turbo_tasks::function]
+    fn read(&self, name: &str) -> OptionStringVc {
+        if name.starts_with(&self.filter) {
+            self.prior.read(name)
+        } else {
+            OptionStringVc::cell(None)
+        }
+    }
+}

crates/turbo-tasks-env/src/lib.rs

@@ -0,0 +1,51 @@
+#![feature(min_specialization)]
+
+mod command_line;
+mod dotenv;
+mod filter;
+
+use std::{env, sync::Mutex};
+
+use anyhow::Result;
+use indexmap::IndexMap;
+use turbo_tasks::primitives::OptionStringVc;
+
+pub use self::{
+    command_line::CommandLineProcessEnvVc, dotenv::DotenvProcessEnvVc, filter::FilterProcessEnvVc,
+};
+
+#[turbo_tasks::value(transparent)]
+pub struct EnvMap(#[turbo_tasks(trace_ignore)] IndexMap<String, String>);
+
+#[turbo_tasks::value_impl]
+impl EnvMapVc {
+    #[turbo_tasks::function]
+    pub fn empty() -> Self {
+        EnvMap(IndexMap::new()).cell()
+    }
+}
+
+#[turbo_tasks::value_trait]
+pub trait ProcessEnv {
+    // TODO SECURITY: From security perspective it's not good that we read *all* env
+    // vars into the cache. This might store secrects into the persistent cache
+    // which we want to avoid.
+    // Instead we should use only `read_prefix` to read all env vars with a specific
+    // prefix.
+    /// Reads all env variables into a Map
+    fn read_all(&self) -> EnvMapVc;
+
+    /// Reads a single env variable
+    async fn read(&self, name: &str) -> Result<OptionStringVc> {
+        Ok(OptionStringVc::cell(
+            self.read_all().await?.get(name).cloned(),
+        ))
+    }
+}
+
+pub static GLOBAL_ENV_LOCK: Mutex<()> = Mutex::new(());
+
+pub fn register() {
+    turbo_tasks::register();
+    include!(concat!(env!("OUT_DIR"), "/register.rs"));
+}