From a22f83d849029080fca225246921c056b71b863a Mon Sep 17 00:00:00 2001 From: Alex Bean Date: Fri, 7 Mar 2025 14:26:41 +0100 Subject: [PATCH] feat: build deterministic runtime (#425) * test: detects_project_type_correctly * feat: parachain deployment in pop up * fix: parachain feature * docs: improve comments * feat: deploy a parachain commands * feat: build specs in pop up * feat: logic to interact with a chain * feat: register parachain * refactor: clean code and improve docs * test: unit tests for pop up methods * refactor: small fixes with visibility and removing logs * feat: return events in submit_signed_extrinsic * feat: get para_id from event * test: fix detects_parachain_correctly * refactor: improve docs and code * test: fix change_working_directory_works * fix: clippy warnings * refactor: move submit_extrinsic_with_wallet in a common file * refactor: remove unnecesary code * refactor: UpChainCommand structure * test: adjust tests to refactored struct * refactor: renaming prepare_register_parachain_call_data and prepare_rerve_parachain_call_data * refactor: move events module * fix: submit_extrinsic_with_wallet under parachain feature * refactor: remove unnecesary code * test: increase coverage with reserve_parachain_id_fails_wrong_chain and resolve_genesis_files_fails_wrong_path * refactor: remove unnecesary clones * refactor: minor improvements * test: refactor tests and include comments * refactor: map errors in submit_extrinsic_with_wallet * test: fix prepare_register_parachain_call_data_works * feat: container_engine to detect container to use * feat: generate_deterministic_runtime logic * feat: srtool logic * feat: build deterministic runtime in build spec command * feat: update_runtime_code * test: include tests for updateing code * refactor: clean unnecesary code * test: in container_engine * refactor: docker warning in container_engine * chore: improve screen messaging * refactor: use profile picked by the user * refactor: docs in ContainerEngine and clean prints * fix: generate_deterministic_runtime parameter * refactor: rename, improve messages and clean empty lines * feat: extract package name automatically from runtime dir * docs: change broken link * refactor: update srtool_lib crate and remove unnecesary code * docs: include srtool in Acknowledgements * refactor: remove unnecesary Error * test: detects_project_type_correctly * feat: parachain deployment in pop up * fix: parachain feature * docs: improve comments * feat: replace index.html with summary costs in ui (#430) * refactor: rename parachain to rollup * docs: improve deprecation message * refactor: rename parachain to rollup in up help * feat: deploy a parachain commands * feat: build specs in pop up * feat: logic to interact with a chain * feat: register parachain * refactor: clean code and improve docs * test: unit tests for pop up methods * refactor: small fixes with visibility and removing logs * feat: return events in submit_signed_extrinsic * feat: get para_id from event * test: fix detects_parachain_correctly * refactor: improve docs and code * test: fix change_working_directory_works * fix: clippy warnings * refactor: move submit_extrinsic_with_wallet in a common file * refactor: remove unnecesary code * refactor: UpChainCommand structure * test: adjust tests to refactored struct * refactor: renaming prepare_register_parachain_call_data and prepare_rerve_parachain_call_data * refactor: move events module * fix: submit_extrinsic_with_wallet under parachain feature * refactor: remove unnecesary code * test: increase coverage with reserve_parachain_id_fails_wrong_chain and resolve_genesis_files_fails_wrong_path * refactor: remove unnecesary clones * refactor: minor improvements * test: refactor tests and include comments * refactor: map errors in submit_extrinsic_with_wallet * test: fix prepare_register_parachain_call_data_works * refactor: move configure_chain into a common folder * refactor: function visibility * fix: error message and include test for it * refactor: build specs removing repetitive code * refactor: use prepare_extrinsic from Call module to prepare a call * docs: improve comments and messages * refactor: rename variables and structs * refactor: relay_chain_url * refactor: rename prepare_reserve_call_data and prepare_register_call_data * test: remove unnecesary test * refactor: remove events module * refactor: rename parachain to rollup * chore: improve succesful message * chore: change intro title to use rollup * docs: comments for Reserved event * chore: always prompt the user to build runtime deterministacally and improve message * docs: improve and add missing comments * refactor: clean code to prompt for the package and runtime dir * docs: improve warning message * refactor: rename variables and structs * fix: throw error message when build deterministic runtime fail * feat: include skip_deterministic_build flag to avoid prompting the user * docs: improve comments --- Cargo.lock | 91 +++++--- Cargo.toml | 1 + README.md | 1 + crates/pop-cli/src/commands/build/spec.rs | 190 +++++++++++++++- crates/pop-cli/tests/parachain.rs | 3 +- crates/pop-parachains/Cargo.toml | 1 + .../src/{build.rs => build/mod.rs} | 54 +++++ crates/pop-parachains/src/build/runtime.rs | 204 ++++++++++++++++++ crates/pop-parachains/src/errors.rs | 3 + crates/pop-parachains/src/lib.rs | 4 +- 10 files changed, 521 insertions(+), 31 deletions(-) rename crates/pop-parachains/src/{build.rs => build/mod.rs} (92%) create mode 100644 crates/pop-parachains/src/build/runtime.rs diff --git a/Cargo.lock b/Cargo.lock index f762f8bb9..443e7003a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -837,7 +837,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.8", + "thiserror 2.0.11", "tokio", "tokio-util", "tower-service", @@ -995,7 +995,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.8", + "thiserror 2.0.11", ] [[package]] @@ -1346,7 +1346,7 @@ dependencies = [ "serde", "serde_json", "strsim 0.11.1", - "thiserror 2.0.8", + "thiserror 2.0.11", "tracing", ] @@ -2821,7 +2821,7 @@ dependencies = [ "http 1.2.0", "hyper 1.5.2", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.23", "rustls-pki-types", "tokio", "tokio-rustls 0.26.1", @@ -3456,7 +3456,7 @@ dependencies = [ "http 1.2.0", "jsonrpsee-core", "pin-project", - "rustls 0.23.20", + "rustls 0.23.23", "rustls-pki-types", "rustls-platform-verifier", "soketto", @@ -3943,9 +3943,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "lru" @@ -4558,7 +4558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.8", + "thiserror 2.0.11", "ucd-trie", ] @@ -4866,6 +4866,7 @@ dependencies = [ "scale-value", "serde_json", "sp-core 32.0.0", + "srtool-lib", "strum 0.26.3", "strum_macros 0.26.4", "subxt", @@ -5377,9 +5378,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "log", "once_cell", @@ -5435,9 +5436,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-platform-verifier" @@ -5450,7 +5451,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.20", + "rustls 0.23.23", "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki 0.102.8", @@ -6006,9 +6007,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "indexmap 2.7.0", "itoa", @@ -6824,6 +6825,18 @@ dependencies = [ "der", ] +[[package]] +name = "srtool-lib" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbb79b4db5ac7e4628e6537551c905661d0525dfe34051d14137cd347f85e21" +dependencies = [ + "log", + "serde_json", + "thiserror 2.0.11", + "ureq", +] + [[package]] name = "ss58-registry" version = "1.51.0" @@ -7266,11 +7279,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.8" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f5383f3e0071702bf93ab5ee99b52d26936be9dedd9413067cbdcddcb6141a" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.8", + "thiserror-impl 2.0.11", ] [[package]] @@ -7286,9 +7299,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.8" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f357fcec90b3caef6623a099691be676d033b40a058ac95d2a6ade6fa0c943" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -7435,7 +7448,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.23", "tokio", ] @@ -7872,6 +7885,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f85f9ee53e53fd3f65b73dc51c5a782785d1a6cc381a06652a1141649537ad8" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "percent-encoding", + "rustls 0.23.23", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e8862e186de3ad6ea1505dd071d179ec27aa7d2cb0798196c83f94625c8e9a" +dependencies = [ + "base64 0.22.1", + "http 1.2.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.4" @@ -8211,9 +8254,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.7" +version = "0.26.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" dependencies = [ "rustls-pki-types", ] @@ -8660,7 +8703,7 @@ dependencies = [ "displaydoc", "indexmap 2.7.0", "memchr", - "thiserror 2.0.8", + "thiserror 2.0.11", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 217357957..e49d0ac01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ toml_edit = { version = "0.22", features = ["serde"] } symlink = "0.1" serde_json = { version = "1.0", features = ["preserve_order"] } serde = { version = "1.0", features = ["derive"] } +srtool-lib = "0.13.2" zombienet-sdk = "0.2.26" git2_credentials = "0.13.0" diff --git a/README.md b/README.md index bdd46d216..b1cd2207f 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ Pop CLI would not be possible without these awesome crates! - Local network deployment powered by [zombienet-sdk](https://github.com/paritytech/zombienet-sdk) - [cargo contract](https://github.com/use-ink/cargo-contract) a setup and deployment tool for developing Wasm based Smart Contracts via ink! +- Build deterministic runtimes powered by [srtool-cli](https://github.com/chevdor/srtool-cli) ## License diff --git a/crates/pop-cli/src/commands/build/spec.rs b/crates/pop-cli/src/commands/build/spec.rs index f2464add3..26ad45447 100644 --- a/crates/pop-cli/src/commands/build/spec.rs +++ b/crates/pop-cli/src/commands/build/spec.rs @@ -10,14 +10,15 @@ use crate::{ }; use clap::{Args, ValueEnum}; use cliclack::spinner; -use pop_common::Profile; +use pop_common::{manifest::from_path, Profile}; use pop_parachains::{ binary_path, build_parachain, export_wasm_file, generate_genesis_state_file, - generate_plain_chain_spec, generate_raw_chain_spec, is_supported, ChainSpec, + generate_plain_chain_spec, generate_raw_chain_spec, is_supported, Builder, ChainSpec, + ContainerEngine, }; use std::{ env::current_dir, - fs::create_dir_all, + fs::{self, create_dir_all}, path::{Path, PathBuf}, }; #[cfg(not(test))] @@ -29,8 +30,10 @@ pub(crate) type CodePathBuf = PathBuf; pub(crate) type StatePathBuf = PathBuf; const DEFAULT_CHAIN: &str = "dev"; +const DEFAULT_PACKAGE: &str = "parachain-template-runtime"; const DEFAULT_PARA_ID: u32 = 2000; const DEFAULT_PROTOCOL_ID: &str = "my-protocol"; +const DEFAULT_RUNTIME_DIR: &str = "./runtime"; const DEFAULT_SPEC_NAME: &str = "chain-spec.json"; #[derive( @@ -172,6 +175,20 @@ pub struct BuildSpecCommand { /// Whether the genesis code file should be generated. #[arg(short = 'C', long = "genesis-code")] pub(crate) genesis_code: bool, + /// Whether to build the runtime deterministically. This requires a containerization solution + /// (Docker/Podman). + #[arg(short, long)] + pub(crate) deterministic: bool, + /// Skips the confirmation prompt for deterministic build. + #[arg(long, conflicts_with = "deterministic")] + pub(crate) skip_deterministic_build: bool, + /// Define the directory path where the runtime is located. + #[clap(name = "runtime", long, requires = "deterministic")] + pub runtime_dir: Option, + /// Specify the runtime package name. If not specified, it will be automatically determined + /// based on `runtime`. + #[clap(long, requires = "deterministic")] + pub package: Option, } impl BuildSpecCommand { @@ -212,6 +229,10 @@ impl BuildSpecCommand { protocol_id, genesis_state, genesis_code, + deterministic, + skip_deterministic_build, + package, + runtime_dir, } = self; // Chain. @@ -418,6 +439,51 @@ impl BuildSpecCommand { true }; + // Prompt the user for deterministic build only if the profile is Production. + let deterministic = if skip_deterministic_build { + false + } else { + deterministic || cli + .confirm("Would you like to build the runtime deterministically? This requires a containerization solution (Docker/Podman) and is recommended for production builds.") + .initial_value(profile == Profile::Production) + .interact()? + }; + + // If deterministic build is selected, use the provided runtime path or prompt the user if + // missing. + let runtime_dir = if deterministic { + runtime_dir.unwrap_or_else(|| { + cli.input("Enter the directory path where the runtime is located:") + .placeholder(DEFAULT_RUNTIME_DIR) + .default_input(DEFAULT_RUNTIME_DIR) + .interact() + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from(DEFAULT_RUNTIME_DIR)) + }) + } else { + DEFAULT_RUNTIME_DIR.into() + }; + + // If deterministic build is selected, extract package name from runtime path provided + // above. Prompt the user if unavailable. + let package = if deterministic { + package + .or_else(|| { + from_path(Some(&runtime_dir)) + .ok() + .and_then(|manifest| manifest.package.map(|pkg| pkg.name)) + }) + .unwrap_or_else(|| { + cli.input("Enter the runtime package name:") + .placeholder(DEFAULT_PACKAGE) + .default_input(DEFAULT_PACKAGE) + .interact() + .unwrap_or_else(|_| DEFAULT_PACKAGE.to_string()) + }) + } else { + DEFAULT_PACKAGE.to_string() + }; + if release { cli.warning("NOTE: release flag is deprecated. Use `--profile` instead.")?; #[cfg(not(test))] @@ -436,12 +502,15 @@ impl BuildSpecCommand { protocol_id, genesis_state, genesis_code, + deterministic, + package, + runtime_dir, }) } } // Represents the configuration for building a chain specification. -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct BuildSpec { output_file: PathBuf, profile: Profile, @@ -453,6 +522,9 @@ pub(crate) struct BuildSpec { protocol_id: String, genesis_state: bool, genesis_code: bool, + deterministic: bool, + package: String, + runtime_dir: PathBuf, } impl BuildSpec { @@ -486,6 +558,18 @@ impl BuildSpec { generate_plain_chain_spec(&binary_path, output_file, default_bootnode, chain)?; // Customize spec based on input. self.customize()?; + // Deterministic build. + if self.deterministic { + spinner.set_message("Building deterministic runtime..."); + cli.warning("NOTE: this may take some time...")?; + spinner.clear(); + let code = self.build_deterministic_runtime(cli).map_err(|e| { + anyhow::anyhow!("Failed to build the deterministic runtime: {}", e.to_string()) + })?; + cli.success("Runtime built successfully.")?; + self.update_code(&code)?; + } + generated_files.push(format!( "Plain text chain specification file generated at: {}", &output_file.display() @@ -551,6 +635,37 @@ impl BuildSpec { chain_spec.to_file(&self.output_file)?; Ok(()) } + + fn build_deterministic_runtime( + &self, + cli: &mut impl cli::traits::Cli, + ) -> anyhow::Result> { + let engine = ContainerEngine::detect().map_err(|_| anyhow::anyhow!("No container engine detected. A supported containerization solution (Docker or Podman) is required."))?; + // Warning from srtool-cli: https://github.com/chevdor/srtool-cli/blob/master/cli/src/main.rs#L28). + if engine == ContainerEngine::Docker { + cli.warning("WARNING: You are using docker. It is recommend to use podman instead.")?; + } + let builder = Builder::new( + engine, + None, + self.package.clone(), + self.profile.clone(), + self.runtime_dir.clone(), + )?; + let wasm_path = builder.build()?; + if !wasm_path.exists() { + return Err(anyhow::anyhow!("Can't find the generated runtime at {:?}", wasm_path)); + } + fs::read(&wasm_path).map_err(anyhow::Error::from) + } + + // Updates the chain specification with the runtime code. + fn update_code(&self, bytes: &[u8]) -> anyhow::Result<()> { + let mut chain_spec = ChainSpec::from(&self.output_file)?; + chain_spec.update_runtime_code(bytes)?; + chain_spec.to_file(&self.output_file)?; + Ok(()) + } } // Locate binary, if it doesn't exist trigger build. @@ -600,6 +715,8 @@ fn prepare_output_path(output_path: impl AsRef) -> anyhow::Result mod tests { use super::{ChainType::*, RelayChain::*, *}; use crate::cli::MockCli; + use serde_json::json; + use sp_core::bytes::from_hex; use std::{fs::create_dir_all, path::PathBuf}; use tempfile::{tempdir, TempDir}; @@ -616,6 +733,9 @@ mod tests { let relay = Polkadot; let release = false; let profile = Profile::Production; + let deterministic = true; + let package = "runtime-name"; + let runtime_dir = PathBuf::from("./new-runtime-dir"); for build_spec_cmd in [ // No flags used. @@ -633,6 +753,10 @@ mod tests { protocol_id: Some(protocol_id.to_string()), genesis_state, genesis_code, + deterministic, + skip_deterministic_build: false, + package: Some(package.to_string()), + runtime_dir: Some(runtime_dir.clone()), }, ] { let mut cli = MockCli::new(); @@ -669,7 +793,10 @@ mod tests { None, ).expect_confirm("Would you like to use local host as a bootnode ?", default_bootnode ).expect_confirm("Should the genesis state file be generated ?", genesis_state - ).expect_confirm("Should the genesis code file be generated ?", genesis_code); + ).expect_confirm("Should the genesis code file be generated ?", genesis_code) + .expect_confirm("Would you like to build the runtime deterministically? This requires a containerization solution (Docker/Podman) and is recommended for production builds.", deterministic) + .expect_input("Enter the directory path where the runtime is located:", runtime_dir.display().to_string()) + .expect_input("Enter the runtime package name:", package.to_string()); } let build_spec = build_spec_cmd.configure_build_spec(&mut cli).await?; assert_eq!(build_spec.chain, chain); @@ -682,6 +809,9 @@ mod tests { assert_eq!(build_spec.protocol_id, protocol_id); assert_eq!(build_spec.genesis_state, genesis_state); assert_eq!(build_spec.genesis_code, genesis_code); + assert_eq!(build_spec.deterministic, deterministic); + assert_eq!(build_spec.package, package); + assert_eq!(build_spec.runtime_dir, runtime_dir); cli.verify()?; } Ok(()) @@ -699,6 +829,9 @@ mod tests { let relay = Polkadot; let release = false; let profile = Profile::Production; + let deterministic = true; + let package = "runtime-name"; + let runtime_dir = PathBuf::from("./new-runtime-dir"); // Create a temporary file to act as the existing chain spec file. let temp_dir = tempdir()?; @@ -726,6 +859,10 @@ mod tests { protocol_id: Some(protocol_id.to_string()), genesis_state, genesis_code, + deterministic, + skip_deterministic_build: false, + package: Some(package.to_string()), + runtime_dir: Some(runtime_dir.clone()), }, ] { let mut cli = MockCli::new().expect_confirm( @@ -794,6 +931,13 @@ mod tests { genesis_code, ); } + if !build_spec_cmd.deterministic { + cli = cli.expect_confirm( + "Would you like to build the runtime deterministically? This requires a containerization solution (Docker/Podman) and is recommended for production builds.", + deterministic, + ).expect_input("Enter the directory path where the runtime is located:", runtime_dir.display().to_string()) + .expect_input("Enter the runtime package name:", package.to_string()); + } } let build_spec = build_spec_cmd.configure_build_spec(&mut cli).await?; if changes && no_flags_used { @@ -805,6 +949,9 @@ mod tests { assert_eq!(build_spec.protocol_id, protocol_id); assert_eq!(build_spec.genesis_state, genesis_state); assert_eq!(build_spec.genesis_code, genesis_code); + assert_eq!(build_spec.deterministic, deterministic); + assert_eq!(build_spec.package, package); + assert_eq!(build_spec.runtime_dir, runtime_dir); } // Assert that the chain spec file is correctly detected and used. assert_eq!(build_spec.chain, chain_spec_path.to_string_lossy()); @@ -839,6 +986,39 @@ mod tests { Ok(()) } + #[test] + fn update_code_works() -> anyhow::Result<()> { + let temp_dir = tempdir()?; + let output_file = temp_dir.path().join("chain_spec.json"); + std::fs::write( + &output_file, + json!({ + "genesis": { + "runtimeGenesis": { + "code": "0x00" + } + } + }) + .to_string(), + )?; + let build_spec = BuildSpec { output_file: output_file.clone(), ..Default::default() }; + build_spec.update_code(&from_hex("0x1234")?)?; + + let updated_output_file: serde_json::Value = + serde_json::from_str(&fs::read_to_string(&output_file)?)?; + assert_eq!( + updated_output_file, + json!({ + "genesis": { + "runtimeGenesis": { + "code": "0x1234" + } + } + }) + ); + Ok(()) + } + #[test] fn prepare_output_path_works() -> anyhow::Result<()> { // Create a temporary directory for testing. diff --git a/crates/pop-cli/tests/parachain.rs b/crates/pop-cli/tests/parachain.rs index d8d3b5b56..783793782 100644 --- a/crates/pop-cli/tests/parachain.rs +++ b/crates/pop-cli/tests/parachain.rs @@ -48,7 +48,7 @@ async fn parachain_lifecycle() -> Result<()> { let temp_parachain_dir = temp_dir.join("test_parachain"); // pop build spec --output ./target/pop/test-spec.json --id 2222 --type development --relay - // paseo-local --protocol-id pop-protocol" --chain local + // paseo-local --protocol-id pop-protocol" --chain local --skip-deterministic-build Command::cargo_bin("pop") .unwrap() .current_dir(&temp_parachain_dir) @@ -71,6 +71,7 @@ async fn parachain_lifecycle() -> Result<()> { "--genesis-code", "--protocol-id", "pop-protocol", + "--skip-deterministic-build", ]) .assert() .success(); diff --git a/crates/pop-parachains/Cargo.toml b/crates/pop-parachains/Cargo.toml index a7bd5b4d8..bd52cbbca 100644 --- a/crates/pop-parachains/Cargo.toml +++ b/crates/pop-parachains/Cargo.toml @@ -29,6 +29,7 @@ scale.workspace = true scale-info.workspace = true scale-value.workspace = true sp-core.workspace = true +srtool-lib.workspace = true symlink.workspace = true toml_edit.workspace = true walkdir.workspace = true diff --git a/crates/pop-parachains/src/build.rs b/crates/pop-parachains/src/build/mod.rs similarity index 92% rename from crates/pop-parachains/src/build.rs rename to crates/pop-parachains/src/build/mod.rs index d84a7c9cd..8f96b13dc 100644 --- a/crates/pop-parachains/src/build.rs +++ b/crates/pop-parachains/src/build/mod.rs @@ -5,12 +5,16 @@ use anyhow::{anyhow, Result}; use duct::cmd; use pop_common::{manifest::from_path, Profile}; use serde_json::{json, Value}; +use sp_core::bytes::to_hex; use std::{ fs, path::{Path, PathBuf}, str::FromStr, }; +/// Build the deterministic runtime. +pub mod runtime; + /// Build the parachain and returns the path to the binary. /// /// # Arguments @@ -326,6 +330,25 @@ impl ChainSpec { fs::write(path, self.to_string()?)?; Ok(()) } + + /// Updates the runtime code in the chain specification. + /// + /// # Arguments + /// * `bytes` - The new runtime code. + pub fn update_runtime_code(&mut self, bytes: &[u8]) -> Result<(), Error> { + // Replace `genesis.runtimeGenesis.code` + let code = self + .0 + .get_mut("genesis") + .ok_or_else(|| Error::Config("expected `genesis`".into()))? + .get_mut("runtimeGenesis") + .ok_or_else(|| Error::Config("expected `runtimeGenesis`".into()))? + .get_mut("code") + .ok_or_else(|| Error::Config("expected `runtimeGenesis.code`".into()))?; + let hex = to_hex(bytes, true); + *code = json!(hex); + Ok(()) + } } #[cfg(test)] @@ -337,6 +360,7 @@ mod tests { }; use anyhow::Result; use pop_common::{manifest::Dependency, set_executable_permission}; + use sp_core::bytes::from_hex; use std::{fs, fs::write, io::Write, path::Path}; use strum::VariantArray; use tempfile::{tempdir, Builder, TempDir}; @@ -778,6 +802,36 @@ mod tests { Ok(()) } + #[test] + fn update_runtime_code_works() -> Result<()> { + let mut chain_spec = + ChainSpec(json!({"genesis": {"runtimeGenesis" : { "code": "0x00" }}})); + + chain_spec.update_runtime_code(&from_hex("0x1234")?)?; + assert_eq!(chain_spec.0, json!({"genesis": {"runtimeGenesis" : { "code": "0x1234" }}})); + Ok(()) + } + + #[test] + fn update_runtime_code_fails() -> Result<()> { + let mut chain_spec = + ChainSpec(json!({"invalidKey": {"runtimeGenesis" : { "code": "0x00" }}})); + assert!( + matches!(chain_spec.update_runtime_code(&from_hex("0x1234")?), Err(Error::Config(error)) if error == "expected `genesis`") + ); + + chain_spec = ChainSpec(json!({"genesis": {"invalidKey" : { "code": "0x00" }}})); + assert!( + matches!(chain_spec.update_runtime_code(&from_hex("0x1234")?), Err(Error::Config(error)) if error == "expected `runtimeGenesis`") + ); + + chain_spec = ChainSpec(json!({"genesis": {"runtimeGenesis" : { "invalidKey": "0x00" }}})); + assert!( + matches!(chain_spec.update_runtime_code(&from_hex("0x1234")?), Err(Error::Config(error)) if error == "expected `runtimeGenesis.code`") + ); + Ok(()) + } + #[test] fn check_command_exists_fails() -> Result<()> { let binary_path = PathBuf::from("/bin"); diff --git a/crates/pop-parachains/src/build/runtime.rs b/crates/pop-parachains/src/build/runtime.rs new file mode 100644 index 000000000..2a247258f --- /dev/null +++ b/crates/pop-parachains/src/build/runtime.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-3.0 + +use crate::Error; +use duct::cmd; +use pop_common::Profile; +pub use srtool_lib::{get_image_digest, get_image_tag, ContainerEngine}; +use std::{env, fs, path::PathBuf}; + +const DEFAULT_IMAGE: &str = "docker.io/paritytech/srtool"; +const TIMEOUT: u64 = 60 * 60; + +/// Builds and executes the command for running a deterministic runtime build process using +/// srtool. +pub struct Builder { + /// Mount point for cargo cache. + cache_mount: String, + /// List of default features to enable during the build process. + default_features: String, + /// Digest of the image for reproducibility. + digest: String, + /// The container engine used to run the build process. + engine: ContainerEngine, + /// Name of the image used for building. + image: String, + /// The runtime package name. + package: String, + /// The path to the project directory. + path: PathBuf, + /// The profile used for building. + profile: Profile, + /// The directory path where the runtime is located. + runtime_dir: PathBuf, + /// The tag of the image to use. + tag: String, +} + +impl Builder { + /// Creates a new instance of `Builder`. + /// + /// # Arguments + /// * `engine` - The container engine to use. + /// * `path` - The path to the project. + /// * `package` - The runtime package name. + /// * `profile` - The profile to build the runtime. + /// * `runtime_dir` - The directory path where the runtime is located. + pub fn new( + engine: ContainerEngine, + path: Option, + package: String, + profile: Profile, + runtime_dir: PathBuf, + ) -> Result { + let default_features = String::new(); + let tag = get_image_tag(Some(TIMEOUT)).map_err(|_| Error::ImageTagRetrievalFailed)?; + let digest = get_image_digest(DEFAULT_IMAGE, &tag).unwrap_or_default(); + let dir = fs::canonicalize(path.unwrap_or_else(|| PathBuf::from("./")))?; + let tmpdir = env::temp_dir().join("cargo"); + + let no_cache = engine == ContainerEngine::Podman; + let cache_mount = + if !no_cache { format!("-v {}:/cargo-home", tmpdir.display()) } else { String::new() }; + + Ok(Self { + cache_mount, + default_features, + digest, + engine, + image: DEFAULT_IMAGE.to_string(), + package, + path: dir, + profile, + runtime_dir, + tag, + }) + } + + /// Executes the runtime build process and returns the path of the generated file. + pub fn build(&self) -> Result { + let command = self.build_command(); + cmd("sh", vec!["-c", &command]).stderr_null().run()?; + let wasm_path = self.get_output_path(); + Ok(wasm_path) + } + + // Builds the srtool runtime container command string. + fn build_command(&self) -> String { + format!( + "{} run --name srtool --rm \ + -e PACKAGE={} \ + -e RUNTIME_DIR={} \ + -e DEFAULT_FEATURES={} \ + -e PROFILE={} \ + -e IMAGE={} \ + -v {}:/build \ + {} \ + {}:{} build --app --json", + self.engine, + self.package, + self.runtime_dir.display(), + self.default_features, + self.profile, + self.digest, + self.path.display(), + self.cache_mount, + self.image, + self.tag + ) + } + + // Returns the expected output path of the compiled runtime `.wasm` file. + fn get_output_path(&self) -> PathBuf { + self.runtime_dir + .join("target") + .join("srtool") + .join(self.profile.to_string()) + .join("wbuild") + .join(&self.package) + .join(format!("{}.compact.compressed.wasm", self.package.replace("-", "_"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + + #[test] + fn srtool_builder_new_works() -> Result<()> { + let srtool_builer = Builder::new( + ContainerEngine::Docker, + None, + "parachain-template-runtime".to_string(), + Profile::Release, + PathBuf::from("./runtime"), + )?; + assert_eq!( + srtool_builer.cache_mount, + format!("-v {}:/cargo-home", env::temp_dir().join("cargo").display()) + ); + assert_eq!(srtool_builer.default_features, ""); + + let tag = get_image_tag(Some(TIMEOUT))?; + let digest = get_image_digest(DEFAULT_IMAGE, &tag).unwrap_or_default(); + assert_eq!(srtool_builer.digest, digest); + assert_eq!(srtool_builer.tag, tag); + + assert!(srtool_builer.engine == ContainerEngine::Docker); + assert_eq!(srtool_builer.image, DEFAULT_IMAGE); + assert_eq!(srtool_builer.package, "parachain-template-runtime"); + assert_eq!(srtool_builer.path, fs::canonicalize(PathBuf::from("./"))?); + assert_eq!(srtool_builer.profile, Profile::Release); + assert_eq!(srtool_builer.runtime_dir, PathBuf::from("./runtime")); + + Ok(()) + } + + #[test] + fn build_command_works() -> Result<()> { + let temp_dir = tempfile::tempdir()?; + let path = temp_dir.path(); + let tag = get_image_tag(Some(TIMEOUT))?; + let digest = get_image_digest(DEFAULT_IMAGE, &tag).unwrap_or_default(); + assert_eq!( + Builder::new( + ContainerEngine::Podman, + Some(path.to_path_buf()), + "parachain-template-runtime".to_string(), + Profile::Production, + PathBuf::from("./runtime"), + )? + .build_command(), + format!( + "podman run --name srtool --rm \ + -e PACKAGE=parachain-template-runtime \ + -e RUNTIME_DIR=./runtime \ + -e DEFAULT_FEATURES= \ + -e PROFILE=production \ + -e IMAGE={} \ + -v {}:/build \ + {} \ + {}:{} build --app --json", + digest, + fs::canonicalize(path)?.display(), + String::new(), + DEFAULT_IMAGE, + tag + ) + ); + Ok(()) + } + + #[test] + fn get_output_path_works() -> Result<()> { + let srtool_builder = Builder::new( + ContainerEngine::Podman, + None, + "template-runtime".to_string(), + Profile::Debug, + PathBuf::from("./runtime-folder"), + )?; + assert_eq!(srtool_builder.get_output_path().display().to_string(), "./runtime-folder/target/srtool/debug/wbuild/template-runtime/template_runtime.compact.compressed.wasm"); + Ok(()) + } +} diff --git a/crates/pop-parachains/src/errors.rs b/crates/pop-parachains/src/errors.rs index 889bd2be9..81f50b9df 100644 --- a/crates/pop-parachains/src/errors.rs +++ b/crates/pop-parachains/src/errors.rs @@ -36,6 +36,9 @@ pub enum Error { /// The dispatchable function is not supported. #[error("The dispatchable function is not supported")] FunctionNotSupported, + /// Failed to retrieve the image tag. + #[error("Failed to retrieve image tag.")] + ImageTagRetrievalFailed, #[error("IO error: {0}")] IO(#[from] std::io::Error), #[error("JSON error: {0}")] diff --git a/crates/pop-parachains/src/lib.rs b/crates/pop-parachains/src/lib.rs index 26bbe6227..7bb282ee4 100644 --- a/crates/pop-parachains/src/lib.rs +++ b/crates/pop-parachains/src/lib.rs @@ -15,7 +15,9 @@ mod utils; pub use build::{ binary_path, build_parachain, export_wasm_file, generate_genesis_state_file, - generate_plain_chain_spec, generate_raw_chain_spec, is_supported, ChainSpec, + generate_plain_chain_spec, generate_raw_chain_spec, is_supported, + runtime::{Builder, ContainerEngine}, + ChainSpec, }; pub use call::{ construct_extrinsic, construct_sudo_extrinsic, decode_call_data, encode_call_data,