WIP on chain heap

demo/cli/src/lib.rs

@@ -155,7 +155,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	init_logger(log_pattern);
 
 	// Create client
-	let executor = NativeExecutor::with_heap_pages(8);
+	let executor = NativeExecutor::new();
 
 	let god_key = hex!["3d866ec8a9190c8343c2fc593d21d8a6d0c5c4763aaab2349de3a6111d64d124"];
 	let genesis_config = GenesisConfig {

demo/executor/src/lib.rs

@@ -98,7 +98,7 @@ mod tests {
 	}
 
 	fn executor() -> ::substrate_executor::NativeExecutor<Executor> {
-		::substrate_executor::NativeExecutor::with_heap_pages(8)
+		::substrate_executor::NativeExecutor::new()
 	}
 
 	#[test]
@@ -114,9 +114,9 @@ mod tests {
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
 		];
 
-		let r = executor().call(&mut t, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+		let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
 		assert!(r.is_ok());
-		let v = executor().call(&mut t, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
+		let v = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
 		let r = ApplyResult::decode(&mut &v[..]).unwrap();
 		assert_eq!(r, Err(ApplyError::CantPay));
 	}
@@ -134,9 +134,9 @@ mod tests {
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
 		];
 
-		let r = executor().call(&mut t, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+		let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
 		assert!(r.is_ok());
-		let v = executor().call(&mut t, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
+		let v = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0.unwrap();
 		let r = ApplyResult::decode(&mut &v[..]).unwrap();
 		assert_eq!(r, Err(ApplyError::CantPay));
 	}
@@ -154,9 +154,9 @@ mod tests {
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
 		];
 
-		let r = executor().call(&mut t, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+		let r = executor().call(&mut t, 8, COMPACT_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
 		assert!(r.is_ok());
-		let r = executor().call(&mut t, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
+		let r = executor().call(&mut t, 8, COMPACT_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
 		assert!(r.is_ok());
 
 		runtime_io::with_externalities(&mut t, || {
@@ -178,9 +178,9 @@ mod tests {
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
 		];
 
-		let r = executor().call(&mut t, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
+		let r = executor().call(&mut t, 8, BLOATY_CODE, "initialise_block", &vec![].and(&from_block_number(1u64)), true).0;
 		assert!(r.is_ok());
-		let r = executor().call(&mut t, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
+		let r = executor().call(&mut t, 8, BLOATY_CODE, "apply_extrinsic", &vec![].and(&xt()), true).0;
 		assert!(r.is_ok());
 
 		runtime_io::with_externalities(&mut t, || {
@@ -312,7 +312,7 @@ mod tests {
 	fn full_native_block_import_works() {
 		let mut t = new_test_ext();
 
-		executor().call(&mut t, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap();
+		executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Balances::total_balance(&alice()), 41);
@@ -329,7 +329,7 @@ mod tests {
 			]);
 		});
 
-		executor().call(&mut t, COMPACT_CODE, "execute_block", &block2().0, true).0.unwrap();
+		executor().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0, true).0.unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Balances::total_balance(&alice()), 30);
@@ -359,14 +359,14 @@ mod tests {
 	fn full_wasm_block_import_works() {
 		let mut t = new_test_ext();
 
-		WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1().0).unwrap();
+		WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Balances::total_balance(&alice()), 41);
 			assert_eq!(Balances::total_balance(&bob()), 69);
 		});
 
-		WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block2().0).unwrap();
+		WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block2().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
 			assert_eq!(Balances::total_balance(&alice()), 30);
@@ -378,23 +378,23 @@ mod tests {
 	fn wasm_big_block_import_fails() {
 		let mut t = new_test_ext();
 
-		let r = WasmExecutor::new(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0);
+		let r = WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0);
 		assert!(!r.is_ok());
 	}
 
 	#[test]
 	fn native_big_block_import_succeeds() {
 		let mut t = new_test_ext();
 
-		let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, true).0;
+		let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, true).0;
 		assert!(r.is_ok());
 	}
 
 	#[test]
 	fn native_big_block_import_fails_on_fallback() {
 		let mut t = new_test_ext();
 
-		let r = Executor::with_heap_pages(8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, false).0;
+		let r = Executor::new().call(&mut t, 8, COMPACT_CODE, "execute_block", &block1big().0, false).0;
 		assert!(!r.is_ok());
 	}
 
@@ -412,9 +412,9 @@ mod tests {
 		];
 
 		let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm");
-		let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
+		let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
 		assert!(r.is_ok());
-		let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
+		let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
 		let r = ApplyResult::decode(&mut &r[..]).unwrap();
 		assert_eq!(r, Err(ApplyError::CantPay));
 	}
@@ -433,9 +433,9 @@ mod tests {
 		];
 
 		let foreign_code = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm");
-		let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
+		let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "initialise_block", &vec![].and(&from_block_number(1u64)));
 		assert!(r.is_ok());
-		let r = WasmExecutor::new(8).call(&mut t, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
+		let r = WasmExecutor::new().call(&mut t, 8, &foreign_code[..], "apply_extrinsic", &vec![].and(&xt())).unwrap();
 		let r = ApplyResult::decode(&mut &r[..]).unwrap();
 		assert_eq!(r, Ok(ApplyOutcome::Success));
 

substrate/cli/src/cli.yml

@@ -125,10 +125,6 @@ args:
       long: execution
       value_name: STRATEGY
       help: The means of execution used when calling into the runtime. Can be either wasm, native or both.
-  - max-heap-pages:
-      long: max-heap-pages
-      value_name: COUNT
-      help: The maximum number of 64KB pages to ever allocate for Wasm execution. Don't alter this unless you know what you're doing.
 subcommands:
   - build-spec:
       about: Build a spec.json file, outputing to stdout

substrate/cli/src/lib.rs

@@ -266,10 +266,6 @@ where
 			service::Roles::FULL
 		};
 
-	if let Some(v) = matches.value_of("max-heap-pages") {
-		config.max_heap_pages = v.parse().map_err(|_| "Invalid --max-heap-pages argument")?;
-	}
-
 	if let Some(s) = matches.value_of("execution") {
 		config.execution_strategy = match s {
 			"both" => service::ExecutionStrategy::Both,
@@ -398,10 +394,6 @@ fn import_blocks<F, E>(matches: &clap::ArgMatches, spec: ChainSpec<FactoryGenesi
 	let mut config = service::Configuration::default_with_spec(spec);
 	config.database_path = db_path(&base_path, config.chain_spec.id()).to_string_lossy().into();
 
-	if let Some(v) = matches.value_of("max-heap-pages") {
-		config.max_heap_pages = v.parse().map_err(|_| "Invalid --max-heap-pages argument")?;
-	}
-
 	if let Some(s) = matches.value_of("execution") {
 		config.execution_strategy = match s {
 			"both" => service::ExecutionStrategy::Both,

substrate/client/src/call_executor.rs

@@ -26,6 +26,7 @@ use patricia_trie::NodeCodec;
 use primitives::{KeccakHasher, RlpCodec};
 use hashdb::Hasher;
 use rlp::Encodable;
+use codec::Decode;
 
 use backend;
 use error;
@@ -145,8 +146,9 @@ where
 		let mut externalities = Ext::new(&mut overlay, &state);
 		let code = externalities.storage(b":code").ok_or(error::ErrorKind::VersionInvalid)?
 			.to_vec();
+		let heap_pages = externalities.storage(b":heappages").and_then(|v| u64::decode(&mut &v[..])).unwrap_or(8) as usize;
 
-		self.executor.runtime_version(&mut externalities, &code)
+		self.executor.runtime_version(&mut externalities, heap_pages, &code)
 			.ok_or(error::ErrorKind::VersionInvalid.into())
 	}
 

substrate/client/src/genesis.rs

@@ -56,7 +56,7 @@ mod tests {
 	native_executor_instance!(Executor, test_client::runtime::api::dispatch, test_client::runtime::VERSION, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
 
 	fn executor() -> ::executor::NativeExecutor<Executor> {
-		NativeExecutionDispatch::with_heap_pages(8)
+		NativeExecutionDispatch::new()
 	}
 
 	fn construct_block(backend: &InMemory<KeccakHasher, RlpCodec>, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec<Transfer>) -> (Vec<u8>, Hash) {
@@ -194,7 +194,7 @@ mod tests {
 		let _ = execute(
 			&backend,
 			&mut overlay,
-			&Executor::with_heap_pages(8),
+			&Executor::new(),
 			"execute_block",
 			&b1data,
 			ExecutionStrategy::NativeWhenPossible,

substrate/client/src/light/backend.rs

@@ -270,7 +270,7 @@ pub mod tests {
 		let remote_read_proof = remote_client.read_proof(&remote_block_id, b":auth:len").unwrap();
 
 		// check remote read proof locally
-		let local_executor = test_client::LocalExecutor::with_heap_pages(8);
+		let local_executor = test_client::LocalExecutor::new();
 		let local_checker = new_fetch_checker::<_, KeccakHasher, RlpCodec>(local_executor);
 		let request = RemoteReadRequest {
 			block: remote_block_hash,

substrate/client/src/light/call_executor.rs

@@ -156,7 +156,7 @@ mod tests {
 		let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1;
 
 		// check remote execution proof locally
-		let local_executor = test_client::LocalExecutor::with_heap_pages(8);
+		let local_executor = test_client::LocalExecutor::new();
 		check_execution_proof::<_, _, _, RlpCodec>(&local_executor, &RemoteCallRequest {
 			block: test_client::runtime::Hash::default(),
 			header: test_client::runtime::Header {

substrate/executor/src/lib.rs

@@ -87,6 +87,7 @@ pub trait RuntimeInfo {
 	fn runtime_version<E: Externalities<KeccakHasher>> (
 		&self,
 		ext: &mut E,
+		heap_pages: usize,
 		code: &[u8]
 	) -> Option<RuntimeVersion>;
 }

substrate/executor/src/native_executor.rs

@@ -55,12 +55,13 @@ fn fetch_cached_runtime_version<'a, E: Externalities<KeccakHasher>>(
 	wasm_executor: &WasmExecutor,
 	cache: &'a mut MutexGuard<CacheType>,
 	ext: &mut E,
+	heap_pages: usize,
 	code: &[u8]
 ) -> Result<(&'a WasmModule, &'a Option<RuntimeVersion>)> {
 	let maybe_runtime_preproc = cache.entry(gen_cache_key(code))
 		.or_insert_with(|| match WasmModule::from_buffer(code) {
 			Ok(module) => {
-				let version = wasm_executor.call_in_wasm_module(ext, &module, "version", &[])
+				let version = wasm_executor.call_in_wasm_module(ext, heap_pages, &module, "version", &[])
 					.ok()
 					.and_then(|v| RuntimeVersion::decode(&mut v.as_slice()));
 				RuntimePreproc::ValidCode(module, version)
@@ -108,9 +109,9 @@ pub trait NativeExecutionDispatch: Send + Sync {
 	/// Get native runtime version.
 	const VERSION: RuntimeVersion;
 
-	/// Construct corresponding `NativeExecutor` with given `heap_pages`.
-	fn with_heap_pages(max_heap_pages: usize) -> NativeExecutor<Self> where Self: Sized {
-		NativeExecutor::with_heap_pages(max_heap_pages)
+	/// Construct corresponding `NativeExecutor`
+	fn new() -> NativeExecutor<Self> where Self: Sized {
+		NativeExecutor::new()
 	}
 }
 
@@ -125,11 +126,11 @@ pub struct NativeExecutor<D: NativeExecutionDispatch> {
 }
 
 impl<D: NativeExecutionDispatch> NativeExecutor<D> {
-	/// Create new instance with specific number of pages for wasm fallback's heap.
-	pub fn with_heap_pages(max_heap_pages: usize) -> Self {
+	/// Create new instance.
+	pub fn new() -> Self {
 		NativeExecutor {
 			_dummy: Default::default(),
-			fallback: WasmExecutor::new(max_heap_pages),
+			fallback: WasmExecutor::new(),
 		}
 	}
 }
@@ -149,9 +150,10 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
 	fn runtime_version<E: Externalities<KeccakHasher>>(
 		&self,
 		ext: &mut E,
+		heap_pages: usize,
 		code: &[u8],
 	) -> Option<RuntimeVersion> {
-		fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, code).ok()?.1.clone()
+		fetch_cached_runtime_version(&self.fallback, &mut RUNTIMES_CACHE.lock(), ext, heap_pages, code).ok()?.1.clone()
 	}
 }
 
@@ -161,23 +163,24 @@ impl<D: NativeExecutionDispatch> CodeExecutor<KeccakHasher> for NativeExecutor<D
 	fn call<E: Externalities<KeccakHasher>>(
 		&self,
 		ext: &mut E,
+		heap_pages: usize,
 		code: &[u8],
 		method: &str,
 		data: &[u8],
 		use_native: bool,
 	) -> (Result<Vec<u8>>, bool) {
 		let mut c = RUNTIMES_CACHE.lock();
-		let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, code) {
+		let (module, onchain_version) = match fetch_cached_runtime_version(&self.fallback, &mut c, ext, heap_pages, code) {
 			Ok((module, onchain_version)) => (module, onchain_version),
 			Err(_) => return (Err(ErrorKind::InvalidCode(code.into()).into()), false),
 		};
 		match (use_native, onchain_version.as_ref().map_or(false, |v| v.can_call_with(&D::VERSION))) {
 			(_, false) => {
 				trace!(target: "executor", "Request for native execution failed (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
-				(self.fallback.call_in_wasm_module(ext, module, method, data), false)
+				(self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
 			}
 			(false, _) => {
-				(self.fallback.call_in_wasm_module(ext, module, method, data), false)
+				(self.fallback.call_in_wasm_module(ext, heap_pages, module, method, data), false)
 			}
 			_ => {
 				trace!(target: "executor", "Request for native execution succeeded (native: {}, chain: {})", D::VERSION, onchain_version.as_ref().map_or_else(||"<None>".into(), |v| format!("{}", v)));
@@ -213,8 +216,8 @@ macro_rules! native_executor_instance {
 					.ok_or_else(|| $crate::error::ErrorKind::MethodNotFound(method.to_owned()).into())
 			}
 
-			fn with_heap_pages(max_heap_pages: usize) -> $crate::NativeExecutor<$name> {
-				$crate::NativeExecutor::with_heap_pages(max_heap_pages)
+			fn new() -> $crate::NativeExecutor<$name> {
+				$crate::NativeExecutor::new()
 			}
 		}
 	}