From 78ff0aba7e671c7534e72955090c2371c843b82a Mon Sep 17 00:00:00 2001
From: Mara Robin B <mara@broda.me>
Date: Mon, 30 May 2022 10:45:32 +0200
Subject: [PATCH] [ci] Upstream benchmark script changes from
 https://github.com/paritytech/substrate/pull/11493 (#5586)

* [ci] Upstream benchmark script changes from https://github.com/paritytech/substrate/pull/11493

* Update scripts/ci/run_benches_for_runtime.sh

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update scripts/ci/run_benches_for_runtime.sh

Co-authored-by: Chevdor <chevdor@users.noreply.github.com>

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: Chevdor <chevdor@users.noreply.github.com>
---
 scripts/ci/run_benches_for_runtime.sh | 73 +++++++++++++++++----------
 1 file changed, 45 insertions(+), 28 deletions(-)

diff --git a/scripts/ci/run_benches_for_runtime.sh b/scripts/ci/run_benches_for_runtime.sh
index b0aa3ed87624..8149195c82c4 100755
--- a/scripts/ci/run_benches_for_runtime.sh
+++ b/scripts/ci/run_benches_for_runtime.sh
@@ -1,54 +1,71 @@
 #!/bin/bash
 
-set -e
-
 # Runs all benchmarks for all pallets, for a given runtime, provided by $1
 # Should be run on a reference machine to gain accurate benchmarks
 # current reference machine: https://github.com/paritytech/substrate/pull/5848
 
 runtime="$1"
 
-echo "[+] Running all benchmarks for $runtime"
+echo "[+] Compiling benchmarks..."
+cargo build --profile production --locked --features=runtime-benchmarks
+
+# Load all pallet names in an array.
+PALLETS=($(
+  ./target/production/polkadot benchmark pallet --list --chain="${runtime}-dev" |\
+    tail -n+2 |\
+    cut -d',' -f1 |\
+    sort |\
+    uniq
+))
 
-cargo +nightly build --profile production --locked --features=runtime-benchmarks
+echo "[+] Benchmarking ${#PALLETS[@]} pallets for runtime $runtime"
 
-./target/production/polkadot benchmark pallet \
-    --chain "${runtime}-dev" \
-    --list |\
-  tail -n+2 |\
-  cut -d',' -f1 |\
-  uniq > "${runtime}_pallets"
+# Define the error file.
+ERR_FILE="benchmarking_errors.txt"
+# Delete the error file before each run.
+rm -f $ERR_FILE
 
-# For each pallet found in the previous command, run benches on each function
-while read -r line; do
-  pallet="$(echo "$line" | cut -d' ' -f1)";
-  echo "Runtime: $runtime. Pallet: $pallet";
-  # '!' has the side effect of bypassing errexit / set -e
-  ! ./target/production/polkadot benchmark pallet \
+# Benchmark each pallet.
+for PALLET in "${PALLETS[@]}"; do
+  echo "[+] Benchmarking $PALLET for $runtime";
+
+  OUTPUT=$(
+    ./target/production/polkadot benchmark pallet \
     --chain="${runtime}-dev" \
     --steps=50 \
     --repeat=20 \
-    --pallet="$pallet" \
+    --pallet="$PALLET" \
     --extrinsic="*" \
     --execution=wasm \
     --wasm-execution=compiled \
-    --heap-pages=4096 \
     --header=./file_header.txt \
-    --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
-done < "${runtime}_pallets"
-rm "${runtime}_pallets"
+    --output="./runtime/${runtime}/src/weights/${PALLET/::/_}.rs" 2>&1
+  )
+  if [ $? -ne 0 ]; then
+    echo "$OUTPUT" >> "$ERR_FILE"
+    echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..."
+  fi
+done
 
-# Benchmark base weights
-! ./target/production/polkadot benchmark overhead \
+# Update the block and extrinsic overhead weights.
+echo "[+] Benchmarking block and extrinsic overheads..."
+OUTPUT=$(
+  ./target/production/polkadot benchmark overhead \
   --chain="${runtime}-dev" \
   --execution=wasm \
   --wasm-execution=compiled \
   --weight-path="runtime/${runtime}/constants/src/weights/" \
   --warmup=10 \
   --repeat=100
+)
+if [ $? -ne 0 ]; then
+  echo "$OUTPUT" >> "$ERR_FILE"
+  echo "[-] Failed to benchmark the block and extrinsic overheads. Error written to $ERR_FILE; continuing..."
+fi
 
-
-# This true makes sure that $? is 0 instead of
-# carrying over a failure which would otherwise cause
-# the whole CI job to abort.
-true
+# Check if the error file exists.
+if [ -f "$ERR_FILE" ]; then
+  echo "[-] Some benchmarks failed. See: $ERR_FILE"
+else
+  echo "[+] All benchmarks passed."
+fi