fix(iota-genesis-builder): Make load from file deterministic for the UnsignedGenesis

[miker83z]

Description of change

• For genesis Delegations a BTreeMap is used instead of a HashMap so that the list of delegations always keeps the same order (this was changing between different runs).
• Given that CheckpointContents uses the OnceCell primitive for its hash digest, we make sure that the digest is always cached before comparing built and loaded UnsignedGenesis data.

Links to any relevant issues

Fixes #5346

Type of change

• Bug fix (a non-breaking change which fixes an issue)

How the change has been tested (COLLAPSED)

Test script by @nonast (collapsed)

#!/bin/bash

##### CONFIG #####
# Validators.
NUM_VALIDATORS=3
ALIAS_PREFIX="validator"
DESCRIPTION_PREFIX="validator-description"

# Paths
GENESIS_FOLDER_DIR="genesis-folder"
TEST_IOTA_CONFIG_DIR="test_iota_config"
HORNET_SNAPSHOT_FILE_RELATIVE_PATH="../../latest-full_snapshot.bin"
NETWORK_YAML_FILE="$TEST_IOTA_CONFIG_DIR/network.yaml"
DELEGATIONS_FILE="delegations.csv"
ADDRESS_SWAP_FILE="address_swap.csv"
ADDRESS_SWAP_SPLIT_FILE="address_swap_split.csv"

# Delegations; for each validator: delegator address, amount of staked IOTA to delegate, amount of IOTA as gas.
# Validator 0
declare -A delegations

# Validator 0
delegations[0]="0x4f72f788cdf4bb478cf9809e878e6163d5b351c82c11f1ea28750430752e7892"
delegations[1]="1500000000000000"
delegations[2]="10000000000"

# Validator 1
delegations[3]="0xece6dd8a970dccc65c3f1057248e5034705ea5a625099fd06113222eddf1238d"
delegations[4]="1500000000000000"
delegations[5]="10000000000"

# Validator 2
delegations[6]="0xf9694dba95d0ec3ff0d350ba7ab03739fc4360a7e62bad9596feeb11b7ca84c2"
delegations[7]="1500000000000000"
delegations[8]="10000000000"

echo "Validator 0:"
echo "${delegations[0]}"
echo "${delegations[1]}"
echo "${delegations[2]}"

echo "Validator 1:"
echo "${delegations[3]}"
echo "${delegations[4]}"
echo "${delegations[5]}"

echo "Validator 2:"
echo "${delegations[6]}"
echo "${delegations[7]}"
echo "${delegations[8]}"

if [[ $NUM_VALIDATORS -ne $((${#delegations[@]} / 3)) ]]; then
    echo "Number of delegations must be equal to the number of validators"
    exit 1
fi


# Clean
rm -rf $GENESIS_FOLDER_DIR
mkdir -p $GENESIS_FOLDER_DIR
cd $GENESIS_FOLDER_DIR
# Remove existing files and directories.
#rm -rf committee signatures $TEST_IOTA_CONFIG_DIR validator_keys parameters unsigned-genesis token-distribution-schedule migration-sources delegator-map

# Address swap map
echo "Origin,Destination" >> $ADDRESS_SWAP_FILE
#echo "iota1qp8h9augeh6tk3uvlxqfapuwv93atv63eqkpru029p6sgvr49eufyz7katr,0x4f72f788cdf4bb478cf9809e878e6163d5b351c82c11f1ea28750430752e7892" >> $ADDRESS_SWAP_FILE
#echo "iota1qrkwdhv2juxue3ju8ug9wfyw2q68qh495cjsn87svyfjytka7y3c6nf3lrk,0xece6dd8a970dccc65c3f1057248e5034705ea5a625099fd06113222eddf1238d" >> $ADDRESS_SWAP_FILE
#echo "iota1qrukjnd6jhgwc0ls6dgt574sxuulcsmq5lnzhtv4jmlwkydhe2zvy69t7jj,0xf9694dba95d0ec3ff0d350ba7ab03739fc4360a7e62bad9596feeb11b7ca84c2" >> $ADDRESS_SWAP_FILE
#

# Address swap split map
echo "Origin,Destination,Tokens,TokensTimelocked" >> $ADDRESS_SWAP_SPLIT_FILE
echo "iota1qp8h9augeh6tk3uvlxqfapuwv93atv63eqkpru029p6sgvr49eufyz7katr,0x4f72f788cdf4bb478cf9809e878e6163d5b351c82c11f1ea28750430752e7892,1500000000000000,1500000000000000" >> $ADDRESS_SWAP_SPLIT_FILE
echo "iota1qp8h9augeh6tk3uvlxqfapuwv93atv63eqkpru029p6sgvr49eufyz7katr,0xece6dd8a970dccc65c3f1057248e5034705ea5a625099fd06113222eddf1238d,1500000000000000,1500000000000000" >> $ADDRESS_SWAP_SPLIT_FILE
echo "iota1qp8h9augeh6tk3uvlxqfapuwv93atv63eqkpru029p6sgvr49eufyz7katr,0xf9694dba95d0ec3ff0d350ba7ab03739fc4360a7e62bad9596feeb11b7ca84c2,1500000000000000,1500000000000000" >> $ADDRESS_SWAP_SPLIT_FILE


##### START #####
echo "[TEST INFO] Start"
echo "[TEST INFO] Build iota-genesis-builder"
cargo build --release --bin iota-genesis-builder
echo "[TEST INFO] Build iota"
cargo build --release --bin iota

if [[ -n "$1" ]]; then
    # Reuse a Stardust Objects snapshot
    cd ..
    cp "$1" $GENESIS_FOLDER_DIR
    cd $GENESIS_FOLDER_DIR
else
    # Build the Stardust Objects snapshot
    echo "[TEST INFO] Build the Stardust Objects snapshot"
    ../target/release/iota-genesis-builder --disable-global-snapshot-verification iota --address-swap-map-path $ADDRESS_SWAP_FILE --address-swap-split-map-path $ADDRESS_SWAP_SPLIT_FILE --snapshot-path $HORNET_SNAPSHOT_FILE_RELATIVE_PATH --target-network mainnet
fi


# Init new delegations file.
echo "delegator,validator,amount-nanos-to-stake,amount-nanos-to-pay-gas" >> $DELEGATIONS_FILE

mkdir -p $TEST_IOTA_CONFIG_DIR

# Initialize genesis ceremony.
echo "[TEST INFO] Genesis Ceremony init"
../target/release/iota genesis-ceremony init

# Create genesis configuration.
echo "[TEST INFO] Init configuration for local network"
../target/release/iota genesis --working-dir ./$TEST_IOTA_CONFIG_DIR -f --committee-size $NUM_VALIDATORS

# Directory to save validator key files.
KEYS_DIR="validator_keys"
mkdir -p "$KEYS_DIR"

# Loop through the validators.
echo "[TEST INFO] Validators info generation"
for i in $(seq 1 $NUM_VALIDATORS); do
  echo "[TEST INFO] Processing validator $i..."

  # Create new address and extract alias and address.
  OUTPUT=$(../target/release/iota client new-address)
  echo "$OUTPUT"

  # Extract alias (2 words separated by dash) and address (starting with 0x).
  ALIAS=$(echo "$OUTPUT" | grep -oE "[a-z]+-[a-z]+")
  ADDRESS=$(echo "$OUTPUT" | grep -oE "0x[0-9a-f]{64}")

  echo "Alias: $ALIAS"
  echo "Address: $ADDRESS"

  # Switch to the new keypair.
  ../target/release/iota client switch --address "$ALIAS"

  # Make validator info.
  ALIAS="$ALIAS_PREFIX$i"
  DESCRIPTION="$DESCRIPTION_PREFIX$i"
  ../target/release/iota validator make-validator-info "$ALIAS" "$DESCRIPTION" \
    https://www.iota.org/favicon.png https://www.iota.org /ip4/127.0.0.1 1000

  # Move key files to dedicated directory.
  VALIDATOR_DIR="$KEYS_DIR/$ALIAS"
  mkdir -p "$VALIDATOR_DIR"
  mv authority.key "$VALIDATOR_DIR/authority.key"
  mv protocol.key "$VALIDATOR_DIR/protocol.key"
  mv account.key "$VALIDATOR_DIR/account.key"
  mv network.key "$VALIDATOR_DIR/network.key"
  mv validator.info "$VALIDATOR_DIR/validator.info"

  # This address can be found in chronological order.
  external_address=$(awk -v idx="$i" '/external-address/ {count++; if (count == idx) print $2}' "$NETWORK_YAML_FILE")
  network_address=$(awk -v idx="$i" '/network-address/ {count++; if (count == idx) print $2}' "$NETWORK_YAML_FILE")
  primary_address=$(awk -v addr="$network_address" '$0 ~ "network_address: " addr {getline; print $2}' "$NETWORK_YAML_FILE")


  # Update network.yaml file with the new key paths.
  for key in authority protocol account network; do
    # Get the real path of each key.
    path=$(realpath "$VALIDATOR_DIR/${key}.key")
    sed -i '' '1,/value:/s|value: [^ ]*|path: '"$path"'|' "$NETWORK_YAML_FILE"
  done

  # Add validator to genesis ceremony.
  ../target/release/iota genesis-ceremony add-validator \
    --name "$ALIAS" \
    --authority-key-file "$VALIDATOR_DIR/authority.key" \
    --protocol-key-file "$VALIDATOR_DIR/protocol.key" \
    --account-key-file "$VALIDATOR_DIR/account.key" \
    --network-key-file "$VALIDATOR_DIR/network.key" \
    --network-address $network_address \
    --p2p-address $external_address \
    --primary-address $primary_address \
    --description "$DESCRIPTION" \
    --image-url https://www.iota.org/favicon.png \
    --project-url https://www.iota.org

  # Add delegation entry to $DELEGATIONS_FILE.
  echo "${delegations[$((i*3-3))]},$ADDRESS,${delegations[$((i*3-2))]},${delegations[$((i*3-1))]}" >> $DELEGATIONS_FILE

  echo "[TEST INFO] Validator $i setup complete. Keys saved in $VALIDATOR_DIR."
    echo "--------------------------"

done

read -p "Press Enter to continue..."

# Init Delegations.
echo "[TEST INFO] Genesis Ceremony init-delegations"
../target/release/iota genesis-ceremony init-delegations --delegations-path $DELEGATIONS_FILE

# Validate the state.
echo "[TEST INFO] Genesis Ceremony validate-state"
../target/release/iota genesis-ceremony validate-state

# Build Unsigned Checkpoint.
echo "[TEST INFO] Genesis Ceremony build-unsigned-checkpoint"
../target/release/iota genesis-ceremony build-unsigned-checkpoint --local-migration-snapshots stardust_object_snapshot.bin

# Examine Genesis Checkpoint.
echo "[TEST INFO] Genesis Ceremony examine-genesis-checkpoint"
../target/release/iota genesis-ceremony examine-genesis-checkpoint

read -p "Press Enter to continue..."

# Sign.
echo "[TEST INFO] Genesis Ceremony verify-and-sign"
for i in $(seq 1 $NUM_VALIDATORS); do
  ALIAS="$ALIAS_PREFIX$i"
  VALIDATOR_DIR="$KEYS_DIR/$ALIAS"
  echo "Verifying and signing for validator $i..."
  ../target/release/iota genesis-ceremony verify-and-sign --key-file "$VALIDATOR_DIR/authority.key"
done

# Finalize genesis.
echo "[TEST INFO] Genesis Ceremony finalize"
../target/release/iota genesis-ceremony finalize

# Move the genesis files to test_iota_config.
cp migration.blob genesis.blob $TEST_IOTA_CONFIG_DIR

echo "[TEST INFO] All validators configured successfully."
../target/release/iota start --network.config $TEST_IOTA_CONFIG_DIR

Change checklist

• I have followed the contribution guidelines for this project
• I have performed a self-review of my own code
• I have commented my code, particularly in hard-to-understand areas
• I have made corresponding changes to the documentation
• I have added tests that prove my fix is effective or that my feature works
• I have checked that new and existing unit tests pass locally with my changes

[vercel]

The latest updates on your projects. Learn more about Vercel for Git ↗︎

Name	Status	Preview	Comments	Updated (UTC)
apps-backend	✅ Ready (Inspect)	Visit Preview	💬 Add feedback	Feb 14, 2025 11:42am
apps-ui-kit	✅ Ready (Inspect)	Visit Preview		Feb 14, 2025 11:42am
rebased-explorer	✅ Ready (Inspect)	Visit Preview	💬 Add feedback	Feb 14, 2025 11:42am
wallet-dashboard	✅ Ready (Inspect)	Visit Preview		Feb 14, 2025 11:42am

[nonast]

This fixed the errors 🙏