Skip to content
This repository was archived by the owner on Jan 22, 2025. It is now read-only.

Pgarg optimize start nodes #588

Closed
wants to merge 2 commits into from
Closed

Pgarg optimize start nodes #588

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
303d0a0
Optimized start_nodes script to speed up GCE nodes start time
pgarg66 Jul 3, 2018
2fe1f99
Start drone on a remote leader node
pgarg66 Jul 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 32 additions & 20 deletions multinode-demo/start_nodes.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,24 @@ if [[ -z "$remote_user" ]]; then
usage
fi

echo "Build started at $(date)"
SECONDS=0
# Build and install locally
PATH="$HOME"/.cargo/bin:"$PATH"
cargo install --force

echo "Build took $SECONDS seconds"

ip_addr_array=()
# Get IP address array
# shellcheck source=/dev/null
source "$ip_addr_file"

# shellcheck disable=SC2089,SC2016
ssh_command_prefix='export PATH="$HOME/.cargo/bin:$PATH"; cd solana; USE_INSTALL=1 ./multinode-demo/'
ssh_command_prefix='export PATH="$HOME/.cargo/bin:$PATH"; cd solana; USE_INSTALL=1'

echo "Deployment started at $(date)"
SECONDS=0
count=0
leader=
for ip_addr in "${ip_addr_array[@]}"; do
Expand All @@ -43,23 +49,12 @@ for ip_addr in "${ip_addr_array[@]}"; do
ssh-keygen -R "$ip_addr"
ssh-keyscan "$ip_addr" >>~/.ssh/known_hosts

ssh "$remote_user@$ip_addr" 'mkdir ~/.ssh'
ssh -n -f "$remote_user@$ip_addr" 'mkdir -p ~/.ssh ~/solana ~/.cargo/bin'

# Killing sshguard for now. TODO: Find a better solution
# sshguard is blacklisting IP address after ssh-keyscan and ssh login attempts
ssh -n -f "$remote_user@$ip_addr" "sudo service sshguard stop"
ssh -n -f "$remote_user@$ip_addr" 'sudo apt-get --assume-yes install rsync'

if [[ -n $leader ]]; then
echo "Adding known hosts for $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "ssh-keygen -R $leader"
ssh -n -f "$remote_user@$ip_addr" "ssh-keyscan $leader >> ~/.ssh/known_hosts"
fi

# Deploy build and scripts to remote node
ssh "$remote_user@$ip_addr" 'mkdir ~/solana'
rsync -vPrz ~/.cargo/bin "$remote_user"@"$ip_addr":~/.cargo
rsync -vPrz ./multinode-demo "$remote_user"@"$ip_addr":~/solana/
ssh -n -f "$remote_user@$ip_addr" 'sudo apt-get --assume-yes install rsync libssl-dev'

# If provided, deploy SSH keys
if [[ -z $ssh_keys ]]; then
Expand All @@ -72,24 +67,41 @@ for ip_addr in "${ip_addr_array[@]}"; do
fi

# Stop current nodes
ssh "$remote_user@$ip_addr" 'pkill -9 solana-fullnode'
ssh "$remote_user@$ip_addr" 'pkill -9 solana-client-demo'
ssh "$remote_user@$ip_addr" 'pkill -9 solana-'

ssh "$remote_user@$ip_addr" 'sudo apt-get --assume-yes install libssl-dev'
if [[ -n $leader ]]; then
echo "Adding known hosts for $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "ssh-keygen -R $leader"
ssh -n -f "$remote_user@$ip_addr" "ssh-keyscan $leader >> ~/.ssh/known_hosts"

ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/.cargo/bin/solana* ~/.cargo/bin/"
ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/solana/multinode-demo ~/solana/"
ssh -n -f "$remote_user@$ip_addr" "rsync -vPrz ""$remote_user@$leader"":~/solana/fetch-perf-libs.sh ~/solana/"
else
# Deploy build and scripts to remote node
rsync -vPrz ~/.cargo/bin/solana* "$remote_user@$ip_addr":~/.cargo/bin/
rsync -vPrz ./multinode-demo "$remote_user@$ip_addr":~/solana/
rsync -vPrz ./fetch-perf-libs.sh "$remote_user@$ip_addr":~/solana/
fi

# Run setup
ssh "$remote_user@$ip_addr" "$ssh_command_prefix"'setup.sh -p "$ip_addr"'
ssh "$remote_user@$ip_addr" "$ssh_command_prefix"' ./multinode-demo/setup.sh -p "$ip_addr"'

if ((!count)); then
# Start the leader on the first node
echo "Starting leader node $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"'leader.sh > leader.log 2>&1'
ssh -n -f "$remote_user@$ip_addr" 'cd solana; ./fetch-perf-libs.sh'
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"' SOLANA_CUDA=1 ./multinode-demo/leader.sh > leader.log 2>&1'
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"' ./multinode-demo/drone.sh > drone.log 2>&1'
leader=${ip_addr_array[0]}
else
# Start validator on all other nodes
echo "Starting validator node $ip_addr"
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix""validator.sh $remote_user@$leader:~/solana $leader > validator.log 2>&1"
ssh -n -f "$remote_user@$ip_addr" "$ssh_command_prefix"" ./multinode-demo/validator.sh $remote_user@$leader:~/solana $leader > validator.log 2>&1"
fi

((count++))
done

echo "Deployment finished at $(date)"
echo "Deployment took $SECONDS seconds"