From 446b6bc84e22be3d03816b1641d0be1eede37ac9 Mon Sep 17 00:00:00 2001 From: yihau Date: Mon, 5 Feb 2024 18:13:17 +0800 Subject: [PATCH 1/7] ci: add new ci docker image --- ci/docker/Dockerfile | 125 +++++++++++++++++++++++++++++++++++++++++++ ci/docker/build.sh | 22 ++++++++ ci/rust-version.sh | 2 + ci/test.sh | 9 ++++ 4 files changed, 158 insertions(+) create mode 100644 ci/docker/Dockerfile create mode 100755 ci/docker/build.sh create mode 100644 ci/test.sh diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile new file mode 100644 index 00000000000000..cee80877c6db5d --- /dev/null +++ b/ci/docker/Dockerfile @@ -0,0 +1,125 @@ +FROM ubuntu:20.04 + +ARG \ + RUST_VERSION= \ + RUST_NIGHTLY_VERSION= \ + GOLANG_VERSION=1.21.3 \ + NODE_MAJOR=18 \ + SCCACHE_VERSION=v0.5.4 \ + GRCOV_VERSION=v0.8.18 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV \ + DEBIAN_FRONTEND=noninteractive \ + TZ=UTC + +# golang +ENV PATH="/usr/local/go/bin:$PATH" + +# rust +ENV \ + RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH="$PATH:/usr/local/cargo/bin" + +RUN \ + if [ -z "$RUST_VERSION" ]; then echo "ERROR: The RUST_VERSION argument is required!" && exit 1; fi && \ + if [ -z "$RUST_NIGHTLY_VERSION" ]; then echo "ERROR: The RUST_NIGHTLY_VERSION argument is required!" && exit 1; fi && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + # basic + tzdata \ + apt-transport-https \ + sudo \ + build-essential \ + git \ + vim \ + jq \ + ca-certificates \ + curl \ + gnupg \ + lld \ + cmake \ + # docs + mscgen \ + # solana compiling + libssl-dev \ + libudev-dev \ + pkg-config \ + zlib1g-dev \ + llvm \ + clang \ + cmake \ + make \ + libprotobuf-dev \ + protobuf-compiler \ + && \ + # buildkite + curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | tee /etc/apt/sources.list.d/buildkite-agent.list && \ + apt-get update && \ + apt-get install -y buildkite-agent && \ + # gh + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ + sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null && \ + apt-get update && \ + apt-get install -y gh && \ + # rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --no-modify-path --profile minimal --default-toolchain $RUST_VERSION -y && \ + rustup component add rustfmt && \ + rustup component add clippy && \ + rustup install $RUST_NIGHTLY_VERSION && \ + rustup component add clippy --toolchain=$RUST_NIGHTLY_VERSION && \ + rustup component add rustfmt --toolchain=$RUST_NIGHTLY_VERSION && \ + rustup target add wasm32-unknown-unknown && \ + cargo install cargo-audit && \ + cargo install cargo-hack && \ + cargo install cargo-sort && \ + cargo install mdbook && \ + cargo install mdbook-linkcheck && \ + cargo install svgbob_cli && \ + cargo install wasm-pack && \ + cargo install rustfilt && \ + rustup show && \ + rustc --version && \ + cargo --version && \ + chmod -R a+w $CARGO_HOME $RUSTUP_HOME && \ + rm -rf $CARGO_HOME/registry && \ + # sccache + curl -LOsS "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ + tar -xzf "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ + mv "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl"/sccache "$CARGO_HOME/bin/" && \ + rm "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ + rm -rf "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl" && \ + # nextest + curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "$CARGO_HOME/bin" && \ + # golang + curl -LOsS "https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz" && \ + tar -C /usr/local -xzf "go$GOLANG_VERSION.linux-amd64.tar.gz" && \ + rm "go$GOLANG_VERSION.linux-amd64.tar.gz" && \ + # nodejs + sudo mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list && \ + sudo apt-get update && \ + sudo apt-get install -y nodejs && \ + # setup path + mkdir /.cache && \ + chmod -R a+w /.cache && \ + mkdir /.config && \ + chmod -R a+w /.config && \ + mkdir /.npm && \ + chmod -R a+w /.npm && \ + # grcov + curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \ + tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \ + mv ./grcov $CARGO_HOME/bin && \ + rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \ + # codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov && \ + chmod +x codecov && \ + mv codecov /usr/bin && \ + # clean lists + rm -rf /var/lib/apt/lists/* diff --git a/ci/docker/build.sh b/ci/docker/build.sh new file mode 100755 index 00000000000000..565e53583043b6 --- /dev/null +++ b/ci/docker/build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source $here/../rust-version.sh + +platform=() +if [[ $(uname -m) = arm64 ]]; then + # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr + platform+=(--platform linux/amd64) +fi + +echo "build image: $ci_docker_image" +docker build "${platform[@]}" \ + -f "$here/Dockerfile" \ + --build-arg "RUST_VERSION=$rust_stable" \ + --build-arg "RUST_NIGHTLY_VERSION=$rust_nightly" \ + -t "$ci_docker_image" . + +docker push "$ci_docker_image" diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 3db1a843fa105b..3e4b567caf2992 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -39,6 +39,8 @@ export rust_stable_docker_image=solanalabs/rust:"$stable_version" export rust_nightly=nightly-"$nightly_version" export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version" +export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" + [[ -z $1 ]] || ( rustup_install() { diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 00000000000000..987f2a6cf36153 --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Get the directory of the current script +script_dir_by_bash_source=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +script_dir_by_0=$(cd "$(dirname "$0")" && pwd) + +echo "script_dir_by_bash_source = $script_dir_by_bash_source" +echo "script_dir_by_0 = $script_dir_by_0" From ca6d83b3a0e0a20ba22d7c21fd3d7e26c551b37c Mon Sep 17 00:00:00 2001 From: yihau Date: Mon, 5 Feb 2024 18:49:10 +0800 Subject: [PATCH 2/7] ci: use the new docker image --- .buildkite/scripts/build-bench.sh | 4 ++-- .buildkite/scripts/build-stable.sh | 6 +++--- ci/buildkite-pipeline.sh | 16 ++++++++-------- ci/buildkite-secondary.yml | 2 +- ci/buildkite-solana-private.sh | 10 +++++----- ci/dependabot-pr.sh | 2 +- ci/docker-run-default-image.sh | 9 +++++++++ ci/publish-crate.sh | 2 +- docs/build.sh | 4 ++-- sdk/docker-solana/build.sh | 3 +-- 10 files changed, 33 insertions(+), 25 deletions(-) create mode 100755 ci/docker-run-default-image.sh diff --git a/.buildkite/scripts/build-bench.sh b/.buildkite/scripts/build-bench.sh index a19e4291bc1426..27f156c141fe03 100755 --- a/.buildkite/scripts/build-bench.sh +++ b/.buildkite/scripts/build-bench.sh @@ -22,5 +22,5 @@ EOF # shellcheck disable=SC2016 group "bench" \ - "$(build_steps "bench-part-1" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part1.sh")" \ - "$(build_steps "bench-part-2" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part2.sh")" + "$(build_steps "bench-part-1" "ci/docker-run-default-image.sh ci/bench/part1.sh")" \ + "$(build_steps "bench-part-2" "ci/docker-run-default-image.sh ci/bench/part2.sh")" diff --git a/.buildkite/scripts/build-stable.sh b/.buildkite/scripts/build-stable.sh index e1d774e1669ab8..f20ca1db358402 100755 --- a/.buildkite/scripts/build-stable.sh +++ b/.buildkite/scripts/build-stable.sh @@ -12,7 +12,7 @@ partitions=$( cat <> "$output_file" <<"EOF" - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh" + - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh" name: "stable-sbf" timeout_in_minutes: 35 artifact_paths: "sbf-dumps.tar.bz2" @@ -226,7 +226,7 @@ EOF ^ci/test-stable.sh \ ^sdk/ \ ; then - command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20 + command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20 else annotate --style info \ "wasm skipped as no relevant files were modified" @@ -258,7 +258,7 @@ EOF ^ci/test-coverage.sh \ ^scripts/coverage.sh \ ; then - command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80 + command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80 else annotate --style info --context test-coverage \ "Coverage skipped as no .rs files were modified" @@ -296,7 +296,7 @@ pull_or_push_steps() { if [ -z "$diff_other_than_version_bump" ]; then echo "Diff only contains version bump." - command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 + command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20 exit 0 fi fi diff --git a/ci/buildkite-secondary.yml b/ci/buildkite-secondary.yml index c8bf7b4fd9fd57..c43c7ee449e758 100644 --- a/ci/buildkite-secondary.yml +++ b/ci/buildkite-secondary.yml @@ -3,7 +3,7 @@ # Pull requests to not run these steps. steps: - name: "cargo audit" - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/do-audit.sh" + command: "ci/docker-run-default-image.sh ci/do-audit.sh" agents: queue: "release-build" timeout_in_minutes: 10 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index eeb087d323ee9a..70d8e4bfe4f59f 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -134,7 +134,7 @@ wait_step() { } all_test_steps() { - command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 + command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20 wait_step # Full test suite @@ -146,7 +146,7 @@ all_test_steps() { ^ci/rust-version.sh \ ^ci/test-docs.sh \ ; then - command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15 + command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15 else annotate --style info --context test-docs \ "Docs skipped as no .rs files were modified" @@ -168,7 +168,7 @@ all_test_steps() { ^sdk/ \ ; then cat >> "$output_file" <<"EOF" - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh" + - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh" name: "stable-sbf" timeout_in_minutes: 35 artifact_paths: "sbf-dumps.tar.bz2" @@ -208,7 +208,7 @@ EOF ^ci/test-stable.sh \ ^sdk/ \ ; then - command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20 + command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20 else annotate --style info \ "wasm skipped as no relevant files were modified" @@ -238,7 +238,7 @@ EOF ^ci/test-coverage.sh \ ^scripts/coverage.sh \ ; then - command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80 + command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80 else annotate --style info --context test-coverage \ "Coverage skipped as no .rs files were modified" diff --git a/ci/dependabot-pr.sh b/ci/dependabot-pr.sh index 9ef6816cec5b96..91ecd5948c9a43 100755 --- a/ci/dependabot-pr.sh +++ b/ci/dependabot-pr.sh @@ -11,7 +11,7 @@ fi source ci/rust-version.sh stable -ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh +ci/docker-run-default-image.sh ci/dependabot-updater.sh if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then echo --- ok diff --git a/ci/docker-run-default-image.sh b/ci/docker-run-default-image.sh new file mode 100755 index 00000000000000..6190d477b3ccf0 --- /dev/null +++ b/ci/docker-run-default-image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$here/rust-version.sh" + +"$here/docker-run.sh" "$ci_docker_image" "$@" diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index 099d02129e3cb8..5d7f3b1e1e1c50 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -72,7 +72,7 @@ for Cargo_toml in $Cargo_tomls; do echo "Attempt ${i} of ${numRetries}" # The rocksdb package does not build with the stock rust docker image so use # the solana rust docker image - if ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"; then + if ci/docker-run-default-image.sh bash -exc "cd $crate; $cargoCommand"; then break fi diff --git a/docs/build.sh b/docs/build.sh index 5cb2ee6eebf7a7..391189a7446245 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -10,8 +10,8 @@ source ../ci/env.sh # shellcheck source=ci/rust-version.sh source ../ci/rust-version.sh -../ci/docker-run.sh "$rust_stable_docker_image" docs/build-cli-usage.sh -../ci/docker-run.sh "$rust_stable_docker_image" docs/convert-ascii-to-svg.sh +../ci/docker-run-default-image.sh docs/build-cli-usage.sh +../ci/docker-run-default-image.sh docs/convert-ascii-to-svg.sh ./set-solana-release-tag.sh # Get current channel diff --git a/sdk/docker-solana/build.sh b/sdk/docker-solana/build.sh index 77160d73edbc38..f1c8ee265d6d56 100755 --- a/sdk/docker-solana/build.sh +++ b/sdk/docker-solana/build.sh @@ -20,8 +20,7 @@ fi cd "$(dirname "$0")" rm -rf usr/ -../../ci/docker-run.sh "$rust_stable_docker_image" \ - scripts/cargo-install-all.sh sdk/docker-solana/usr +../../ci/docker-run-default-image.sh scripts/cargo-install-all.sh sdk/docker-solana/usr cp -f ../../scripts/run.sh usr/bin/solana-run.sh cp -f ../../fetch-spl.sh usr/bin/ From 0bd2c85e7184a5413470f6de86258030556d2621 Mon Sep 17 00:00:00 2001 From: yihau Date: Mon, 5 Feb 2024 19:00:29 +0800 Subject: [PATCH 3/7] fix shellcheck --- ci/docker-run-default-image.sh | 3 ++- ci/docker/build.sh | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/docker-run-default-image.sh b/ci/docker-run-default-image.sh index 6190d477b3ccf0..927167cc8ef1ac 100755 --- a/ci/docker-run-default-image.sh +++ b/ci/docker-run-default-image.sh @@ -4,6 +4,7 @@ set -e here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 source "$here/rust-version.sh" -"$here/docker-run.sh" "$ci_docker_image" "$@" +"$here/docker-run.sh" "${ci_docker_image:?}" "$@" diff --git a/ci/docker/build.sh b/ci/docker/build.sh index 565e53583043b6..0c20c5e928d94d 100755 --- a/ci/docker/build.sh +++ b/ci/docker/build.sh @@ -4,7 +4,8 @@ set -e here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source $here/../rust-version.sh +# shellcheck disable=SC1091 +source "$here/../rust-version.sh" platform=() if [[ $(uname -m) = arm64 ]]; then @@ -12,11 +13,11 @@ if [[ $(uname -m) = arm64 ]]; then platform+=(--platform linux/amd64) fi -echo "build image: $ci_docker_image" +echo "build image: ${ci_docker_image:?}" docker build "${platform[@]}" \ -f "$here/Dockerfile" \ - --build-arg "RUST_VERSION=$rust_stable" \ - --build-arg "RUST_NIGHTLY_VERSION=$rust_nightly" \ + --build-arg "RUST_VERSION=${rust_stable:?}" \ + --build-arg "RUST_NIGHTLY_VERSION=${rust_nightly:?}" \ -t "$ci_docker_image" . docker push "$ci_docker_image" From be93655352495e3af75092e66d702cb48250db50 Mon Sep 17 00:00:00 2001 From: yihau Date: Tue, 6 Feb 2024 12:39:07 +0800 Subject: [PATCH 4/7] add readme for the new docker image --- ci/docker/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ci/docker/README.md diff --git a/ci/docker/README.md b/ci/docker/README.md new file mode 100644 index 00000000000000..58bd9accdb14a1 --- /dev/null +++ b/ci/docker/README.md @@ -0,0 +1,11 @@ +Docker image containing rust, rust nightly and some preinstalled packages used in CI + +This image is manually maintained: + +#### CLI + +1. Edit + 1. `ci/rust-version.sh` for rust and rust nightly version + 2. `ci/docker/Dockerfile` for other packages +2. Ensure you're a member of the [Solana Docker Hub Organization](https://hub.docker.com/u/solanalabs/) and already `docker login` +3. Run `ci/docker/build.sh` to build/publish the new image From feff06d100eef6bc640af9a79547b85573fd15dd Mon Sep 17 00:00:00 2001 From: yihau Date: Tue, 6 Feb 2024 12:43:52 +0800 Subject: [PATCH 5/7] remove old docker images --- ci/docker-rust-nightly/Dockerfile | 21 ------ ci/docker-rust-nightly/README.md | 42 ------------ ci/docker-rust-nightly/build.sh | 20 ------ ci/docker-rust/Dockerfile | 105 ------------------------------ ci/docker-rust/README.md | 11 ---- ci/docker-rust/build.sh | 19 ------ ci/rust-version.sh | 2 - 7 files changed, 220 deletions(-) delete mode 100644 ci/docker-rust-nightly/Dockerfile delete mode 100644 ci/docker-rust-nightly/README.md delete mode 100755 ci/docker-rust-nightly/build.sh delete mode 100644 ci/docker-rust/Dockerfile delete mode 100644 ci/docker-rust/README.md delete mode 100755 ci/docker-rust/build.sh diff --git a/ci/docker-rust-nightly/Dockerfile b/ci/docker-rust-nightly/Dockerfile deleted file mode 100644 index 60d48cc22594f4..00000000000000 --- a/ci/docker-rust-nightly/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM solanalabs/rust:1.75.0 - -ARG date -ARG GRCOV_VERSION=v0.8.18 - -RUN \ - rustup install nightly-$date && \ - rustup component add clippy --toolchain=nightly-$date && \ - rustup component add rustfmt --toolchain=nightly-$date && \ - rustup show && \ - rustc --version && \ - cargo --version && \ - # grcov - curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \ - tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \ - mv ./grcov $CARGO_HOME/bin && \ - rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \ - # codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov && \ - chmod +x codecov && \ - mv codecov /usr/bin diff --git a/ci/docker-rust-nightly/README.md b/ci/docker-rust-nightly/README.md deleted file mode 100644 index f4e7931f3d8511..00000000000000 --- a/ci/docker-rust-nightly/README.md +++ /dev/null @@ -1,42 +0,0 @@ -Docker image containing rust nightly and some preinstalled crates used in CI. - -This image may be manually updated by running `CI=true ./build.sh` if you are a member -of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub -organization. - -## Moving to a newer nightly - -NOTE: Follow instructions in docker-rust/README.md before this when updating the stable -rust version as well. - -We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`) -to avoid the build breaking at unexpected times, as occasionally nightly will -introduce breaking changes. - -To update the pinned version: -1. Edit `Dockerfile` to match the desired stable rust version to base on if needed. -1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally, - or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a - specific YYYY-MM-DD that is desired (default is today's build). - Check https://rust-lang.github.io/rustup-components-history/ for build - status -1. Update `ci/rust-version.sh` to reflect the new nightly `YYYY-MM-DD` -1. Run `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-checks.sh` - and `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh [args]...` - to confirm the new nightly image builds. Fix any issues as needed -1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized. -1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com. -1. Send a PR with the `ci/rust-version.sh` change and any codebase adjustments needed. - -## Troubleshooting - -### Resource is denied - -When running `CI=true ci/docker-rust-nightly/build.sh`, you see: - -``` -denied: requested access to the resource is denied -``` - -Run `docker login` to enable pushing images to Docker Hub. Contact @mvines or @garious -to get write access. diff --git a/ci/docker-rust-nightly/build.sh b/ci/docker-rust-nightly/build.sh deleted file mode 100755 index dad7221e5dbecf..00000000000000 --- a/ci/docker-rust-nightly/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd "$(dirname "$0")" - -platform=() -if [[ $(uname -m) = arm64 ]]; then - # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr - platform+=(--platform linux/amd64) -fi - -nightlyDate=${1:-$(date +%Y-%m-%d)} -docker build "${platform[@]}" -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" . - -maybeEcho= -if [[ -z $CI ]]; then - echo "Not CI, skipping |docker push|" - maybeEcho="echo" -fi -$maybeEcho docker push solanalabs/rust-nightly:"$nightlyDate" diff --git a/ci/docker-rust/Dockerfile b/ci/docker-rust/Dockerfile deleted file mode 100644 index 227d5f55d7753b..00000000000000 --- a/ci/docker-rust/Dockerfile +++ /dev/null @@ -1,105 +0,0 @@ -FROM ubuntu:20.04 - -ARG \ - RUST_VERSION=1.75.0 \ - GOLANG_VERSION=1.21.3 \ - NODE_MAJOR=18 \ - SCCACHE_VERSION=v0.5.4 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV \ - DEBIAN_FRONTEND=noninteractive \ - TZ=UTC - -# golang -ENV PATH="/usr/local/go/bin:$PATH" - -# rust -ENV \ - RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH="$PATH:/usr/local/cargo/bin" - -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - # basic - tzdata \ - apt-transport-https \ - sudo \ - build-essential \ - git \ - vim \ - jq \ - ca-certificates \ - curl \ - gnupg \ - lld \ - cmake \ - # docs - mscgen \ - # solana compiling - libssl-dev \ - libudev-dev \ - pkg-config \ - zlib1g-dev \ - llvm \ - clang \ - cmake \ - make \ - libprotobuf-dev \ - protobuf-compiler \ - && \ - # buildkite - curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg && \ - echo "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | tee /etc/apt/sources.list.d/buildkite-agent.list && \ - apt-get update && \ - apt-get install -y buildkite-agent && \ - # gh - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg && \ - sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg && \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null && \ - apt-get update && \ - apt-get install -y gh && \ - # rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --no-modify-path --profile minimal --default-toolchain $RUST_VERSION -y && \ - rustup component add rustfmt && \ - rustup component add clippy && \ - rustup target add wasm32-unknown-unknown && \ - cargo install cargo-audit && \ - cargo install cargo-hack && \ - cargo install cargo-sort && \ - cargo install mdbook && \ - cargo install mdbook-linkcheck && \ - cargo install svgbob_cli && \ - cargo install wasm-pack && \ - cargo install rustfilt && \ - chmod -R a+w $CARGO_HOME $RUSTUP_HOME && \ - rm -rf $CARGO_HOME/registry && \ - # sccache - curl -LOsS "https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ - tar -xzf "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ - mv "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl"/sccache "$CARGO_HOME/bin/" && \ - rm "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz" && \ - rm -rf "sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl" && \ - # nextest - curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C "$CARGO_HOME/bin" && \ - # golang - curl -LOsS "https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz" && \ - tar -C /usr/local -xzf "go$GOLANG_VERSION.linux-amd64.tar.gz" && \ - rm "go$GOLANG_VERSION.linux-amd64.tar.gz" && \ - # nodejs - sudo mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list && \ - sudo apt-get update && \ - sudo apt-get install -y nodejs && \ - # setup path - mkdir /.cache && \ - chmod -R a+w /.cache && \ - mkdir /.config && \ - chmod -R a+w /.config && \ - mkdir /.npm && \ - chmod -R a+w /.npm && \ - # clean lists - rm -rf /var/lib/apt/lists/* diff --git a/ci/docker-rust/README.md b/ci/docker-rust/README.md deleted file mode 100644 index 3f818476867be3..00000000000000 --- a/ci/docker-rust/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Docker image containing rust and some preinstalled packages used in CI. - -NOTE: Recreate rust-nightly docker image after this when updating the stable rust -version! Both docker images must be updated in tandem. - -This image is manually maintained: -1. Edit `Dockerfile` to match the desired rust version -1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized. -1. Run `./build.sh` to publish the new image, if you are a member of the [Solana - Labs](https://hub.docker.com/u/solanalabs/) Docker Hub organization. - diff --git a/ci/docker-rust/build.sh b/ci/docker-rust/build.sh deleted file mode 100755 index 360bbbcbe3bcb3..00000000000000 --- a/ci/docker-rust/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd "$(dirname "$0")" - - -platform=() -if [[ $(uname -m) = arm64 ]]; then - # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr - platform+=(--platform linux/amd64) -fi - -docker build "${platform[@]}" -t solanalabs/rust . - -read -r rustc version _ < <(docker run solanalabs/rust rustc --version) -[[ $rustc = rustc ]] -docker tag solanalabs/rust:latest solanalabs/rust:"$version" -docker push solanalabs/rust:"$version" -docker push solanalabs/rust:latest diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 3e4b567caf2992..3321f1d5ecb6a1 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -34,10 +34,8 @@ fi export rust_stable="$stable_version" -export rust_stable_docker_image=solanalabs/rust:"$stable_version" export rust_nightly=nightly-"$nightly_version" -export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version" export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" From 88a18e5d3dd6ea58b003cb3cb8ed0daa4d43a9f3 Mon Sep 17 00:00:00 2001 From: yihau Date: Tue, 6 Feb 2024 12:44:09 +0800 Subject: [PATCH 6/7] remove unused check in docs/build.sh --- docs/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 391189a7446245..6269eabdbb78b0 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -6,8 +6,6 @@ cd "$(dirname "$0")" # shellcheck source=ci/env.sh source ../ci/env.sh -: "${rust_stable_docker_image:=}" # Pacify shellcheck - # shellcheck source=ci/rust-version.sh source ../ci/rust-version.sh ../ci/docker-run-default-image.sh docs/build-cli-usage.sh From ec6ad00998fbe4efa25507ff1b29623ed58f27fb Mon Sep 17 00:00:00 2001 From: yihau Date: Tue, 6 Feb 2024 12:44:21 +0800 Subject: [PATCH 7/7] use the new image in net.sh --- net/net.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/net.sh b/net/net.sh index fd25d429be3aa6..fe52116250545d 100755 --- a/net/net.sh +++ b/net/net.sh @@ -191,7 +191,7 @@ build() { if [[ $(uname) != Linux || ! " ${supported[*]} " =~ $(lsb_release -sr) ]]; then # shellcheck source=ci/rust-version.sh source "$SOLANA_ROOT"/ci/rust-version.sh - MAYBE_DOCKER="ci/docker-run.sh $rust_stable_docker_image" + MAYBE_DOCKER="ci/docker-run.sh ${ci_docker_image:?}" fi SECONDS=0 (