diff --git a/.buildkite/scripts/build-bench.sh b/.buildkite/scripts/build-bench.sh index a19e4291bc1426..27f156c141fe03 100755 --- a/.buildkite/scripts/build-bench.sh +++ b/.buildkite/scripts/build-bench.sh @@ -22,5 +22,5 @@ EOF # shellcheck disable=SC2016 group "bench" \ - "$(build_steps "bench-part-1" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part1.sh")" \ - "$(build_steps "bench-part-2" ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/bench/part2.sh")" + "$(build_steps "bench-part-1" "ci/docker-run-default-image.sh ci/bench/part1.sh")" \ + "$(build_steps "bench-part-2" "ci/docker-run-default-image.sh ci/bench/part2.sh")" diff --git a/.buildkite/scripts/build-stable.sh b/.buildkite/scripts/build-stable.sh index e1d774e1669ab8..f20ca1db358402 100755 --- a/.buildkite/scripts/build-stable.sh +++ b/.buildkite/scripts/build-stable.sh @@ -12,7 +12,7 @@ partitions=$( cat <> "$output_file" <<"EOF" - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh" + - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh" name: "stable-sbf" timeout_in_minutes: 35 artifact_paths: "sbf-dumps.tar.bz2" @@ -226,7 +226,7 @@ EOF ^ci/test-stable.sh \ ^sdk/ \ ; then - command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20 + command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20 else annotate --style info \ "wasm skipped as no relevant files were modified" @@ -258,7 +258,7 @@ EOF ^ci/test-coverage.sh \ ^scripts/coverage.sh \ ; then - command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80 + command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80 else annotate --style info --context test-coverage \ "Coverage skipped as no .rs files were modified" @@ -296,7 +296,7 @@ pull_or_push_steps() { if [ -z "$diff_other_than_version_bump" ]; then echo "Diff only contains version bump." - command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 + command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20 exit 0 fi fi diff --git a/ci/buildkite-secondary.yml b/ci/buildkite-secondary.yml index c8bf7b4fd9fd57..c43c7ee449e758 100644 --- a/ci/buildkite-secondary.yml +++ b/ci/buildkite-secondary.yml @@ -3,7 +3,7 @@ # Pull requests to not run these steps. steps: - name: "cargo audit" - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/do-audit.sh" + command: "ci/docker-run-default-image.sh ci/do-audit.sh" agents: queue: "release-build" timeout_in_minutes: 10 diff --git a/ci/buildkite-solana-private.sh b/ci/buildkite-solana-private.sh index eeb087d323ee9a..70d8e4bfe4f59f 100755 --- a/ci/buildkite-solana-private.sh +++ b/ci/buildkite-solana-private.sh @@ -134,7 +134,7 @@ wait_step() { } all_test_steps() { - command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20 + command_step checks "ci/docker-run-default-image.sh ci/test-checks.sh" 20 wait_step # Full test suite @@ -146,7 +146,7 @@ all_test_steps() { ^ci/rust-version.sh \ ^ci/test-docs.sh \ ; then - command_step doctest ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-docs.sh" 15 + command_step doctest "ci/docker-run-default-image.sh ci/test-docs.sh" 15 else annotate --style info --context test-docs \ "Docs skipped as no .rs files were modified" @@ -168,7 +168,7 @@ all_test_steps() { ^sdk/ \ ; then cat >> "$output_file" <<"EOF" - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable-sbf.sh" + - command: "ci/docker-run-default-image.sh ci/test-stable-sbf.sh" name: "stable-sbf" timeout_in_minutes: 35 artifact_paths: "sbf-dumps.tar.bz2" @@ -208,7 +208,7 @@ EOF ^ci/test-stable.sh \ ^sdk/ \ ; then - command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20 + command_step wasm "ci/docker-run-default-image.sh ci/test-wasm.sh" 20 else annotate --style info \ "wasm skipped as no relevant files were modified" @@ -238,7 +238,7 @@ EOF ^ci/test-coverage.sh \ ^scripts/coverage.sh \ ; then - command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 80 + command_step coverage "ci/docker-run-default-image.sh ci/test-coverage.sh" 80 else annotate --style info --context test-coverage \ "Coverage skipped as no .rs files were modified" diff --git a/ci/dependabot-pr.sh b/ci/dependabot-pr.sh index 9ef6816cec5b96..91ecd5948c9a43 100755 --- a/ci/dependabot-pr.sh +++ b/ci/dependabot-pr.sh @@ -11,7 +11,7 @@ fi source ci/rust-version.sh stable -ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh +ci/docker-run-default-image.sh ci/dependabot-updater.sh if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then echo --- ok diff --git a/ci/docker-run-default-image.sh b/ci/docker-run-default-image.sh new file mode 100755 index 00000000000000..927167cc8ef1ac --- /dev/null +++ b/ci/docker-run-default-image.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +source "$here/rust-version.sh" + +"$here/docker-run.sh" "${ci_docker_image:?}" "$@" diff --git a/ci/docker-rust-nightly/Dockerfile b/ci/docker-rust-nightly/Dockerfile deleted file mode 100644 index 60d48cc22594f4..00000000000000 --- a/ci/docker-rust-nightly/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM solanalabs/rust:1.75.0 - -ARG date -ARG GRCOV_VERSION=v0.8.18 - -RUN \ - rustup install nightly-$date && \ - rustup component add clippy --toolchain=nightly-$date && \ - rustup component add rustfmt --toolchain=nightly-$date && \ - rustup show && \ - rustc --version && \ - cargo --version && \ - # grcov - curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \ - tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \ - mv ./grcov $CARGO_HOME/bin && \ - rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \ - # codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov && \ - chmod +x codecov && \ - mv codecov /usr/bin diff --git a/ci/docker-rust-nightly/README.md b/ci/docker-rust-nightly/README.md deleted file mode 100644 index f4e7931f3d8511..00000000000000 --- a/ci/docker-rust-nightly/README.md +++ /dev/null @@ -1,42 +0,0 @@ -Docker image containing rust nightly and some preinstalled crates used in CI. - -This image may be manually updated by running `CI=true ./build.sh` if you are a member -of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub -organization. - -## Moving to a newer nightly - -NOTE: Follow instructions in docker-rust/README.md before this when updating the stable -rust version as well. - -We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`) -to avoid the build breaking at unexpected times, as occasionally nightly will -introduce breaking changes. - -To update the pinned version: -1. Edit `Dockerfile` to match the desired stable rust version to base on if needed. -1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally, - or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a - specific YYYY-MM-DD that is desired (default is today's build). - Check https://rust-lang.github.io/rustup-components-history/ for build - status -1. Update `ci/rust-version.sh` to reflect the new nightly `YYYY-MM-DD` -1. Run `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-checks.sh` - and `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh [args]...` - to confirm the new nightly image builds. Fix any issues as needed -1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized. -1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com. -1. Send a PR with the `ci/rust-version.sh` change and any codebase adjustments needed. - -## Troubleshooting - -### Resource is denied - -When running `CI=true ci/docker-rust-nightly/build.sh`, you see: - -``` -denied: requested access to the resource is denied -``` - -Run `docker login` to enable pushing images to Docker Hub. Contact @mvines or @garious -to get write access. diff --git a/ci/docker-rust-nightly/build.sh b/ci/docker-rust-nightly/build.sh deleted file mode 100755 index dad7221e5dbecf..00000000000000 --- a/ci/docker-rust-nightly/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd "$(dirname "$0")" - -platform=() -if [[ $(uname -m) = arm64 ]]; then - # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr - platform+=(--platform linux/amd64) -fi - -nightlyDate=${1:-$(date +%Y-%m-%d)} -docker build "${platform[@]}" -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" . - -maybeEcho= -if [[ -z $CI ]]; then - echo "Not CI, skipping |docker push|" - maybeEcho="echo" -fi -$maybeEcho docker push solanalabs/rust-nightly:"$nightlyDate" diff --git a/ci/docker-rust/README.md b/ci/docker-rust/README.md deleted file mode 100644 index 3f818476867be3..00000000000000 --- a/ci/docker-rust/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Docker image containing rust and some preinstalled packages used in CI. - -NOTE: Recreate rust-nightly docker image after this when updating the stable rust -version! Both docker images must be updated in tandem. - -This image is manually maintained: -1. Edit `Dockerfile` to match the desired rust version -1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized. -1. Run `./build.sh` to publish the new image, if you are a member of the [Solana - Labs](https://hub.docker.com/u/solanalabs/) Docker Hub organization. - diff --git a/ci/docker-rust/build.sh b/ci/docker-rust/build.sh deleted file mode 100755 index 360bbbcbe3bcb3..00000000000000 --- a/ci/docker-rust/build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -ex - -cd "$(dirname "$0")" - - -platform=() -if [[ $(uname -m) = arm64 ]]; then - # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr - platform+=(--platform linux/amd64) -fi - -docker build "${platform[@]}" -t solanalabs/rust . - -read -r rustc version _ < <(docker run solanalabs/rust rustc --version) -[[ $rustc = rustc ]] -docker tag solanalabs/rust:latest solanalabs/rust:"$version" -docker push solanalabs/rust:"$version" -docker push solanalabs/rust:latest diff --git a/ci/docker-rust/Dockerfile b/ci/docker/Dockerfile similarity index 78% rename from ci/docker-rust/Dockerfile rename to ci/docker/Dockerfile index 227d5f55d7753b..cee80877c6db5d 100644 --- a/ci/docker-rust/Dockerfile +++ b/ci/docker/Dockerfile @@ -1,10 +1,12 @@ FROM ubuntu:20.04 ARG \ - RUST_VERSION=1.75.0 \ + RUST_VERSION= \ + RUST_NIGHTLY_VERSION= \ GOLANG_VERSION=1.21.3 \ NODE_MAJOR=18 \ - SCCACHE_VERSION=v0.5.4 + SCCACHE_VERSION=v0.5.4 \ + GRCOV_VERSION=v0.8.18 SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -21,7 +23,10 @@ ENV \ CARGO_HOME=/usr/local/cargo \ PATH="$PATH:/usr/local/cargo/bin" -RUN apt-get update && \ +RUN \ + if [ -z "$RUST_VERSION" ]; then echo "ERROR: The RUST_VERSION argument is required!" && exit 1; fi && \ + if [ -z "$RUST_NIGHTLY_VERSION" ]; then echo "ERROR: The RUST_NIGHTLY_VERSION argument is required!" && exit 1; fi && \ + apt-get update && \ apt-get install --no-install-recommends -y \ # basic tzdata \ @@ -65,6 +70,9 @@ RUN apt-get update && \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --no-modify-path --profile minimal --default-toolchain $RUST_VERSION -y && \ rustup component add rustfmt && \ rustup component add clippy && \ + rustup install $RUST_NIGHTLY_VERSION && \ + rustup component add clippy --toolchain=$RUST_NIGHTLY_VERSION && \ + rustup component add rustfmt --toolchain=$RUST_NIGHTLY_VERSION && \ rustup target add wasm32-unknown-unknown && \ cargo install cargo-audit && \ cargo install cargo-hack && \ @@ -74,6 +82,9 @@ RUN apt-get update && \ cargo install svgbob_cli && \ cargo install wasm-pack && \ cargo install rustfilt && \ + rustup show && \ + rustc --version && \ + cargo --version && \ chmod -R a+w $CARGO_HOME $RUSTUP_HOME && \ rm -rf $CARGO_HOME/registry && \ # sccache @@ -101,5 +112,14 @@ RUN apt-get update && \ chmod -R a+w /.config && \ mkdir /.npm && \ chmod -R a+w /.npm && \ + # grcov + curl -LOsS "https://github.com/mozilla/grcov/releases/download/$GRCOV_VERSION/grcov-x86_64-unknown-linux-musl.tar.bz2" && \ + tar -xf grcov-x86_64-unknown-linux-musl.tar.bz2 && \ + mv ./grcov $CARGO_HOME/bin && \ + rm grcov-x86_64-unknown-linux-musl.tar.bz2 && \ + # codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov && \ + chmod +x codecov && \ + mv codecov /usr/bin && \ # clean lists rm -rf /var/lib/apt/lists/* diff --git a/ci/docker/README.md b/ci/docker/README.md new file mode 100644 index 00000000000000..58bd9accdb14a1 --- /dev/null +++ b/ci/docker/README.md @@ -0,0 +1,11 @@ +Docker image containing rust, rust nightly and some preinstalled packages used in CI + +This image is manually maintained: + +#### CLI + +1. Edit + 1. `ci/rust-version.sh` for rust and rust nightly version + 2. `ci/docker/Dockerfile` for other packages +2. Ensure you're a member of the [Solana Docker Hub Organization](https://hub.docker.com/u/solanalabs/) and already `docker login` +3. Run `ci/docker/build.sh` to build/publish the new image diff --git a/ci/docker/build.sh b/ci/docker/build.sh new file mode 100755 index 00000000000000..0c20c5e928d94d --- /dev/null +++ b/ci/docker/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +source "$here/../rust-version.sh" + +platform=() +if [[ $(uname -m) = arm64 ]]; then + # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr + platform+=(--platform linux/amd64) +fi + +echo "build image: ${ci_docker_image:?}" +docker build "${platform[@]}" \ + -f "$here/Dockerfile" \ + --build-arg "RUST_VERSION=${rust_stable:?}" \ + --build-arg "RUST_NIGHTLY_VERSION=${rust_nightly:?}" \ + -t "$ci_docker_image" . + +docker push "$ci_docker_image" diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index 099d02129e3cb8..5d7f3b1e1e1c50 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -72,7 +72,7 @@ for Cargo_toml in $Cargo_tomls; do echo "Attempt ${i} of ${numRetries}" # The rocksdb package does not build with the stock rust docker image so use # the solana rust docker image - if ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"; then + if ci/docker-run-default-image.sh bash -exc "cd $crate; $cargoCommand"; then break fi diff --git a/ci/rust-version.sh b/ci/rust-version.sh index 3db1a843fa105b..3321f1d5ecb6a1 100644 --- a/ci/rust-version.sh +++ b/ci/rust-version.sh @@ -34,10 +34,10 @@ fi export rust_stable="$stable_version" -export rust_stable_docker_image=solanalabs/rust:"$stable_version" export rust_nightly=nightly-"$nightly_version" -export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version" + +export ci_docker_image="solanalabs/ci:rust_${rust_stable}_${rust_nightly}" [[ -z $1 ]] || ( diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 00000000000000..987f2a6cf36153 --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Get the directory of the current script +script_dir_by_bash_source=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +script_dir_by_0=$(cd "$(dirname "$0")" && pwd) + +echo "script_dir_by_bash_source = $script_dir_by_bash_source" +echo "script_dir_by_0 = $script_dir_by_0" diff --git a/docs/build.sh b/docs/build.sh index 5cb2ee6eebf7a7..6269eabdbb78b0 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -6,12 +6,10 @@ cd "$(dirname "$0")" # shellcheck source=ci/env.sh source ../ci/env.sh -: "${rust_stable_docker_image:=}" # Pacify shellcheck - # shellcheck source=ci/rust-version.sh source ../ci/rust-version.sh -../ci/docker-run.sh "$rust_stable_docker_image" docs/build-cli-usage.sh -../ci/docker-run.sh "$rust_stable_docker_image" docs/convert-ascii-to-svg.sh +../ci/docker-run-default-image.sh docs/build-cli-usage.sh +../ci/docker-run-default-image.sh docs/convert-ascii-to-svg.sh ./set-solana-release-tag.sh # Get current channel diff --git a/net/net.sh b/net/net.sh index fd25d429be3aa6..fe52116250545d 100755 --- a/net/net.sh +++ b/net/net.sh @@ -191,7 +191,7 @@ build() { if [[ $(uname) != Linux || ! " ${supported[*]} " =~ $(lsb_release -sr) ]]; then # shellcheck source=ci/rust-version.sh source "$SOLANA_ROOT"/ci/rust-version.sh - MAYBE_DOCKER="ci/docker-run.sh $rust_stable_docker_image" + MAYBE_DOCKER="ci/docker-run.sh ${ci_docker_image:?}" fi SECONDS=0 ( diff --git a/sdk/docker-solana/build.sh b/sdk/docker-solana/build.sh index 77160d73edbc38..f1c8ee265d6d56 100755 --- a/sdk/docker-solana/build.sh +++ b/sdk/docker-solana/build.sh @@ -20,8 +20,7 @@ fi cd "$(dirname "$0")" rm -rf usr/ -../../ci/docker-run.sh "$rust_stable_docker_image" \ - scripts/cargo-install-all.sh sdk/docker-solana/usr +../../ci/docker-run-default-image.sh scripts/cargo-install-all.sh sdk/docker-solana/usr cp -f ../../scripts/run.sh usr/bin/solana-run.sh cp -f ../../fetch-spl.sh usr/bin/