diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 76aa425633..1daf774fce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,80 +101,82 @@ jobs: KAGOME_IN_DOCKER: 0 KAGOME_MAC_CI: 1 - Linux: - if: false # Need to fix - # ${{ - # !(github.ref == 'refs/heads/master' || - # startsWith( github.ref, 'refs/tags/' )) }} - strategy: - fail-fast: false - matrix: - options: - - name: "Linux: clang-19 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake - name: "${{ matrix.options.name }}" - runs-on: ubuntu-24.04 - timeout-minutes: 120 - container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - if: ${{ env.USE_CACHE == 'true' }} - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: df du - run: | - df -m || true - du -hd1 /__w /github || true - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - - name: df du - if: ${{ always() }} - run: | - df -m || true - du -hd1 /__w /github || true - - coverage-self-hosted: - if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} - name: "Self-hosted: Linux: gcc-13 coverage/sonar" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - if: ${{ env.USE_CACHE == 'true' }} - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: makeBuild - env: - BUILD_FINAL_TARGET: ctest_coverage - run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON - - if: ${{ - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository }} - name: Submit Coverage - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi - - if: ${{ - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository }} - name: Sonar - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH_NAME: ${{ github.ref }} - run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi +# Linux: +# if: false # Need to fix +# # ${{ +# # !(github.ref == 'refs/heads/master' || +# # startsWith( github.ref, 'refs/tags/' )) }} +# strategy: +# fail-fast: false +# matrix: +# options: +# - name: "Linux: clang-19 External Project" +# run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake +# name: "${{ matrix.options.name }}" +# runs-on: ubuntu-24.04 +# timeout-minutes: 120 +# container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/cache@v4 +# if: ${{ env.USE_CACHE == 'true' }} +# with: +# path: ${{ env.CACHE_PATHS }} +# key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} +# - name: df du +# run: | +# df -m || true +# du -hd1 /__w /github || true +# - name: Install mold +# run: ./housekeeping/ci_install_mold.sh --make-default +# - name: "${{ matrix.options.name }}" +# run: "${{ matrix.options.run }}" +# - name: df du +# if: ${{ always() }} +# run: | +# df -m || true +# du -hd1 /__w /github || true +# +# coverage-self-hosted: +# if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} +# name: "Self-hosted: Linux: gcc-13 coverage/sonar" +# runs-on: [ actions-runner-controller ] +# timeout-minutes: 120 +# container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/cache@v4 +# if: ${{ env.USE_CACHE == 'true' }} +# with: +# path: ${{ env.CACHE_PATHS }} +# key: ${{ github.job }}-${{ env.CACHE_VERSION }} +# - name: Install mold +# run: ./housekeeping/ci_install_mold.sh --make-default +# - name: makeBuild +# env: +# BUILD_FINAL_TARGET: ctest_coverage +# run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON +# - if: ${{ +# github.event_name != 'pull_request' || +# github.event.pull_request.head.repo.full_name == github.repository }} +# name: Submit Coverage +# env: +# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} +# run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi +# - if: ${{ +# github.event_name != 'pull_request' || +# github.event.pull_request.head.repo.full_name == github.repository }} +# name: Sonar +# env: +# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} +# GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# BRANCH_NAME: ${{ github.ref }} +# run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi kagome_dev_docker_build_sanitizers: + # temporary disabled + if: false strategy: fail-fast: false matrix: @@ -242,6 +244,8 @@ jobs: key: ${{ steps.cache-restore.outputs.cache-primary-key }} kagome_dev_docker_build_tidy: + # temporary disabled + if: false runs-on: ubuntu-24.04 timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208 name: "Linux: clang-tidy" @@ -452,13 +456,15 @@ jobs: Zombie-Tests: + # temporary disabled + if: false runs-on: [ actions-runner-controller ] timeout-minutes: 60 needs: [kagome_dev_docker_build] - if: ${{ - github.ref == 'refs/heads/master' || - startsWith( github.ref, 'refs/tags/') || - github.event.inputs.run_all_steps == 'true' }} +# if: ${{ +# github.ref == 'refs/heads/master' || +# startsWith( github.ref, 'refs/tags/') || +# github.event.inputs.run_all_steps == 'true' }} strategy: fail-fast: false matrix: diff --git a/.gitignore b/.gitignore index fabb982e03..4766059592 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ **/build/ **/build-*/ +**/build_*/ **/cmake-build-*/ *.DS_Store *.vscode/ diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index b2f35e7a3e..8d3c7690f5 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -3,17 +3,34 @@ VERSION_FILES := commit_hash.txt kagome_version.txt ifneq ($(filter configure,$(MAKECMDGOALS)),configure) missing_files := $(filter-out $(wildcard $(VERSION_FILES)),$(VERSION_FILES)) ifneq ($(missing_files),) - $(error Missing version files: $(missing_files). Use 'make configure' to generate them.) + $(error Missing version files: $(missing_files). Use 'make configure' to generate them.) endif # Check if the necessary environment variables are set before running any targets ENV_VARS := DOCKER_REGISTRY_PATH PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS missing_env := $(strip $(foreach var,$(ENV_VARS),$(if $(strip $($(var))),, $(var)))) ifneq ($(missing_env),) - $(error Missing environment variables: $(missing_env). Please set them before running make.) + $(error Missing environment variables: $(missing_env). Please set them before running make.) endif endif +define safe_rm + PATH_TO_RM="$$(readlink -f '$(1)')"; \ + for F in / /home /etc /usr; do \ + if [ "$$PATH_TO_RM" = "$$F" ]; then \ + echo "Forbidden path: $$PATH_TO_RM"; \ + exit 1; \ + fi; \ + done; \ + echo "Checking $$PATH_TO_RM..."; \ + if [ -d "$$PATH_TO_RM" ]; then \ + echo "Deleting $$PATH_TO_RM"; \ + rm -rv --preserve-root "$$PATH_TO_RM"; \ + else \ + echo "Directory $$PATH_TO_RM does not exist"; \ + fi +endef + # Include all the necessary files include Makefile.include/variables.mk include Makefile.include/versions.mk diff --git a/housekeeping/docker/kagome-dev/Makefile.include/kagome_bin.mk b/housekeeping/docker/kagome-dev/Makefile.include/kagome_bin.mk index 8bfd9f92fa..268bc6b396 100644 --- a/housekeeping/docker/kagome-dev/Makefile.include/kagome_bin.mk +++ b/housekeeping/docker/kagome-dev/Makefile.include/kagome_bin.mk @@ -1,72 +1,112 @@ -kagome_dev_docker_build: - $(MAKE) get_versions +kagome_docker_build: + @echo "-- Building Kagome binary for $(ARCHITECTURE) architecture..." ; \ + $(MAKE) docker_run ; \ + if $(MAKE) docker_exec; then \ + echo "-- Build successful."; \ + else \ + echo "-- Build failed. Cleaning up..."; \ + $(MAKE) clean_container; \ + exit 1; \ + fi ; \ + $(MAKE) clean_container + +stop_container: + @echo "-- Stopping and removing polkadot_builder container..." + docker stop $(KAGOME_BUILD_CONTAINER_NAME) || true + +clean_container: stop_container + @echo "-- Removing previous build container..." + docker rm $(KAGOME_BUILD_CONTAINER_NAME) || true + +reset_build_state: clean_container + @echo "-- Resetting build state..." ; \ + rm -f ./commit_hash.txt ./kagome_version.txt || true; \ + $(call safe_rm,$(BUILD_DIR)) || true + +docker_run: clean_container + @echo "-- Running Kagome build container..." ; \ mkdir -p \ - $(CACHE_DIR)/.cargo/git \ - $(CACHE_DIR)/.cargo/registry \ + $(CACHE_DIR)/.cargo \ $(CACHE_DIR)/.hunter \ $(CACHE_DIR)/.cache/ccache ; \ - CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ - SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ - docker run -d --name $$CONTAINER_NAME \ + docker run -d --name $(CONTAINER_NAME) \ --platform $(PLATFORM) \ - --entrypoint "/bin/bash" \ - -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ + -e SHORT_COMMIT_HASH=$(SHORT_COMMIT_HASH) \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ + -e ARCHITECTURE=$(ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ -e CTEST_OUTPUT_ON_FAILURE=$(CTEST_OUTPUT_ON_FAILURE) \ - -v $$(pwd)/../../../../kagome:/opt/kagome \ - -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ - -v $(CACHE_DIR)/.cargo/git:/root/.cargo/git \ - -v $(CACHE_DIR)/.cargo/registry:/root/.cargo/registry \ - -v $(CACHE_DIR)/.hunter:/root/.hunter \ - -v $(CACHE_DIR)/.cache/ccache:/root/.cache/ccache \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):$(IN_DOCKER_HOME)/.gcp/google_creds.json \ + -v ./build_apt_package.sh:$(IN_DOCKER_HOME)/build_apt_package.sh \ + -v $(WORKING_DIR):$(IN_DOCKER_HOME)/kagome \ + -v $(BUILD_DIR)/pkg:$(IN_DOCKER_HOME)/pkg \ + -v $(BUILD_DIR)/kagome_binary:$(IN_DOCKER_HOME)/kagome_binary \ + -v $(CACHE_DIR)/.cargo:$(IN_DOCKER_HOME)/.cargo \ + -v $(CACHE_DIR)/.hunter:$(IN_DOCKER_HOME)/.hunter \ + -v $(CACHE_DIR)/.cache/ccache:$(IN_DOCKER_HOME)/.cache/ccache \ $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(BUILDER_IMAGE_TAG) \ - -c "tail -f /dev/null"; \ - docker exec -t $$CONTAINER_NAME /bin/bash -c \ - "cd /opt/kagome && \ - git config --global --add safe.directory /opt/kagome && \ - git config --global --add safe.directory /root/.hunter/_Base/Cache/meta && \ + tail -f /dev/null ; \ + until [ "$$(docker inspect --format='{{.State.Health.Status}}' $(CONTAINER_NAME))" = "healthy" ]; do \ + echo "Health is: \"$$(docker inspect --format='{{.State.Health.Status}}' $(CONTAINER_NAME))\"" ; \ + sleep 5 ; \ + done + +docker_exec: + echo "-- Executing kagome_builder container..." ; \ + docker exec -t $(CONTAINER_NAME) gosu $(USER_ID):$(GROUP_ID) /bin/bash -c \ + "echo \"-- Setting up environment...\"; \ + env ; \ + cd $(IN_DOCKER_HOME)/kagome && \ + git config --global --add safe.directory $(IN_DOCKER_HOME)/kagome && \ + git config --global --add safe.directory $(IN_DOCKER_HOME)/.hunter/_Base/Cache/meta && \ source /venv/bin/activate && \ git submodule update --init && \ - echo \"Building in $$(pwd)\" && \ - cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target kagome -- -j$(BUILD_THREADS) && \ - mkdir -p /tmp/kagome && \ - cp /opt/kagome/$(BUILD_DIR)/node/kagome /tmp/kagome/kagome && \ - cd /opt/kagome/housekeeping/docker/kagome-dev && \ - ./build_apt_package.sh \ - \"$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)\" \ - $(PACKAGE_ARCHITECTURE) \ + echo \"-- Recent commits:\" && git log --oneline -n 5 && \ + echo \"Building Kagome in $$(pwd)\" && \ + $(BUILD_COMMANDS) && \ + echo \"-- Copying Kagome binary...\" && \ + cp $(IN_DOCKER_HOME)/kagome/$(DOCKER_BUILD_DIR_NAME)/node/kagome $(IN_DOCKER_HOME)/kagome_binary/kagome && \ + echo \"-- Building apt package...\" && \ + cd $(IN_DOCKER_HOME) && ./build_apt_package.sh \ + $(KAGOME_DEB_PACKAGE_VERSION) \ + $(ARCHITECTURE) \ kagome-dev \ - /tmp/kagome \ + $(IN_DOCKER_HOME)/kagome_binary \ 'Kagome Dev Ubuntu Package' \ '$(DEPENDENCIES)' && \ if [ "$(IS_MAIN_OR_TAG)" = "true" ] && [ "$(GIT_REF_NAME)" != "master" ] && [ "$(BUILD_TYPE)" = "Release" ]; then \ ./build_apt_package.sh \ - \"$(KAGOME_SANITIZED_VERSION)-$(BUILD_TYPE)\" \ - $(PACKAGE_ARCHITECTURE) \ + $(KAGOME_DEB_RELEASE_PACKAGE_VERSION) \ + $(ARCHITECTURE) \ kagome \ - /tmp/kagome \ + $(IN_DOCKER_HOME)/kagome_binary \ 'Kagome Ubuntu Package' \ '$(DEPENDENCIES)' ; \ fi; \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ - docker stop $$CONTAINER_NAME; \ exit $$DOCKER_EXEC_RESULT; \ - fi; \ - docker stop $$CONTAINER_NAME + fi upload_apt_package: - SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + @echo "-- Uploading Kagome binary to Google Cloud Storage..." ; \ gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ gcloud config set artifacts/location $(REGION); \ - gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb ; \ + gcloud artifacts versions delete $(KAGOME_DEB_PACKAGE_VERSION) --package=kagome-dev --quiet || true ; \ + gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./$(BUILD_DIR)/pkg/$(KAGOME_DEB_PACKAGE_NAME) ; \ if [ "$(IS_MAIN_OR_TAG)" = "true" ] && [ "$(GIT_REF_NAME)" != "master" ] && [ "$(BUILD_TYPE)" = "Release" ]; then \ - gcloud artifacts apt upload $(PUBLIC_ARTIFACTS_REPO) --source=./pkg/kagome_$(KAGOME_SANITIZED_VERSION)-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb ; \ + echo "-- Uploading Kagome release binary to Google Cloud Storage..." ; \ + gcloud artifacts versions delete $(KAGOME_DEB_RELEASE_PACKAGE_VERSION) --package=kagome-dev --quiet || true ; \ + gcloud artifacts apt upload $(PUBLIC_ARTIFACTS_REPO) --source=./$(BUILD_DIR)/pkg/KAGOME_DEB_RELEASE_PACKAGE_NAME ; \ fi; + +kagome_deb_package_info: + @echo "Kagome version: $(KAGOME_SANITIZED_VERSION)" + @echo "Kagome deb package version: $(KAGOME_DEB_PACKAGE_VERSION)" + @echo "Kagome deb package name: $(KAGOME_DEB_PACKAGE_NAME)" + @echo "Kagome deb release package version: $(KAGOME_DEB_RELEASE_PACKAGE_VERSION)" + @echo "Kagome deb release package name: $(KAGOME_DEB_RELEASE_PACKAGE_NAME)" diff --git a/housekeeping/docker/kagome-dev/Makefile.include/variables.mk b/housekeeping/docker/kagome-dev/Makefile.include/variables.mk index 6fb6a3a09e..c1d478edfb 100644 --- a/housekeeping/docker/kagome-dev/Makefile.include/variables.mk +++ b/housekeeping/docker/kagome-dev/Makefile.include/variables.mk @@ -1,9 +1,23 @@ -# Shared Environment Variables +# CI Variables +IS_MAIN_OR_TAG ?= false +GIT_REF_NAME ?= +CI ?= false +BUILD_THREADS=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu) +CURRENT_DATE := $(shell TZ=UTC-3 date +"%Y%m%d") + +# Shared Environment Variables OS_IMAGE_NAME ?= ubuntu OS_IMAGE_TAG ?= 24.04 OS_IMAGE_TAG_CLEAN ?= $(subst .,,$(OS_IMAGE_TAG)) OS_IMAGE_HASH ?= 72297848456d5d37d1262630108ab308d3e9ec7ed1c3286a32fe09856619a782 OS_REL_VERSION ?= noble +OS_IMAGE_SHORT_HASH := $(shell echo $(OS_IMAGE_HASH) | cut -c1-7) +USER_ID ?= $(shell id -u) +GROUP_ID ?= $(shell id -g) +DIR_PATH_HASH := $(shell echo -n $(CURDIR) | sha1sum | cut -c1-8 ) +# For Docker build +# USER_ID = 5555 +# GROUP_ID = 5555 # kagome_builder Variables RUST_VERSION ?= 1.81.0 @@ -22,10 +36,12 @@ ARCHITECTURE ?= $(subst linux/,,$(PLATFORM)) # Debug, Release, RelWithDebInfo BUILD_TYPE ?= Release -# kagome_dev_docker_build Variables -BUILD_DIR ?= build -CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache +# kagome_docker_build Variables +CONTAINER_NAME := kagome_build-$(DIR_PATH_HASH) +DOCKER_BUILD_DIR_NAME := build_docker_$(ARCHITECTURE) WORKING_DIR := $(shell pwd)/../../../../kagome +BUILD_DIR := $(WORKING_DIR)/$(DOCKER_BUILD_DIR_NAME) +CACHE_DIR := $(BUILD_DIR)/cache DEPENDENCIES ?= libstdc++6, zlib1g, libgcc-s1, libc6, libtinfo6, libseccomp2, libatomic1, libnsl2 GITHUB_HUNTER_USERNAME ?= GITHUB_HUNTER_TOKEN ?= @@ -33,6 +49,9 @@ CTEST_OUTPUT_ON_FAILURE ?= 1 WERROR ?= OFF SAN_PARAMS ?= -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON UBSAN_OPTIONS ?= print_stacktrace=1 +BUILD_COMMANDS = \ + time cmake . -B\"$(DOCKER_BUILD_DIR_NAME)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ + time cmake --build \"$(DOCKER_BUILD_DIR_NAME)\" --target kagome -- -j$(BUILD_THREADS) # Generated versions OS_IMAGE ?= $(OS_IMAGE_NAME):$(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH) @@ -43,6 +62,7 @@ BUILDER_LATEST_TAG ?= latest TESTER_LATEST_TAG ?= latest DOCKERHUB_BUILDER_PATH ?= $(DOCKERHUB_REGISTRY_PATH)_builder KAGOME_SANITIZED_VERSION = $(shell cd $(WORKING_DIR) && ./get_version.sh --sanitized) +SHORT_COMMIT_HASH=$(shell grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) # kagome_runtime_cache and kagome_image_build Variables KAGOME_PACKAGE_VERSION ?= @@ -53,12 +73,6 @@ ARTIFACTS_REPO ?= kagome-apt PUBLIC_ARTIFACTS_REPO ?= kagome REGION ?= europe-north1 -# CI Variables -IS_MAIN_OR_TAG ?= false -GIT_REF_NAME ?= -CI ?= false -BUILD_THREADS=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu) - export DOCKER_BUILDKIT=1 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson export BUILDKIT_PROGRESS=auto diff --git a/housekeeping/docker/kagome-dev/Makefile.include/versions.mk b/housekeeping/docker/kagome-dev/Makefile.include/versions.mk index 4fa31befd1..16be935ab1 100644 --- a/housekeeping/docker/kagome-dev/Makefile.include/versions.mk +++ b/housekeeping/docker/kagome-dev/Makefile.include/versions.mk @@ -1,6 +1,19 @@ ifeq ($(filter configure,$(MAKECMDGOALS)),) +ifeq ($(KAGOME_DEB_PACKAGE_VERSION_NO_ARCH),) +KAGOME_DEB_PACKAGE_VERSION_NO_ARCH=$(CURRENT_DATE)-$(SHORT_COMMIT_HASH)-$(BUILD_TYPE) +endif + +ifeq ($(KAGOME_DEB_PACKAGE_VERSION),) +KAGOME_DEB_PACKAGE_VERSION=$(KAGOME_DEB_PACKAGE_VERSION_NO_ARCH)-$(ARCHITECTURE) +endif + +ifeq ($(KAGOME_DEB_RELEASE_PACKAGE_VERSION),) + KAGOME_DEB_RELEASE_PACKAGE_VERSION=$(KAGOME_SANITIZED_VERSION)-$(BUILD_TYPE)-$(ARCHITECTURE) +endif +KAGOME_DEB_PACKAGE_NAME=kagome-dev_$(KAGOME_DEB_PACKAGE_VERSION)_$(ARCHITECTURE).deb +KAGOME_DEB_RELEASE_PACKAGE_NAME=kagome_$(KAGOME_DEB_RELEASE_PACKAGE_VERSION)_$(ARCHITECTURE).deb endif diff --git a/housekeeping/docker/kagome-dev/entrypoint.sh b/housekeeping/docker/kagome-dev/entrypoint.sh index 9208660798..257f0c8d42 100644 --- a/housekeeping/docker/kagome-dev/entrypoint.sh +++ b/housekeeping/docker/kagome-dev/entrypoint.sh @@ -7,9 +7,13 @@ GROUP_ID=${GROUP_ID:-1000} USER_NAME=${USER_NAME:-builder} USER_GROUP=${USER_GROUP:-builder} -MOUNTED_DIRS=( - "/home/builder" -) +DEFAULT_DIRS="/home/builder" + +if [ -z "$MOUNTED_DIRS" ]; then + read -r -a MOUNTED_DIRS_ARRAY <<< "$DEFAULT_DIRS" +else + read -r -a MOUNTED_DIRS_ARRAY <<< "$MOUNTED_DIRS" +fi echo "Current IDs for $USER_NAME: UID=$(id -u "$USER_NAME"), GID=$(id -g "$USER_NAME")" @@ -27,7 +31,7 @@ else USER_NAME=$(getent passwd "$USER_ID" | cut -d: -f1) fi -for dir in "${MOUNTED_DIRS[@]}"; do +for dir in "${MOUNTED_DIRS_ARRAY[@]}"; do if [ -d "$dir" ]; then echo "Changing ownership of $dir to $USER_NAME:$USER_GROUP" chown -R "$USER_ID":"$GROUP_ID" "$dir" diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile index e2fbec7dd4..952b2b7619 100644 --- a/zombienet/docker/Makefile +++ b/zombienet/docker/Makefile @@ -17,6 +17,7 @@ endif # Include all the necessary files include Makefile.include/variables.mk include Makefile.include/run_zombietest.mk +include Makefile.include/polkadot_image.mk include Makefile.include/polkadot.mk include Makefile.include/zombie_tester.mk include Makefile.include/zombietests.mk diff --git a/zombienet/docker/Makefile.include/polkadot.mk b/zombienet/docker/Makefile.include/polkadot.mk index a0a3b86e40..aa535eefde 100644 --- a/zombienet/docker/Makefile.include/polkadot.mk +++ b/zombienet/docker/Makefile.include/polkadot.mk @@ -1,51 +1,3 @@ -polkadot_builder: - echo "-- Building polkadot_builder image for $(ARCHITECTURE) architecture..." ; \ - docker build --progress=plain --platform $(PLATFORM) \ - -t $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-$(ARCHITECTURE) \ - -f polkadot_builder.Dockerfile \ - --build-arg USER_ID="5555" \ - --build-arg GROUP_ID="5555" \ - --build-arg USER_NAME=$(IN_DOCKER_USERNAME) \ - --build-arg RUST_VERSION=$(RUST_VERSION) \ - --build-arg SCCACHE_VERSION=$(SCCACHE_VERSION) \ - --build-arg BASE_IMAGE=$(OS_IMAGE_NAME) \ - --build-arg BASE_IMAGE_TAG=$(OS_IMAGE_TAG_WITH_HASH) . - -polkadot_builder_push: - echo "-- Pushing polkadot_builder image for $(ARCHITECTURE) architecture..." ; \ - docker push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-$(ARCHITECTURE) - -polkadot_builder_push_manifest: - echo "-- Creating and pushing manifest for polkadot_builder image..." ; \ - docker manifest create $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) \ - --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 \ - --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 && \ - docker manifest create $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) \ - --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 \ - --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 && \ - docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) && \ - docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) - -polkadot_builder_all_arch: - echo "-- Building and pushing polkadot_builder image for all architectures..." ; \ - $(MAKE) polkadot_builder PLATFORM=linux/amd64 ARCHITECTURE=amd64 && \ - $(MAKE) polkadot_builder_push PLATFORM=linux/amd64 ARCHITECTURE=amd64 && \ - $(MAKE) polkadot_builder PLATFORM=linux/arm64 ARCHITECTURE=arm64 && \ - $(MAKE) polkadot_builder_push PLATFORM=linux/arm64 ARCHITECTURE=arm64 && \ - $(MAKE) polkadot_builder_push_manifest - -polkadot_builder_image_info: - @echo "-- Polkadot Builder Image Info --" - @echo POLKADOT_BUILDER_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) - @echo POLKADOT_BUILDER_AMD64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 - @echo POLKADOT_BUILDER_ARM64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 - -polkadot_builder_check_tag: - @docker manifest inspect $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) > /dev/null 2>&1 && echo "true" || echo "false" - -polkadot_builder_check_latest_tag: - @docker manifest inspect $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) > /dev/null 2>&1 && echo "true" || echo "false" - polkadot_binary: echo "-- Building Polkadot binary for $(ARCHITECTURE) architecture..." ; \ $(MAKE) docker_run ; \ @@ -92,7 +44,6 @@ docker_run: clean_container sleep 5 ; \ done - docker_exec: echo "-- Executing polkadot_builder container..." ; \ docker exec -t $(POLKADOT_BUILD_CONTAINER_NAME) gosu $(USER_ID):$(GROUP_ID) /bin/bash -c \ @@ -123,7 +74,12 @@ docker_exec: polkadot-binary \ $(IN_DOCKER_HOME)/polkadot_binary \ 'Polkadot binaries: $(RESULT_BIN_NAMES)' \ - '$(POLKADOT_BINARY_DEPENDENCIES)'" + '$(POLKADOT_BINARY_DEPENDENCIES)' ; \ + " || DOCKER_EXEC_RESULT=$$? ; \ + if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ + echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ + exit $$DOCKER_EXEC_RESULT; \ + fi stop_container: @echo "-- Stopping and removing polkadot_builder container..." diff --git a/zombienet/docker/Makefile.include/polkadot_image.mk b/zombienet/docker/Makefile.include/polkadot_image.mk new file mode 100644 index 0000000000..4199ac3600 --- /dev/null +++ b/zombienet/docker/Makefile.include/polkadot_image.mk @@ -0,0 +1,47 @@ +polkadot_builder: + echo "-- Building polkadot_builder image for $(ARCHITECTURE) architecture..." ; \ + docker build --progress=plain --platform $(PLATFORM) \ + -t $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-$(ARCHITECTURE) \ + -f polkadot_builder.Dockerfile \ + --build-arg USER_ID="5555" \ + --build-arg GROUP_ID="5555" \ + --build-arg USER_NAME=$(IN_DOCKER_USERNAME) \ + --build-arg RUST_VERSION=$(RUST_VERSION) \ + --build-arg SCCACHE_VERSION=$(SCCACHE_VERSION) \ + --build-arg BASE_IMAGE=$(OS_IMAGE_NAME) \ + --build-arg BASE_IMAGE_TAG=$(OS_IMAGE_TAG_WITH_HASH) . + +polkadot_builder_push: + echo "-- Pushing polkadot_builder image for $(ARCHITECTURE) architecture..." ; \ + docker push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-$(ARCHITECTURE) + +polkadot_builder_push_manifest: + echo "-- Creating and pushing manifest for polkadot_builder image..." ; \ + docker manifest create $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) \ + --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 \ + --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 && \ + docker manifest create $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) \ + --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 \ + --amend $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 && \ + docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) && \ + docker manifest push $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) + +polkadot_builder_all_arch: + echo "-- Building and pushing polkadot_builder image for all architectures..." ; \ + $(MAKE) polkadot_builder PLATFORM=linux/amd64 ARCHITECTURE=amd64 && \ + $(MAKE) polkadot_builder_push PLATFORM=linux/amd64 ARCHITECTURE=amd64 && \ + $(MAKE) polkadot_builder PLATFORM=linux/arm64 ARCHITECTURE=arm64 && \ + $(MAKE) polkadot_builder_push PLATFORM=linux/arm64 ARCHITECTURE=arm64 && \ + $(MAKE) polkadot_builder_push_manifest + +polkadot_builder_image_info: + @echo "-- Polkadot Builder Image Info --" + @echo POLKADOT_BUILDER_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) + @echo POLKADOT_BUILDER_AMD64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-amd64 + @echo POLKADOT_BUILDER_ARM64_IMAGE: $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION)-arm64 + +polkadot_builder_check_tag: + @docker manifest inspect $(DOCKER_REGISTRY_PATH)polkadot_builder:$(CURRENT_DATE)-rust$(RUST_VERSION) > /dev/null 2>&1 && echo "true" || echo "false" + +polkadot_builder_check_latest_tag: + @docker manifest inspect $(DOCKER_REGISTRY_PATH)polkadot_builder:$(BUILDER_LATEST_TAG) > /dev/null 2>&1 && echo "true" || echo "false"