Workflow for Linux portability CI #36
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Workflow for Linux portability CI | |
| on: | |
| workflow_call: | |
| inputs: | |
| targets_pre: | |
| default: all-sage-local | |
| type: string | |
| targets: | |
| default: build | |
| type: string | |
| targets_optional: | |
| default: ptest | |
| type: string | |
| tox_system_factors: | |
| description: 'Stringified JSON object listing tox system factors' | |
| type: string | |
| # 'tox -e update_docker_platforms' updates below | |
| default: >- | |
| [ | |
| "ubuntu-xenial-toolchain-gcc_9", | |
| "ubuntu-bionic-gcc_8", | |
| "ubuntu-focal", | |
| "ubuntu-jammy", | |
| "ubuntu-noble", | |
| "debian-bullseye", | |
| "debian-bookworm", | |
| "debian-trixie", | |
| "debian-sid", | |
| "linuxmint-21", | |
| "linuxmint-21.1", | |
| "linuxmint-21.2", | |
| "linuxmint-21.3", | |
| "linuxmint-22", | |
| "linuxmint-22.1", | |
| "fedora-30", | |
| "fedora-31", | |
| "fedora-32", | |
| "fedora-33", | |
| "fedora-34", | |
| "fedora-35", | |
| "fedora-36", | |
| "fedora-37", | |
| "fedora-38", | |
| "fedora-39", | |
| "fedora-40", | |
| "fedora-41", | |
| "centos-stream-9", | |
| "centos-stream-9-python3.12", | |
| "almalinux-8-python3.9", | |
| "almalinux-9-python3.11", | |
| "gentoo-python3.10", | |
| "gentoo-python3.11", | |
| "gentoo-python3.12", | |
| "archlinux-latest", | |
| "opensuse-15.5-gcc_11-python3.11", | |
| "opensuse-tumbleweed-python3.10", | |
| ] | |
| # 'tox -e update_docker_platforms' updates above | |
| tox_packages_factors: | |
| description: 'Stringified JSON object listing tox packages factors' | |
| type: string | |
| default: >- | |
| ["minimal", | |
| "standard", | |
| ] | |
| extra_sage_packages: | |
| description: 'Extra Sage packages to install as system packages' | |
| type: string | |
| default: "" | |
| max_parallel: | |
| type: number | |
| default: 30 | |
| free_disk_space: | |
| default: false | |
| type: boolean | |
| timeout: | |
| description: 'Elapsed time (seconds) at which to kill the build' | |
| default: 20000 | |
| type: number | |
| logs_artifact: | |
| description: 'Switch for uploading logs artifact' | |
| default: true | |
| type: boolean | |
| logs_artifact_postfix: | |
| description: 'Postfix (default: none) for logs artifact name' | |
| default: "" | |
| type: string | |
| # | |
| # Publishing to GitHub Packages | |
| # | |
| docker_push_repository: | |
| required: false | |
| type: string | |
| # | |
| # Incremental builds | |
| # | |
| docker_targets: | |
| default: "with-system-packages configured with-targets-pre with-targets with-targets-optional" | |
| type: string | |
| incremental: | |
| description: 'Whether to build Sage from scratch or from prebuilt Sage' | |
| default: false | |
| type: boolean | |
| from_docker_repository: | |
| required: false | |
| type: string | |
| from_docker_target: | |
| required: false | |
| type: string | |
| from_docker_tag: | |
| required: false | |
| default: "$BUILD_TAG" | |
| type: string | |
| # | |
| # For use in upstream CIs. sage_trac_* are now ignored and will be removed later. | |
| # | |
| upstream_artifact: | |
| required: false | |
| type: string | |
| sage_repo: | |
| required: false | |
| type: string | |
| sage_trac_git: | |
| required: false | |
| type: string | |
| sage_trac_ticket: | |
| required: false | |
| type: string | |
| sage_ref: | |
| required: false | |
| type: string | |
| workflow_dispatch: | |
| inputs: | |
| tox_system_factors: | |
| default: >- | |
| [ "ubuntu-jammy"] | |
| type: string | |
| tox_packages_factors: | |
| type: string | |
| default: >- | |
| ["standard"] | |
| targets_pre: | |
| default: "all-sage-local" | |
| type: string | |
| targets: | |
| default: "build" | |
| type: string | |
| targets_optional: | |
| default: "ptest" | |
| type: string | |
| docker_targets: | |
| default: "with-targets" | |
| type: string | |
| incremental: | |
| default: true | |
| type: boolean | |
| from_docker_repository: | |
| default: "ghcr.io/sagemath/sage/" | |
| type: string | |
| from_docker_target: | |
| default: "with-targets-pre" | |
| type: string | |
| from_docker_tag: | |
| default: "dev" | |
| type: string | |
| jobs: | |
| linux: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ inputs.max_parallel || 10 }} | |
| matrix: | |
| tox_system_factor: ${{ fromJson(inputs.tox_system_factors) }} | |
| tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }} | |
| env: | |
| # See tox.ini to see how these environment variables define the | |
| # Dockerfile used to build the docker image, written by the | |
| # .ci/write-dockerfile.sh | |
| TOX_ENV: "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}" | |
| LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.logs_artifact_postfix }} | |
| DOCKER_TARGETS: ${{ inputs.docker_targets || github.event.inputs.docker_targets }} | |
| TARGETS_PRE: ${{ inputs.targets_pre }} | |
| TARGETS: ${{ inputs.targets }} | |
| TARGETS_OPTIONAL: ${{ inputs.targets_optional }} | |
| FROM_DOCKER_REPOSITORY: ${{ inputs.from_docker_repository }} | |
| FROM_DOCKER_TARGET: ${{ inputs.from_docker_target }} | |
| FROM_DOCKER_TAG: ${{ inputs.from_docker_tag }} | |
| EXTRA_CONFIGURE_ARGS: --enable-fat-binary | |
| EXTRA_SAGE_PACKAGES: ${{ inputs.extra_sage_packages }} | |
| steps: | |
| - name: Maximize build disk space | |
| uses: easimon/maximize-build-space@v10 | |
| with: | |
| # need space in /var for Docker images | |
| root-reserve-mb: 30000 | |
| remove-dotnet: true | |
| remove-android: true | |
| remove-haskell: true | |
| remove-codeql: true | |
| remove-docker-images: true | |
| continue-on-error: true | |
| if: inputs.free_disk_space | |
| - name: Check out SageMath | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ inputs.sage_repo }} | |
| ref: ${{ inputs.sage_ref }} | |
| fetch-depth: 10000 | |
| - name: Download upstream artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: upstream | |
| name: ${{ inputs.upstream_artifact }} | |
| if: inputs.upstream_artifact | |
| - name: Install test prerequisites | |
| run: | | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install tox | |
| sudo apt-get clean | |
| df -h | |
| - name: Update Sage packages from upstream artifact | |
| run: | | |
| export PATH=$(pwd)/build/bin:$PATH | |
| # The sed command removes any lines containing 'upstream' from .dockerignore | |
| (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore | |
| # Handle both the old and new location of write-dockerfile.sh, because docker.yml is a reusable workflow | |
| for script_path in build/bin/write-dockerfile.sh .ci/write-dockerfile.sh; do | |
| if [ -r "$script_path" ]; then | |
| # Add 'ADD upstream upstream' before the line containing ':toolchain:' in the script | |
| echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - "$script_path" | |
| fi | |
| done | |
| # Show the changes made to the repository for review | |
| git diff | |
| if: inputs.upstream_artifact | |
| - name: Try to login to ghcr.io | |
| if: (inputs.docker_push_repository == null || inputs.docker_push_repository != '') | |
| # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable | |
| run: | | |
| TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" | |
| if [ -z "$TOKEN" ]; then | |
| TOKEN="${{ secrets.GITHUB_TOKEN }}" | |
| fi | |
| if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then | |
| if [ -z "${{ inputs.docker_push_repository }}" ]; then | |
| echo "DOCKER_PUSH_REPOSITORY=ghcr.io/${{ github.repository }}/" >> $GITHUB_ENV | |
| else | |
| echo "DOCKER_PUSH_REPOSITORY=$(echo ${{ inputs.docker_push_repository }} | tr "[:upper:]" "[:lower:]")" >> $GITHUB_ENV | |
| fi | |
| echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV | |
| fi | |
| - name: Determine Docker tags to use | |
| run: | | |
| # This line needs to be run before the step "Merge CI fixes from sagemath/sage". | |
| # | |
| DOCKER_TAG="$(git describe --dirty --always)" | |
| echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV | |
| # | |
| # From the docker documentation via .ci/update-env.sh: | |
| # "A tag name must be valid ASCII and may | |
| # contain lowercase and uppercase letters, digits, underscores, periods and | |
| # dashes. A tag name may not start with a period or a dash and may contain a | |
| # maximum of 128 characters." | |
| # | |
| EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128` | |
| shopt -s extglob | |
| case "$GITHUB_REF_NAME" in | |
| +([0-9]).+([0-9])?(.+([0-9])) ) | |
| EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";; | |
| +([0-9]).+([0-9])?(.+([0-9]))?(.)@(a|alpha|b|beta|rc)+([0-9]) ) | |
| EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";; | |
| esac | |
| echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV | |
| - name: Merge CI fixes from sagemath/sage | |
| # This step needs to happen after the commit sha is put in DOCKER_TAG | |
| # so that multi-stage builds can work correctly. | |
| run: | | |
| .ci/merge-fixes.sh | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} | |
| - name: Show disk space | |
| run: | | |
| df -h | |
| if: inputs.free_disk_space | |
| - name: Configure and build Sage distribution within a Docker container | |
| run: | | |
| # The first command below is a self-destruct sequence, | |
| # which preempts the GitHub Actions 6-hour job cancellation. | |
| # | |
| # Using "docker exec", we enter the temporary containers used by | |
| # "docker build" and kill the "make" processes of the Sage distribution. | |
| ( | |
| sleep ${{ inputs.timeout || 20000 }} | |
| for container_id in $(docker ps -q); do | |
| # | |
| # The arcane "find" command is a replacement for "pkill make", | |
| # which we use because pkill is not installed in the "minimal" package | |
| # configuration on many platforms. | |
| # | |
| docker exec "$container_id" find /proc -maxdepth 2 -name cmdline \ | |
| -exec bash -c 'grep -l "[m][a][k][e]" {} | cut -d/ -f3 | xargs --no-run-if-empty kill' \; | |
| done | |
| ) & | |
| # Set pipeline to fail if any command fails | |
| set -o pipefail | |
| # | |
| EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=9 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=5\"" | |
| # | |
| # The tox command starts to build and test Sage within a Docker container. | |
| # | |
| # After the build, the sed command strips away timestamps from | |
| # "docker build" (buildkit) such as "#25 1211.0" at the beginning of | |
| # each line. The timestamps are redundant because GH Actions provides | |
| # timestamps for each line already. Stripping the timestamps from | |
| # the beginnings of lines also allows GitHub Actions to recognize | |
| # workflow commands such as ::error etc. The other sed commands | |
| # annotate configuration notices/warnings/errors as GitHub | |
| # ::warning/::warning/::error respectively. | |
| # | |
| tox -e $TOX_ENV -- $TARGETS 2>&1 | sed -E --unbuffered " | |
| s/^#[0-9]+ [0-9]+[.][0-9]+ // | |
| /^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::| | |
| /^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::| | |
| /^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::| | |
| " | |
| - name: Copy logs from the Docker image or build container | |
| run: | | |
| mkdir -p "artifacts/$LOGS_ARTIFACT_NAME" | |
| cp -r .tox/$TOX_ENV/* "artifacts/$LOGS_ARTIFACT_NAME" | |
| rm -rf "artifacts/$LOGS_ARTIFACT_NAME"/{bin,lib,pyvenv.cfg} | |
| if: always() | |
| - name: Upload logs artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: artifacts | |
| name: ${{ env.LOGS_ARTIFACT_NAME }} | |
| if: always() && (inputs.logs_artifact == true || inputs.logs_artifact == null) | |
| - name: Print out logs for immediate inspection | |
| # and markup the output with GitHub Actions logging commands | |
| run: | | |
| .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" | |
| if: always() | |
| - name: List Docker images | |
| run: | | |
| if [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags.pushed ]; then | |
| set -- $(cat .tox/$TOX_ENV/Dockertags.pushed) | |
| case $# in | |
| 1) images="image"; one_image="the image";; | |
| *) images="images"; one_image="one of the images";; | |
| esac | |
| echo "::notice title=Docker $images pushed::Pushed $images $*)" | |
| echo | |
| echo "To pull $one_image and enter the container, type:" | |
| echo | |
| for TAG in $*; do | |
| echo " \$ docker run -it $TAG bash" | |
| done | |
| echo | |
| echo "To use $one_image as the base for an incremental build, type:" | |
| echo | |
| TOX_ENV_SANS_INCREMENTAL=${TOX_ENV/-incremental/} | |
| DOCKER_IMAGE=${TOX_ENV_SANS_INCREMENTAL#docker-} | |
| for TAG in $*; do | |
| echo -n " \$" | |
| if [ "$DOCKER_PUSH_REPOSITORY" != "ghcr.io/sagemath/sage/" ]; then | |
| echo -n " FROM_DOCKER_REPOSITORY=$DOCKER_PUSH_REPOSITORY" | |
| fi | |
| eval DOCKER_TARGET=\${TAG#*$DOCKER_IMAGE-} | |
| DOCKER_TARGET=${DOCKER_TARGET%:*} | |
| if [ "$DOCKER_TARGET" != "with-targets" ]; then | |
| echo -n " FROM_DOCKER_TARGET=$DOCKER_TARGET" | |
| fi | |
| echo " FROM_DOCKER_TAG=${TAG#*:} tox -e $TOX_ENV_SANS_INCREMENTAL-incremental" | |
| done | |
| elif [ -n "$DOCKER_PUSH_REPOSITORY" -a -f .tox/$TOX_ENV/Dockertags ]; then | |
| echo "Unable to push Docker images to $DOCKER_PUSH_REPOSITORY." | |
| echo "This is normal in a pull request to sagemath/sage or to another user's repository." | |
| echo | |
| echo "If you need Docker images, " | |
| echo " - either run this GitHub Actions workflow in your repository fork" | |
| echo " - or use the method described in https://doc.sagemath.org/html/en/developer/portability_testing.html#automatic-docker-based-build-testing-using-tox" | |
| else | |
| echo "No Docker images created." | |
| fi | |
| if: always() && ${{ inputs.docker_push_repository }} |