From 217bfcf660b0fbc0f5521fc3830edd2bcc281e39 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 6 Jul 2022 13:37:35 +0800 Subject: [PATCH 1/4] Merge dev --- azure-pipelines.yml | 59 ++++++++++++++++++----- scripts/ci/check_pull_request.py | 2 +- scripts/release/rpm/azure-cli.spec | 4 ++ scripts/release/rpm/fedora.dockerfile | 9 ++-- scripts/release/rpm/pipeline.sh | 6 ++- scripts/release/rpm/test_rpm_in_docker.sh | 12 +++-- scripts/release/rpm/test_rpm_package.py | 5 +- 7 files changed, 71 insertions(+), 26 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bef01b1f73..7731b95e77c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -627,14 +627,27 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: rpm-mariner$(tag) - # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackageUbi8 - displayName: Build Rpm Package Red Hat Universal Base Image 8 +- job: BuildRpmPackages + displayName: Build Rpm Packages # Do not run this job for Pull Requests due to the slowness condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: vmImage: 'ubuntu-20.04' + strategy: + matrix: + ubi8: + image: ubi8 + tag: 8.4 + artifact: rpm-ubi8 + Fedora35: + image: fedora + tag: 35 + artifact: rpm-fedora35 + Fedora36: + image: fedora + tag: 36 + artifact: rpm-fedora36 steps: - task: Bash@3 displayName: 'Build Rpm Package' @@ -642,7 +655,8 @@ jobs: targetType: 'filePath' filePath: scripts/release/rpm/pipeline.sh env: - IMAGE: ubi8 + IMAGE: $(image) + TAG: $(tag) - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' inputs: @@ -651,16 +665,39 @@ jobs: displayName: 'Publish Artifact: rpm' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-ubi8 + ArtifactName: $(artifact) - job: TestRpmPackage displayName: Test Rpm Package timeoutInMinutes: 120 - dependsOn: BuildRpmPackageUbi8 + dependsOn: BuildRpmPackages condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: vmImage: 'ubuntu-20.04' + strategy: + matrix: + ubi8: + artifact: rpm-ubi8 + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python39 + python_cmd: python3.9 + pip_cmd: pip3.9 + Fedora35: + artifact: rpm-fedora35 + distro: fc35 + image: fedora:35 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + Fedora36: + artifact: rpm-fedora36 + distro: fc36 + image: fedora:36 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -673,21 +710,19 @@ jobs: displayName: 'Download Build Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: rpm-ubi8 + artifactName: $(artifact) - bash: | set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm + RPM_NAME=azure-cli-$CLI_VERSION-1.${DISTRO}.x86_64.rpm RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME - echo "== Test rpm package on Red Hat UBI 8 ==" - - IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 + echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" displayName: 'Test Rpm Package' diff --git a/scripts/ci/check_pull_request.py b/scripts/ci/check_pull_request.py index c76d2b62f4d..69c629936b8 100644 --- a/scripts/ci/check_pull_request.py +++ b/scripts/ci/check_pull_request.py @@ -121,7 +121,7 @@ def check_line(line): logger.error(' ' * idx + '↑') error_flag = True # az xxx commands must be enclosed in `, e.g., `az vm` - if i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ': + if idx + 2 < len(line) and i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ': command = 'az ' index = idx + 3 while index < len(line) and line[index] != ':': diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index 1adc7df502b..a2c1adf4a49 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -37,6 +37,10 @@ BuildRequires: %{python_package}-devel %global _python_bytecompile_errors_terminate_build 0 +# To get rid of `cannot open linker script file` error on Fedora36. If %_package_note_file is undefined, the +# linker script will not be generated. Related bug: https://bugzilla.redhat.com/show_bug.cgi?id=2043092 +%undefine _package_note_file + %description A great cloud needs great tools; we're excited to introduce Azure CLI, our next generation multi-platform command line experience for Azure. diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index 8b92bb8922f..fb3c8f75de9 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -4,20 +4,19 @@ FROM fedora:${tag} AS build-env ARG cli_version=dev RUN dnf update -y -RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel python3-virtualenv openssl-devel make bash coreutils diffutils patch dos2unix perl +RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel openssl-devel make bash coreutils diffutils patch dos2unix perl WORKDIR /azure-cli COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ - REPO_PATH=$(pwd) CLI_VERSION=$cli_version rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ + REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ + rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ cp /root/rpmbuild/RPMS/x86_64/azure-cli-${cli_version}-1.*.x86_64.rpm /azure-cli-dev.rpm FROM fedora:${tag} AS execution-env -RUN dnf install -y python3 python3-virtualenv - COPY --from=build-env /azure-cli-dev.rpm ./ RUN rpm -i ./azure-cli-dev.rpm && \ - az --version + az --version \ No newline at end of file diff --git a/scripts/release/rpm/pipeline.sh b/scripts/release/rpm/pipeline.sh index f6f2d5384a1..d1d6ec84331 100755 --- a/scripts/release/rpm/pipeline.sh +++ b/scripts/release/rpm/pipeline.sh @@ -6,8 +6,10 @@ set -exv : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}" -# IMAGE should be one of 'centos7', 'ubi8' +# IMAGE should be one of 'centos7', 'ubi8' or 'fedora' : "${IMAGE:?IMAGE environment variable not set.}" +# TAG should be 'centos7', '8.4' or Fedora version number +: "${TAG:?TAG environment variable not set.}" CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` @@ -15,6 +17,7 @@ CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/ docker build \ --target build-env \ --build-arg cli_version=${CLI_VERSION} \ + --build-arg tag=${TAG} \ -f ./scripts/release/rpm/${IMAGE}.dockerfile \ -t azure/azure-cli:${IMAGE}-builder \ . @@ -22,6 +25,7 @@ docker build \ # Continue the previous build, and create a container that has the current azure-cli build but not the source code. docker build \ --build-arg cli_version=${CLI_VERSION} \ + --build-arg tag=${TAG} \ -f ./scripts/release/rpm/${IMAGE}.dockerfile \ -t azure/azure-cli:${IMAGE} \ . diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index cf806a49c3d..29647f22f26 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -5,13 +5,9 @@ set -exv export USERNAME=azureuser -PYTHON_PACKAGE=python39 -PYTHON_CMD=python3.9 -PIP_CMD=pip3.9 - dnf --nogpgcheck install /mnt/rpm/$RPM_NAME -y -dnf install git gcc $PYTHON_PACKAGE-devel -y +dnf install git gcc $PYTHON_PACKAGE-devel findutils -y ln -s -f /usr/bin/$PYTHON_CMD /usr/bin/python ln -s -f /usr/bin/$PIP_CMD /usr/bin/pip @@ -21,6 +17,12 @@ time az --version cd /azure-cli/ pip install wheel ./scripts/ci/build.sh + +# From Fedora36, when using `pip install --prefix`with root, the package is installed into `{prefix}/local/lib`. +# In order to keep the original installation path, I have to set RPM_BUILD_ROOT +# Ref https://docs.fedoraproject.org/en-US/fedora/latest/release-notes/developers/Development_Python/ +export RPM_BUILD_ROOT=/ + pip install pytest --prefix /usr/lib64/az pip install pytest-xdist --prefix /usr/lib64/az diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 95845cae4d1..4bf8d9a8d83 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -7,10 +7,11 @@ import sys import subprocess -root_dir = '/usr/lib64/az/lib/python3.9/site-packages/azure/cli/command_modules' +python_version = os.listdir('/usr/lib64/az/lib/')[0] +root_dir = f'/usr/lib64/az/lib/{python_version}/site-packages/azure/cli/command_modules' mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] -pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.9/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' +pytest_base_cmd = f'PYTHONPATH=/usr/lib64/az/lib/{python_version}/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd) serial_test_modules = ['botservice', 'network', 'cloud', 'appservice'] From fa071604eed2f46c1ca4ea6a32214060ec2d6943 Mon Sep 17 00:00:00 2001 From: Hang Date: Thu, 7 Jul 2022 10:38:47 +0800 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> --- scripts/release/rpm/test_rpm_in_docker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index 29647f22f26..45e9a2e07cc 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -18,9 +18,9 @@ cd /azure-cli/ pip install wheel ./scripts/ci/build.sh -# From Fedora36, when using `pip install --prefix`with root, the package is installed into `{prefix}/local/lib`. +# From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. # In order to keep the original installation path, I have to set RPM_BUILD_ROOT -# Ref https://docs.fedoraproject.org/en-US/fedora/latest/release-notes/developers/Development_Python/ +# Ref https://docs.fedoraproject.org/en-US/fedora/latest/release-notes/developers/Development_Python/#_pipsetup_py_installation_with_prefix export RPM_BUILD_ROOT=/ pip install pytest --prefix /usr/lib64/az From a57bc891d29aeae13d8dcdb318032d9b8afd2642 Mon Sep 17 00:00:00 2001 From: Hang Date: Thu, 7 Jul 2022 11:05:49 +0800 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7731b95e77c..6b108c8b5d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -636,7 +636,7 @@ jobs: vmImage: 'ubuntu-20.04' strategy: matrix: - ubi8: + Red Hat Universal Base Image 8: image: ubi8 tag: 8.4 artifact: rpm-ubi8 @@ -677,7 +677,7 @@ jobs: vmImage: 'ubuntu-20.04' strategy: matrix: - ubi8: + Red Hat Universal Base Image 8: artifact: rpm-ubi8 distro: el8 image: registry.access.redhat.com/ubi8/ubi:8.4 From 0a28054f1280da138e35c4e897f82c23ad935d91 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Thu, 7 Jul 2022 11:26:02 +0800 Subject: [PATCH 4/4] Minor fix --- scripts/release/rpm/azure-cli.spec | 2 ++ scripts/release/rpm/fedora.dockerfile | 2 +- scripts/release/rpm/test_rpm_package.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index a2c1adf4a49..040618f1cf2 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -39,6 +39,8 @@ BuildRequires: %{python_package}-devel # To get rid of `cannot open linker script file` error on Fedora36. If %_package_note_file is undefined, the # linker script will not be generated. Related bug: https://bugzilla.redhat.com/show_bug.cgi?id=2043092 +# Ref: https://src.fedoraproject.org/rpms/ruby/c/a0bcb33eaa666d3e1d08ca45e77161ca05611487?branch=rawhide +# https://src.fedoraproject.org/rpms/redhat-rpm-config//blob/rawhide/f/buildflags.md %undefine _package_note_file %description diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index fb3c8f75de9..531216f0d2e 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -19,4 +19,4 @@ FROM fedora:${tag} AS execution-env COPY --from=build-env /azure-cli-dev.rpm ./ RUN rpm -i ./azure-cli-dev.rpm && \ - az --version \ No newline at end of file + az --version diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 4bf8d9a8d83..f3ee8ffb0de 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -11,7 +11,7 @@ root_dir = f'/usr/lib64/az/lib/{python_version}/site-packages/azure/cli/command_modules' mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] -pytest_base_cmd = f'PYTHONPATH=/usr/lib64/az/lib/{python_version}/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' +pytest_base_cmd = f'PYTHONPATH=/usr/lib64/az/lib/{python_version}/site-packages python -m pytest -x -v --boxed -p no:warnings --log-level=WARN' pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd) serial_test_modules = ['botservice', 'network', 'cloud', 'appservice']