From 8a5d6d081cf075cd9da8001c2119ce43d2427285 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 7 Feb 2025 11:29:17 +0300 Subject: [PATCH 1/2] Fix EKS Cluster conversion by injecting index key when list Manual cherry pick of https://github.com/crossplane-contrib/provider-upjet-aws/pull/1669 Signed-off-by: Hasan Turken (cherry picked from commit c14e7160791a6d0cdb1fc9bb2876a3765a8db428) --- config/registry.go | 33 +++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/config/registry.go b/config/registry.go index ea0f176ea9..3b03f96162 100644 --- a/config/registry.go +++ b/config/registry.go @@ -236,14 +236,43 @@ func configureSingletonListAPIConverters(r *config.Resource) error { // no need for a Terraform conversion. r.ControllerReconcileVersion = currentVer + // This block is to fix the issue described in the following PR: https://github.com/crossplane/upjet/pull/465 + // In EKS Cluster object v1beta1, for spec.vpcConfig field, we mark + // +listType as "map" and +listMapKey as "index". During conversion between + // v1beta1 to v1beta2, we convert the that field from array to object, + // losing the index field since it is not in the schema. This is fine in + // most cases since in v1beta1 of the object schema index defaults to "0", + // even though conversion doesn't output the index field in the object. + // + // However, with Server Side Apply, apparently some on the fly conversions + // happening when different managers using different api versions and losing + // index field causing unexpected merging results and drop of the whole + // spec.forProvider.vpcConfig object. This is surfaced with an error like + // below: + /// cannot patch the managed resource via server-side apply: Cluster.eks.aws.upbound.io + // "some-eks-cluster" is invalid: [spec.forProvider.vpcConfig: Invalid + // value: "null": spec.forProvider.vpcConfig in body must be of type array: "null", + // : Invalid value: "null": some validation rules were not checked because + // the object was invalid; correct the existing errors to complete validation] + var opts []conversion.SingletonListConversionOption + if r.Name == "aws_eks_cluster" { + opts = append(opts, conversion.WithConvertOptions(&conversion.ConvertOptions{ + ListInjectKeys: map[string]conversion.SingletonListInjectKey{ + "vpcConfig": { + Key: "index", + Value: "0", + }, + }, + })) + } // assumes the first element is the identity conversion from // the default resource and removes it because we will register another // identity converter below. r.Conversions = r.Conversions[1:] r.Conversions = append([]conversion.Conversion{ conversion.NewIdentityConversionExpandPaths(conversion.AllVersions, conversion.AllVersions, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths()...), - conversion.NewSingletonListConversion(conversion.AllVersions, bumped, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToEmbeddedObject), - conversion.NewSingletonListConversion(bumped, conversion.AllVersions, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToSingletonList), + conversion.NewSingletonListConversion(conversion.AllVersions, bumped, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToEmbeddedObject, opts...), + conversion.NewSingletonListConversion(bumped, conversion.AllVersions, conversion.DefaultPathPrefixes(), r.CRDListConversionPaths(), conversion.ToSingletonList, opts...), }, r.Conversions...) return nil diff --git a/go.mod b/go.mod index dc3782a178..45a286aa67 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/aws/smithy-go v1.20.3 github.com/crossplane/crossplane-runtime v1.16.0-rc.2.0.20240510094504-3f697876fa57 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 - github.com/crossplane/upjet v1.4.1-0.20240612123927-37c7f4e91d57 + github.com/crossplane/upjet v1.4.1-0.20250207082618-16771915af9e github.com/go-ini/ini v1.46.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/awspolicyequivalence v1.6.0 diff --git a/go.sum b/go.sum index d8e8592810..f39ed2478c 100644 --- a/go.sum +++ b/go.sum @@ -478,8 +478,8 @@ github.com/crossplane/crossplane-runtime v1.16.0-rc.2.0.20240510094504-3f697876f github.com/crossplane/crossplane-runtime v1.16.0-rc.2.0.20240510094504-3f697876fa57/go.mod h1:Pz2tdGVMF6KDGzHZOkvKro0nKc8EzK0sb/nSA7pH4Dc= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 h1:HigXs5tEQxWz0fcj8hzbU2UAZgEM7wPe0XRFOsrtF8Y= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79/go.mod h1:+e4OaFlOcmr0JvINHl/yvEYBrZawzTgj6pQumOH1SS0= -github.com/crossplane/upjet v1.4.1-0.20240612123927-37c7f4e91d57 h1:hyKXccOb8BepVJj79KsvwYsid5Lhlr2DojO29Sqi+rw= -github.com/crossplane/upjet v1.4.1-0.20240612123927-37c7f4e91d57/go.mod h1:wkdZf/Cvhr6PI30VdHIOjg4dX39Z5uijqnLWFk5PbGM= +github.com/crossplane/upjet v1.4.1-0.20250207082618-16771915af9e h1:BvV3I9iao6YURrAie/VQfJ4Z5AwKlNvEhe+4Vt2ugdM= +github.com/crossplane/upjet v1.4.1-0.20250207082618-16771915af9e/go.mod h1:wkdZf/Cvhr6PI30VdHIOjg4dX39Z5uijqnLWFk5PbGM= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= From 76ac1e3598c2a7904c4b7e682fbc9af216d57975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fatih=20T=C3=BCrken?= Date: Fri, 7 Feb 2025 11:47:49 +0300 Subject: [PATCH 2/2] Stop using deprecated reusable workflows and use new ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fatih Türken (cherry picked from commit 631850af3efb9efc6fe9dc343969dda5e009f345) --- .github/workflows/backport-trigger.yml | 30 +++ .github/workflows/backport.yml | 15 +- .github/workflows/ci.yml | 211 ++++++++++++++---- .github/workflows/commands.yml | 11 - .github/workflows/e2e.yaml | 19 -- .github/workflows/native-provider-bump.yml | 17 -- .../workflows/publish-service-artifacts.yml | 49 ---- .github/workflows/scan.yml | 52 ----- .github/workflows/tag.yaml | 20 +- .github/workflows/updoc.yml | 17 -- .github/workflows/uptest-all.yml | 161 ------------- .github/workflows/uptest-trigger.yaml | 180 +++++++++++++++ 12 files changed, 403 insertions(+), 379 deletions(-) create mode 100644 .github/workflows/backport-trigger.yml delete mode 100644 .github/workflows/commands.yml delete mode 100644 .github/workflows/e2e.yaml delete mode 100644 .github/workflows/native-provider-bump.yml delete mode 100644 .github/workflows/publish-service-artifacts.yml delete mode 100644 .github/workflows/scan.yml delete mode 100644 .github/workflows/updoc.yml delete mode 100644 .github/workflows/uptest-all.yml create mode 100644 .github/workflows/uptest-trigger.yaml diff --git a/.github/workflows/backport-trigger.yml b/.github/workflows/backport-trigger.yml new file mode 100644 index 0000000000..7594f698cd --- /dev/null +++ b/.github/workflows/backport-trigger.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2024 The Crossplane Authors +# +# SPDX-License-Identifier: CC0-1.0 + +name: Backport comment trigger + +on: issue_comment + +jobs: + backport: + runs-on: ubuntu-latest + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/backport') + steps: + - name: Extract Command + id: command + uses: xt0rted/slash-command-action@bf51f8f5f4ea3d58abc7eca58f77104182b23e88 # v2.0.0 + with: + command: backport + reaction: "true" + reaction-type: "eyes" + allow-edits: "false" + permission-level: write + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Open Backport PR + uses: zeebe-io/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index c3881aa399..e5736dd6f5 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -12,8 +12,17 @@ on: # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ pull_request_target: types: [closed] - # See also commands.yml for the /backport triggered variant of this workflow. + # See also backport-trigger.yml for the /backport triggered variant of this workflow. jobs: - backport: - uses: upbound/official-providers-ci/.github/workflows/provider-backport.yml@standard-runners + open-pr: + runs-on: ubuntu-latest + if: github.event.pull_request.merged + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - name: Open Backport PR + uses: zeebe-io/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75b131dcf4..ccf5706b0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,19 +12,12 @@ on: pull_request: {} workflow_dispatch: {} -jobs: - ci: - uses: upbound/official-providers-ci/.github/workflows/provider-ci.yml@standard-runners - with: - go-version: "1.22" - golangci-skip: true # we will run the linter via "make lint" - cleanup-disk: true - secrets: - UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} - UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} +env: + GO_VERSION: "1.22" +jobs: detect-noop: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: noop: ${{ steps.noop.outputs.should_skip }} steps: @@ -32,68 +25,73 @@ jobs: id: noop uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 with: - github_token: ${{ secrets.GITHUB_TOKEN }} paths_ignore: '["**.md", "**.png", "**.jpg"]' do_not_skip: '["workflow_dispatch", "schedule", "push"]' - lint: - runs-on: ubuntu-22.04 + report-breaking-changes: + runs-on: ubuntu-latest needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + - name: Get modified CRDs + id: modified-crds + uses: tj-actions/changed-files@039afcd1024c210363c9d3fc8fd07e1f3fcf2867 # v35.9.3 (breaks the 'Report native schema version changes' step on newer versions) + with: + files: | + package/crds/** + + - name: Report breaking CRD OpenAPI v3 schema changes + if: steps.modified-crds.outputs.any_changed == 'true' + env: + MODIFIED_CRD_LIST: ${{ steps.modified-crds.outputs.all_changed_files }} + run: | + make crddiff + + - name: Report native schema version changes + run: | + make schema-version-diff + + lint: + runs-on: ubuntu-latest + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' steps: - name: Cleanup Disk uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: - android: true - dotnet: true - haskell: true - tool-cache: true large-packages: false swap-storage: false - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: true - name: Setup Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: - go-version: "1.22" + go-version: ${{ env.GO_VERSION }} - - name: Find the Go Build Cache - id: go_cache + - name: Find the Analysis Cache + id: analysis_cache run: | - echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \ - echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \ echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \ echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \ echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT - - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go_cache.outputs.cache }} - key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-lint- - - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go_cache.outputs.mod_cache }} - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - name: Cache Linter Analysis - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 id: cache-analysis with: - path: ${{ steps.go_cache.outputs.analysis_cache }} - key: ${{ steps.go_cache.outputs.analysis_cache_key }} + path: ${{ steps.analysis_cache.outputs.analysis_cache }} + key: ${{ steps.analysis_cache.outputs.analysis_cache_key }} restore-keys: | - ${{ steps.go_cache.outputs.analysis_cache_key_int }} + ${{ steps.analysis_cache.outputs.analysis_cache_key_int }} - name: Vendor Dependencies run: make vendor vendor.check @@ -105,3 +103,128 @@ jobs: RUN_BUILDTAGGER: true GOGC: "50" run: make lint + + check-diff: + runs-on: ubuntu-latest + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + large-packages: false + swap-storage: false + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install goimports + run: go install golang.org/x/tools/cmd/goimports + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Check Diff + id: check-diff + run: | + mkdir _output + make check-diff + env: + # check-diff depends on the generate Make target, and we would like + # to save a skipped resource list + SKIPPED_RESOURCES_CSV: ../_output/skipped_resources.csv + + - name: Show diff + if: failure() && steps.check-diff.outcome == 'failure' + run: git diff + + - name: Report Statistics + run: head -1 _output/skipped_resources.csv + + - name: Publish skipped resources CSV to Github + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: skipped_resources + path: _output/skipped_resources.csv + + unit-tests: + runs-on: ubuntu-latest + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + large-packages: false + swap-storage: false + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Fetch History + run: git fetch --prune --unshallow + + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Run Unit Tests + run: make -j2 test + + local-deploy: + runs-on: ubuntu-latest + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + large-packages: false + swap-storage: false + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Fetch History + run: git fetch --prune --unshallow + + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Deploying locally built provider package + run: make local-deploy + + check-examples: + runs-on: ubuntu-latest + needs: detect-noop + if: ${{ needs.detect-noop.outputs.noop != 'true' }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Check Example Manifests + run: | + ./scripts/check-examples.py package/crds examples diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml deleted file mode 100644 index 3874e6d7d5..0000000000 --- a/.github/workflows/commands.yml +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Comment Commands - -on: issue_comment - -jobs: - comment-commands: - uses: upbound/official-providers-ci/.github/workflows/provider-commands.yml@standard-runners diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml deleted file mode 100644 index 294339206a..0000000000 --- a/.github/workflows/e2e.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: End to End Testing - -on: - issue_comment: - types: [created] - -jobs: - e2e: - uses: upbound/official-providers-ci/.github/workflows/pr-comment-trigger.yml@standard-runners - with: - go-version: 1.22 - cleanup-disk: true - secrets: - UPTEST_CLOUD_CREDENTIALS: ${{ secrets.UPTEST_CLOUD_CREDENTIALS }} - UPTEST_DATASOURCE: ${{ secrets.UPTEST_DATASOURCE }} diff --git a/.github/workflows/native-provider-bump.yml b/.github/workflows/native-provider-bump.yml deleted file mode 100644 index f01bf0b442..0000000000 --- a/.github/workflows/native-provider-bump.yml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Native Provider Version Bump - -on: - workflow_dispatch: {} - -jobs: - open-bump-pr: - uses: upbound/official-providers-ci/.github/workflows/native-provider-bump.yml@standard-runners - with: - provider-source: hashicorp/aws - go-version: 1.22 - secrets: - TOKEN: ${{ secrets.OFFICIAL_PROVIDERS_GA_TOKEN }} diff --git a/.github/workflows/publish-service-artifacts.yml b/.github/workflows/publish-service-artifacts.yml deleted file mode 100644 index 44f9cf5330..0000000000 --- a/.github/workflows/publish-service-artifacts.yml +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Publish Service Artifacts - -on: - workflow_dispatch: - inputs: - subpackages: - description: 'Subpackages to be built individually (e.g. monolith config ec2)' - default: 'monolith' - required: false - size: - description: "Number of smaller provider packages to build and push with each build job" - default: '30' - required: true - concurrency: - description: "Number of parallel package builds within each build job" - default: '1' - required: false - regorg: - description: 'Package registry and organization where the packages will be pushed or (e.g. xpkg.upbound.io/upbound)' - default: 'xpkg.upbound.io/upbound' - required: false - branch_name: - description: "Branch name to use while publishing the packages (e.g. main)" - default: '' - required: false - version: - description: "Version string to use while publishing the packages (e.g. v1.0.0-alpha.1)" - default: '' - required: false - -jobs: - publish-service-artifacts: - uses: upbound/official-providers-ci/.github/workflows/provider-publish-service-artifacts.yml@standard-runners - with: - subpackages: ${{ github.event.inputs.subpackages }} - size: ${{ github.event.inputs.size }} - concurrency: ${{ github.event.inputs.concurrency }} - regorg: ${{ github.event.inputs.regorg }} - branch_name: ${{ github.event.inputs.branch_name }} - version: ${{ github.event.inputs.version }} - go-version: 1.22 - cleanup-disk: true - secrets: - UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR_RC }} - UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW_RC }} diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml deleted file mode 100644 index 6777955aa9..0000000000 --- a/.github/workflows/scan.yml +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Scan - - -on: - workflow_dispatch: - inputs: - supported_releases_number: - description: 'Number of supported releases' - type: number - default: 1 - schedule: - # run every day at 3:07am UTC - - cron: '7 3 * * *' - -permissions: - security-events: write - -env: - SUPPORTED_RELEASES_NUMBER: '1' - # comma separated list of images, without tag - IMAGES: "xpkg.upbound.io/upbound/provider-family-aws" - -jobs: - setup-vars: - runs-on: ubuntu-22.04 - outputs: - supported_releases_number: ${{ steps.setup.outputs.supported_releases_number }} - images: ${{ steps.setup.outputs.images }} - steps: - - name: Setup outputs - shell: bash - id: setup - run: | - supported_releases_number="${{ fromJSON(inputs.supported_releases_number || env.SUPPORTED_RELEASES_NUMBER) }}" - echo "supported_releases_number=${supported_releases_number}" >> $GITHUB_OUTPUT - - images="${{ env.IMAGES }}" - echo "images=${images}" >> $GITHUB_OUTPUT - - echo "We are going to scan the last ${supported_releases_number} releases for: ${images}" - - scan: - uses: upbound/official-providers-ci/.github/workflows/scan.yml@standard-runners - needs: - - setup-vars - with: - images: ${{ needs.setup-vars.outputs.images }} - supported_releases: ${{ fromJSON(needs.setup-vars.outputs.supported_releases_number) }} diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml index 6ae5106fcd..def498a14b 100644 --- a/.github/workflows/tag.yaml +++ b/.github/workflows/tag.yaml @@ -5,7 +5,7 @@ name: Tag on: - workflow_dispatch: + workflow_dispatch: inputs: version: description: 'Release version (e.g. v0.1.0)' @@ -15,8 +15,16 @@ on: required: true jobs: - tag: - uses: upbound/official-providers-ci/.github/workflows/provider-tag.yml@standard-runners - with: - version: ${{ github.event.inputs.version }} - message: ${{ github.event.inputs.message }} + create-tag: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Create Tag + uses: negz/create-tag@39bae1e0932567a58c20dea5a1a0d18358503320 # v1 + with: + version: ${{ github.event.inputs.version }} + message: ${{ github.event.inputs.message }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/updoc.yml b/.github/workflows/updoc.yml deleted file mode 100644 index cdf2e52a8f..0000000000 --- a/.github/workflows/updoc.yml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Updoc - -on: - workflow_dispatch: {} - -jobs: - publish-docs: - uses: upbound/official-providers-ci/.github/workflows/provider-updoc.yml@standard-runners - with: - providers: "config" - go-version: 1.22 - secrets: - UPBOUND_CI_PROD_BUCKET_SA: ${{ secrets.UPBOUND_CI_PROD_BUCKET_SA }} diff --git a/.github/workflows/uptest-all.yml b/.github/workflows/uptest-all.yml deleted file mode 100644 index 19f1175305..0000000000 --- a/.github/workflows/uptest-all.yml +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2024 The Crossplane Authors -# -# SPDX-License-Identifier: CC0-1.0 - -name: Uptest All - -on: - workflow_dispatch: {} - -env: - # Common versions - GO_VERSION: '1.22' - DOCKER_BUILDX_VERSION: 'v0.8.2' - -jobs: - detect-api-groups: - runs-on: ubuntu-22.04 - outputs: - api_groups: ${{ steps.api-groups.outputs.list }} - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - submodules: true - - name: Detect Api Groups - id: api-groups - run: | - export API_GROUPS="" - for i in $(ls -d examples/*/); do API_GROUPS="${API_GROUPS},${i}"; done - - OUTPUT=$(python3 -c "\ - import os, sys; \ - value = os.getenv('API_GROUPS'); \ - list = value.split(','); \ - print(list[1:]); \ - ") - echo "Api Groups: $OUTPUT" - echo "list=${OUTPUT}" >> $GITHUB_OUTPUT - - uptest: - runs-on: ubuntu-22.04 - needs: detect-api-groups - strategy: - fail-fast: false - matrix: - api-groups: ${{ fromJson(needs.detect-api-groups.outputs.api_groups) }} - exclude: - - api-groups: examples/providerconfig/ - steps: - - name: Setup QEMU - uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v3 - with: - platforms: all - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@aa33708b10e362ff993539393ff100fa93ed6a27 # v3 - with: - version: ${{ env.DOCKER_BUILDX_VERSION }} - install: true - - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - submodules: true - - - name: Fetch History - run: git fetch --prune --unshallow - - - name: Setup Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Find the Go Build Cache - id: go - run: echo "cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT - - - name: Cache the Go Build Cache - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: ${{ steps.go.outputs.cache }} - key: ${{ runner.os }}-build-uptest-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-build-uptest- - - - name: Cache Go Dependencies - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4 - with: - path: .work/pkg - key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} - restore-keys: ${{ runner.os }}-pkg- - - - name: Vendor Dependencies - run: make vendor vendor.check - - - name: Build Artifacts - run: make -j2 build.all - env: - # We're using docker buildx, which doesn't actually load the images it - # builds by default. Specifying --load does so. - BUILD_ARGS: "--load" - - - name: Prepare The Example List - id: prepare-example-list - run: | - cd .. - EXAMPLE_LIST="" - EXAMPLE_LIST="${EXAMPLE_LIST},$(find ${{ matrix.api-groups }} -name '*.yaml' | tr '\n' ',')" - echo "example-list=${EXAMPLE_LIST:1}" >> $GITHUB_OUTPUT - echo "Examples: ${EXAMPLE_LIST:1}" - - - name: Run Uptest - id: run-uptest - env: - UPTEST_AWS_CREDS: ${{ secrets.UPTEST_AWS_CREDS }} - EXAMPLE_LIST: ${{ needs.get-example-list.outputs.example_list }} - UPTEST_EXAMPLE_VALUE_REPLACEMENTS: ${{ secrets.UPTEST_EXAMPLE_VALUE_REPLACEMENTS }} - DUMP_DIRECTORY: "./_output/cluster-dump" - run: make uptest - - - name: Collect Cluster Dump - if: always() - run: | - export DUMP_DIRECTORY=./_output/cluster-dump - make cluster_dump - - - name: Upload Cluster Dump - if: always() - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4 - with: - name: cluster-dump-aws - path: ./_output/cluster-dump - - - name: Cleanup - if: always() - run: | - kubectl delete managed --all - - report: - runs-on: ubuntu-22.04 - needs: uptest - if: always() - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - with: - submodules: true - - name: Manual Intervention Report - run: | - echo "Total Resources for AWS: $(cat examples/*/* | grep -o "kind: " | wc -l)" - echo "Number of Skipped Resources: $(grep -r -o "upjet.upbound.io/manual-intervention" examples/* | wc -l)" - echo "Skipped Manifests (Manual Intervention): $(grep -r -o "upjet.upbound.io/manual-intervention" examples/*)" - - name: Coverage Report - run: | - OUTPUT=$(gh api -H "Accept: application/vnd.github+json" /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs | jq '.jobs | .[] | select(.name!="detect-api-groups" and .name!="report") | .name + " -> " + .conclusion') - FAILURE=$(echo ${OUTPUT} | grep -o 'failure' | wc -l) - SUCCESS=$(echo ${OUTPUT} | grep -o 'success' | wc -l) - echo "Success: ${SUCCESS}" - echo "Failure: ${FAILURE}" - echo "${OUTPUT}" diff --git a/.github/workflows/uptest-trigger.yaml b/.github/workflows/uptest-trigger.yaml new file mode 100644 index 0000000000..f8af1938cb --- /dev/null +++ b/.github/workflows/uptest-trigger.yaml @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2024 The Crossplane Authors +# +# SPDX-License-Identifier: CC0-1.0 + +name: End to End Testing + +on: + issue_comment: + types: [created] + +env: + GO_VERSION: "1.22" + +jobs: + debug: + runs-on: ubuntu-latest + steps: + - name: Debug + run: | + echo "Trigger keyword: '/test-examples'" + echo "Go version: ${{ env.GO_VERSION }}" + echo "github.event.comment.author_association: ${{ github.event.comment.author_association }}" + + get-example-list: + if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'CONTRIBUTOR' ) && + github.event.issue.pull_request && + contains(github.event.comment.body, '/test-examples' ) }} + runs-on: ubuntu-latest + outputs: + example_list: ${{ steps.get-example-list-name.outputs.example-list }} + example_hash: ${{ steps.get-example-list-name.outputs.example-hash }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Checkout PR + id: checkout-pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr checkout ${{ github.event.issue.number }} + git submodule update --init --recursive + OUTPUT=$(git log -1 --format='%H') + echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Prepare The Example List + env: + COMMENT: ${{ github.event.comment.body }} + id: get-example-list-name + run: | + PATHS=$(echo $COMMENT | sed 's/^.*\/test-examples="//g' | cut -d '"' -f 1 | sed 's/,/ /g') + EXAMPLE_LIST="" + for P in $PATHS; do EXAMPLE_LIST="${EXAMPLE_LIST},$(find $P -name '*.yaml' | tr '\n' ',')"; done + + sudo apt-get -y install coreutils + COUNT=$(echo ${EXAMPLE_LIST:1} | grep -o ".yaml" | wc -l) + if [ $COUNT -gt 1 ]; then EXAMPLE_HASH=$(echo ${EXAMPLE_LIST} | md5sum | cut -f1 -d" "); else EXAMPLE_HASH=$(echo ${EXAMPLE_LIST:1} | sed 's/.$//'); fi + + echo "Examples: ${EXAMPLE_LIST:1}" + echo "Example Hash: ${EXAMPLE_HASH}" + + echo "example-list=${EXAMPLE_LIST:1}" >> $GITHUB_OUTPUT + echo "example-hash=${EXAMPLE_HASH}" >> $GITHUB_OUTPUT + + - name: Create Pending Status Check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='pending' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Running...' \ + -f context="Uptest-${{ steps.get-example-list-name.outputs.example-hash }}" + + uptest: + if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'CONTRIBUTOR' ) && + github.event.issue.pull_request && + contains(github.event.comment.body, '/test-examples' ) }} + runs-on: ubuntu-latest + needs: get-example-list + + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + large-packages: false + swap-storage: false + + - name: Setup QEMU + uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0 + with: + platforms: all + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Checkout PR + id: checkout-pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr checkout ${{ github.event.issue.number }} + git submodule update --init --recursive + OUTPUT=$(git log -1 --format='%H') + echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Run Uptest + id: run-uptest + env: + UPTEST_CLOUD_CREDENTIALS: ${{ secrets.UPTEST_CLOUD_CREDENTIALS }} + UPTEST_EXAMPLE_LIST: ${{ needs.get-example-list.outputs.example_list }} + UPTEST_TEST_DIR: ./_output/controlplane-dump + UPTEST_DATASOURCE_PATH: .work/uptest-datasource.yaml + UPTEST_UPDATE_PARAMETER: "" + run: | + mkdir -p .work && echo "${{ secrets.UPTEST_DATASOURCE }}" > .work/uptest-datasource.yaml + make e2e + + - name: Create Successful Status Check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='success' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Passed' \ + -f context="Uptest-${EXAMPLE_HASH}" + + - name: Collect Cluster Dump + if: always() + run: | + make controlplane.dump + + - name: Upload Cluster Dump + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: controlplane-dump + path: ./_output/controlplane-dump + + - name: Cleanup + if: always() + run: | + eval $(make --no-print-directory build.vars) + ${KUBECTL} delete managed --all || true + + - name: Create Unsuccessful Status Check + if: failure() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='failure' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Failed' \ + -f context="Uptest-${EXAMPLE_HASH}"