diff --git a/.buildkite/package-lock.json b/.buildkite/package-lock.json index e9017b545e468..66e6b85a1ab4c 100644 --- a/.buildkite/package-lock.json +++ b/.buildkite/package-lock.json @@ -13,12 +13,14 @@ "globby": "^11.1.0", "js-yaml": "^4.1.0", "minimatch": "^5.0.1", + "minimist": "^1.2.8", "tslib": "*" }, "devDependencies": { "@types/chai": "^4.3.3", "@types/js-yaml": "^4.0.9", "@types/minimatch": "^3.0.5", + "@types/minimist": "^1.2.5", "@types/mocha": "^10.0.1", "@types/node": "^15.12.2", "chai": "^4.3.10", @@ -365,6 +367,12 @@ "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", "dev": true }, + "node_modules/@types/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", + "dev": true + }, "node_modules/@types/mocha": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz", @@ -1224,6 +1232,14 @@ "node": ">=10" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -2226,6 +2242,12 @@ "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", "dev": true }, + "@types/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", + "dev": true + }, "@types/mocha": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz", @@ -2841,6 +2863,11 @@ "brace-expansion": "^2.0.1" } }, + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" + }, "minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", diff --git a/.buildkite/package.json b/.buildkite/package.json index 3c7fe28d0c064..521c72a950d9e 100644 --- a/.buildkite/package.json +++ b/.buildkite/package.json @@ -15,12 +15,14 @@ "globby": "^11.1.0", "js-yaml": "^4.1.0", "minimatch": "^5.0.1", + "minimist": "^1.2.8", "tslib": "*" }, "devDependencies": { "@types/chai": "^4.3.3", "@types/js-yaml": "^4.0.9", "@types/minimatch": "^3.0.5", + "@types/minimist": "^1.2.5", "@types/mocha": "^10.0.1", "@types/node": "^15.12.2", "chai": "^4.3.10", diff --git a/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml b/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml new file mode 100644 index 0000000000000..0453d137558ca --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-deploy-cloud.yml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-deploy-cloud-from-pr + description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR' + links: + - url: 'https://buildkite.com/elastic/kibana-deploy-cloud-from-pr' + title: Pipeline link +spec: + type: buildkite-pipeline + system: buildkite + owner: 'group:kibana-operations' + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / deploy cloud from PR + description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR' + spec: + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' + + allow_rebuilds: false + branch_configuration: main + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/build_pr_and_deploy_cloud.yml + provider_settings: + build_pull_requests: true + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: true + trigger_mode: none + cancel_intermediate_builds: true + teams: + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + tags: + - kibana diff --git a/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml b/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml index f098ff82f322f..dddc0f974b213 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml @@ -25,7 +25,7 @@ spec: branch_configuration: main default_branch: main repository: elastic/kibana - pipeline_file: .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml + pipeline_file: .buildkite/pipelines/serverless_deployment/build_pr_and_deploy_project.yml provider_settings: build_pull_requests: true prefix_pull_request_fork_branch_names: false diff --git a/.buildkite/pipeline-utils/github/github.ts b/.buildkite/pipeline-utils/github/github.ts index eb9a240386bbc..a201457b09ede 100644 --- a/.buildkite/pipeline-utils/github/github.ts +++ b/.buildkite/pipeline-utils/github/github.ts @@ -9,6 +9,8 @@ import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; +export const KIBANA_COMMENT_SIGIL = 'kbn-message-context'; + const github = new Octokit({ auth: process.env.GITHUB_TOKEN, }); @@ -113,6 +115,56 @@ export function addComment( }); } +export async function upsertComment( + messageOpts: { + commentBody: string; + commentContext: string; + clearPrevious: boolean; + }, + owner = process.env.GITHUB_PR_BASE_OWNER, + repo = process.env.GITHUB_PR_BASE_REPO, + prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER +) { + const { commentBody, commentContext, clearPrevious } = messageOpts; + if (!owner || !repo || !prNumber) { + throw Error( + "Couldn't retrieve Github PR info from environment variables in order to add a comment" + ); + } + if (!commentContext) { + throw Error('Comment context is required when updating a comment'); + } + + const commentMarker = ``; + const body = `${commentMarker}\n${commentBody}`; + + const existingComment = ( + await github.paginate(github.issues.listComments, { + owner, + repo, + issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10), + }) + ).find((comment) => comment.body?.includes(commentMarker)); + + if (!existingComment) { + return addComment(body, owner, repo, prNumber); + } else if (clearPrevious) { + await github.issues.deleteComment({ + owner, + repo, + comment_id: existingComment.id, + }); + return addComment(body, owner, repo, prNumber); + } else { + return github.issues.updateComment({ + owner, + repo, + comment_id: existingComment.id, + body, + }); + } +} + export function getGithubClient() { return github; } diff --git a/.buildkite/pipelines/build_pr_and_deploy_cloud.yml b/.buildkite/pipelines/build_pr_and_deploy_cloud.yml new file mode 100644 index 0000000000000..0db8196c65d09 --- /dev/null +++ b/.buildkite/pipelines/build_pr_and_deploy_cloud.yml @@ -0,0 +1,82 @@ +env: + ELASTIC_PR_COMMENTS_ENABLED: 'true' + ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true' + GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-cloud-from-pr + +steps: + - group: 'Cloud Deployment' + if: "build.env('GITHUB_PR_LABELS') =~ /(ci:cloud-deploy|ci:cloud-redeploy)/" + + steps: + - command: .buildkite/scripts/lifecycle/pre_build.sh + label: Pre-Build + timeout_in_minutes: 10 + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + retry: + automatic: + - exit_status: '*' + limit: 1 + + - command: | + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \ + --message "PR Cloud deployment started at: $BUILDKITE_BUILD_URL" \ + --context "cloud-deploy-job" \ + --clear-previous + label: Comment with job URL + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + timeout_in_minutes: 5 + + - command: .buildkite/scripts/steps/build_kibana.sh + label: Build Kibana Distribution + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-8 + preemptible: true + diskSizeGb: 125 + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + timeout_in_minutes: 90 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - wait: ~ + + - command: .buildkite/scripts/steps/cloud/build_and_deploy.sh + label: 'Build and Deploy to Cloud' + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + preemptible: true + timeout_in_minutes: 30 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - wait: ~ + + - command: | + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \ + --message "Cloud deployment initiated, see credentials at: $BUILDKITE_BUILD_URL" \ + --context "cloud-deploy-job" \ + --clear-previous + label: Comment with job URL + agents: + provider: gcp + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + machineType: n2-standard-2 + timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml b/.buildkite/pipelines/serverless_deployment/build_pr_and_deploy_project.yml similarity index 87% rename from .buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml rename to .buildkite/pipelines/serverless_deployment/build_pr_and_deploy_project.yml index cfc6e1dd451d9..d42aae49fa82a 100644 --- a/.buildkite/pipelines/serverless_deployment/project-build-and-deploy-pr.yml +++ b/.buildkite/pipelines/serverless_deployment/build_pr_and_deploy_project.yml @@ -22,7 +22,10 @@ steps: limit: 1 - command: | - ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "PR Project deployment started at: $BUILDKITE_BUILD_URL" + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \ + --message "PR Project deployment started at: $BUILDKITE_BUILD_URL" \ + --context "project-deploy-job" \ + --clear-previous label: Comment with job URL agents: provider: gcp @@ -62,7 +65,10 @@ steps: - wait: ~ - command: | - ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts "Project deployed, see credentials at: $BUILDKITE_BUILD_URL" + ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \ + --message "Project deployed, see credentials at: $BUILDKITE_BUILD_URL" \ + --context "project-deploy-job" \ + --clear-previous label: Comment with job URL agents: provider: gcp diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json index e88982ec00d9d..d0d8e8a561b3f 100644 --- a/.buildkite/pull_requests.json +++ b/.buildkite/pull_requests.json @@ -70,6 +70,31 @@ "/__snapshots__/", "\\.test\\.(ts|tsx|js|jsx)" ] + }, + { + "repoOwner": "elastic", + "repoName": "kibana", + "pipelineSlug": "kibana-deploy-cloud-from-pr", + "skip_ci_labels": [], + "enabled": true, + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": ["elastic-vault-github-plugin-prod[bot]"], + "set_commit_status": true, + "commit_status_context": "kibana-deploy-cloud-from-pr", + "build_on_commit": false, + "build_on_comment": true, + "build_drafts": false, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:cloud))$", + "kibana_versions_check": true, + "kibana_build_reuse": true, + "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-cloud-from-pr"], + "kibana_build_reuse_regexes": [ + "^test/", + "^x-pack/test/", + "/__snapshots__/", + "\\.test\\.(ts|tsx|js|jsx)" + ] } ] } diff --git a/.buildkite/scripts/lifecycle/comment_on_pr.ts b/.buildkite/scripts/lifecycle/comment_on_pr.ts index 39ebd511d8410..f44f6330c121c 100644 --- a/.buildkite/scripts/lifecycle/comment_on_pr.ts +++ b/.buildkite/scripts/lifecycle/comment_on_pr.ts @@ -7,7 +7,8 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { addComment } from '#pipeline-utils'; +import parseArgs from 'minimist'; +import { upsertComment, addComment } from '#pipeline-utils'; const ALLOWED_ENV_VARS = [ 'BUILDKITE_BRANCH', @@ -31,20 +32,16 @@ const ALLOWED_ENV_VARS = [ 'GITHUB_PR_TRIGGER_USER', 'GITHUB_PR_USER', ]; -const DEFAULT_MESSAGE_TEMPLATE = - '🚀 Buildkite job started for PR #${GITHUB_PR_NUMBER}: ${BUILDKITE_BUILD_URL}'; - -export function commentOnPR() { - const messageTemplate = - process.argv.slice(2)?.join(' ') || - process.env.JOB_START_COMMENT_TEMPLATE || - DEFAULT_MESSAGE_TEMPLATE; - if (messageTemplate === DEFAULT_MESSAGE_TEMPLATE) { - console.log('No message template provided, using default message'); - } else { - console.log(`Using message template: ${messageTemplate}`); - } +export function commentOnPR({ + messageTemplate, + context, + clearPrevious, +}: { + messageTemplate: string; + context?: string; + clearPrevious: boolean; +}) { const message = messageTemplate.replace(/\${([^}]+)}/g, (_, envVar) => { if (ALLOWED_ENV_VARS.includes(envVar)) { return process.env[envVar] || ''; @@ -53,11 +50,39 @@ export function commentOnPR() { } }); - return addComment(message); + if (context) { + return upsertComment({ commentBody: message, commentContext: context, clearPrevious }); + } else { + return addComment(message); + } } if (require.main === module) { - commentOnPR().catch((error) => { + const args = parseArgs<{ + context?: string; + message: string; + 'clear-previous'?: boolean | string; + }>(process.argv.slice(2), { + string: ['message', 'context'], + boolean: ['clear-previous'], + }); + + if (!args.message) { + throw new Error( + `No message template provided for ${process.argv[1]}, use --message to provide one.` + ); + } else { + console.log(`Using message template: ${args.message}`); + } + + commentOnPR({ + messageTemplate: args.message, + context: args.context, + clearPrevious: + typeof args['clear-previous'] === 'string' + ? !!args['clear-previous'].match(/(1|true)/i) + : !!args['clear-previous'], + }).catch((error) => { console.error(error); process.exit(1); });