From 0671abddba7b5b25542948729bd941aaf7956f6c Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 8 Nov 2022 09:51:35 +0100 Subject: [PATCH 1/5] Create PRs --- .../utils/config_migration.py | 37 ----- .../strictness_level_migration/README.md | 28 ++++ .../config_migration.py | 58 +++++++ .../create_issues.py | 1 - .../strictness_level_migration/create_prs.py | 144 ++++++++++++++++++ .../requirements.txt | 62 +++++++- .../templates/pr.md.j2 | 17 +++ 7 files changed, 308 insertions(+), 39 deletions(-) delete mode 100644 airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/config_migration.py create mode 100644 airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/config_migration.py create mode 100644 airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py create mode 100644 airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/config_migration.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/config_migration.py deleted file mode 100644 index b0a4f4f4ee153..0000000000000 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/config_migration.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import argparse -from pathlib import Path - -import yaml -from source_acceptance_test.config import Config -from yaml import load - -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader - -parser = argparse.ArgumentParser(description="Migrate legacy acceptance-test-config.yml to the latest configuration format.") -parser.add_argument("config_path", type=str, help="Path to the acceptance-test-config.yml to migrate.") - - -def migrate_legacy_configuration(config_path: Path): - - with open(config_path, "r") as file: - to_migrate = load(file, Loader=Loader) - - if Config.is_legacy(to_migrate): - migrated_config = Config.migrate_legacy_to_current_config(to_migrate) - with open(config_path, "w") as output_file: - yaml.dump(migrated_config, output_file) - print(f"Your configuration was successfully migrated to the latest configuration format: {config_path}") - else: - print("Your configuration is not in a legacy format.") - - -if __name__ == "__main__": - args = parser.parse_args() - migrate_legacy_configuration(Path(args.config_path)) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/README.md b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/README.md index 84373d328955d..5d229f7508aa9 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/README.md +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/README.md @@ -34,3 +34,31 @@ Issues are added to the following project: `SAT-high-test-strictness-level` **Real execution**: `python create_issues.py --dry False` +## Create migration PRs for GA connectors (`create_prs.py`) +This script will create one PR per GA connectors to migrate to `high` test strictness level. + +### What it does: +1. Iterate on all GA connectors in `../../../../../airbyte-config/init/src/main/resources/seed/source_definitions.yaml` +2. Create a branch for each GA connector +3. Locally migrate `acceptance_test_config.yml` to the latest format +4. Commit and push the changes on this branch +5. Open a PR for this branch +6. Run a SAT on this branch by posting a `/test` comment on the PR + +An example of the PR it creates can be found [here](https://github.com/airbytehq/airbyte/pull/19136) + +PR get created with the following labels: +* `area/connectors` +* `team/connectors-python` +* `type/enhancement` +* `test-strictness-level` + +PR are added to the following project: `SAT-high-test-strictness-level` + +### How to run: +**Dry run**: +`python create_prs.py` + +**Real execution**: +`python create_prs.py --dry False` + diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/config_migration.py b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/config_migration.py new file mode 100644 index 0000000000000..55f68267ead00 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/config_migration.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import argparse +import logging +from pathlib import Path + +import yaml +from source_acceptance_test.config import Config +from yaml import load + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +parser = argparse.ArgumentParser(description="Migrate legacy acceptance-test-config.yml to the latest configuration format.") +parser.add_argument("config_path", type=str, help="Path to the acceptance-test-config.yml to migrate.") + + +def get_new_config_format(config_path: Path): + + with open(config_path, "r") as file: + to_migrate = load(file, Loader=Loader) + + if Config.is_legacy(to_migrate): + return Config.migrate_legacy_to_current_config(to_migrate) + else: + logging.warn("The configuration is not in a legacy format.") + return to_migrate + + +def set_high_test_strictness_level(config): + config["test_strictness_level"] = "high" + for basic_read_test in config["acceptance_tests"].get("basic_read", {"tests": []})["tests"]: + basic_read_test.pop("configured_catalog_path", None) + return config + + +def write_new_config(new_config, output_path): + with open(output_path, "w") as output_file: + yaml.dump(new_config, output_file) + logging.info("Saved the configuration in its new format") + + +def migrate_configuration(config_path): + new_config = get_new_config_format(config_path) + new_config = set_high_test_strictness_level(new_config) + write_new_config(new_config, config_path) + logging.info(f"The configuration was successfully migrated to the latest configuration format: {config_path}") + return config_path + + +if __name__ == "__main__": + args = parser.parse_args() + config_path = Path(args.config_path) + migrate_configuration(config_path) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_issues.py b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_issues.py index aa15a3b46d56f..25cd305ba4730 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_issues.py +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_issues.py @@ -32,7 +32,6 @@ def get_issue_content(source_definition): file_definition, issue_body_path = tempfile.mkstemp() with os.fdopen(file_definition, "w") as tmp: - # do stuff with temp file tmp.write(issue_body) return {"title": issue_title, "body_file": issue_body_path, "labels": COMMON_ISSUE_LABELS} diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py new file mode 100644 index 0000000000000..183e701b2a395 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py @@ -0,0 +1,144 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# +import argparse +import json +import logging +import os +import subprocess +import tempfile +from pathlib import Path + +from config_migration import migrate_configuration +from create_issues import COMMON_ISSUE_LABELS as COMMON_PR_LABELS +from create_issues import GITHUB_PROJECT_NAME +from definitions import GA_DEFINITIONS +from git import Repo +from jinja2 import Environment, FileSystemLoader + +CONNECTORS_DIRECTORY = "../../../../connectors" +REPO_ROOT = "../../../../../" +AIRBYTE_REPO = Repo(REPO_ROOT) +environment = Environment(loader=FileSystemLoader("./templates/")) +PR_TEMPLATE = environment.get_template("pr.md.j2") + +parser = argparse.ArgumentParser(description="Create PRs for migration of GA connectors to high test strictness level in SAT") +parser.add_argument("-d", "--dry", default=True) + + +logging.basicConfig(level=logging.DEBUG) + + +def migrate_acceptance_test_config(connector_name): + acceptance_test_config_path = Path(CONNECTORS_DIRECTORY) / connector_name / "acceptance-test-config.yml" + return migrate_configuration(acceptance_test_config_path) + + +def checkout_new_branch(connector_name): + AIRBYTE_REPO.heads.master.checkout() + new_branch_name = f"{connector_name}/sat/migrate-to-high-test-strictness-level" + new_branch = AIRBYTE_REPO.create_head(new_branch_name) + new_branch.checkout() + return new_branch + + +def commit_push_migrated_config(config_path, connector_name, new_branch, dry_run): + process = subprocess.Popen(["pre-commit", "run", "--files", config_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.communicate() + relative_config_path = f"airbyte-integrations/connectors/{connector_name}/acceptance-test-config.yml" + AIRBYTE_REPO.git.add(relative_config_path) + AIRBYTE_REPO.git.commit(m=f"Migrated config for {connector_name}") + logging.info(f"Committed migrated config on {new_branch}") + if not dry_run: + AIRBYTE_REPO.git.push("--set-upstream", "origin", new_branch) + logging.info(f"Pushed branch {new_branch} to origin") + + +def get_pr_content(definition): + pr_title = f"Source {definition['name']}: enable `high` test strictness level in SAT" + + pr_body = PR_TEMPLATE.render(connector_name=definition["name"], release_stage=definition["releaseStage"]) + file_definition, pr_body_path = tempfile.mkstemp() + + with os.fdopen(file_definition, "w") as tmp: + tmp.write(pr_body) + + return {"title": pr_title, "body_file": pr_body_path, "labels": COMMON_PR_LABELS} + + +def open_pr(definition, new_branch, dry_run): + pr_content = get_pr_content(definition) + list_command_arguments = ["gh", "pr", "list", "--state", "open", "--head", new_branch.name, "--json", "url"] + create_command_arguments = [ + "gh", + "pr", + "create", + "--draft", + "--title", + pr_content["title"], + "--body-file", + pr_content["body_file"], + "--project", + GITHUB_PROJECT_NAME, + ] + for label in pr_content["labels"]: + create_command_arguments += ["--label", label] + list_existing_pr_process = subprocess.Popen(list_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = list_existing_pr_process.communicate() + existing_prs = json.loads(stdout.decode()) + already_created = len(existing_prs) > 0 + if already_created: + logging.warning(f"A PR was already created for this definition: {existing_prs[0]}") + if not already_created: + if not dry_run: + process = subprocess.Popen(create_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stderr: + logging.error(stderr.decode()) + else: + created_pr_url = stdout.decode() + logging.info(f"Created PR for {definition['name']}: {created_pr_url}") + else: + logging.info(f"[DRY RUN]: {' '.join(create_command_arguments)}") + os.remove(pr_content["body_file"]) + + +def add_test_comment(definition, new_branch, dry_run): + connector_name = definition["dockerRepository"].replace("airbyte/", "") + comment = f"/test connector=connectors/{connector_name}" + comment_command_arguments = ["gh", "pr", "comment", new_branch.name, "--body", comment] + if not dry_run: + process = subprocess.Popen(comment_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = process.communicate() + if stderr: + logging.error(stderr.decode()) + else: + logging.info("Added test comment") + else: + logging.info(f"[DRY RUN]: {' '.join(comment_command_arguments)}") + + +def migrate_config_on_new_branch(definition, dry_run): + AIRBYTE_REPO.heads.master.checkout() + connector_name = definition["dockerRepository"].replace("airbyte/", "") + new_branch = checkout_new_branch(connector_name) + config_path = migrate_acceptance_test_config(connector_name) + commit_push_migrated_config(config_path, connector_name, new_branch, dry_run) + return new_branch + + +def migrate_definition_and_open_pr(definition, dry_run): + original_branch = AIRBYTE_REPO.active_branch + new_branch = migrate_config_on_new_branch(definition, dry_run) + open_pr(definition, new_branch, dry_run) + add_test_comment(definition, new_branch, dry_run) + original_branch.checkout() + AIRBYTE_REPO.git.branch(D=new_branch) + logging.info(f"Deleted branch {new_branch}") + + +if __name__ == "__main__": + args = parser.parse_args() + dry_run = False if args.dry == "False" or args.dry == "false" else True + for definition in GA_DEFINITIONS[:1]: + migrate_definition_and_open_pr(definition, dry_run=dry_run) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/requirements.txt b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/requirements.txt index 046f20eb6cc26..b30299006be8a 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/requirements.txt +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/requirements.txt @@ -1,4 +1,64 @@ +airbyte-cdk==0.7.0 +appdirs==1.4.4 +attrs==22.1.0 +backoff==2.2.1 +cattrs==22.2.0 +certifi==2022.9.24 +charset-normalizer==2.1.1 +coverage==6.5.0 +dataclasses-jsonschema==2.15.1 +deepdiff==5.8.1 +Deprecated==1.2.13 +docker==5.0.3 +dpath==2.0.6 +exceptiongroup==1.0.1 +fancycompleter==0.9.1 +gitdb==4.0.9 +GitPython==3.1.29 +hypothesis==6.54.6 +hypothesis-jsonschema==0.20.1 +icdiff==1.9.1 +idna==3.4 +inflection==0.5.1 +iniconfig==1.1.1 Jinja2==3.1.2 +jsonref==0.2 +jsonschema==3.2.0 MarkupSafe==2.1.1 +ordered-set==4.1.0 +packaging==21.3 +pdbpp==0.10.3 +pendulum==2.1.2 +pluggy==1.0.0 +pprintpp==0.4.0 +py==1.11.0 pyaml==21.10.1 -PyYAML==6.0 +pydantic==1.9.2 +Pygments==2.13.0 +pyparsing==3.0.9 +pyrepl==0.9.0 +pyrsistent==0.19.2 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-mock==3.6.1 +pytest-sugar==0.9.6 +pytest-timeout==1.4.2 +python-dateutil==2.8.2 +pytzdata==2020.1 +PyYAML==5.4.1 +requests==2.28.1 +requests-cache==0.9.7 +requests-mock==1.9.3 +six==1.16.0 +smmap==5.0.0 +sortedcontainers==2.4.0 +-e git+ssh://git@github.com/airbytehq/airbyte.git@70679775b55c5bb1be7384114155924772885be0#egg=source_acceptance_test&subdirectory=airbyte-integrations/bases/source-acceptance-test +termcolor==2.1.0 +toml==0.10.2 +tomli==2.0.1 +typing_extensions==4.4.0 +url-normalize==1.4.3 +urllib3==1.26.12 +websocket-client==1.4.2 +wmctrl==0.4 +wrapt==1.14.1 diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 new file mode 100644 index 0000000000000..da0c877331028 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 @@ -0,0 +1,17 @@ +## What +A `test_strictness_level` field was introduced to Source Acceptance Tests (SAT). +{{ connector_name }} is a {{ release_stage }} connector, we want it to have a `high` test strictness level. + +**This will help**: +- maximize the SAT coverage on this connector. +- document its potential weaknesses in term of test coverage. + +## How +1. Migrate the existing `acceptance-test-config.yml` file to the latest configuration format. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/source-acceptance-test/README.md#L61)) +2. Enable `high` test strictness level in `acceptance-test-config.yml`. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md#L240)) + +**If tests are failing please fix the failing test by changing the `acceptance-test-config.yml` file or use `bypass_reason` fields to explain why a specific test can't be run.** +Please open a new PR ff the new enabled tests help discover a new bug. +Once this bug fix is merged please rebase this branch and run `/test` again. + +You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/). \ No newline at end of file From d77741f8b2dad3c062853fd95833adf59dab7fe3 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 8 Nov 2022 20:55:02 +0100 Subject: [PATCH 2/5] clean --- .../tools/strictness_level_migration/templates/pr.md.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 index da0c877331028..5890f7dee41b5 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 @@ -14,4 +14,4 @@ A `test_strictness_level` field was introduced to Source Acceptance Tests (SAT). Please open a new PR ff the new enabled tests help discover a new bug. Once this bug fix is merged please rebase this branch and run `/test` again. -You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/). \ No newline at end of file +You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/). From 27152382bfdec479462d2bc1e6cbdcd036b402d5 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 8 Nov 2022 21:02:01 +0100 Subject: [PATCH 3/5] clean --- airbyte-integrations/bases/source-acceptance-test/README.md | 2 +- .../tools/strictness_level_migration/templates/pr.md.j2 | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/README.md b/airbyte-integrations/bases/source-acceptance-test/README.md index b527589398f0d..d240d26387fc8 100644 --- a/airbyte-integrations/bases/source-acceptance-test/README.md +++ b/airbyte-integrations/bases/source-acceptance-test/README.md @@ -66,5 +66,5 @@ To migrate a legacy configuration to the latest configuration format please run: ```bash python -m venv .venv # If you don't have a virtualenv already source ./.venv/bin/activate # If you're not in your virtualenv already -python source_acceptance_test/utils/config_migration.py ../../connectors/source-to-migrate/acceptance-test-config.yml +python source_acceptance_test/tools/strictness_level_migration/config_migration.py ../../connectors/source-to-migrate/acceptance-test-config.yml ``` \ No newline at end of file diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 index 5890f7dee41b5..347bc6cb41161 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 @@ -15,3 +15,6 @@ Please open a new PR ff the new enabled tests help discover a new bug. Once this bug fix is merged please rebase this branch and run `/test` again. You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/). + +## Review process +Please ask the `connector-operations` teams for review. \ No newline at end of file From 3461db4416c92b9cd111fdcd79f36750cb592232 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 8 Nov 2022 21:04:13 +0100 Subject: [PATCH 4/5] update PR template --- .../tools/strictness_level_migration/templates/pr.md.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 index 347bc6cb41161..7b8c52abe710e 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/templates/pr.md.j2 @@ -10,8 +10,10 @@ A `test_strictness_level` field was introduced to Source Acceptance Tests (SAT). 1. Migrate the existing `acceptance-test-config.yml` file to the latest configuration format. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/source-acceptance-test/README.md#L61)) 2. Enable `high` test strictness level in `acceptance-test-config.yml`. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md#L240)) +⚠️ ⚠️ ⚠️ **If tests are failing please fix the failing test by changing the `acceptance-test-config.yml` file or use `bypass_reason` fields to explain why a specific test can't be run.** -Please open a new PR ff the new enabled tests help discover a new bug. + +Please open a new PR if the new enabled tests help discover a new bug. Once this bug fix is merged please rebase this branch and run `/test` again. You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference/). From 7e198186c6cfc6760ec9310241fb816fe3b33104 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 8 Nov 2022 22:09:21 +0100 Subject: [PATCH 5/5] format --- .../tools/strictness_level_migration/create_prs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py index 183e701b2a395..86004efb55d3d 100644 --- a/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py +++ b/airbyte-integrations/bases/source-acceptance-test/tools/strictness_level_migration/create_prs.py @@ -1,6 +1,7 @@ # # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + import argparse import json import logging