diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 551681633ed7c..d935c78ae0f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -637,9 +637,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Static checks" run: breeze static-checks --all-files --show-diff-on-failure --color always env: - VERBOSE: false + VERBOSE: "false" SKIP: "identity" - COLUMNS: 250 + COLUMNS: "250" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -654,8 +654,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" needs: [build-info] env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - SKIP: "build,run-mypy,run-flake8,lint-javascript,update-migration-references,identity" - MOUNT_SELECTED_LOCAL_SOURCES: "true" if: needs.build-info.outputs.basic-checks-only == 'true' steps: - name: Cleanup repo @@ -695,7 +693,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze static-checks --all-files --show-diff-on-failure --color always --commit-ref "${{ github.sha }}" env: - VERBOSE: false + VERBOSE: "false" + SKIP_IMAGE_PRE_COMMITS: "true" + SKIP: "identity" + COLUMNS: "250" - name: "Fix ownership" run: breeze fix-ownership if: always() diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 71f132e7d5d10..e2b57eae6c69b 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -111,8 +111,8 @@ For details on advanced usage of the install method, use: Available pre-commit checks ........................... -This table lists pre-commit hooks used by Airflow. The ``Breeze`` column indicates which hooks -require Breeze Docker images to be installed locally. +This table lists pre-commit hooks used by Airflow. The ``Image`` column indicates which hooks +require Breeze Docker image to be build locally. .. note:: Disabling particular checks @@ -123,6 +123,10 @@ require Breeze Docker images to be installed locally. ``export SKIP=run-flake8,run-mypy``. You can also add this to your ``.bashrc`` or ``.zshrc`` if you do not want to set it manually every time you enter the terminal. + In case you do not have breeze image configured locally, you can also disable all checks that require + the image by setting ``SKIP_IMAGE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically + when run locally (note that those checks will anyway run in CI). + .. BEGIN AUTO-GENERATED STATIC CHECK LIST +--------------------------------------------------------+------------------------------------------------------------------+---------+ @@ -242,7 +246,7 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-helm-chart | Lint Helm Chart | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| lint-javascript | * ESLint against airflow/ui | | +| lint-javascript | * ESLint against airflow/ui | * | | | * ESLint against current UI JavaScript files | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | lint-json-schema | * Lint JSON Schema files with JSON Schema | | @@ -269,9 +273,9 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | rst-backticks | Check if RST files use double backticks for code | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| run-flake8 | Run flake8 | | +| run-flake8 | Run flake8 | * | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| run-mypy | * Run mypy for dev | | +| run-mypy | * Run mypy for dev | * | | | * Run mypy for core | | | | * Run mypy for providers | | | | * Run mypy for /docs/ folder | | @@ -294,7 +298,7 @@ require Breeze Docker images to be installed locally. +--------------------------------------------------------+------------------------------------------------------------------+---------+ | update-local-yml-file | Update mounts in the local yml file | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ -| update-migration-references | Update migration ref doc | | +| update-migration-references | Update migration ref doc | * | +--------------------------------------------------------+------------------------------------------------------------------+---------+ | update-providers-dependencies | Update cross-dependencies for providers packages | | +--------------------------------------------------------+------------------------------------------------------------------+---------+ @@ -387,49 +391,63 @@ Run the ``mypy`` check for the currently staged changes: .. code-block:: bash - breeze static-check --type run-mypy + breeze static-checks --type run-mypy Run the ``mypy`` check for all files: .. code-block:: bash - breeze static-check --type run-mypy --all-files + breeze static-checks --type run-mypy --all-files Run the ``flake8`` check for the ``tests.core.py`` file with verbose output: .. code-block:: bash - breeze static-check --type run-flake8 --file tests/core.py --verbose + breeze static-checks --type run-flake8 --file tests/core.py --verbose Run the ``flake8`` check for the ``tests.core`` package with verbose output: .. code-block:: bash - breeze static-check --type run-flake8 --file tests/core/* --verbose + breeze static-checks --type run-flake8 --file tests/core/* --verbose Run all tests for the currently staged files: .. code-block:: bash - breeze static-check --type all + breeze static-checks --type all Run all tests for all files: .. code-block:: bash - breeze static-check --type all --all-files + breeze static-checks --type all --all-files Run all tests for last commit : .. code-block:: bash - breeze static-check --type all --last-commit + breeze static-checks -type all --last-commit + +Debugging pre-commit check scripts requiring image +-------------------------------------------------- + +Those commits that use Breeze docker image might sometimes fail, depending on your operating system and +docker setup, so sometimes it might be required to run debugging with the commands. This is done via +two environment variables ``VERBOSE`` and ``DRY_RUN``. Setting them to "true" will respectively show the +commands to run before running them or skip running the commands. + +Note that you need to run pre-commit with --verbose command to get the output regardless of the status +of the static check (normally it will only show output on failure). + +Printing the commands while executing: + +.. code-block:: bash + VERBOSE="true" pre-commit run --verbose run-flake8 -The ``license`` check is run via a separate script and a separate Docker image containing the -Apache RAT verification tool that checks for Apache-compatibility of licenses within the codebase. -It does not take pre-commit parameters as extra arguments. +Just performing dry run: .. code-block:: bash - breeze static-check licenses + DRY_RUN="true" pre-commit run --verbose run-flake8 diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index bb1d974e7ce9e..1719bab5fc7a8 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -438,7 +438,7 @@ def build_docs( type=BetterChoice(PRE_COMMIT_LIST), multiple=True, ) -@click.option('-a', '--all-files', help="Run checks on all files.") +@click.option('-a', '--all-files', help="Run checks on all files.", is_flag=True) @click.option('-f', '--file', help="List of files to run the checks on.", type=click.Path(), multiple=True) @click.option( '-s', '--show-diff-on-failure', help="Show diff for files modified by the checks.", is_flag=True diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 49d9d3bf9ceef..759c3e09a7d6a 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -62,13 +62,7 @@ def in_help() -> bool: def skip_upgrade_check(): - return ( - in_self_upgrade() - or in_autocomplete() - or in_help() - or hasattr(sys, '_called_from_test') - or os.environ.get('SKIP_BREEZE_UPGRADE_CHECK') - ) + return in_self_upgrade() or in_autocomplete() or in_help() or hasattr(sys, '_called_from_test') def get_package_setup_metadata_hash() -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index bbfdfcb4326c4..86b84be4c076f 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -27,6 +27,7 @@ from re import match from typing import Dict, List, Mapping, Optional, Union +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import get_console @@ -375,3 +376,32 @@ def filter_out_none(**kwargs) -> dict: if kwargs[key] is None: kwargs.pop(key) return kwargs + + +def fail_if_image_missing(image: str, verbose: bool, dry_run: bool, instruction: str) -> None: + skip_image_pre_commits = os.environ.get('SKIP_IMAGE_PRE_COMMITS', "false") + if skip_image_pre_commits[0].lower() == "t": + get_console().print( + f"[info]Skipping image check as SKIP_IMAGE_PRE_COMMITS is set to {skip_image_pre_commits}[/]" + ) + sys.exit(0) + cmd_result = run_command( + ["docker", "inspect", image], stdout=subprocess.DEVNULL, check=False, verbose=verbose, dry_run=dry_run + ) + if cmd_result.returncode != 0: + print(f'[red]The image {image} is not available.[/]\n') + print(f"\n[yellow]Please run at the earliest convenience:[/]\n\n{instruction}\n\n") + sys.exit(1) + + +def get_runnable_ci_image(verbose: bool, dry_run: bool) -> str: + github_repository = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") + python_version = "3.7" + airflow_image = f"ghcr.io/{github_repository}/{AIRFLOW_BRANCH}/ci/python{python_version}" + fail_if_image_missing( + image=airflow_image, + verbose=verbose, + dry_run=dry_run, + instruction=f"breeze build-image --python {python_version}", + ) + return airflow_image diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 4e134adeefe01..4d58a2969e4be 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -5b79e317c30d2dc10bc28f425557af1b +969f1b4101773456e832c03b8d4e89ef diff --git a/images/breeze/output-static-checks.svg b/images/breeze/output-static-checks.svg index d9188e5fe8863..e76f1a3f27ea3 100644 --- a/images/breeze/output-static-checks.svg +++ b/images/breeze/output-static-checks.svg @@ -19,85 +19,85 @@ font-weight: 700; } - .terminal-2026671122-matrix { + .terminal-158108476-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 26.400000000000002px; font-variant-east-asian: full-width; } - .terminal-2026671122-title { + .terminal-158108476-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2026671122-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-2026671122-r2 { fill: #c5c8c6 } -.terminal-2026671122-r3 { fill: #d0b344;font-weight: bold } -.terminal-2026671122-r4 { fill: #868887 } -.terminal-2026671122-r5 { fill: #68a0b3;font-weight: bold } -.terminal-2026671122-r6 { fill: #98a84b;font-weight: bold } -.terminal-2026671122-r7 { fill: #8d7b39 } + .terminal-158108476-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-158108476-r2 { fill: #c5c8c6 } +.terminal-158108476-r3 { fill: #d0b344;font-weight: bold } +.terminal-158108476-r4 { fill: #868887 } +.terminal-158108476-r5 { fill: #68a0b3;font-weight: bold } +.terminal-158108476-r6 { fill: #98a84b;font-weight: bold } +.terminal-158108476-r7 { fill: #8d7b39 } - Command: static-checks + Command: static-checks -                                                                                                                          - Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]...                                                               -                                                                                                                         - Run static checks.                                                                                                      -                                                                                                                         -╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --type                  -t  Type(s) of the static checks to run (multiple can be added).                            │ -│                              (all | black | blacken-docs | check-airflow-2-1-compatibility |                         │ -│                              check-airflow-config-yaml-consistent | check-airflow-providers-have-extras |            │ -│                              check-apache-license-rat | check-base-operator-usage |                                  │ -│                              check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ -│                              check-builtin-literals | check-changelog-has-no-duplicates |                            │ -│                              check-daysago-import-from-utils | check-docstring-param-types |                         │ -│                              check-executables-have-shebangs | check-extra-packages-references | check-extras-order  │ -│                              | check-for-inclusive-language | check-hooks-apply |                                    │ -│                              check-incorrect-use-of-LoggingMixin | check-integrations-are-consistent |               │ -│                              check-merge-conflict | check-newsfragments-are-valid |                                  │ -│                              check-no-providers-in-core-examples | check-no-relative-imports |                       │ -│                              check-persist-credentials-disabled-in-github-workflows |                                │ -│                              check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ -│                              check-provider-yaml-valid | check-providers-init-file-missing |                         │ -│                              check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ -│                              check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ -│                              check-start-date-not-used-in-defaults | check-system-tests-present | check-xml |        │ -│                              codespell | debug-statements | detect-private-key | doctoc | end-of-file-fixer |        │ -│                              fix-encoding-pragma | flynt | forbid-tabs | identity | insert-license | isort |         │ -│                              lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart | lint-javascript |    │ -│                              lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |                   │ -│                              pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade | rst-backticks |      │ -│                              run-flake8 | run-mypy | run-shellcheck | static-check-autoflake | trailing-whitespace   │ -│                              | update-breeze-file | update-breeze-readme-config-hash | update-extras |               │ -│                              update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ -│                              update-local-yml-file | update-migration-references | update-providers-dependencies |   │ -│                              update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ -│                              update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ -│                              yamllint | yesqa)                                                                       │ -│  --file                  -f  List of files to run the checks on. (PATH)                                              │ -│  --all-files             -a  Run checks on all files. (TEXT)                                                         │ -│  --show-diff-on-failure  -s  Show diff for files modified by the checks.                                             │ -│  --last-commit           -c  Run checks for all files in last commit. Mutually exclusive with --commit-ref.          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --commit-ref         -r  Run checks for this commit reference only (can be any git commit-ish reference). Mutually  │ -│                           exclusive with --last-commit.                                                              │ -│                           (TEXT)                                                                                     │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze static-checks [OPTIONS] [PRECOMMIT_ARGS]...                                                               +                                                                                                                         + Run static checks.                                                                                                      +                                                                                                                         +╭─ Pre-commit flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --type                  -t  Type(s) of the static checks to run (multiple can be added).                            │ +│                              (all | black | blacken-docs | check-airflow-2-1-compatibility |                         │ +│                              check-airflow-config-yaml-consistent | check-airflow-providers-have-extras |            │ +│                              check-apache-license-rat | check-base-operator-usage |                                  │ +│                              check-boring-cyborg-configuration | check-breeze-top-dependencies-limited |             │ +│                              check-builtin-literals | check-changelog-has-no-duplicates |                            │ +│                              check-daysago-import-from-utils | check-docstring-param-types |                         │ +│                              check-executables-have-shebangs | check-extra-packages-references | check-extras-order  │ +│                              | check-for-inclusive-language | check-hooks-apply |                                    │ +│                              check-incorrect-use-of-LoggingMixin | check-integrations-are-consistent |               │ +│                              check-merge-conflict | check-newsfragments-are-valid |                                  │ +│                              check-no-providers-in-core-examples | check-no-relative-imports |                       │ +│                              check-persist-credentials-disabled-in-github-workflows |                                │ +│                              check-pre-commit-information-consistent | check-provide-create-sessions-imports |       │ +│                              check-provider-yaml-valid | check-providers-init-file-missing |                         │ +│                              check-providers-subpackages-init-file-exist | check-pydevd-left-in-code |               │ +│                              check-revision-heads-map | check-safe-filter-usage-in-html | check-setup-order |        │ +│                              check-start-date-not-used-in-defaults | check-system-tests-present | check-xml |        │ +│                              codespell | debug-statements | detect-private-key | doctoc | end-of-file-fixer |        │ +│                              fix-encoding-pragma | flynt | forbid-tabs | identity | insert-license | isort |         │ +│                              lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart | lint-javascript |    │ +│                              lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending |                   │ +│                              pretty-format-json | pydocstyle | python-no-log-warn | pyupgrade | rst-backticks |      │ +│                              run-flake8 | run-mypy | run-shellcheck | static-check-autoflake | trailing-whitespace   │ +│                              | update-breeze-file | update-breeze-readme-config-hash | update-extras |               │ +│                              update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |                   │ +│                              update-local-yml-file | update-migration-references | update-providers-dependencies |   │ +│                              update-setup-cfg-file | update-spelling-wordlist-to-be-sorted |                         │ +│                              update-supported-versions | update-vendored-in-k8s-json-schema | update-version |       │ +│                              yamllint | yesqa)                                                                       │ +│  --file                  -f  List of files to run the checks on. (PATH)                                              │ +│  --all-files             -a  Run checks on all files.                                                                │ +│  --show-diff-on-failure  -s  Show diff for files modified by the checks.                                             │ +│  --last-commit           -c  Run checks for all files in last commit. Mutually exclusive with --commit-ref.          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --commit-ref         -r  Run checks for this commit reference only (can be any git commit-ish reference). Mutually  │ +│                           exclusive with --last-commit.                                                              │ +│                           (TEXT)                                                                                     │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py index 9855091ba8085..2a5a25639b6c5 100755 --- a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py +++ b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py @@ -56,7 +56,7 @@ def get_errors_and_hooks(content: Any, max_length: int) -> Tuple[List[str], Dict else: errors.append(f"The id is missing in {hook}") continue - if hook_id == 'mypy': + if hook_id == 'run-mypy': needs_image = True if 'name' not in hook: errors.append( diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 7adba2ffc656b..59bd356d24df0 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,22 +27,20 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) create_static_check_volumes() - return_code = subprocess.call( - args=[ + cmd_result = run_command( + [ "docker", "run", "-t", @@ -58,9 +53,12 @@ "BACKEND=sqlite", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "/opt/airflow/scripts/in_container/run_flake8.sh", *sys.argv[1:], ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index e72a7a33265fc..154a795f3e84a 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,31 +27,36 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", "python3 /opt/airflow/scripts/in_container/run_migration_reference.py", ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 9d92b99e80edf..74d511a65b7ed 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -31,22 +28,20 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) create_static_check_volumes() - return_code = subprocess.call( - args=[ + cmd_result = run_command( + [ "docker", "run", "-t", @@ -59,9 +54,12 @@ "BACKEND=sqlite", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "/opt/airflow/scripts/in_container/run_mypy.sh", *sys.argv[1:], ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index 93a5c3bf72ff0..7755e3d204bd8 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -29,33 +26,36 @@ ) AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() -GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") -AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7" +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", 'cd airflow/ui && yarn --frozen-lockfile --non-interactive && yarn run lint', ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode) diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index c2807d4082c3f..65553c3d81345 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -16,12 +16,9 @@ # specific language governing permissions and limitations # under the License. import os -import subprocess import sys from pathlib import Path -from rich import print - if __name__ not in ("__main__", "__mp_main__"): raise SystemExit( "This file is intended to be executed as an executable program. You cannot use it as a module." @@ -30,32 +27,36 @@ AIRFLOW_SOURCES = Path(__file__).parents[3].resolve() GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow") -AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7" +# allow "False", "false", "True", "true", "f", "F", "t", "T" and the like +VERBOSE = os.environ.get('VERBOSE', "false")[0].lower() == "t" +DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t" if __name__ == '__main__': - sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src")) - from airflow_breeze.branch_defaults import AIRFLOW_BRANCH + sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) + from airflow_breeze.global_constants import MOUNT_SELECTED + from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command - AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7" - if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0: - print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n') - print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n") - sys.exit(1) - return_code = subprocess.call( - args=[ + airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) + create_static_check_volumes() + cmd_result = run_command( + [ "docker", "run", - "-v", - f"{AIRFLOW_SOURCES}:/opt/airflow/", + "-t", + *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", - AIRFLOW_CI_IMAGE, + airflow_image, "-c", 'cd airflow/www && yarn --frozen-lockfile --non-interactive && yarn run lint', ], + check=False, + verbose=VERBOSE, + dry_run=DRY_RUN, ) - sys.exit(return_code) + sys.exit(cmd_result.returncode)