Standard provider bash operator (apache#42252)

.github/workflows/basic-tests.yml

@@ -200,6 +200,11 @@ jobs:
             breeze release-management prepare-provider-packages fab --package-format wheel --skip-tag-check
       - name: "Install Airflow with fab for webserver tests"
         run: pip install . dist/apache_airflow_providers_fab-*.whl
+      - name: "Prepare Standard provider packages: wheel"
+        run: >
+          breeze release-management prepare-provider-packages standard --package-format wheel --skip-tag-check
+      - name: "Install Airflow with standard provider for webserver tests"
+        run: pip install . dist/apache_airflow_providers_standard-*.whl
       - name: "Install Python client"
         run: pip install ./dist/apache_airflow_client-*.whl
       - name: "Initialize Airflow DB and start webserver"

.pre-commit-config.yaml

@@ -550,17 +550,9 @@ repos:
       - id: check-no-providers-in-core-examples
         language: pygrep
         name: No providers imports in core example DAGs
-        description: The core example DAGs have no dependencies other than core Airflow
-        entry: "^\\s*from airflow\\.providers.*"
+        description: The core example DAGs have no dependencies other than standard provider or core Airflow
+        entry: "^\\s*from airflow\\.providers.(?!standard.)"
         pass_filenames: true
-        exclude: >
-          (?x)
-          ^airflow/example_dags/example_branch_datetime_operator.py|
-          ^airflow/example_dags/example_branch_day_of_week_operator.py|
-          ^airflow/example_dags/example_sensors.py|
-          ^airflow/example_dags/example_sensors.py|
-          ^airflow/example_dags/example_sensors.py|
-          ^airflow/example_dags/example_time_delta_sensor_async.py
         files: ^airflow/example_dags/.*\.py$
       - id: check-no-airflow-deprecation-in-providers
         language: pygrep
@@ -717,7 +709,7 @@ repos:
         files: >
           (?x)
           ^airflow/providers/.*\.py$
-        exclude: ^.*/.*_vendor/
+        exclude: ^.*/.*_vendor/|airflow/providers/standard/operators/bash.py
       - id: check-get-lineage-collector-providers
         language: python
         name: Check providers import hook lineage code from compat

airflow/decorators/bash.py

@@ -21,7 +21,7 @@
 from typing import Any, Callable, Collection, Mapping, Sequence
 
 from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 from airflow.utils.context import Context, context_merge
 from airflow.utils.operator_helpers import determine_kwargs
 from airflow.utils.types import NOTSET

airflow/example_dags/example_assets.py

@@ -56,7 +56,7 @@
 
 from airflow.assets import Asset
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 from airflow.timetables.assets import AssetOrTimeSchedule
 from airflow.timetables.trigger import CronTriggerTimetable
 

airflow/example_dags/example_bash_operator.py

@@ -24,8 +24,8 @@
 import pendulum
 
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
 from airflow.operators.empty import EmptyOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 with DAG(
     dag_id="example_bash_operator",

airflow/example_dags/example_complex.py

@@ -25,7 +25,7 @@
 
 from airflow.models.baseoperator import chain
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 with DAG(
     dag_id="example_complex",

airflow/example_dags/example_inlet_event_extra.py

@@ -28,7 +28,7 @@
 from airflow.assets import Asset
 from airflow.decorators import task
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 asset = Asset("s3://output/1.txt")
 

airflow/example_dags/example_outlet_event_extra.py

@@ -29,7 +29,7 @@
 from airflow.assets.metadata import Metadata
 from airflow.decorators import task
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 ds = Asset("s3://output/1.txt")
 

airflow/example_dags/example_passing_params_via_test_command.py

@@ -27,7 +27,7 @@
 
 from airflow.decorators import task
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 
 @task(task_id="run_this")

airflow/example_dags/example_sensors.py

@@ -22,11 +22,11 @@
 import pendulum
 
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
+from airflow.providers.standard.sensors.bash import BashSensor
 from airflow.providers.standard.sensors.time import TimeSensor, TimeSensorAsync
 from airflow.providers.standard.sensors.time_delta import TimeDeltaSensor, TimeDeltaSensorAsync
 from airflow.providers.standard.sensors.weekday import DayOfWeekSensor
-from airflow.sensors.bash import BashSensor
 from airflow.sensors.filesystem import FileSensor
 from airflow.sensors.python import PythonSensor
 from airflow.utils.trigger_rule import TriggerRule

airflow/example_dags/example_setup_teardown.py

@@ -22,7 +22,7 @@
 import pendulum
 
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 from airflow.utils.task_group import TaskGroup
 
 with DAG(

airflow/example_dags/example_task_group.py

@@ -22,8 +22,8 @@
 import pendulum
 
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
 from airflow.operators.empty import EmptyOperator
+from airflow.providers.standard.operators.bash import BashOperator
 from airflow.utils.task_group import TaskGroup
 
 # [START howto_task_group]

airflow/example_dags/example_trigger_target_dag.py

@@ -27,7 +27,7 @@
 
 from airflow.decorators import task
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 
 @task(task_id="run_this")

airflow/example_dags/example_xcom.py

@@ -24,7 +24,7 @@
 from airflow.decorators import task
 from airflow.models.dag import DAG
 from airflow.models.xcom_arg import XComArg
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 value_1 = [1, 2, 3]
 value_2 = {"a": "b"}

airflow/example_dags/example_xcomargs.py

@@ -25,7 +25,7 @@
 
 from airflow.decorators import task
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 log = logging.getLogger(__name__)
 

airflow/example_dags/tutorial.py

@@ -32,7 +32,7 @@
 from airflow.models.dag import DAG
 
 # Operators; we need this to operate!
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 # [END import_module]
 

airflow/providers/celery/executors/celery_executor_utils.py

@@ -111,8 +111,12 @@ def on_celery_import_modules(*args, **kwargs):
 
     import airflow.jobs.local_task_job_runner
     import airflow.macros
-    import airflow.operators.bash
-    import airflow.operators.python  # noqa: F401
+    import airflow.operators.python
+
+    try:
+        import airflow.providers.standard.operators.bash
+    except ImportError:
+        import airflow.operators.bash  # noqa: F401
 
     with contextlib.suppress(ImportError):
         import numpy  # noqa: F401

airflow/providers/edge/example_dags/integration_test.py

@@ -32,10 +32,14 @@
 from airflow.models.dag import DAG
 from airflow.models.param import Param
 from airflow.models.variable import Variable
-from airflow.operators.bash import BashOperator
 from airflow.operators.empty import EmptyOperator
 from airflow.operators.python import PythonOperator
 
+try:
+    from airflow.providers.standard.operators.bash import BashOperator
+except ImportError:
+    from airflow.operators.bash import BashOperator  # type: ignore[no-redef,attr-defined]
+
 with DAG(
     dag_id="integration_test",
     dag_display_name="Integration Test",

airflow/providers/openlineage/provider.yaml

@@ -84,7 +84,8 @@ config:
           Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated
           full import paths of Operators to disable.
         type: string
-        example: "airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator"
+        example: "airflow.providers.standard.operators.bash.BashOperator;
+          airflow.operators.python.PythonOperator"
         default: ""
         version_added: 1.1.0
       selective_enable:

airflow/providers/standard/provider.yaml

@@ -42,6 +42,7 @@ operators:
     python-modules:
       - airflow.providers.standard.operators.datetime
       - airflow.providers.standard.operators.weekday
+      - airflow.providers.standard.operators.bash
 
 sensors:
   - integration-name: Standard
@@ -50,3 +51,4 @@ sensors:
       - airflow.providers.standard.sensors.time_delta
       - airflow.providers.standard.sensors.time
       - airflow.providers.standard.sensors.weekday
+      - airflow.providers.standard.sensors.bash

dev/breeze/tests/test_selective_checks.py

@@ -703,9 +703,9 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
             id="Only Always and common providers tests should run when only common.io and tests/always changed",
         ),
         pytest.param(
-            ("airflow/operators/bash.py",),
+            ("airflow/providers/standard/operators/bash.py",),
             {
-                "affected-providers-list-as-string": None,
+                "affected-providers-list-as-string": "celery edge standard",
                 "all-python-versions": "['3.9']",
                 "all-python-versions-list-as-string": "3.9",
                 "python-versions": "['3.9']",
@@ -717,14 +717,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                 "run-amazon-tests": "false",
                 "docs-build": "true",
                 "run-kubernetes-tests": "false",
-                "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,"
+                "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,"
                 "ts-compile-format-lint-ui,ts-compile-format-lint-www",
                 "upgrade-to-newer-dependencies": "false",
-                "parallel-test-types-list-as-string": "Always Core Operators Serialization",
+                "parallel-test-types-list-as-string": "Always Core Providers[celery,edge,standard] Serialization",
                 "needs-mypy": "true",
-                "mypy-folders": "['airflow']",
+                "mypy-folders": "['providers']",
             },
-            id="Force Core and Serialization tests to run when airflow bash.py changed",
+            id="Providers standard tests and Serialization tests to run when airflow bash.py changed",
         ),
         pytest.param(
             ("tests/operators/bash.py",),
@@ -1720,7 +1720,7 @@ def test_upgrade_to_newer_dependencies(
         ),
         pytest.param(
             ("airflow/providers/celery/file.py",),
-            {"docs-list-as-string": "apache-airflow celery cncf.kubernetes"},
+            {"docs-list-as-string": "apache-airflow celery cncf.kubernetes standard"},
             id="Celery python files changed",
         ),
         pytest.param(

dev/perf/dags/elastic_dag.py

@@ -24,7 +24,7 @@
 
 from airflow.models.baseoperator import chain
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 # DAG File used in performance tests. Its shape can be configured by environment variables.
 RE_TIME_DELTA = re.compile(

dev/perf/dags/perf_dag_2.py

@@ -24,7 +24,7 @@
 import datetime
 
 from airflow.models.dag import DAG
-from airflow.operators.bash import BashOperator
+from airflow.providers.standard.operators.bash import BashOperator
 
 args = {
     "owner": "airflow",

docs/apache-airflow-providers-amazon/notifications/chime_notifier_howto_guide.rst

@@ -31,7 +31,7 @@ Example Code:
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.amazon.aws.notifications.chime import send_chime_notification
 
     with DAG(

docs/apache-airflow-providers-amazon/notifications/sns.rst

@@ -33,7 +33,7 @@ Example Code:
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.amazon.aws.notifications.sns import send_sns_notification
 
     dag_failure_sns_notification = send_sns_notification(

docs/apache-airflow-providers-amazon/notifications/sqs.rst

@@ -33,7 +33,7 @@ Example Code:
 
     from datetime import datetime, timezone
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.amazon.aws.notifications.sqs import send_sqs_notification
 
     dag_failure_sqs_notification = send_sqs_notification(

docs/apache-airflow-providers-apprise/notifications/apprise_notifier_howto_guide.rst

@@ -30,7 +30,7 @@ Example Code:
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.apprise.notifications.apprise import send_apprise_notification
     from apprise import NotifyType
 

docs/apache-airflow-providers-atlassian-jira/notifications/jira-notifier-howto-guide.rst

@@ -31,7 +31,7 @@ Example Code
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.atlassian.jira.notifications.jira import send_jira_notification
 
     with DAG(

docs/apache-airflow-providers-google/operators/cloud/mlengine.rst

@@ -96,7 +96,7 @@ instead.
 You can use :ref:`Jinja templating <concepts:jinja-templating>` with the ``project_id`` and ``model``
 fields to dynamically determine their values. The result are saved to :ref:`XCom <concepts:xcom>`,
 allowing them to be used by other operators. In this case, the
-:class:`~airflow.operators.bash.BashOperator` is used to print the model information.
+:class:`~airflow.providers.standard.operators.bash.BashOperator` is used to print the model information.
 
 .. exampleinclude:: /../../tests/system/providers/google/cloud/ml_engine/example_mlengine.py
     :language: python

docs/apache-airflow-providers-google/operators/cloud/pubsub.rst

@@ -101,7 +101,7 @@ Also for this action you can use sensor in the deferrable mode:
     :start-after: [START howto_operator_gcp_pubsub_pull_message_with_operator]
     :end-before: [END howto_operator_gcp_pubsub_pull_message_with_operator]
 
-To pull messages from XCom use the :class:`~airflow.operators.bash.BashOperator`.
+To pull messages from XCom use the :class:`~airflow.providers.standard.operators.bash.BashOperator`.
 
 .. exampleinclude:: /../../tests/system/providers/google/cloud/pubsub/example_pubsub.py
     :language: python

docs/apache-airflow-providers-openlineage/guides/developer.rst

@@ -390,7 +390,7 @@ An Operator inside the Airflow DAG can be annotated with inlets and outlets like
     import pendulum
 
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.lineage.entities import Table, File, Column, User
 
 

docs/apache-airflow-providers-openlineage/guides/user.rst

@@ -257,13 +257,13 @@ full import paths of Airflow Operators to disable as ``disabled_for_operators``
 
     [openlineage]
     transport = {"type": "http", "url": "http://example.com:5000", "endpoint": "api/v1/lineage"}
-    disabled_for_operators = 'airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator'
+    disabled_for_operators = 'airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator'
 
 ``AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS`` environment variable is an equivalent.
 
 .. code-block:: ini
 
-  AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator'
+  AIRFLOW__OPENLINEAGE__DISABLED_FOR_OPERATORS='airflow.providers.standard.operators.bash.BashOperator;airflow.operators.python.PythonOperator'
 
 Full Task Info
 ^^^^^^^^^^^^^^

docs/apache-airflow-providers-pagerduty/notifications/pagerduty_notifier_howto_guide.rst

@@ -31,7 +31,7 @@ Example Code:
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.pagerduty.notifications.pagerduty import send_pagerduty_notification
 
     with DAG(

docs/apache-airflow-providers-slack/notifications/slack_notifier_howto_guide.rst

@@ -31,7 +31,7 @@ Example Code:
 
     from datetime import datetime
     from airflow import DAG
-    from airflow.operators.bash import BashOperator
+    from airflow.providers.standard.operators.bash import BashOperator
     from airflow.providers.slack.notifications.slack import send_slack_notification
 
     with DAG(