Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Small fixes for Google provider's system tests #42614

Merged
merged 1 commit into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/system/providers/google/ads/example_ads.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

# [START howto_google_ads_env_variables]
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "example_google_ads"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "example_automl_video_clss"
DAG_ID = "automl_video_clss"
REGION = "us-central1"
VIDEO_DISPLAY_NAME = f"auto-ml-video-clss-{ENV_ID}"
MODEL_DISPLAY_NAME = f"auto-ml-video-clss-model-{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "example_automl_video_track"
DAG_ID = "automl_video_track"
REGION = "us-central1"
VIDEO_DISPLAY_NAME = f"auto-ml-video-tracking-{ENV_ID}"
MODEL_DISPLAY_NAME = f"auto-ml-video-tracking-model-{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "azure_fileshare_to_gcs_example"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_dataset"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "example_gcp_bigquery_dts"
DAG_ID = "gcp_bigquery_dts"

BUCKET_NAME = f"bucket-{DAG_ID}-{ENV_ID}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_operations"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_operations_location"

BQ_LOCATION = "europe-north1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
LOCATION = "us-east1"
QUERY_SQL_PATH = "resources/example_bigquery_query.sql"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")

DAG_ID = "example_bigquery_queries_async"
DAG_ID = "bigquery_queries_async"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}".replace("-", "_")
LOCATION = "us"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "example_bigquery_sensors"
DAG_ID = "bigquery_sensors"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}".replace("-", "_")
TABLE_NAME = f"partitioned_table_{DAG_ID}_{ENV_ID}".replace("-", "_")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_tables"

BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from airflow.providers.google.cloud.transfers.bigquery_to_bigquery import BigQueryToBigQueryOperator
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_to_bigquery"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_to_gcs"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_to_gcs_async"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
except ImportError:
pytest.skip("MsSQL not available", allow_module_level=True)

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "example-project")
DAG_ID = "bigquery_to_mssql"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
except ImportError:
pytest.skip("MySQL not available", allow_module_level=True)

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "bigquery_to_mysql"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}".replace("-", "_")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "example_bigquery_transfer"
DAG_ID = "bigquery_transfer"

DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "bigtable"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
)
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "gcp_cloud_build"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "gcp_cloud_build_trigger"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
DAG_ID = "example_gcp_function"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "gcp_function"
LOCATION = "europe-west1"

# make sure there are no dashes in function name (!)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "google_project_id")

DAG_ID = "cloud_memorystore_memcached"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
ENV_ID_LOWER = ENV_ID.lower() if ENV_ID else ""
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)

PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")


# [START howto_operator_cloud_run_service_creation]
Expand All @@ -50,11 +50,11 @@ def _create_service():


with DAG(
"example_cloud_run_service",
"cloud_run_service",
schedule="@once",
start_date=datetime(2021, 1, 1),
catchup=False,
tags=["cloud-run-service-example"],
tags=["cloud-run", "service", "example"],
) as dag:
# [START howto_operator_cloud_run_create_service]
create_cloud_run_service = CloudRunCreateServiceOperator(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "cloudsql"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "cloudsql_query"
REGION = "us-central1"
Expand Down Expand Up @@ -378,7 +378,7 @@ def cloud_sql_database_create_body(instance: str) -> dict[str, Any]:
with DAG(
dag_id=DAG_ID,
start_date=datetime(2021, 1, 1),
schedule=None,
schedule="@once",
catchup=False,
tags=["example", "cloudsql", "postgres"],
) as dag:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
DAG_ID = "cloudsql_query_ssl"
REGION = "us-central1"
Expand Down Expand Up @@ -257,7 +257,7 @@ def cloud_sql_database_create_body(instance: str) -> dict[str, Any]:
with DAG(
dag_id=DAG_ID,
start_date=datetime(2021, 1, 1),
schedule=None,
schedule="@once",
catchup=False,
tags=["example", "cloudsql", "postgres"],
) as dag:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")

DAG_ID = "example_composer"
DAG_ID = "composer"
REGION = "us-central1"

# [START howto_operator_composer_simple_environment]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "dlp_deidentify_content"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
CRYPTO_KEY_NAME = f"{DAG_ID}_{ENV_ID}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "dlp_info_types"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

TEMPLATE_ID = f"dlp-inspect-info-{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "dlp_inspect_template"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

TEMPLATE_ID = f"dlp-inspect-{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "dlp_job"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

JOB_ID = f"dlp_job_{ENV_ID}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from tests.system.providers.google import DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

DAG_ID = "dlp_job_trigger"
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID

JOB_TRIGGER = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "dataflow_native_go_async"
BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@
},
job_class="org.apache.beam.examples.WordCount",
dataflow_config={
"job_name": "test-java-pipeline-job",
"job_name": f"java-pipeline-job-{ENV_ID}",
"check_if_running": CheckJobRunning.IgnoreJob,
"location": LOCATION,
"poll_sleep": 10,
Expand All @@ -124,7 +124,7 @@
},
job_class="org.apache.beam.examples.WordCount",
dataflow_config={
"job_name": "test-deferrable-java-pipeline-job",
"job_name": f"deferrable-java-pipeline-job-{ENV_ID}",
"check_if_running": CheckJobRunning.WaitForRun,
"location": LOCATION,
"poll_sleep": 10,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "dataflow_native_python_async"

RESOURCE_DATA_BUCKET = "airflow-system-tests-resources"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
)
from airflow.utils.trigger_rule import TriggerRule

ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
DAG_ID = "dataflow_sensors_deferrable"

RESOURCE_DATA_BUCKET = "airflow-system-tests-resources"
Expand Down
Loading