Skip to content

Commit 548c16c

Browse files
authored
Live/regression tests: add support for selecting from a subset of connections (#42044)
1 parent 990d1e3 commit 548c16c

File tree

11 files changed

+62
-8
lines changed

11 files changed

+62
-8
lines changed

.github/workflows/live_tests.yml

+13-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ on:
2929
description: Use the local CDK when building the target connector
3030
default: "false"
3131
type: boolean
32+
connection_subset:
33+
description: The subset of connections to select from.
34+
required: true
35+
type: choice
36+
options:
37+
- sandboxes
38+
- all
3239

3340
jobs:
3441
live_tests:
@@ -86,6 +93,11 @@ jobs:
8693
echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
8794
fi
8895
96+
- name: Setup Connection Subset Option
97+
if: github.event_name == 'workflow_dispatch'
98+
run: |
99+
echo "CONNECTION_SUBSET=--connector_live_tests.connection-subset=${{ github.event.inputs.connection_subset }}" >> $GITHUB_ENV
100+
89101
- name: Run Live Tests [WORKFLOW DISPATCH]
90102
if: github.event_name == 'workflow_dispatch' # TODO: consider using the matrix strategy (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs). See https://github.com/airbytehq/airbyte/pull/37659#discussion_r1583380234 for details.
91103
uses: ./.github/actions/run-airbyte-ci
@@ -102,4 +114,4 @@ jobs:
102114
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
103115
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
104116
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
105-
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}
117+
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}

.github/workflows/regression_tests.yml

+13-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ on:
2929
description: Use the local CDK when building the target connector
3030
default: "false"
3131
type: boolean
32+
connection_subset:
33+
description: The subset of connections to select from.
34+
required: true
35+
type: choice
36+
options:
37+
- sandboxes
38+
- all
3239

3340
jobs:
3441
regression_tests:
@@ -86,6 +93,11 @@ jobs:
8693
echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
8794
fi
8895
96+
- name: Setup Connection Subset Option
97+
if: github.event_name == 'workflow_dispatch'
98+
run: |
99+
echo "CONNECTION_SUBSET=--connector_live_tests.connection-subset=${{ github.event.inputs.connection_subset }}" >> $GITHUB_ENV
100+
89101
- name: Run Regression Tests [WORKFLOW DISPATCH]
90102
if: github.event_name == 'workflow_dispatch' # TODO: consider using the matrix strategy (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs). See https://github.com/airbytehq/airbyte/pull/37659#discussion_r1583380234 for details.
91103
uses: ./.github/actions/run-airbyte-ci
@@ -102,4 +114,4 @@ jobs:
102114
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
103115
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
104116
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
105-
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}
117+
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}

airbyte-ci/connectors/live-tests/README.md

+4
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,10 @@ The traffic recorded on the control connector is passed to the target connector
280280

281281
## Changelog
282282

283+
### 0.18.0
284+
285+
Add support for selecting from a subset of connections.
286+
283287
### 0.17.8
284288

285289
Fix the self-signed certificate path we bind to Python connectors.

airbyte-ci/connectors/live-tests/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "live-tests"
7-
version = "0.17.8"
7+
version = "0.18.0"
88
description = "Contains utilities for testing connectors against live data."
99
authors = ["Airbyte <contact@airbyte.io>"]
1010
license = "MIT"

airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py

+8
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import rich
1111
from connection_retriever import ConnectionObject, retrieve_objects # type: ignore
1212
from connection_retriever.errors import NotPermittedError # type: ignore
13+
from live_tests.commons.models import ConnectionSubset
1314

1415
from .models import AirbyteCatalog, Command, ConfiguredAirbyteCatalog, ConnectionObjects, SecretDict
1516

@@ -95,6 +96,7 @@ def get_connection_objects(
9596
connector_version: Optional[str] = None,
9697
auto_select_connection: bool = False,
9798
selected_streams: Optional[set[str]] = None,
99+
connection_subset: ConnectionSubset = ConnectionSubset.SANDBOXES,
98100
) -> ConnectionObjects:
99101
"""This function retrieves the connection objects values.
100102
It checks that the required objects are available and raises a UsageError if they are not.
@@ -113,6 +115,7 @@ def get_connection_objects(
113115
connector_version (Optional[str]): The version for the connector under test.
114116
auto_select_connection (bool, optional): Whether to automatically select a connection if no connection id is passed. Defaults to False.
115117
selected_streams (Optional[Set[str]]): The set of selected streams to use when auto selecting a connection.
118+
connection_subset (ConnectionSubset): The subset of connections to select from.
116119
Raises:
117120
click.UsageError: If a required object is missing for the command.
118121
click.UsageError: If a retrieval reason is missing when passing a connection id.
@@ -146,6 +149,7 @@ def get_connection_objects(
146149
custom_config=custom_config,
147150
custom_configured_catalog=custom_configured_catalog,
148151
custom_state=custom_state,
152+
connection_subset=connection_subset,
149153
)
150154

151155
else:
@@ -160,6 +164,7 @@ def get_connection_objects(
160164
custom_config=custom_config,
161165
custom_configured_catalog=custom_configured_catalog,
162166
custom_state=custom_state,
167+
connection_subset=connection_subset,
163168
)
164169

165170
else:
@@ -175,6 +180,7 @@ def get_connection_objects(
175180
destination_id=None,
176181
connection_id=None,
177182
source_docker_image=None,
183+
connection_subset=connection_subset,
178184
)
179185

180186
if fail_if_missing_objects:
@@ -198,6 +204,7 @@ def _get_connection_objects_from_retrieved_objects(
198204
custom_config: Optional[Dict] = None,
199205
custom_configured_catalog: Optional[ConfiguredAirbyteCatalog] = None,
200206
custom_state: Optional[Dict] = None,
207+
connection_subset: ConnectionSubset = ConnectionSubset.SANDBOXES,
201208
):
202209
LOGGER.info("Retrieving connection objects from the database...")
203210
connection_id, retrieved_objects = retrieve_objects(
@@ -208,6 +215,7 @@ def _get_connection_objects_from_retrieved_objects(
208215
prompt_for_connection_selection=prompt_for_connection_selection,
209216
with_streams=selected_streams,
210217
connection_id=connection_id,
218+
connection_subset=connection_subset,
211219
)
212220

213221
retrieved_source_config = parse_config(retrieved_objects.get(ConnectionObject.SOURCE_CONFIG))

airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py

+5
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,11 @@ class ActorType(Enum):
148148
DESTINATION = "destination"
149149

150150

151+
class ConnectionSubset(Enum):
152+
SANDBOXES = "sandboxes"
153+
ALL = "all"
154+
155+
151156
@dataclass
152157
class ConnectorUnderTest:
153158
image_name: str

airbyte-ci/connectors/live-tests/src/live_tests/conftest.py

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
ActorType,
2424
Command,
2525
ConnectionObjects,
26+
ConnectionSubset,
2627
ConnectorUnderTest,
2728
ExecutionInputs,
2829
ExecutionResult,
@@ -90,6 +91,12 @@ def pytest_addoption(parser: Parser) -> None:
9091
default=TestEvaluationMode.STRICT.value,
9192
help='If "diagnostic" mode is selected, all tests will pass as long as there is no exception; warnings will be logged. In "strict" mode, tests may fail.',
9293
)
94+
parser.addoption(
95+
"--connection-subset",
96+
choices=[c.value for c in ConnectionSubset],
97+
default=ConnectionSubset.SANDBOXES.value,
98+
help="Whether to select from sandbox accounts only.",
99+
)
93100

94101

95102
def pytest_configure(config: Config) -> None:
@@ -128,6 +135,7 @@ def pytest_configure(config: Config) -> None:
128135
config.stash[stash_keys.CONNECTOR_IMAGE] = get_option_or_fail(config, "--connector-image")
129136
config.stash[stash_keys.TARGET_VERSION] = get_option_or_fail(config, "--target-version")
130137
config.stash[stash_keys.CONTROL_VERSION] = get_control_version(config)
138+
config.stash[stash_keys.CONNECTION_SUBSET] = ConnectionSubset(get_option_or_fail(config, "--connection-subset"))
131139
custom_source_config_path = config.getoption("--config-path")
132140
custom_configured_catalog_path = config.getoption("--catalog-path")
133141
custom_state_path = config.getoption("--state-path")
@@ -165,6 +173,7 @@ def pytest_configure(config: Config) -> None:
165173
connector_version=config.stash[stash_keys.CONTROL_VERSION],
166174
auto_select_connection=config.stash[stash_keys.AUTO_SELECT_CONNECTION],
167175
selected_streams=config.stash[stash_keys.SELECTED_STREAMS],
176+
connection_subset=config.stash[stash_keys.CONNECTION_SUBSET],
168177
)
169178
config.stash[stash_keys.IS_PERMITTED_BOOL] = True
170179
except (ConnectionNotFoundError, NotPermittedError) as exc:

airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import pytest
77
from live_tests.commons.evaluation_modes import TestEvaluationMode
8-
from live_tests.commons.models import ConnectionObjects
8+
from live_tests.commons.models import ConnectionObjects, ConnectionSubset
99
from live_tests.report import Report
1010

1111
AIRBYTE_API_KEY = pytest.StashKey[str]()
@@ -15,6 +15,7 @@
1515
CONNECTION_URL = pytest.StashKey[str | None]()
1616
CONNECTOR_IMAGE = pytest.StashKey[str]()
1717
CONTROL_VERSION = pytest.StashKey[str]()
18+
CONNECTION_SUBSET = pytest.StashKey[ConnectionSubset]()
1819
DAGGER_LOG_PATH = pytest.StashKey[Path]()
1920
DUCKDB_PATH = pytest.StashKey[Path]()
2021
HTTP_DUMP_CACHE_VOLUMES = pytest.StashKey[list]()

airbyte-ci/connectors/pipelines/README.md

+3-2
Original file line numberDiff line numberDiff line change
@@ -766,13 +766,14 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
766766
## Changelog
767767

768768
| Version | PR | Description |
769-
| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
769+
|---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
770+
| 4.25.0 | [#42044](https://github.com/airbytehq/airbyte/pull/42044) | Live/regression tests: add support for selecting from a subset of connections |
770771
| 4.24.3 | [#42040](https://github.com/airbytehq/airbyte/pull/42040) | Always send regression test approval status check; skip on auto-merge PRs. |
771772
| 4.24.2 | [#41676](https://github.com/airbytehq/airbyte/pull/41676) | Send regression test approval status check when skipped. |
772773
| 4.24.1 | [#41642](https://github.com/airbytehq/airbyte/pull/41642) | Use the AIRBYTE_GITHUB_REPO environment variable to run airbyte-ci in other repos. |
773774
| 4.24.0 | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors |
774775
| 4.23.1 | [#41541](https://github.com/airbytehq/airbyte/pull/41541) | Add support for submodule use-case. |
775-
| 4.23.0 | [#39906](https://github.com/airbytehq/airbyte/pull/39906) | Add manifest only build pipeline |
776+
| 4.23.0 | [#39906](https://github.com/airbytehq/airbyte/pull/39906) | Add manifest only build pipeline |
776777
| 4.22.0 | [#41623](https://github.com/airbytehq/airbyte/pull/41623) | Make `airbyte-ci` run on private forks. |
777778
| 4.21.1 | [#41029](https://github.com/airbytehq/airbyte/pull/41029) | `up-to-date`: mount local docker config to `Syft` to pull private images and benefit from increased DockerHub rate limits. |
778779
| 4.21.0 | [#40547](https://github.com/airbytehq/airbyte/pull/40547) | Make bump-version accept a `--pr-number` option. |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,7 @@ def _get_command_options(self) -> List[str]:
516516
command_options += ["--test-evaluation-mode", self.test_evaluation_mode]
517517
if self.selected_streams:
518518
command_options += ["--stream", self.selected_streams]
519+
command_options += ["--connection-subset", self.connection_subset]
519520
return command_options
520521

521522
def _run_command_with_proxy(self, command: str) -> List[str]:
@@ -565,11 +566,12 @@ def __init__(self, context: ConnectorContext) -> None:
565566

566567
self.test_suite = self.context.run_step_options.get_item_or_default(options, "test-suite", LiveTestSuite.ALL.value)
567568
self.test_dir = self.test_suite_to_dir[LiveTestSuite(self.test_suite)]
568-
self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", "latest")
569+
self.control_version = self.context.run_step_options.get_item_or_default(options, "control-version", None)
569570
self.target_version = self.context.run_step_options.get_item_or_default(options, "target-version", "dev")
570571
self.should_read_with_state = self.context.run_step_options.get_item_or_default(options, "should-read-with-state", "1")
571572
self.selected_streams = self.context.run_step_options.get_item_or_default(options, "selected-streams", None)
572573
self.test_evaluation_mode = "strict" if self.context.connector.metadata.get("supportLevel") == "certified" else "diagnostic"
574+
self.connection_subset = self.context.run_step_options.get_item_or_default(options, "connection-subset", "sandboxes")
573575
self.run_id = os.getenv("GITHUB_RUN_ID") or str(int(time.time()))
574576

575577
async def _run(self, connector_under_test_container: Container) -> StepResult:

airbyte-ci/connectors/pipelines/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
66
name = "pipelines"
7-
version = "4.24.3"
7+
version = "4.25.0"
88
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
99
authors = ["Airbyte <contact@airbyte.io>"]
1010

0 commit comments

Comments
 (0)