From 588ce5627e75430c16bc09f336f5c7e17e2b68bc Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Fri, 16 Feb 2024 13:04:06 -0800 Subject: [PATCH] Archive `destination-kvdb` (#35370) --- .../connectors/destination-kvdb/.dockerignore | 5 - .../connectors/destination-kvdb/Dockerfile | 16 -- .../connectors/destination-kvdb/README.md | 118 --------------- .../destination_kvdb/__init__.py | 26 ---- .../destination_kvdb/client.py | 78 ---------- .../destination_kvdb/destination.py | 72 --------- .../destination_kvdb/spec.json | 26 ---- .../destination_kvdb/writer.py | 46 ------ .../integration_tests/integration_test.py | 138 ------------------ .../connectors/destination-kvdb/main.py | 11 -- .../destination-kvdb/requirements.txt | 1 - .../connectors/destination-kvdb/setup.py | 26 ---- .../destination-kvdb/unit_tests/unit_test.py | 7 - docs/integrations/destinations/kvdb.md | 2 +- 14 files changed, 1 insertion(+), 571 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-kvdb/.dockerignore delete mode 100644 airbyte-integrations/connectors/destination-kvdb/Dockerfile delete mode 100644 airbyte-integrations/connectors/destination-kvdb/README.md delete mode 100644 airbyte-integrations/connectors/destination-kvdb/destination_kvdb/__init__.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/destination_kvdb/client.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/destination_kvdb/spec.json delete mode 100644 airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/main.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/requirements.txt delete mode 100644 airbyte-integrations/connectors/destination-kvdb/setup.py delete mode 100644 airbyte-integrations/connectors/destination-kvdb/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/destination-kvdb/.dockerignore b/airbyte-integrations/connectors/destination-kvdb/.dockerignore deleted file mode 100644 index 1b4b5767b5549..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!Dockerfile -!main.py -!destination_kvdb -!setup.py diff --git a/airbyte-integrations/connectors/destination-kvdb/Dockerfile b/airbyte-integrations/connectors/destination-kvdb/Dockerfile deleted file mode 100644 index fba4f5cb3d41a..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9.11-alpine3.15 - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY destination_kvdb ./destination_kvdb -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-kvdb diff --git a/airbyte-integrations/connectors/destination-kvdb/README.md b/airbyte-integrations/connectors/destination-kvdb/README.md deleted file mode 100644 index b834894111b6b..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Kvdb Destination - -This is the repository for the [Kvdb](https://kvdb.io) destination connector, written in Python. It is intended to be an example for how to write a Python destination. KvDB is a very simple key value store, which makes it great for the purposes of illustrating how to write a Python destination connector. - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-kvdb:build -``` - -#### Create credentials -**If you are a community contributor**, generate the necessary credentials from [Kvdb](https://kvdb.io/docs/api/), and then create a file `secrets/config.json` conforming to the `destination_kvdb/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination kvdb test creds` -and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - - - -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** -```bash -airbyte-ci connectors --name=destination-kvdb build -``` - -An image will be built with the tag `airbyte/destination-kvdb:dev`. - -**Via `docker build`:** -```bash -docker build -t airbyte/destination-kvdb:dev . -``` -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/destination-kvdb:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-kvdb:dev check --config /secrets/config.json -# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages -cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-kvdb:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=destination-kvdb test -``` - - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-kvdb test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/kvdb.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/__init__.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/__init__.py deleted file mode 100644 index 5f3b041035bf9..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from .destination import DestinationKvdb - -__all__ = ["DestinationKvdb"] diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/client.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/client.py deleted file mode 100644 index 74d9f41176f57..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/client.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Iterable, List, Mapping, Tuple, Union - -import requests - - -class KvDbClient: - base_url = "https://kvdb.io" - PAGE_SIZE = 1000 - - def __init__(self, bucket_id: str, secret_key: str = None): - self.secret_key = secret_key - self.bucket_id = bucket_id - - def write(self, key: str, value: Mapping[str, Any]): - return self.batch_write([(key, value)]) - - def batch_write(self, keys_and_values: List[Tuple[str, Mapping[str, Any]]]): - """ - https://kvdb.io/docs/api/#execute-transaction - """ - request_body = {"txn": [{"set": key, "value": value} for key, value in keys_and_values]} - return self._request("POST", json=request_body) - - def list_keys(self, list_values: bool = False, prefix: str = None) -> Iterable[Union[str, List]]: - """ - https://kvdb.io/docs/api/#list-keys - """ - # TODO handle rate limiting - pagination_complete = False - offset = 0 - - while not pagination_complete: - response = self._request( - "GET", - params={ - "limit": self.PAGE_SIZE, - "skip": offset, - "format": "json", - "prefix": prefix or "", - "values": "true" if list_values else "false", - }, - endpoint="/", # the "list" endpoint doesn't work without adding a trailing slash to the URL - ) - - response_json = response.json() - yield from response_json - - pagination_complete = len(response_json) < self.PAGE_SIZE - offset += self.PAGE_SIZE - - def delete(self, key: Union[str, List[str]]): - """ - https://kvdb.io/docs/api/#execute-transaction - """ - key_list = key if isinstance(key, List) else [key] - request_body = {"txn": [{"delete": k} for k in key_list]} - return self._request("POST", json=request_body) - - def _get_base_url(self) -> str: - return f"{self.base_url}/{self.bucket_id}" - - def _get_auth_headers(self) -> Mapping[str, Any]: - return {"Authorization": f"Bearer {self.secret_key}"} if self.secret_key else {} - - def _request( - self, http_method: str, endpoint: str = None, params: Mapping[str, Any] = None, json: Mapping[str, Any] = None - ) -> requests.Response: - url = self._get_base_url() + (endpoint or "") - headers = {"Accept": "application/json", **self._get_auth_headers()} - - response = requests.request(method=http_method, params=params, url=url, headers=headers, json=json) - - response.raise_for_status() - return response diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py deleted file mode 100644 index 33ab8565fae4b..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import time -import traceback -import uuid -from typing import Any, Iterable, Mapping - -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type -from destination_kvdb.client import KvDbClient -from destination_kvdb.writer import KvDbWriter - - -class DestinationKvdb(Destination): - def write( - self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] - ) -> Iterable[AirbyteMessage]: - - """ - Reads the input stream of messages, config, and catalog to write data to the destination. - - This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received - in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been - successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, - then the source is given the last state message output from this method as the starting point of the next sync. - """ - writer = KvDbWriter(KvDbClient(**config)) - - for configured_stream in configured_catalog.streams: - if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: - writer.delete_stream_entries(configured_stream.stream.name) - - for message in input_messages: - if message.type == Type.STATE: - # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush - # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state - writer.flush() - yield message - elif message.type == Type.RECORD: - record = message.record - writer.queue_write_operation( - record.stream, record.data, time.time_ns() / 1_000_000 - ) # convert from nanoseconds to milliseconds - else: - # ignore other message types for now - continue - - # Make sure to flush any records still in the queue - writer.flush() - - def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the destination with the needed permissions - e.g: if a provided API token or password can be used to connect and write to the destination. - """ - try: - # Verify write access by attempting to write and then delete to a random key - client = KvDbClient(**config) - random_key = str(uuid.uuid4()) - client.write(random_key, {"value": "_airbyte_connection_check"}) - client.delete(random_key) - except Exception as e: - traceback.print_exc() - return AirbyteConnectionStatus( - status=Status.FAILED, message=f"An exception occurred: {e}. \nStacktrace: \n{traceback.format_exc()}" - ) - else: - return AirbyteConnectionStatus(status=Status.SUCCEEDED) diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/spec.json b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/spec.json deleted file mode 100644 index 0ced52c17a227..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/spec.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "documentationUrl": "https://kvdb.io/docs/api/", - "supported_destination_sync_modes": ["overwrite", "append"], - "supportsIncremental": true, - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination KVdb", - "type": "object", - "required": ["bucket_id", "secret_key"], - "additionalProperties": false, - "properties": { - "bucket_id": { - "title": "Bucket ID", - "type": "string", - "description": "The ID of your KVdb bucket.", - "order": 1 - }, - "secret_key": { - "title": "Secret Key", - "type": "string", - "description": "Your bucket Secret Key.", - "order": 2 - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py b/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py deleted file mode 100644 index 33acbf8a22fb3..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/writer.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from collections import Mapping - -from destination_kvdb.client import KvDbClient - - -class KvDbWriter: - """ - Data is written to KvDB in the following format: - key: stream_name__ab__ - value: a JSON object representing the record's data - - This is because unless a data source explicitly designates a primary key, we don't know what to key the record on. - Since KvDB allows reading records with certain prefixes, we treat it more like a message queue, expecting the reader to - read messages with a particular prefix e.g: name__ab__123, where 123 is the timestamp they last read data from. - """ - - write_buffer = [] - flush_interval = 1000 - - def __init__(self, client: KvDbClient): - self.client = client - - def delete_stream_entries(self, stream_name: str): - """Deletes all the records belonging to the input stream""" - keys_to_delete = [] - for key in self.client.list_keys(prefix=f"{stream_name}__ab__"): - keys_to_delete.append(key) - if len(keys_to_delete) == self.flush_interval: - self.client.delete(keys_to_delete) - keys_to_delete.clear() - if len(keys_to_delete) > 0: - self.client.delete(keys_to_delete) - - def queue_write_operation(self, stream_name: str, record: Mapping, written_at: int): - kv_pair = (f"{stream_name}__ab__{written_at}", record) - self.write_buffer.append(kv_pair) - if len(self.write_buffer) == self.flush_interval: - self.flush() - - def flush(self): - self.client.batch_write(self.write_buffer) - self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py deleted file mode 100644 index 5e083acc53518..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from typing import Any, Dict, List, Mapping - -import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStateMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - Status, - SyncMode, - Type, -) -from destination_kvdb import DestinationKvdb -from destination_kvdb.client import KvDbClient - - -@pytest.fixture(name="config") -def config_fixture() -> Mapping[str, Any]: - with open("secrets/config.json", "r") as f: - return json.loads(f.read()) - - -@pytest.fixture(name="configured_catalog") -def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} - - append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.append, - ) - - overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - - return ConfiguredAirbyteCatalog(streams=[append_stream, overwrite_stream]) - - -@pytest.fixture(autouse=True) -def teardown(config: Mapping): - yield - client = KvDbClient(**config) - client.delete(list(client.list_keys())) - - -@pytest.fixture(name="client") -def client_fixture(config) -> KvDbClient: - return KvDbClient(**config) - - -def test_check_valid_config(config: Mapping): - outcome = DestinationKvdb().check(AirbyteLogger(), config) - assert outcome.status == Status.SUCCEEDED - - -def test_check_invalid_config(): - outcome = DestinationKvdb().check(AirbyteLogger(), {"bucket_id": "not_a_real_id"}) - assert outcome.status == Status.FAILED - - -def _state(data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) - - -def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: - return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) - ) - - -def retrieve_all_records(client: KvDbClient) -> List[AirbyteRecordMessage]: - """retrieves and formats all records in kvdb as Airbyte messages""" - all_records = client.list_keys(list_values=True) - out = [] - for record in all_records: - key = record[0] - stream = key.split("__ab__")[0] - value = record[1] - out.append(_record(stream, value["str_col"], value["int_col"])) - return out - - -def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: KvDbClient): - """ - This test verifies that: - 1. writing a stream in "overwrite" mode overwrites any existing data for that stream - 2. writing a stream in "append" mode appends new records without deleting the old ones - 3. The correct state message is output by the connector at the end of the sync - """ - append_stream, overwrite_stream = configured_catalog.streams[0].stream.name, configured_catalog.streams[1].stream.name - first_state_message = _state({"state": "1"}) - first_record_chunk = [_record(append_stream, str(i), i) for i in range(5)] + [_record(overwrite_stream, str(i), i) for i in range(5)] - - second_state_message = _state({"state": "2"}) - second_record_chunk = [_record(append_stream, str(i), i) for i in range(5, 10)] + [ - _record(overwrite_stream, str(i), i) for i in range(5, 10) - ] - - destination = DestinationKvdb() - - expected_states = [first_state_message, second_state_message] - output_states = list( - destination.write( - config, configured_catalog, [*first_record_chunk, first_state_message, *second_record_chunk, second_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - expected_records = [_record(append_stream, str(i), i) for i in range(10)] + [_record(overwrite_stream, str(i), i) for i in range(10)] - records_in_destination = retrieve_all_records(client) - assert expected_records == records_in_destination, "Records in destination should match records expected" - - # After this sync we expect the append stream to have 15 messages and the overwrite stream to have 5 - third_state_message = _state({"state": "3"}) - third_record_chunk = [_record(append_stream, str(i), i) for i in range(10, 15)] + [ - _record(overwrite_stream, str(i), i) for i in range(10, 15) - ] - - output_states = list(destination.write(config, configured_catalog, [*third_record_chunk, third_state_message])) - assert [third_state_message] == output_states - - records_in_destination = retrieve_all_records(client) - expected_records = [_record(append_stream, str(i), i) for i in range(15)] + [ - _record(overwrite_stream, str(i), i) for i in range(10, 15) - ] - assert expected_records == records_in_destination diff --git a/airbyte-integrations/connectors/destination-kvdb/main.py b/airbyte-integrations/connectors/destination-kvdb/main.py deleted file mode 100644 index 178789589e5af..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/main.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from destination_kvdb import DestinationKvdb - -if __name__ == "__main__": - DestinationKvdb().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-kvdb/requirements.txt b/airbyte-integrations/connectors/destination-kvdb/requirements.txt deleted file mode 100644 index d6e1198b1ab1f..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/airbyte-integrations/connectors/destination-kvdb/setup.py b/airbyte-integrations/connectors/destination-kvdb/setup.py deleted file mode 100644 index dab5520718aba..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "requests", -] - -TEST_REQUIREMENTS = ["pytest~=6.1"] - -setup( - name="destination_kvdb", - description="Destination implementation for Kvdb.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/destination-kvdb/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-kvdb/unit_tests/unit_test.py deleted file mode 100644 index 219ae0142c724..0000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/docs/integrations/destinations/kvdb.md b/docs/integrations/destinations/kvdb.md index 1d9a4341e8fc6..5ca1dfa0526b9 100644 --- a/docs/integrations/destinations/kvdb.md +++ b/docs/integrations/destinations/kvdb.md @@ -1,4 +1,4 @@ -# KVDB +# KVDB [ARCHIVED] The KVDB destination for Airbyte