From 0e71677105b90d8841cef3214b6ef31c9433b823 Mon Sep 17 00:00:00 2001 From: Jianhao Zhang Date: Mon, 15 Aug 2022 04:18:09 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Bigcommerce:=20fix=20in?= =?UTF-8?q?finite=20loop=20in=20Page=20stream=20(#14940)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(source-bigcommerce): infinite loop by overloaded request_params() in class Page * bump connector version to 0.1.6 in Dockerfile * update docs/integrations/sources/bigcommerce.md * fix: tests are failing * docs: updated connector definitions * auto-bump connector version [ci skip] Co-authored-by: Harshith Mullapudi Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 4 +- .../connectors/source-bigcommerce/Dockerfile | 2 +- .../source_bigcommerce/source.py | 6 --- .../source_bigcommerce/spec.json | 2 +- docs/integrations/sources/bigcommerce.md | 52 +++++++++---------- 6 files changed, 31 insertions(+), 37 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 889a5eedc5358..f8f8bd47cfbca 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -88,7 +88,7 @@ - name: BigCommerce sourceDefinitionId: 59c5501b-9f95-411e-9269-7143c939adbd dockerRepository: airbyte/source-bigcommerce - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/bigcommerce icon: bigcommerce.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 80d2d5138cee8..6bce86b8145b9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -795,7 +795,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigcommerce:0.1.5" +- dockerImage: "airbyte/source-bigcommerce:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bigcommerce" connectionSpecification: @@ -806,7 +806,7 @@ - "start_date" - "store_hash" - "access_token" - additionalProperties: false + additionalProperties: true properties: start_date: type: "string" diff --git a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile index 192ee3073b47c..28103d20be7a7 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile +++ b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-bigcommerce diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py index 4c23e6d72779d..91a5bfb8293a4 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py @@ -184,12 +184,6 @@ def path(self, **kwargs) -> str: def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - return params - def read_records( self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs ) -> Iterable[Mapping[str, Any]]: diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json index 2f4560a860fcd..9cc4bc6f21543 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json @@ -5,7 +5,7 @@ "title": "BigCommerce Source CDK Specifications", "type": "object", "required": ["start_date", "store_hash", "access_token"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "start_date": { "type": "string", diff --git a/docs/integrations/sources/bigcommerce.md b/docs/integrations/sources/bigcommerce.md index 5b256b8da8467..946c9fed2cedd 100644 --- a/docs/integrations/sources/bigcommerce.md +++ b/docs/integrations/sources/bigcommerce.md @@ -12,28 +12,28 @@ This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/connec This Source is capable of syncing the following core Streams: -* [Customers](https://developer.bigcommerce.com/api-reference/store-management/customers-v3/customers/customersget) -* [Orders](https://developer.bigcommerce.com/api-reference/store-management/orders/orders/getallorders) -* [Transactions](https://developer.bigcommerce.com/api-reference/store-management/order-transactions/transactions/gettransactions) -* [Pages](https://developer.bigcommerce.com/api-reference/store-management/store-content/pages/getallpages) -* [Products](https://developer.bigcommerce.com/api-reference/store-management/catalog/products/getproducts) +- [Customers](https://developer.bigcommerce.com/api-reference/store-management/customers-v3/customers/customersget) +- [Orders](https://developer.bigcommerce.com/api-reference/store-management/orders/orders/getallorders) +- [Transactions](https://developer.bigcommerce.com/api-reference/store-management/order-transactions/transactions/gettransactions) +- [Pages](https://developer.bigcommerce.com/api-reference/store-management/store-content/pages/getallpages) +- [Products](https://developer.bigcommerce.com/api-reference/store-management/catalog/products/getproducts) ### Data type mapping | Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| :--------------- | :----------- | :---- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | ### Performance considerations @@ -44,18 +44,18 @@ BigCommerce has some [rate limit restrictions](https://developer.bigcommerce.com 1. Navigate to your store’s control panel \(Advanced Settings > API Accounts > Create API Account\) 2. Create an API account. 3. Select the resources you want to allow access to. Airbyte only needs read-level access. - * Note: The UI will show all possible data sources and will show errors when syncing if it doesn't have permissions to access a resource. -4. The generated `Access Token` is what you'll use as the `access_token` for the integration. + - Note: The UI will show all possible data sources and will show errors when syncing if it doesn't have permissions to access a resource. +4. The generated `Access Token` is what you'll use as the `access_token` for the integration. 5. You're ready to set up BigCommerce in Airbyte! ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.5 | 2022-01-31 | [9935](https://github.com/airbytehq/airbyte/pull/9935) | Correct date-time columns for `orders` (v2 stream) | -| 0.1.4 | 2022-01-13 | [9516](https://github.com/airbytehq/airbyte/pull/9516) | Add Catalog Products Stream and fix date-time parsing | -| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | -| 0.1.2 | 2021-12-07 | [8416](https://github.com/airbytehq/airbyte/pull/8416) | Correct Incremental Function | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-08-19 | [5521](https://github.com/airbytehq/airbyte/pull/5521) | Initial Release. Source BigCommerce | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.6 | 2022-07-27 | [14940](https://github.com/airbytehq/airbyte/pull/14940) | Fix infinite loop when the page stream goes beyond one page | +| 0.1.5 | 2022-01-31 | [9935](https://github.com/airbytehq/airbyte/pull/9935) | Correct date-time columns for `orders` (v2 stream) | +| 0.1.4 | 2022-01-13 | [9516](https://github.com/airbytehq/airbyte/pull/9516) | Add Catalog Products Stream and fix date-time parsing | +| 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| 0.1.2 | 2021-12-07 | [8416](https://github.com/airbytehq/airbyte/pull/8416) | Correct Incremental Function | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-08-19 | [5521](https://github.com/airbytehq/airbyte/pull/5521) | Initial Release. Source BigCommerce |