Skip to content

Commit 9da9343

Browse files
bazarnovrobbinhan
bazarnov
authored and
robbinhan
committed
🐛 Source Recharge: data types should be transformed based on schema types (airbytehq#16959)
1 parent d0eb791 commit 9da9343

File tree

11 files changed

+25
-125
lines changed

11 files changed

+25
-125
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -853,7 +853,7 @@
853853
- name: Recharge
854854
sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e
855855
dockerRepository: airbyte/source-recharge
856-
dockerImageTag: 0.1.8
856+
dockerImageTag: 0.2.0
857857
documentationUrl: https://docs.airbyte.io/integrations/sources/recharge
858858
icon: recharge.svg
859859
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -8772,7 +8772,7 @@
87728772
supportsNormalization: false
87738773
supportsDBT: false
87748774
supported_destination_sync_modes: []
8775-
- dockerImage: "airbyte/source-recharge:0.1.8"
8775+
- dockerImage: "airbyte/source-recharge:0.2.0"
87768776
spec:
87778777
documentationUrl: "https://docs.airbyte.com/integrations/sources/recharge"
87788778
connectionSpecification:

‎airbyte-integrations/connectors/source-recharge/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ RUN pip install .
1212
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
1313
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1414

15-
LABEL io.airbyte.version=0.1.9
15+
LABEL io.airbyte.version=0.2.0
1616
LABEL io.airbyte.name=airbyte/source-recharge

‎airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml

+4
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ tests:
88
- config_path: "integration_tests/invalid_config.json"
99
status: "failed"
1010
discovery:
11+
# schemas were changed in `0.1.9` compare to `0.1.8`,
12+
# plase remove this bypass, once updated to the newer version!
1113
- config_path: "secrets/config.json"
14+
backward_compatibility_tests_config:
15+
disable_for_version: "0.1.8"
1216
basic_read:
1317
- config_path: "secrets/config.json"
1418
configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"

‎airbyte-integrations/connectors/source-recharge/source_recharge/api.py

+3-16
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ class RechargeStream(HttpStream, ABC):
2020
limit = 250
2121
page_num = 1
2222

23+
# regestring the default schema transformation
24+
transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
25+
2326
@property
2427
def data_path(self):
2528
return self.name
@@ -109,20 +112,6 @@ class Charges(IncrementalRechargeStream):
109112
Charges Stream: https://developer.rechargepayments.com/v1-shopify?python#list-charges
110113
"""
111114

112-
def get_stream_data(self, response_data: Any) -> List[dict]:
113-
# We expect total_weight to be an integer, but the API is returning numbers like 42.0
114-
# Cast these down to int if possible, and error if not.
115-
data = super().get_stream_data(response_data)
116-
for record in data:
117-
if "total_weight" in record:
118-
total_weight = record["total_weight"]
119-
int_total_weight = int(total_weight)
120-
if total_weight == int_total_weight:
121-
record["total_weight"] = int_total_weight
122-
else:
123-
raise ValueError(f"Expected total_weight to be an integer, got {total_weight}")
124-
return data
125-
126115

127116
class Collections(RechargeStream):
128117
"""
@@ -164,8 +153,6 @@ class Orders(IncrementalRechargeStream):
164153
Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders
165154
"""
166155

167-
transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization)
168-
169156

170157
class Products(RechargeStream):
171158
"""

‎airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json

+5-33
Original file line numberDiff line numberDiff line change
@@ -226,34 +226,13 @@
226226
"type": ["null", "string"]
227227
},
228228
"tax_lines": {
229-
"oneOf": [
230-
{
231-
"type": ["null", "number"]
232-
},
233-
{
234-
"type": ["null", "string"]
235-
}
236-
]
229+
"type": ["null", "string"]
237230
},
238231
"total_discounts": {
239-
"oneOf": [
240-
{
241-
"type": ["null", "number"]
242-
},
243-
{
244-
"type": ["null", "string"]
245-
}
246-
]
232+
"type": ["null", "string"]
247233
},
248234
"total_line_items_price": {
249-
"oneOf": [
250-
{
251-
"type": ["null", "number"]
252-
},
253-
{
254-
"type": ["null", "string"]
255-
}
256-
]
235+
"type": ["null", "string"]
257236
},
258237
"total_price": {
259238
"type": ["null", "string"]
@@ -262,17 +241,10 @@
262241
"type": ["null", "string"]
263242
},
264243
"total_tax": {
265-
"oneOf": [
266-
{
267-
"type": ["null", "number"]
268-
},
269-
{
270-
"type": ["null", "string"]
271-
}
272-
]
244+
"type": ["null", "string"]
273245
},
274246
"total_weight": {
275-
"type": ["null", "integer"]
247+
"type": ["null", "number"]
276248
},
277249
"transaction_id": {
278250
"type": ["null", "string"]

‎airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json

+2-16
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,7 @@
1919
"type": ["null", "string"]
2020
},
2121
"owner_id": {
22-
"oneOf": [
23-
{
24-
"type": ["null", "number"]
25-
},
26-
{
27-
"type": ["null", "string"]
28-
}
29-
]
22+
"type": ["null", "string"]
3023
},
3124
"owner_resource": {
3225
"type": ["null", "string"]
@@ -36,14 +29,7 @@
3629
"format": "date-time"
3730
},
3831
"value": {
39-
"oneOf": [
40-
{
41-
"type": ["null", "number"]
42-
},
43-
{
44-
"type": ["null", "string"]
45-
}
46-
]
32+
"type": ["null", "string"]
4733
},
4834
"value_type": {
4935
"type": ["null", "string"]

‎airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json

+3-24
Original file line numberDiff line numberDiff line change
@@ -6,42 +6,21 @@
66
"type": ["null", "integer"]
77
},
88
"address_id": {
9-
"oneOf": [
10-
{
11-
"type": ["null", "number"]
12-
},
13-
{
14-
"type": ["null", "string"]
15-
}
16-
]
9+
"type": ["null", "string"]
1710
},
1811
"created_at": {
1912
"type": ["null", "string"],
2013
"format": "date-time"
2114
},
2215
"customer_id": {
23-
"oneOf": [
24-
{
25-
"type": ["null", "number"]
26-
},
27-
{
28-
"type": ["null", "string"]
29-
}
30-
]
16+
"type": ["null", "string"]
3117
},
3218
"next_charge_scheduled_at": {
3319
"type": ["null", "string"],
3420
"format": "date-time"
3521
},
3622
"price": {
37-
"oneOf": [
38-
{
39-
"type": ["null", "number"]
40-
},
41-
{
42-
"type": ["null", "string"]
43-
}
44-
]
23+
"type": ["null", "string"]
4524
},
4625
"product_title": {
4726
"type": ["null", "string"]

‎airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json

+3-24
Original file line numberDiff line numberDiff line change
@@ -219,24 +219,10 @@
219219
"type": ["null", "integer"]
220220
},
221221
"shopify_product_id": {
222-
"oneOf": [
223-
{
224-
"type": ["null", "number"]
225-
},
226-
{
227-
"type": ["null", "string"]
228-
}
229-
]
222+
"type": ["null", "string"]
230223
},
231224
"shopify_variant_id": {
232-
"oneOf": [
233-
{
234-
"type": ["null", "number"]
235-
},
236-
{
237-
"type": ["null", "string"]
238-
}
239-
]
225+
"type": ["null", "string"]
240226
},
241227
"sku": {
242228
"type": ["null", "string"]
@@ -381,14 +367,7 @@
381367
"type": ["null", "string"]
382368
},
383369
"total_tax": {
384-
"oneOf": [
385-
{
386-
"type": ["null", "number"]
387-
},
388-
{
389-
"type": ["null", "string"]
390-
}
391-
]
370+
"type": ["null", "string"]
392371
},
393372
"total_weight": {
394373
"type": ["null", "integer"]

‎airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json

+1-8
Original file line numberDiff line numberDiff line change
@@ -85,14 +85,7 @@
8585
"type": "string"
8686
},
8787
"value": {
88-
"oneOf": [
89-
{
90-
"type": ["null", "number"]
91-
},
92-
{
93-
"type": ["null", "string"]
94-
}
95-
]
88+
"type": ["null", "string"]
9689
}
9790
}
9891
}

‎docs/integrations/sources/recharge.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n
6767

6868
| Version | Date | Pull Request | Subject |
6969
| :--- | :--- | :--- | :--- |
70-
| 0.1.9 | 2022-09-12 | [16592](https://github.com/airbytehq/airbyte/pull/16592) | Use TypeTransformer to reliably convert to int values in Orders stream
70+
| 0.2.0 | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types
7171
| 0.1.8 | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer |
7272
| 0.1.7 | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities |
7373
| 0.1.6 | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint |

0 commit comments

Comments
 (0)