diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json index 5e76551413b44..b307efc884257 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", "name": "HubSpot", "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.30", + "dockerImageTag": "0.1.32", "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", "icon": "hubspot.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 74993fe19c8cb..17551d21c8e43 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -287,7 +287,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.31 + dockerImageTag: 0.1.32 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8a42a527dbb1c..a95b9511c84db 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2846,7 +2846,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-hubspot:0.1.31" +- dockerImage: "airbyte/source-hubspot:0.1.32" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index b8bd788b20cda..e51182b850dd4 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.31 +LABEL io.airbyte.version=0.1.32 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index d10181d818662..7a320596a7ca1 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -18,14 +18,14 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/full_refresh_catalog.json" - empty_streams: ["workflows"] + empty_streams: ["workflows", "form_submissions"] - config_path: "secrets/config_oauth.json" configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json" # The `campaigns` stream is empty in this case, because we use a catalog with # incremental streams: subscription_changes and email_events (it takes a long time to read) # and therefore the start date is set at 2021-10-10 for `config_oauth.json`, # but the campaign was created on 2021-01-11 - empty_streams: ["campaigns", "workflows", "contacts_list_memberships"] + empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions"] incremental: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json index 5f9e42c80d30b..b6157948c2968 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json @@ -96,6 +96,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "line_items", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json index 3bc43efac7647..673f27f562138 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json @@ -93,6 +93,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "line_items", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json index 19f4049c4ac17..98d568d1cc022 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json @@ -81,6 +81,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "form_submissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "line_items", diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 708836d4d3b80..8dedd91d82004 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -732,6 +732,34 @@ class FormStream(Stream): created_at_field = "createdAt" +class FormSubmissionStream(Stream): + """Marketing Forms, API v1 + This endpoint requires the forms scope. + Docs: https://legacydocs.hubspot.com/docs/methods/forms/get-submissions-for-a-form + """ + + url = "/form-integrations/v1/submissions/forms" + limit = 50 + updated_at_field = "updatedAt" + + def _transform(self, records: Iterable) -> Iterable: + for record in super()._transform(records): + keys = record.keys() + + # There's no updatedAt field in the submission however forms fetched by using this field, + # so it has to be added to the submissions otherwise it would fail when calling _filter_old_records + if "updatedAt" not in keys: + record["updatedAt"] = record["submittedAt"] + + yield record + + def list(self, fields) -> Iterable: + for form in self.read(getter=partial(self._api.get, url="/marketing/v3/forms")): + for submission in self.read(getter=partial(self._api.get, url=f"{self.url}/{form['id']}")): + submission["formId"] = form["id"] + yield submission + + class MarketingEmailStream(Stream): """Marketing Email, API v1 Docs: https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-emails diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py index ca17ea698f025..7768755fc387d 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py @@ -19,6 +19,7 @@ EmailEventStream, EngagementStream, FormStream, + FormSubmissionStream, MarketingEmailStream, OwnerStream, SubscriptionChangeStream, @@ -45,6 +46,7 @@ def __init__(self, start_date, credentials, **kwargs): "email_events": EmailEventStream(**common_params), "engagements": EngagementStream(**common_params), "forms": FormStream(**common_params), + "form_submissions": FormSubmissionStream(**common_params), "line_items": CRMObjectIncrementalStream(entity="line_item", **common_params), "marketing_emails": MarketingEmailStream(**common_params), "owners": OwnerStream(**common_params), diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/form_submissions.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/form_submissions.json new file mode 100644 index 0000000000000..3ee1cf9ea6788 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/form_submissions.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": ["null", "object"], + "properties": { + "submittedAt": { + "type": ["null", "integer"] + }, + "updatedAt": { + "type": ["null", "integer"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "objectTypeId": { + "type": ["null", "string"] + } + } + } + }, + "pageUrl": { + "type": ["null", "string"] + }, + "formId": { + "type": ["null", "string"] + } + } +} diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 90ac117be7bf0..79e463b79434e 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -110,6 +110,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.32 | 2022-01-13 | [8011](https://github.com/airbytehq/airbyte/pull/8011) | Add new stream form_submissions | | 0.1.31 | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream | | 0.1.30 | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams | | 0.1.29 | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams |