🎉Source HubSpot: Adds form_submission and property_history streams

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c",
   "name": "HubSpot",
   "dockerRepository": "airbyte/source-hubspot",
-  "dockerImageTag": "0.1.35",
+  "dockerImageTag": "0.1.36",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot",
   "icon": "hubspot.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -307,7 +307,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.35
+  dockerImageTag: 0.1.36
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -3067,7 +3067,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.1.35"
+- dockerImage: "airbyte/source-hubspot:0.1.36"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.35
+LABEL io.airbyte.version=0.1.36
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/README.md

@@ -47,10 +47,10 @@ and place them into `secrets/config.json`.
 
 ### Locally running the connector
 ```
-python main_dev.py spec
-python main_dev.py check --config secrets/config.json
-python main_dev.py discover --config secrets/config.json
-python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
+python main.py spec
+python main.py check --config secrets/config.json
+python main.py discover --config secrets/config.json
+python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
 ## Testing

airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml

@@ -17,15 +17,17 @@ tests:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
+      timeout_seconds: 600
       configured_catalog_path: "sample_files/full_refresh_catalog.json"
-      empty_streams: ["workflows", "form_submissions", "ticket_pipelines"]
+      empty_streams: ["workflows", "form_submissions", "ticket_pipelines", "property_history"]
     - config_path: "secrets/config_oauth.json"
+      timeout_seconds: 600
       configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
       # The `campaigns` stream is empty in this case, because we use a catalog with
       # incremental streams: subscription_changes and email_events (it takes a long time to read)
       # and therefore the start date is set at 2021-10-10 for `config_oauth.json`,
       # but the campaign was created on 2021-01-11
-      empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines"]
+      empty_streams: ["campaigns", "workflows", "contacts_list_memberships", "form_submissions", "ticket_pipelines", "property_history"]
   incremental:
     - config_path: "secrets/config.json"
       configured_catalog_path: "sample_files/configured_catalog.json"
@@ -34,6 +36,7 @@ tests:
         subscription_changes: ["timestamp"]
         email_events: ["timestamp"]
         contact_lists: ["timestamp"]
+        property_history: ["timestamp"]
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "sample_files/full_refresh_catalog.json"
@@ -68,6 +71,8 @@ tests:
         "tickets": ["properties", "hs_time_in_2"]
         "tickets": ["properties", "hs_time_in_3"]
         "tickets": ["properties", "hs_time_in_4"]
+        "property_history": ["property", "hs_time_in_lead"]
+        "property_history": ["property", "hs_time_in_subscriber"]
     - config_path: "secrets/config_oauth.json"
       configured_catalog_path: "sample_files/configured_catalog_for_oauth_config.json"
       ignored_fields:
@@ -101,3 +106,5 @@ tests:
         "tickets": ["properties", "hs_time_in_2"]
         "tickets": ["properties", "hs_time_in_3"]
         "tickets": ["properties", "hs_time_in_4"]
+        "property_history": ["property", "hs_time_in_lead"]
+        "property_history": ["property", "hs_time_in_subscriber"]

airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json

@@ -120,6 +120,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "form_submissions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "line_items",
@@ -165,6 +174,17 @@
       "cursor_field": ["updatedAt"],
       "destination_sync_mode": "append"
     },
+    {
+      "stream": {
+        "name": "property_history",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "default_cursor_field": ["timestamp"]
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": ["timestamp"],
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "quotes",

airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog_for_oauth_config.json

@@ -84,6 +84,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "form_submissions",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "forms",
@@ -141,6 +150,17 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "property_history",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "default_cursor_field": ["timestamp"]
+      },
+      "default_cursor_field": ["timestamp"],
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "subscription_changes",

airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json

@@ -117,6 +117,15 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
+    {
+      "stream": {
+        "name": "property_history",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "quotes",

airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json

@@ -1,6 +1,7 @@
 {
   "start_date": "2020-01-01T00:00:00Z",
   "credentials": {
+    "credentials_title": "API Key Credentials",
     "api_key": "demo"
   }
 }

airbyte-integrations/connectors/source-hubspot/sample_files/sample_config_oauth.json

@@ -0,0 +1,10 @@
+{
+    "start_date": "2021-10-01T00:00:00Z",
+    "credentials": {
+      "credentials_title": "OAuth Credentials",
+      "client_id": "123456789_client_id_hubspot",
+      "client_secret": "123456789_client_secret_hubspot",
+      "refresh_token": "123456789_some_refresh_token"
+    }
+  }
+

airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py

@@ -9,7 +9,7 @@
 from abc import ABC, abstractmethod
 from functools import lru_cache, partial
 from http import HTTPStatus
-from typing import Any, Callable, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union
 
 import backoff
 import pendulum as pendulum
@@ -919,6 +919,50 @@ class OwnerStream(Stream):
     created_at_field = "createdAt"
 
 
+class PropertyHistoryStream(IncrementalStream):
+    """Contacts Endpoint, API v1
+    Is used to get all Contacts and the history of their respective
+    Properties. Whenever a property is changed it is added here.
+    Docs: https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts
+    """
+
+    more_key = "has-more"
+    url = "/contacts/v1/lists/recently_updated/contacts/recent"
+    updated_at_field = "timestamp"
+    created_at_field = "timestamp"
+    data_field = "contacts"
+    page_field = "vid-offset"
+    page_filter = "vidOffset"
+    limit = 100
+
+    def list(self, fields) -> Iterable:
+        properties = self._api.get("/properties/v2/contact/properties")
+        properties_list = [single_property["name"] for single_property in properties]
+        params = {"propertyMode": "value_and_history", "property": properties_list}
+        yield from self.read(partial(self._api.get, url=self.url), params)
+
+    def _transform(self, records: Iterable) -> Iterable:
+        for record in records:
+            properties = record.get("properties")
+            vid = record.get("vid")
+            value_dict: Dict
+            for key, value_dict in properties.items():
+                versions = value_dict.get("versions")
+                if key == "lastmodifieddate":
+                    # Skipping the lastmodifieddate since it only returns the value
+                    # when one field of a contact was changed no matter which
+                    # field was changed. It therefore creates overhead, since for
+                    # every changed property there will be the date it was changed in itself
+                    # and a change in the lastmodifieddate field.
+                    continue
+                if versions:
+                    for version in versions:
+                        version["timestamp"] = self._field_to_datetime(version["timestamp"]).to_datetime_string()
+                        version["property"] = key
+                        version["vid"] = vid
+                        yield version
+
+
 class SubscriptionChangeStream(IncrementalStream):
     """Subscriptions timeline for a portal, API v1
     Docs: https://legacydocs.hubspot.com/docs/methods/email/get_subscriptions_timeline

airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py

@@ -23,6 +23,7 @@
     FormSubmissionStream,
     MarketingEmailStream,
     OwnerStream,
+    PropertyHistoryStream,
     SubscriptionChangeStream,
     TicketPipelineStream,
     WorkflowStream,
@@ -58,6 +59,7 @@ def __init__(self, start_date, credentials, **kwargs):
             "marketing_emails": MarketingEmailStream(**common_params),
             "owners": OwnerStream(**common_params),
             "products": CRMObjectIncrementalStream(entity="product", **common_params),
+            "property_history": PropertyHistoryStream(**common_params),
             "subscription_changes": SubscriptionChangeStream(**common_params),
             "tickets": CRMObjectIncrementalStream(entity="ticket", **common_params),
             "ticket_pipelines": TicketPipelineStream(**common_params),

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json

@@ -0,0 +1,34 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "type": ["null", "object"],
+    "properties": {
+      "value": {
+        "type": ["null", "string"]
+      },
+      "source-type": {
+        "type": ["null", "string"]
+      },
+      "source-id": {
+        "type": ["null", "string"]
+      },
+      "source-label": {
+        "type": ["null", "string"]
+      },
+      "updated-by-user-id" : {
+        "type": ["null", "integer"]
+      },
+      "timestamp": {
+        "type": ["null", "string"],
+        "format": "date-time"
+      },
+      "selected": {
+        "type": ["null", "boolean"]
+      },
+      "property": {
+        "type": ["null", "string"]
+      },
+      "vid": {
+        "type": ["null", "integer"]
+      }
+    }
+}

docs/integrations/sources/hubspot.md

@@ -98,9 +98,11 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
 | `email_events` | `content` |
 | `engagements` | `contacts` |
 | `forms` | `forms` |
+| `form_submissions`| `forms` |
 | `line_items` | `e-commerce` |
 | `owners` | `contacts` |
 | `products` | `e-commerce` |
+| `property_history` | `contacts` |
 | `quotes` | no scope required |
 | `subscription_changes` | `content` |
 | `tickets` | `tickets` |
@@ -110,6 +112,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
 
 | Version | Date       | Pull Request | Subject                                                                                                                                        |
 |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.36  | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784) | Add Property History Stream |
 | 0.1.35  | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081) | Add Feedback Submissions stream and update Ticket Pipelines stream |
 | 0.1.34  | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641) | Add more fields for `email_events` stream |
 | 0.1.33  | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887) | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams |