🎉 Source HubSpot: engagements stream, remove auto-generated properties

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -287,7 +287,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.30
+  dockerImageTag: 0.1.31
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2846,7 +2846,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-hubspot:0.1.30"
+- dockerImage: "airbyte/source-hubspot:0.1.31"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.30
+LABEL io.airbyte.version=0.1.31
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py

@@ -265,10 +265,7 @@ def _cast_value(declared_field_types: List, field_name: str, field_value: Any, d
 
     def _cast_record_fields_if_needed(self, record: Mapping, properties: Mapping[str, Any] = None) -> Mapping:
 
-        if self.entity not in {"contact", "engagement", "product", "quote", "ticket", "company", "deal", "line_item"}:
-            return record
-
-        if not record.get("properties"):
+        if not self.entity or not record.get("properties"):
             return record
 
         properties = properties or self.properties
@@ -363,7 +360,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter
                     'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])',
                     'correlationId': '111111-2222-3333-4444-55555555555'}
                 """
-                logger.warning(f"Stream `{self.entity}` cannot be procced. {response.get('message')}")
+                logger.warning(f"Stream `{self.name}` cannot be procced. {response.get('message')}")
                 return
 
             if response.get(self.data_field) is None:
@@ -713,7 +710,6 @@ class EngagementStream(Stream):
     Docs: https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements
     """
 
-    entity = "engagement"
     url = "/engagements/v1/engagements/paged"
     more_key = "hasMore"
     limit = 250

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json

@@ -5,6 +5,12 @@
     "id": {
       "type": ["null", "integer"]
     },
+    "uid": {
+      "type": ["null", "string"]
+    },
+    "teamId": {
+      "type": ["null", "integer"]
+    },
     "portalId": {
       "type": ["null", "integer"]
     },
@@ -14,6 +20,12 @@
     "createdAt": {
       "type": ["null", "integer"]
     },
+    "createdBy": {
+      "type": ["null", "integer"]
+    },
+    "modifiedBy": {
+      "type": ["null", "integer"]
+    },
     "lastUpdated": {
       "type": ["null", "integer"]
     },
@@ -26,6 +38,33 @@
     "timestamp": {
       "type": ["null", "integer"]
     },
+    "bodyPreview": {
+      "type": ["null", "string"]
+    },
+    "bodyPreviewHtml": {
+      "type": ["null", "string"]
+    },
+    "bodyPreviewIsTruncated": {
+      "type": ["null", "boolean"]
+    },
+    "allAccessibleTeamIds": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "integer"]
+      }
+    },
+    "activityType": {
+      "type": ["null", "string"]
+    },
+    "gdprDeleted": {
+      "type": ["null", "boolean"]
+    },
+    "source": {
+      "type": ["null", "string"]
+    },
+    "sourceId": {
+      "type": ["null", "string"]
+    },
     "associations": {
       "type": ["null", "object"],
       "properties": {
@@ -46,6 +85,24 @@
           "items": {
             "type": ["null", "integer"]
           }
+        },
+        "ownerIds": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "integer"]
+          }
+        },
+        "workflowIds": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "integer"]
+          }
+        },
+        "ticketIds": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "integer"]
+          }
         }
       }
     },
@@ -77,6 +134,17 @@
             },
             "lastName": {
               "type": ["null", "string"]
+            },
+            "raw": {
+              "type": ["null", "string"]
+            }
+          }
+        },
+        "sender": {
+          "type": ["null", "object"],
+          "properties": {
+            "email": {
+              "type": ["null", "string"]
             }
           }
         },
@@ -87,6 +155,15 @@
             "properties": {
               "email": {
                 "type": ["null", "string"]
+              },
+              "firstName": {
+                "type": ["null", "string"]
+              },
+              "lastName": {
+                "type": ["null", "string"]
+              },
+              "raw": {
+                "type": ["null", "string"]
               }
             }
           }
@@ -98,6 +175,15 @@
             "properties": {
               "email": {
                 "type": ["null", "string"]
+              },
+              "firstName": {
+                "type": ["null", "string"]
+              },
+              "lastName": {
+                "type": ["null", "string"]
+              },
+              "raw": {
+                "type": ["null", "string"]
               }
             }
           }
@@ -157,6 +243,84 @@
         },
         "disposition": {
           "type": ["null", "string"]
+        },
+        "completionDate": {
+          "type": ["null", "integer"]
+        },
+        "taskType": {
+          "type": ["null", "string"]
+        },
+        "reminders": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "integer"]
+          }
+        },
+        "threadId": {
+          "type": ["null", "string", "integer"]
+        },
+        "messageId": {
+          "type": ["null", "string"]
+        },
+        "loggedFrom": {
+          "type": ["null", "string"]
+        },
+        "attachedVideoOpened": {
+          "type": ["null", "boolean"]
+        },
+        "attachedVideoWatched": {
+          "type": ["null", "boolean"]
+        },
+        "trackerKey": {
+          "type": ["null", "string"]
+        },
+        "sendDefaultReminder": {
+          "type": ["null", "boolean"]
+        },
+        "source": {
+          "type": ["null", "string"]
+        },
+        "unknownVisitorConversation": {
+          "type": ["null", "boolean"]
+        },
+        "facsimileSendId": {
+          "type": ["null", "string"]
+        },
+        "sentVia": {
+          "type": ["null", "string"]
+        },
+        "sequenceStepOrder": {
+          "type": ["null", "integer"]
+        },
+        "externalUrl": {
+          "type": ["null", "string"]
+        },
+        "postSendStatus": {
+          "type": ["null", "string"]
+        },
+        "errorMessage": {
+          "type": ["null", "string"]
+        },
+        "recipientDropReasons": {
+          "type": ["null", "string"]
+        },
+        "calleeObjectId": {
+          "type": ["null", "integer"]
+        },
+        "calleeObjectType": {
+          "type": ["null", "string"]
+        },
+        "mediaProcessingStatus": {
+          "type": ["null", "string"]
+        },
+        "sourceId": {
+          "type": ["null", "string"]
+        },
+        "priority": {
+          "type": ["null", "string"]
+        },
+        "isAllDay": {
+          "type": ["null", "boolean"]
         }
       }
     }

docs/integrations/sources/hubspot.md

@@ -37,15 +37,15 @@ This source is capable of syncing the following tables and their data:
 * [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows)
 
 ### A note on the `engagements` stream
-Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. 
+Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`.
 
-Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination. 
+Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination.
 
 * A `call` engagement will have a corresponding `engagements_metadata` object with non-null values in the `toNumber`, `fromNumber`, `status`, `externalId`, `durationMilliseconds`, `externalAccountId`, `recordingUrl`, `body`, and `disposition` columns.
-* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`. 
-* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns. 
-* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column. 
-* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns. 
+* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`.
+* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns.
+* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column.
+* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns.
 
 
 **Note**: HubSpot API currently only supports `quotes` endpoint using API Key, using Oauth it is impossible to access this stream (as reported by [community.hubspot.com](https://community.hubspot.com/t5/APIs-Integrations/Help-with-using-Feedback-CRM-API-and-Quotes-CRM-API/m-p/449104/highlight/true#M44411)).
@@ -110,6 +110,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
 
 | Version | Date       | Pull Request | Subject                                                                                                                                        |
 |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.31  | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream                                                                                   |
 | 0.1.30  | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams                                                                                                      |
 | 0.1.29  | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams      |
 | 0.1.28  | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update fields and descriptions                                                                                                                 |
@@ -134,4 +135,3 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
 | 0.1.9   | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype                                                                                                        |
 | 0.1.8   | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions                                                                                                             |
 | 0.1.7   | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema                                                                                                                           |
-