From 1b2595167c9eb05af8bf771644d22086e78d51e7 Mon Sep 17 00:00:00 2001 From: Baz Date: Mon, 12 Feb 2024 15:13:22 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Slack:=20Join=20to=20th?= =?UTF-8?q?e=20channels=20while=20`read`=20instead=20of=20`discovery`=20(#?= =?UTF-8?q?35131)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-slack/README.md | 20 +-- .../source-slack/acceptance-test-config.yml | 4 + .../integration_tests/expected_records.jsonl | 44 ++--- .../connectors/source-slack/metadata.yaml | 2 +- .../schemas/channel_messages.json | 30 ++++ .../source_slack/schemas/threads.json | 33 ++++ .../source_slack/schemas/users.json | 22 +++ .../source-slack/source_slack/source.py | 157 ++++++++++-------- .../source-slack/source_slack/utils.py | 24 +++ .../source-slack/unit_tests/conftest.py | 8 +- .../source-slack/unit_tests/test_source.py | 3 +- .../source-slack/unit_tests/test_streams.py | 17 +- docs/integrations/sources/slack.md | 1 + 13 files changed, 245 insertions(+), 120 deletions(-) create mode 100644 airbyte-integrations/connectors/source-slack/source_slack/utils.py diff --git a/airbyte-integrations/connectors/source-slack/README.md b/airbyte-integrations/connectors/source-slack/README.md index 1b407f1aed598..f61cf7ebbdb59 100644 --- a/airbyte-integrations/connectors/source-slack/README.md +++ b/airbyte-integrations/connectors/source-slack/README.md @@ -1,17 +1,17 @@ # Orbit Source This is the repository for the Orbit configuration based source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/orbit). +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/slack). ## Local development #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/orbit) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/slack) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orbit/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source orbit test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source slack test creds` and place them into `secrets/config.json`. ### Locally running the connector docker image @@ -20,23 +20,23 @@ and place them into `secrets/config.json`. #### Build **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name source-orbit build +airbyte-ci connectors --name source-slack build ``` -An image will be built with the tag `airbyte/source-orbit:dev`. +An image will be built with the tag `airbyte/source-slack:dev`. **Via `docker build`:** ```bash -docker build -t airbyte/source-orbit:dev . +docker build -t airbyte/source-slack:dev . ``` #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-orbit:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orbit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-slack:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-slack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 2fd1ba9af04bd..b94b7cf70bc01 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -26,6 +26,10 @@ acceptance_tests: expect_records: path: "integration_tests/expected_records.jsonl" timeout_seconds: 4800 + ignored_fields: + channels: + - name: updated + bypass_reason: Value can change while interacting with data full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl index 9ddbf42168e51..e5b6c79113767 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl @@ -1,29 +1,15 @@ -{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123063, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123075, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123086, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171463} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171464} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04L3M4PTJ6"}, "emitted_at": 1683105171464} -{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171629} -{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} -{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} -{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695111199104} -{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695111707952} -{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695110320605} -{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1695110320606} -{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1695110320606} -{"stream": "threads", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695112005658} -{"stream": "threads", "data": {"client_msg_id": "e1e2d142-a0dd-4587-86e3-2dcb439ead82", "type": "message", "text": "<@U04LY6NARHU> Test test", "user": "U04L65GPMKN", "ts": "1683104515.919709", "blocks": [{"type": "rich_text", "block_id": "xVnQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " Test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104515.919709}, "emitted_at": 1695112005659} -{"stream": "threads", "data": {"client_msg_id": "ffccbb24-8dd6-476d-87bf-65e5fa033cb9", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "Lvl", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104528.084359}, "emitted_at": 1695112005659} -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354906} -{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} -{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} -{"stream": "users", "data": {"id": "U04L2LC770E", "team_id": "T04KX3KDDU6", "name": "deactivateduser521176", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090821, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} -{"stream": "users", "data": {"id": "U04L69BPZFX", "team_id": "T04KX3KDDU6", "name": "deactivateduser839125", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png", "image_32": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png", "image_48": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png", "image_72": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png", "image_192": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png", "image_512": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811889, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} -{"stream": "users", "data": {"id": "U04L94Y2JPM", "team_id": "T04KX3KDDU6", "name": "deactivateduser962255", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png", "image_32": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png", "image_48": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png", "image_72": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png", "image_192": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png", "image_512": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090815, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04LMS8F7JM", "team_id": "T04KX3KDDU6", "name": "deactivateduser421996", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811683, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354911} -{"stream": "users", "data": {"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354912} +{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826841, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735549} +{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826857, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735550} +{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1706890826875, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1707568735550} +{"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736171} +{"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} +{"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} +{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707568738170} +{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1707569060525} +{"stream": "channel_messages", "data": {"type": "message", "subtype": "reminder_add", "text": " set up a reminder \u201ctest reminder\u201d in this channel at 9AM tomorrow, Eastern European Summer Time.", "user": "U04L65GPMKN", "ts": "1695814864.744249", "channel_id": "C04LTCM2Y56", "float_ts": 1695814864.744249}, "emitted_at": 1707569208689} +{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707569354932} +{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1707569354933} +{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1707569354933} +{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1707569357949} +{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} +{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} diff --git a/airbyte-integrations/connectors/source-slack/metadata.yaml b/airbyte-integrations/connectors/source-slack/metadata.yaml index 349c09cd31dc0..9028678c9fae0 100644 --- a/airbyte-integrations/connectors/source-slack/metadata.yaml +++ b/airbyte-integrations/connectors/source-slack/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd - dockerImageTag: 0.3.7 + dockerImageTag: 0.3.8 dockerRepository: airbyte/source-slack documentationUrl: https://docs.airbyte.com/integrations/sources/slack githubIssueLabel: source-slack diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 38e48b0dd61c4..f85b210d0d40b 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -10,6 +10,36 @@ "properties": { "type": { "type": ["null", "string"] + }, + "block_id": { + "type": ["null", "string"] + }, + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "text": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "type": { + "type": ["null", "string"] + } + } + } } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index dee131fed53f3..2571351507c52 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -49,6 +49,39 @@ "properties": { "type": { "type": ["null", "string"] + }, + "block_id": { + "type": ["null", "string"] + }, + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "elements": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "text": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + } + } + } + }, + "type": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json index d13dce63ee136..47a1d6e9da1a0 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json @@ -10,6 +10,9 @@ "type": "object", "additionalProperties": true, "properties": { + "always_active": { + "type": ["null", "boolean"] + }, "avatar_hash": { "type": "string" }, @@ -34,6 +37,13 @@ "email": { "type": "string" }, + "fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "huddle_state": { + "type": "string" + }, "image_24": { "type": "string" }, @@ -75,6 +85,18 @@ }, "skype": { "type": "string" + }, + "status_emoji_display_info": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "status_expiration": { + "type": ["null", "integer"] + }, + "status_text_canonical": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index d59a46f5ae580..e785114f865f4 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import copy + from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -14,7 +14,9 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from pendulum import DateTime, Period +from pendulum import DateTime + +from .utils import chunk_date_range class SlackStream(HttpStream, ABC): @@ -80,13 +82,67 @@ def should_retry(self, response: requests.Response) -> bool: return response.status_code == requests.codes.REQUEST_TIMEOUT or super().should_retry(response) +class JoinChannelsStream(HttpStream): + """ + This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. + Its responses should only be logged for debugging reasons, not read as records. + """ + + url_base = "https://slack.com/api/" + http_method = "POST" + primary_key = "id" + + def __init__(self, channel_filter: List[str] = None, **kwargs): + self.channel_filter = channel_filter or [] + super().__init__(**kwargs) + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable: + """ + Override to simply indicate that the specific channel was joined successfully. + This method should not return any data, but should return an empty iterable. + """ + self.logger.info(f"Successfully joined channel: {stream_slice['channel_name']}") + return [] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + The pagination is not applicable to this Service Stream. + """ + return None + + def path(self, **kwargs) -> str: + return "conversations.join" + + def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: + return {"channel": stream_slice["channel"]} + + class ChanneledStream(SlackStream, ABC): """Slack stream with channel filter""" - def __init__(self, channel_filter: List[str] = [], **kwargs): + def __init__(self, channel_filter: List[str] = [], join_channels: bool = False, **kwargs): self.channel_filter = channel_filter + self.join_channels = join_channels + self.kwargs = kwargs super().__init__(**kwargs) + @property + def join_channels_stream(self) -> JoinChannelsStream: + return JoinChannelsStream(authenticator=self.kwargs.get("authenticator"), channel_filter=self.channel_filter) + + def should_join_to_channel(self, channel: Mapping[str, Any]) -> bool: + """ + The `is_member` property indicates whether or not the API Bot is already assigned / joined to the channel. + https://api.slack.com/types/conversation#booleans + """ + return self.join_channels and not channel.get("is_member") + + def make_join_channel_slice(self, channel: Mapping[str, Any]) -> Mapping[str, Any]: + channel_id: str = channel.get("id") + channel_name: str = channel.get("name") + self.logger.info(f"Joining Slack Channel: `{channel_name}`") + return {"channel": channel_id, "channel_name": channel_name} + class Channels(ChanneledStream): data_field = "channels" @@ -103,19 +159,30 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: params["types"] = "public_channel" return params - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[MutableMapping]: + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[MutableMapping]: json_response = response.json() channels = json_response.get(self.data_field, []) if self.channel_filter: channels = [channel for channel in channels if channel["name"] in self.channel_filter] yield from channels + def read_records(self, sync_mode: SyncMode, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + Override the default `read_records` method to provide the `JoinChannelsStream` functionality, + and be able to read all the channels, not just the ones that already has the API Bot joined. + """ + for channel in super().read_records(sync_mode=sync_mode): + # check the channel should be joined before reading + if self.should_join_to_channel(channel): + # join the channel before reading it + yield from self.join_channels_stream.read_records( + sync_mode=sync_mode, + stream_slice=self.make_join_channel_slice(channel), + ) + # reading the channel data + self.logger.info(f"Reading the channel: `{channel.get('name')}`") + yield channel + class ChannelMembers(ChanneledStream): data_field = "members" @@ -134,7 +201,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, # Slack just returns raw IDs as a string, so we want to put them in a "join table" format yield {"member_id": member_id, "channel_id": stream_slice["channel_id"]} - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) for channel_record in channels_stream.read_records(sync_mode=SyncMode.full_refresh): yield {"channel_id": channel_record["id"]} @@ -148,21 +215,6 @@ def path(self, **kwargs) -> str: # Incremental Streams -def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1), end_date: Optional[DateTime] = None) -> Iterable[Period]: - """ - Yields a list of the beginning and ending timestamps of each day between the start date and now. - The return value is a pendulum.period - """ - - end_date = end_date or pendulum.now() - # Each stream_slice contains the beginning and ending timestamp for a 24 hour period - chunk_start_date = start_date - while chunk_start_date < end_date: - chunk_end_date = min(chunk_start_date + interval, end_date) - yield pendulum.period(chunk_start_date, chunk_end_date) - chunk_start_date = chunk_end_date - - class IncrementalMessageStream(ChanneledStream, ABC): data_field = "messages" cursor_field = "float_ts" @@ -179,8 +231,7 @@ def set_sub_primary_key(self): for index, value in enumerate(self.primary_key): setattr(self, f"sub_primary_key_{index + 1}", value) else: - logger = AirbyteLogger() - logger.error("Failed during setting sub primary keys. Primary key should be list.") + self.logger.error("Failed during setting sub primary keys. Primary key should be list.") def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) @@ -223,7 +274,7 @@ def path(self, **kwargs) -> str: def use_cache(self) -> bool: return True - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: stream_state = stream_state or {} start_date = pendulum.from_timestamp(stream_state.get(self.cursor_field, self._start_ts)) end_date = self._end_ts and pendulum.from_timestamp(self._end_ts) @@ -246,14 +297,14 @@ def __init__(self, lookback_window: Mapping[str, int], **kwargs): def path(self, **kwargs) -> str: return "conversations.replies" - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: """ The logic for incrementally syncing threads is not very obvious, so buckle up. To get all messages in a thread, one must specify the channel and timestamp of the parent (first) message of that thread, basically its ID. - One complication is that threads can be updated at any time in the future. Therefore, if we wanted to comprehensively sync data + One complication is that threads can be updated at Any time in the future. Therefore, if we wanted to comprehensively sync data i.e: get every single response in a thread, we'd have to read every message in the slack instance every time we ran a sync, because otherwise there is no way to guarantee that a thread deep in the past didn't receive a new message. @@ -297,38 +348,6 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite yield {} -class JoinChannelsStream(HttpStream): - """ - This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. - Its responses should only be logged for debugging reasons, not read as records. - """ - - url_base = "https://slack.com/api/" - http_method = "POST" - primary_key = "id" - - def __init__(self, channel_filter: List[str] = None, **kwargs): - self.channel_filter = channel_filter or [] - super().__init__(**kwargs) - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - return [{"message": f"Successfully joined channel: {stream_slice['channel_name']}"}] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None # No pagination - - def path(self, **kwargs) -> str: - return "conversations.join" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) - for channel in channels_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"channel": channel["id"], "channel_name": channel["name"]} - - def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: - return {"channel": stream_slice["channel"]} - - class SourceSlack(AbstractSource): def _get_authenticator(self, config: Mapping[str, Any]): # Added to maintain backward compatibility with previous versions @@ -365,8 +384,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: end_date = end_date and pendulum.parse(end_date) threads_lookback_window = pendulum.Duration(days=config["lookback_window"]) channel_filter = config.get("channel_filter", []) + should_join_to_channels = config.get("join_channels") - channels = Channels(authenticator=authenticator, channel_filter=channel_filter) + channels = Channels(authenticator=authenticator, join_channels=should_join_to_channels, channel_filter=channel_filter) streams = [ channels, ChannelMembers(authenticator=authenticator, channel_filter=channel_filter), @@ -387,13 +407,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Users(authenticator=authenticator), ] - # To sync data from channels, the bot backed by this token needs to join all those channels. This operation is idempotent. - if config["join_channels"]: - logger = AirbyteLogger() - logger.info("joining Slack channels") - join_channels_stream = JoinChannelsStream(authenticator=authenticator, channel_filter=channel_filter) - for stream_slice in join_channels_stream.stream_slices(): - for message in join_channels_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): - logger.info(message["message"]) - return streams diff --git a/airbyte-integrations/connectors/source-slack/source_slack/utils.py b/airbyte-integrations/connectors/source-slack/source_slack/utils.py new file mode 100644 index 0000000000000..7507dbab35657 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/utils.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Iterable, Optional + +import pendulum +from pendulum import DateTime, Period + + +def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1), end_date: Optional[DateTime] = None) -> Iterable[Period]: + """ + Yields a list of the beginning and ending timestamps of each day between the start date and now. + The return value is a pendulum.period + """ + + end_date = end_date or pendulum.now() + # Each stream_slice contains the beginning and ending timestamp for a 24 hour period + chunk_start_date = start_date + while chunk_start_date < end_date: + chunk_end_date = min(chunk_start_date + interval, end_date) + yield pendulum.period(chunk_start_date, chunk_end_date) + chunk_start_date = chunk_end_date diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 2fa9d3d332fee..6d9254730d5f6 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -18,10 +18,10 @@ def conversations_list(requests_mock): "https://slack.com/api/conversations.list?limit=1000&types=public_channel", json={ "channels": [ - {"name": "advice-data-architecture", "id": 1}, - {"name": "advice-data-orchestration", "id": 2}, - {"name": "airbyte-for-beginners", "id": 3}, - {"name": "good-reads", "id": 4}, + {"name": "advice-data-architecture", "id": 1, "is_member": False}, + {"name": "advice-data-orchestration", "id": 2, "is_member": True}, + {"name": "airbyte-for-beginners", "id": 3, "is_member": False}, + {"name": "good-reads", "id": 4, "is_member": True}, ] }, ) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index 01d1bf1af9bec..bef3bf26651f9 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -11,12 +11,11 @@ @parametrized_configs -def test_streams(conversations_list, join_channels, config, is_valid): +def test_streams(conversations_list, config, is_valid): source = SourceSlack() if is_valid: streams = source.streams(config) assert len(streams) == 5 - assert join_channels.call_count == 2 else: with pytest.raises(Exception) as exc_info: _ = source.streams(config) diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index b783be1817c91..d0327093318f8 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -7,7 +7,7 @@ import pendulum import pytest from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_slack.source import Threads, Users +from source_slack.source import Channels, Threads, Users @pytest.fixture @@ -93,3 +93,18 @@ def test_get_updated_state(authenticator, legacy_token_config, current_state, la def test_backoff(authenticator, headers, expected_result): stream = Users(authenticator=authenticator) assert stream.backoff_time(Mock(headers=headers)) == expected_result + + +def test_channels_stream_with_autojoin(authenticator) -> None: + """ + The test uses the `conversations_list` fixture(autouse=true) as API mocker. + """ + expected = [ + {'name': 'advice-data-architecture', 'id': 1, 'is_member': False}, + {'name': 'advice-data-orchestration', 'id': 2, 'is_member': True}, + {'name': 'airbyte-for-beginners', 'id': 3, 'is_member': False}, + {'name': 'good-reads', 'id': 4, 'is_member': True}, + ] + stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) + assert list(stream.read_records(None)) == expected + \ No newline at end of file diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 2edb9eb6fcc67..4b1e906642abe 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -163,6 +163,7 @@ Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------| +| 0.3.8 | 2024-02-09 | [35131](https://github.com/airbytehq/airbyte/pull/35131) | Fixed the issue when `schema discovery` fails with `502` due to the platform timeout | | 0.3.7 | 2024-01-10 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | | 0.3.6 | 2023-11-21 | [32707](https://github.com/airbytehq/airbyte/pull/32707) | Threads: do not use client-side record filtering | | 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image |