diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 08bd8d53c9496..d0a83faf69652 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -562,7 +562,7 @@ - name: Gitlab sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 dockerRepository: airbyte/source-gitlab - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab icon: gitlab.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 36e887c0dc2e8..f941c7025b46e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4684,7 +4684,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-gitlab:0.1.9" +- dockerImage: "airbyte/source-gitlab:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/gitlab" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index 99765fc1d8ac4..9644fab5adcf8 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml index 1612db7f211fb..0e8f1d78e7206 100644 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml @@ -15,6 +15,11 @@ tests: empty_streams: ["group_issue_boards"] expect_records: path: "integration_tests/expected_records.txt" + - config_path: "secrets/config_with_ids.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["group_issue_boards", "epic_issues"] + expect_records: + path: "integration_tests/expected_records_with_ids.txt" # We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately, # and the Acceptance Tests at this stage do not support this functionality. # Therefore, we hardcode the cursor_paths for our config. diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt index ea9493b16b8c1..e68e1ae2e4ef1 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.txt @@ -1 +1,11 @@ -{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "assignees": [], "type": "ISSUE", "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "19 days remaining"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1670761695320} \ No newline at end of file +{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "assignees": [], "type": "ISSUE", "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "17 days remaining"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1670761695320} +{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1670873255735} +{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873256216} +{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1670873256571} +{"stream": "groups", "data": {"id": 61015232, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2", "name": "Test Private SubSubG 2", "path": "test-private-subsubg-2", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG / Test Private SubSubG 2", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2", "created_at": "2022-12-02T08:56:21.783Z", "parent_id": 61014882, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941bnTmxzY-5zek69yJ7s4r", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873256904} +{"stream": "groups", "data": {"id": 61015239, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2/test-private-subsubg-3", "name": "Test Private SubSubG 3", "path": "test-private-subsubg-3", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG / Test Private SubSubG 2 / Test Private SubSubG 3", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2/test-private-subsubg-3", "created_at": "2022-12-02T08:56:34.202Z", "parent_id": 61015232, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941fxQ648-7Mt4f2K11VUwm", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873257186} +{"stream": "groups", "data": {"id": 61014902, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg", "name": "Test Public SG", "path": "test-public-sg", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG", "full_path": "new-group-airbute/test-public-sg", "created_at": "2022-12-02T08:47:02.412Z", "parent_id": 11329647, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941iDo59yBXTTUMQUQztr_x", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541906, "path_with_namespace": "new-group-airbute/test-public-sg/test-public-project-1"}]}, "emitted_at": 1670873257575} +{"stream": "groups", "data": {"id": 61015202, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-public-subsubg-1", "name": "Test Public SubSubG 1", "path": "test-public-subsubg-1", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Public SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-public-subsubg-1", "created_at": "2022-12-02T08:55:27.046Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941K9JsRDnw2J5ZQxgCDr6d", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873257855} +{"stream": "groups", "data": {"id": 61014943, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2", "name": "Test SG Public 2", "path": "test-sg-public-2", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2", "created_at": "2022-12-02T08:48:04.727Z", "parent_id": 61014902, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941eGbxua89EPU8uu4snVuj", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541858, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-project-1"}]}, "emitted_at": 1670873258194} +{"stream": "groups", "data": {"id": 61014863, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte", "name": "Test Subgroup Airbyte", "path": "test-subgroup-airbyte", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte", "full_path": "new-group-airbute/test-subgroup-airbyte", "created_at": "2022-12-02T08:45:53.479Z", "parent_id": 11329647, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941yL2Jtfwss88thzmUPZv5", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541892, "path_with_namespace": "new-group-airbute/test-subgroup-airbyte/test-project-2"}]}, "emitted_at": 1670873258580} +{"stream": "groups", "data": {"id": 11266951, "web_url": "https://gitlab.com/groups/airbyte.io", "name": "airbyte.io", "path": "airbyte.io", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "airbyte.io", "full_path": "airbyte.io", "created_at": "2021-03-10T17:16:37.549Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "marked_for_deletion_on": null, "shared_with_groups": [], "runners_token": "GR1348941bzmDjXx-Cz48snUcJfK8", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": 10000, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": false, "membership_lock": false, "ip_restriction_ranges": null, "projects": [{"id": 25156633, "path_with_namespace": "airbyte.io/ci-test-project"}, {"id": 25032440, "path_with_namespace": "airbyte.io/learn-gitlab"}, {"id": 25032439, "path_with_namespace": "airbyte.io/documentation"}]}, "emitted_at": 1670873259080} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.txt b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.txt new file mode 100644 index 0000000000000..25a12fce7f738 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.txt @@ -0,0 +1,9 @@ +{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1670873655927} +{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873656251} +{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1670873656646} +{"stream": "groups", "data": {"id": 61015232, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2", "name": "Test Private SubSubG 2", "path": "test-private-subsubg-2", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG / Test Private SubSubG 2", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2", "created_at": "2022-12-02T08:56:21.783Z", "parent_id": 61014882, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941bnTmxzY-5zek69yJ7s4r", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873657003} +{"stream": "groups", "data": {"id": 61015239, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2/test-private-subsubg-3", "name": "Test Private SubSubG 3", "path": "test-private-subsubg-3", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG / Test Private SubSubG 2 / Test Private SubSubG 3", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg/test-private-subsubg-2/test-private-subsubg-3", "created_at": "2022-12-02T08:56:34.202Z", "parent_id": 61015232, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941fxQ648-7Mt4f2K11VUwm", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873657286} +{"stream": "groups", "data": {"id": 61014902, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg", "name": "Test Public SG", "path": "test-public-sg", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG", "full_path": "new-group-airbute/test-public-sg", "created_at": "2022-12-02T08:47:02.412Z", "parent_id": 11329647, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941iDo59yBXTTUMQUQztr_x", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541906, "path_with_namespace": "new-group-airbute/test-public-sg/test-public-project-1"}]}, "emitted_at": 1670873657672} +{"stream": "groups", "data": {"id": 61015202, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-public-subsubg-1", "name": "Test Public SubSubG 1", "path": "test-public-subsubg-1", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Public SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-public-subsubg-1", "created_at": "2022-12-02T08:55:27.046Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941K9JsRDnw2J5ZQxgCDr6d", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1670873657974} +{"stream": "groups", "data": {"id": 61014943, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2", "name": "Test SG Public 2", "path": "test-sg-public-2", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2", "created_at": "2022-12-02T08:48:04.727Z", "parent_id": 61014902, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941eGbxua89EPU8uu4snVuj", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541858, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-project-1"}]}, "emitted_at": 1670873658361} +{"stream": "groups", "data": {"id": 61014863, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte", "name": "Test Subgroup Airbyte", "path": "test-subgroup-airbyte", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte", "full_path": "new-group-airbute/test-subgroup-airbyte", "created_at": "2022-12-02T08:45:53.479Z", "parent_id": 11329647, "ldap_cn": null, "ldap_access": null, "shared_with_groups": [], "runners_token": "GR1348941yL2Jtfwss88thzmUPZv5", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41541892, "path_with_namespace": "new-group-airbute/test-subgroup-airbyte/test-project-2"}]}, "emitted_at": 1670873658771} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py index c1975a634c423..c9fd74b8641a8 100644 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ b/airbyte-integrations/connectors/source-gitlab/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "vcrpy==4.1.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.12.4", "vcrpy==4.1.1"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index ca162994301a3..a7b0b0b8d35a4 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -3,9 +3,8 @@ # -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union -import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -23,6 +22,8 @@ GroupMilestones, GroupProjects, Groups, + GroupsList, + IncludeDescendantGroups, Issues, Jobs, MergeRequestCommits, @@ -40,61 +41,65 @@ class SourceGitlab(AbstractSource): - def _generate_main_streams(self, config: Mapping[str, Any]) -> Tuple[GitlabStream, GitlabStream]: - auth = TokenAuthenticator(token=config["private_token"]) - auth_params = dict(authenticator=auth, api_url=config["api_url"]) - - pids = list(filter(None, config.get("projects", "").split(" "))) - gids = config.get("groups") - - if gids: - gids = list(filter(None, gids.split(" "))) - else: - gids = self._get_group_list(**auth_params) - - groups = Groups(group_ids=gids, **auth_params) - if gids: - projects = GroupProjects(project_ids=pids, parent_stream=groups, **auth_params) - else: - projects = Projects(project_ids=pids, **auth_params) - - return groups, projects - - def _get_group_list(self, **kwargs): - headers = kwargs["authenticator"].get_auth_header() - - ids = [] - has_next = True - # First request params - per_page = 50 - next_page = 1 - - while has_next: - response = requests.get(f'https://{kwargs["api_url"]}/api/v4/groups?page={next_page}&per_page={per_page}', headers=headers) - next_page = response.headers.get("X-Next-Page") - per_page = response.headers.get("X-Per-Page") - results = response.json() - - items = map(lambda i: i["full_path"].replace("/", "%2f"), results) - ids.extend(items) - has_next = "X-Next-Page" in response.headers and response.headers["X-Next-Page"] != "" - - return ids + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.__auth_params: Mapping[str, Any] = {} + self.__groups_stream: Optional[GitlabStream] = None + self.__projects_stream: Optional[GitlabStream] = None + + def _groups_stream(self, config: MutableMapping[str, Any]) -> Groups: + if not self.__groups_stream: + auth_params = self._auth_params(config) + group_ids = list(map(lambda x: x["id"], self._get_group_list(config))) + self.__groups_stream = Groups(group_ids=group_ids, **auth_params) + return self.__groups_stream + + def _projects_stream(self, config: MutableMapping[str, Any]) -> Union[Projects, GroupProjects]: + if not self.__projects_stream: + auth_params = self._auth_params(config) + project_ids = list(filter(None, config.get("projects", "").split(" "))) + groups_stream = self._groups_stream(config) + if groups_stream.group_ids: + self.__projects_stream = GroupProjects(project_ids=project_ids, parent_stream=groups_stream, **auth_params) + return self.__projects_stream + self.__projects_stream = Projects(project_ids=project_ids, **auth_params) + return self.__projects_stream + + def _auth_params(self, config: MutableMapping[str, Any]) -> Mapping[str, Any]: + if not self.__auth_params: + auth = TokenAuthenticator(token=config["private_token"]) + self.__auth_params = dict(authenticator=auth, api_url=config["api_url"]) + return self.__auth_params + + def _get_group_list(self, config: MutableMapping[str, Any]) -> List[str]: + group_ids = list(filter(None, config.get("groups", "").split(" "))) + # Gitlab exposes different APIs to get a list of groups. + # We use https://docs.gitlab.com/ee/api/groups.html#list-groups in case there's no group IDs in the input config. + # This API provides full information about all available groups, including subgroups. + # + # In case there is a definitive list of groups IDs in the input config, the above API can not be used since + # it does not support filtering by group ID, so we use + # https://docs.gitlab.com/ee/api/groups.html#details-of-a-group and + # https: //docs.gitlab.com/ee/api/groups.html#list-a-groups-descendant-groups for each group ID. The latter one does not + # provide full group info so can only be used to retrieve alist of group IDs and pass it further to init a corresponding stream. + auth_params = self._auth_params(config) + stream = GroupsList(**auth_params) if not group_ids else IncludeDescendantGroups(group_ids=group_ids, **auth_params) + for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): + yield from stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) def check_connection(self, logger, config) -> Tuple[bool, any]: try: - groups, projects = self._generate_main_streams(config) - for stream in projects.stream_slices(sync_mode=SyncMode.full_refresh): - next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream)) - return True, None + projects = self._projects_stream(config) + for stream_slice in projects.stream_slices(sync_mode=SyncMode.full_refresh): + next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) + return True, None except Exception as error: return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}" - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["private_token"]) - auth_params = dict(authenticator=auth, api_url=config["api_url"]) + def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: + auth_params = self._auth_params(config) - groups, projects = self._generate_main_streams(config) + groups, projects = self._groups_stream(config), self._projects_stream(config) pipelines = Pipelines(parent_stream=projects, start_date=config["start_date"], **auth_params) merge_requests = MergeRequests(parent_stream=projects, start_date=config["start_date"], **auth_params) epics = Epics(parent_stream=groups, **auth_params) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 4128194b8ef33..4e82f13298643 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -179,6 +179,21 @@ def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): return record +class IncludeDescendantGroups(Groups): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return stream_slice["path"] + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for gid in self.group_ids: + yield {"path": f"groups/{gid}"} + yield {"path": f"groups/{gid}/descendant_groups"} + + +class GroupsList(GitlabStream): + def path(self, **kwargs) -> str: + return "groups" + + class Projects(GitlabStream): stream_base_params = {"statistics": 1} use_cache = True diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index 637d2188a82ac..d538c712b89be 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -63,6 +63,7 @@ GitLab source is working with GitLab API v4. It can also work with self-hosted G | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------| +| 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | | 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | | 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream | | 0.1.7 | 2022-12-01 | [19986](https://github.com/airbytehq/airbyte/pull/19986) | Fix `GroupMilestones` stream schema |