Merge branch 'master' into leti/experiment-speedy-connection-for-acti…

…vation

* master: (26 commits)
  supply a source id for schema discovery in connector integration tests (#17662)
  Source Iterable: Add permission check for stream (#17602)
  Moving TrackingClientSingleton.initialize into the bean itself (#17631)
  Handle null workspace IDs in tracking/reporting methods gracefully (#17641)
  Bump Airbyte version from 0.40.11 to 0.40.12 (#17653)
  Revert "Do not wait the end of a reset to return an update (#17591)" (#17640)
  Standardize HttpRequester's url_base and path format (#17524)
  Create geography_type enum and add geography column in connection and workspace table (#16818)
  airbyte-cron: update connector definitions from remote (#16438)
  Do not wait the end of a reset to return an update (#17591)
  Remove redundant title labels from connector specs (#17544)
  Updated GA4 status
  support large schema discovery (#17394)
  🪟 🐛 Fixes connector checks not properly ending their loading state (#17620)
  🪟🧪 [Experiment] add hideOnboarding experiment (#17605)
  Source Recharge: change releaseStage to GA (#17606)
  Source Recharge: skip stream if 403 received (#17608)
  remove sonar-scan workflow (#17609)
  Mark/tables should be full width on all pages (#17401)
  Auto fail all workfow if there is a Versioning issue (#17562)
  ...

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.10
+current_version = 0.40.12
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.10
+VERSION=0.40.12
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

airbyte-analytics/src/main/java/io/airbyte/analytics/LoggingTrackingClient.java

@@ -10,6 +10,7 @@
 import java.util.Optional;
 import java.util.UUID;
 import java.util.function.Function;
+import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -34,15 +35,21 @@ public void alias(final UUID workspaceId, final String previousCustomerId) {
   }
 
   @Override
-  public void track(final UUID workspaceId, final String action) {
+  public void track(@Nullable final UUID workspaceId, final String action) {
     track(workspaceId, action, Collections.emptyMap());
   }
 
   @Override
-  public void track(final UUID workspaceId, final String action, final Map<String, Object> metadata) {
+  public void track(@Nullable final UUID workspaceId, final String action, final Map<String, Object> metadata) {
+    String version = null;
+    UUID userId = null;
+    if (workspaceId != null) {
+      version = Optional.ofNullable(identityFetcher.apply(workspaceId).getAirbyteVersion()).map(AirbyteVersion::serialize).orElse(null);
+      userId = identityFetcher.apply(workspaceId).getCustomerId();
+    }
     LOGGER.info("track. version: {}, userId: {}, action: {}, metadata: {}",
-        Optional.ofNullable(identityFetcher.apply(workspaceId).getAirbyteVersion()).map(AirbyteVersion::serialize).orElse(null),
-        identityFetcher.apply(workspaceId).getCustomerId(),
+        version,
+        userId,
         action,
         metadata);
   }

airbyte-analytics/src/main/java/io/airbyte/analytics/SegmentTrackingClient.java

@@ -17,6 +17,9 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.function.Function;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is a wrapper around the Segment backend Java SDK.
@@ -40,6 +43,8 @@
  */
 public class SegmentTrackingClient implements TrackingClient {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(SegmentTrackingClient.class);
+
   public static final String CUSTOMER_ID_KEY = "user_id";
   private static final String SEGMENT_WRITE_KEY = "7UDdp5K55CyiGgsauOr2pNNujGvmhaeu";
   private static final String AIRBYTE_VERSION_KEY = "airbyte_version";
@@ -106,12 +111,16 @@ public void alias(final UUID workspaceId, final String previousCustomerId) {
   }
 
   @Override
-  public void track(final UUID workspaceId, final String action) {
+  public void track(@Nullable final UUID workspaceId, final String action) {
     track(workspaceId, action, Collections.emptyMap());
   }
 
   @Override
-  public void track(final UUID workspaceId, final String action, final Map<String, Object> metadata) {
+  public void track(@Nullable final UUID workspaceId, final String action, final Map<String, Object> metadata) {
+    if (workspaceId == null) {
+      LOGGER.error("Could not track action {} due to null workspaceId", action);
+      return;
+    }
     final Map<String, Object> mapCopy = new HashMap<>(metadata);
     final TrackingIdentity trackingIdentity = identityFetcher.apply(workspaceId);
 

airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingClient.java

@@ -6,6 +6,7 @@
 
 import java.util.Map;
 import java.util.UUID;
+import javax.annotation.Nullable;
 
 /**
  * General interface for user level Airbyte usage reporting. We use Segment for behavioural
@@ -28,8 +29,8 @@ public interface TrackingClient {
 
   void alias(UUID workspaceId, String previousCustomerId);
 
-  void track(UUID workspaceId, String action);
+  void track(@Nullable UUID workspaceId, String action);
 
-  void track(UUID workspaceId, String action, Map<String, Object> metadata);
+  void track(@Nullable UUID workspaceId, String action, Map<String, Object> metadata);
 
 }

airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java

@@ -6,7 +6,9 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -135,6 +137,13 @@ void testTrackWithMetadata() {
     assertEquals(metadata, filterTrackedAtProperty(Objects.requireNonNull(actual.properties())));
   }
 
+  @Test
+  void testTrackNullWorkspace() {
+    segmentTrackingClient.track(null, JUMP);
+
+    verify(analytics, never()).enqueue(any());
+  }
+
   private static ImmutableMap<String, Object> filterTrackedAtProperty(final Map<String, ?> properties) {
     final String trackedAtKey = "tracked_at";
     assertTrue(properties.containsKey(trackedAtKey));

airbyte-bootloader/Dockerfile

@@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.4
 ARG JDK_IMAGE=amazoncorretto:${JDK_VERSION}
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.40.10
+ARG VERSION=0.40.12
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -13,6 +13,9 @@
 import io.airbyte.config.Configs;
 import io.airbyte.config.EnvConfigs;
 import io.airbyte.config.StandardWorkspace;
+import io.airbyte.config.init.ApplyDefinitionsHelper;
+import io.airbyte.config.init.DefinitionsProvider;
+import io.airbyte.config.init.LocalDefinitionsProvider;
 import io.airbyte.config.init.YamlSeedConfigPersistence;
 import io.airbyte.config.persistence.ConfigPersistence;
 import io.airbyte.config.persistence.ConfigRepository;
@@ -66,7 +69,7 @@ public class BootloaderApp {
   private final FeatureFlags featureFlags;
   private final SecretMigrator secretMigrator;
   private ConfigPersistence configPersistence;
-  private ConfigPersistence yamlSeedConfigPersistence;
+  private DefinitionsProvider localDefinitionsProvider;
   private Database configDatabase;
   private Database jobDatabase;
   private JobPersistence jobPersistence;
@@ -125,7 +128,11 @@ public BootloaderApp(final Configs configs,
 
     postLoadExecution = () -> {
       try {
-        configPersistence.loadData(yamlSeedConfigPersistence);
+        final ConfigRepository configRepository =
+            new ConfigRepository(configPersistence, configDatabase);
+
+        final ApplyDefinitionsHelper applyDefinitionsHelper = new ApplyDefinitionsHelper(configRepository, localDefinitionsProvider);
+        applyDefinitionsHelper.apply();
 
         if (featureFlags.forceSecretMigration() || !jobPersistence.isSecretMigrated()) {
           if (this.secretMigrator != null) {
@@ -197,8 +204,8 @@ private static ConfigPersistence getConfigPersistence(final Database configDatab
     return DatabaseConfigPersistence.createWithValidation(configDatabase, jsonSecretsProcessor);
   }
 
-  private static ConfigPersistence getYamlSeedConfigPersistence() throws IOException {
-    return new YamlSeedConfigPersistence(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
+  private static DefinitionsProvider getLocalDefinitionsProvider() throws IOException {
+    return new LocalDefinitionsProvider(YamlSeedConfigPersistence.DEFAULT_SEED_DEFINITION_RESOURCE_CLASS);
   }
 
   private static Database getJobDatabase(final DSLContext dslContext) throws IOException {
@@ -213,7 +220,7 @@ private void initPersistences(final DSLContext configsDslContext, final DSLConte
     try {
       configDatabase = getConfigDatabase(configsDslContext);
       configPersistence = getConfigPersistence(configDatabase);
-      yamlSeedConfigPersistence = getYamlSeedConfigPersistence();
+      localDefinitionsProvider = getLocalDefinitionsProvider();
       jobDatabase = getJobDatabase(jobsDslContext);
       jobPersistence = getJobPersistence(jobDatabase);
     } catch (final IOException e) {

airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java

@@ -138,7 +138,7 @@ void testBootloaderAppBlankDb() throws Exception {
       val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
       // this line should change with every new migration
       // to show that you meant to make a new migration to the prod database
-      assertEquals("0.40.3.002", configsMigrator.getLatestMigration().getVersion().getVersion());
+      assertEquals("0.40.11.001", configsMigrator.getLatestMigration().getVersion().getVersion());
 
       val jobsPersistence = new DefaultJobPersistence(jobDatabase);
       assertEquals(VERSION_0330_ALPHA, jobsPersistence.getVersion().get());

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.93
+- Low-code: Standardize slashes in url_base and path
+
 ## 0.1.92
 - Low-code: Properly propagate $options to array items
 - Low-code: Log request and response when running check operation in debug mode

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import os
 from dataclasses import InitVar, dataclass
 from functools import lru_cache
 from typing import Any, Mapping, MutableMapping, Optional, Union
@@ -72,14 +73,14 @@ def get_authenticator(self):
         return self.authenticator
 
     def get_url_base(self):
-        return self.url_base.eval(self.config)
+        return os.path.join(self.url_base.eval(self.config), "")
 
     def get_path(
         self, *, stream_state: Optional[StreamState], stream_slice: Optional[StreamSlice], next_page_token: Optional[Mapping[str, Any]]
     ) -> str:
         kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token}
         path = self.path.eval(self.config, **kwargs)
-        return path
+        return path.strip("/")
 
     def get_method(self):
         return self._method

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.92",
+    version="0.1.93",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py

@@ -4,6 +4,7 @@
 
 from unittest.mock import MagicMock
 
+import pytest as pytest
 import requests
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.http_requester import HttpMethod, HttpRequester
@@ -51,7 +52,7 @@ def test_http_requester():
         options={},
     )
 
-    assert requester.get_url_base() == "https://airbyte.io"
+    assert requester.get_url_base() == "https://airbyte.io/"
     assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234"
     assert requester.get_authenticator() == authenticator
     assert requester.get_method() == HttpMethod.GET
@@ -60,3 +61,51 @@ def test_http_requester():
     assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json
     assert requester.should_retry(requests.Response()) == should_retry
     assert {} == requester.request_kwargs(stream_state={}, stream_slice=None, next_page_token=None)
+
+
+@pytest.mark.parametrize(
+    "test_name, base_url, expected_base_url",
+    [
+        ("test_no_trailing_slash", "https://example.com", "https://example.com/"),
+        ("test_with_trailing_slash", "https://example.com/", "https://example.com/"),
+        ("test_with_v1_no_trailing_slash", "https://example.com/v1", "https://example.com/v1/"),
+        ("test_with_v1_with_trailing_slash", "https://example.com/v1/", "https://example.com/v1/"),
+    ],
+)
+def base_url_has_a_trailing_slash(test_name, base_url, expected_base_url):
+    requester = HttpRequester(
+        name="name",
+        url_base=base_url,
+        path="deals",
+        http_method=HttpMethod.GET,
+        request_options_provider=MagicMock(),
+        authenticator=MagicMock(),
+        error_handler=MagicMock(),
+        config={},
+        options={},
+    )
+    assert requester.get_url_base() == expected_base_url
+
+
+@pytest.mark.parametrize(
+    "test_name, base_url, expected_base_url",
+    [
+        ("test_no_leading_slash", "deals", "deals"),
+        ("test_with_leading_slash", "/deals", "deals"),
+        ("test_with_v1_no_leading_slash", "v1/deals", "v1/deals"),
+        ("test_with_v1_with_trailing_slash", "/v1/deals", "v1/deals"),
+    ],
+)
+def path_has_no_leading_slash(test_name, path, expected_path):
+    requester = HttpRequester(
+        name="name",
+        url_base="https://example.com",
+        path=path,
+        http_method=HttpMethod.GET,
+        request_options_provider=MagicMock(),
+        authenticator=MagicMock(),
+        error_handler=MagicMock(),
+        config={},
+        options={},
+    )
+    assert requester.get_path(stream_state={}, stream_slice={}, next_page_token={}) == expected_path