diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index ceb15ad955b88..c5c208b8a8bad 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -37,6 +37,7 @@ import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.State; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.enums.ActorType; @@ -970,4 +971,15 @@ private Condition includeTombstones(final Field tombstoneField, final b } } + public WorkspaceServiceAccount getWorkspaceServiceAccountNoSecrets(final UUID workspaceId) + throws JsonValidationException, IOException, ConfigNotFoundException { + return persistence.getConfig(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceId.toString(), WorkspaceServiceAccount.class); + } + + public void writeWorkspaceServiceAccountNoSecrets(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + persistence.writeConfig(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceServiceAccount.getWorkspaceId().toString(), + workspaceServiceAccount); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java index 525743e559073..6040734e31544 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @@ -10,6 +10,7 @@ import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -96,4 +97,18 @@ private void hydrateValuesIfKeyPresent(final String key, final Map> configs, final } } + public void writeServiceAccountJsonCredentials(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + final WorkspaceServiceAccount workspaceServiceAccountForDB = getWorkspaceServiceAccountWithSecretCoordinate(workspaceServiceAccount); + configRepository.writeWorkspaceServiceAccountNoSecrets(workspaceServiceAccountForDB); + } + + /** + * This method is to encrypt the secret JSON key and HMAC key of a GCP service account a associated + * with a workspace. If in future we build a similar feature i.e. an AWS account associated with a + * workspace, we will have to build new implementation for it + */ + private WorkspaceServiceAccount getWorkspaceServiceAccountWithSecretCoordinate(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + if (longLivedSecretPersistence.isPresent()) { + final WorkspaceServiceAccount clonedWorkspaceServiceAccount = Jsons.clone(workspaceServiceAccount); + final Optional optionalWorkspaceServiceAccount = getOptionalWorkspaceServiceAccount( + workspaceServiceAccount.getWorkspaceId()); + // Convert the JSON key of Service Account into secret co-oridnate. Ref : + // https://cloud.google.com/iam/docs/service-accounts#key-types + if (workspaceServiceAccount.getJsonCredential() != null) { + final SecretCoordinateToPayload jsonCredSecretCoordinateToPayload = + SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getJsonCredential().toPrettyString(), + longLivedSecretPersistence.get(), + workspaceServiceAccount.getWorkspaceId(), + UUID::randomUUID, + optionalWorkspaceServiceAccount.map(WorkspaceServiceAccount::getJsonCredential).orElse(null), + "json"); + longLivedSecretPersistence.get().write(jsonCredSecretCoordinateToPayload.secretCoordinate(), jsonCredSecretCoordinateToPayload.payload()); + clonedWorkspaceServiceAccount.setJsonCredential(jsonCredSecretCoordinateToPayload.secretCoordinateForDB()); + } + // Convert the HMAC key of Service Account into secret co-oridnate. Ref : + // https://cloud.google.com/storage/docs/authentication/hmackeys + if (workspaceServiceAccount.getHmacKey() != null) { + final SecretCoordinateToPayload hmackKeySecretCoordinateToPayload = + SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getHmacKey().toString(), + longLivedSecretPersistence.get(), + workspaceServiceAccount.getWorkspaceId(), + UUID::randomUUID, + optionalWorkspaceServiceAccount.map(WorkspaceServiceAccount::getHmacKey).orElse(null), + "hmac"); + longLivedSecretPersistence.get().write(hmackKeySecretCoordinateToPayload.secretCoordinate(), hmackKeySecretCoordinateToPayload.payload()); + clonedWorkspaceServiceAccount.setHmacKey(hmackKeySecretCoordinateToPayload.secretCoordinateForDB()); + } + return clonedWorkspaceServiceAccount; + } + return workspaceServiceAccount; + } + + public Optional getOptionalWorkspaceServiceAccount(final UUID workspaceId) + throws JsonValidationException, IOException { + try { + return Optional.of(configRepository.getWorkspaceServiceAccountNoSecrets(workspaceId)); + } catch (ConfigNotFoundException e) { + return Optional.empty(); + } + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java index b2185c56093cd..bb02572bc1bbf 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java @@ -16,4 +16,9 @@ public JsonNode hydrate(final JsonNode partialConfig) { return partialConfig; } + @Override + public JsonNode hydrateSecretCoordinate(JsonNode secretCoordinate) { + return secretCoordinate; + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java index cfdffb0f0057d..69e3d07bdd08a 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java @@ -22,4 +22,9 @@ public JsonNode hydrate(final JsonNode partialConfig) { return SecretsHelpers.combineConfig(partialConfig, readOnlySecretPersistence); } + @Override + public JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinate) { + return SecretsHelpers.hydrateSecretCoordinate(secretCoordinate, readOnlySecretPersistence); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java new file mode 100644 index 0000000000000..41b9ca584809a --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence.split_secrets; + +import com.fasterxml.jackson.databind.JsonNode; + +public record SecretCoordinateToPayload(SecretCoordinate secretCoordinate, + String payload, + JsonNode secretCoordinateForDB) { + +} diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index 579ec254dd651..e4a3406228ad4 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -116,7 +116,7 @@ public static JsonNode combineConfig(final JsonNode partialConfig, final ReadOnl if (config.has(COORDINATE_FIELD)) { final var coordinateNode = config.get(COORDINATE_FIELD); final var coordinate = getCoordinateFromTextNode(coordinateNode); - return getOrThrowSecretValueNode(secretPersistence, coordinate); + return new TextNode(getOrThrowSecretValue(secretPersistence, coordinate)); } // otherwise iterate through all object fields @@ -336,16 +336,16 @@ private static JsonNode getFieldOrEmptyNode(final JsonNode node, final int field * @param secretPersistence storage layer for secrets * @param coordinate reference to a secret in the persistence * @throws RuntimeException when a secret at that coordinate is not available in the persistence - * @return a json text node containing the secret value + * @return a json string containing the secret value or a JSON */ - private static TextNode getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, final SecretCoordinate coordinate) { + private static String getOrThrowSecretValue(final ReadOnlySecretPersistence secretPersistence, + final SecretCoordinate coordinate) { final var secretValue = secretPersistence.read(coordinate); if (secretValue.isEmpty()) { throw new RuntimeException(String.format("That secret was not found in the store! Coordinate: %s", coordinate.getFullCoordinate())); } - - return new TextNode(secretValue.get()); + return secretValue.get(); } private static SecretCoordinate getCoordinateFromTextNode(final JsonNode node) { @@ -379,6 +379,15 @@ protected static SecretCoordinate getCoordinate( final UUID workspaceId, final Supplier uuidSupplier, final @Nullable String oldSecretFullCoordinate) { + return getSecretCoordinate("airbyte_workspace_", newSecret, secretReader, workspaceId, uuidSupplier, oldSecretFullCoordinate); + } + + private static SecretCoordinate getSecretCoordinate(final String secretBasePrefix, + final String newSecret, + final ReadOnlySecretPersistence secretReader, + final UUID secretBaseId, + final Supplier uuidSupplier, + final @Nullable String oldSecretFullCoordinate) { String coordinateBase = null; Long version = null; @@ -398,7 +407,7 @@ protected static SecretCoordinate getCoordinate( if (coordinateBase == null) { // IMPORTANT: format of this cannot be changed without introducing migrations for secrets // persistences - coordinateBase = "airbyte_workspace_" + workspaceId + "_secret_" + uuidSupplier.get(); + coordinateBase = secretBasePrefix + secretBaseId + "_secret_" + uuidSupplier.get(); } if (version == null) { @@ -408,4 +417,54 @@ protected static SecretCoordinate getCoordinate( return new SecretCoordinate(coordinateBase, version); } + /** + * This method takes in the key (JSON key or HMAC key) of a workspace service account as a secret + * and generates a co-ordinate for the secret so that the secret can be written in secret + * persistence at the generated co-ordinate + * + * @param newSecret The JSON key or HMAC key value + * @param secretReader To read the value from secret persistence for comparison with the new value + * @param workspaceId of the service account + * @param uuidSupplier provided to allow a test case to produce known UUIDs in order for easy * + * fixture creation. + * @param oldSecretCoordinate a nullable full coordinate (base+version) retrieved from the * + * previous config + * @param keyType HMAC ot JSON key + * @return a coordinate (versioned reference to where the secret is stored in the persistence) + */ + public static SecretCoordinateToPayload convertServiceAccountCredsToSecret(final String newSecret, + final ReadOnlySecretPersistence secretReader, + final UUID workspaceId, + final Supplier uuidSupplier, + final @Nullable JsonNode oldSecretCoordinate, + final String keyType) { + final String oldSecretFullCoordinate = + (oldSecretCoordinate != null && oldSecretCoordinate.has(COORDINATE_FIELD)) ? oldSecretCoordinate.get(COORDINATE_FIELD).asText() + : null; + final SecretCoordinate coordinateForStagingConfig = getSecretCoordinate("service_account_" + keyType + "_", + newSecret, + secretReader, + workspaceId, + uuidSupplier, + oldSecretFullCoordinate); + return new SecretCoordinateToPayload(coordinateForStagingConfig, + newSecret, + Jsons.jsonNode(Map.of(COORDINATE_FIELD, + coordinateForStagingConfig.getFullCoordinate()))); + } + + /** + * Takes in the secret coordinate in form of a JSON and fetches the secret from the store + * + * @param secretCoordinateAsJson The co-ordinate at which we expect the secret value to be present + * in the secret persistence + * @param readOnlySecretPersistence The secret persistence + * @return Original secret value as JsonNode + */ + public static JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinateAsJson, + final ReadOnlySecretPersistence readOnlySecretPersistence) { + final var secretCoordinate = getCoordinateFromTextNode(secretCoordinateAsJson.get(COORDINATE_FIELD)); + return Jsons.deserialize(getOrThrowSecretValue(readOnlySecretPersistence, secretCoordinate)); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java index 3c61c0b45638b..808e6a7327a1e 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java @@ -19,4 +19,12 @@ public interface SecretsHydrator { */ JsonNode hydrate(JsonNode partialConfig); + /** + * Takes in the secret coordinate in form of a JSON and fetches the secret from the store + * + * @param secretCoordinate The co-ordinate of the secret in the store in JSON format + * @return original secret value + */ + JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinate); + } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java index dcf2b0c950f49..dcd6c6416e1eb 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ActorCatalog; import io.airbyte.config.ActorCatalogFetchEvent; @@ -45,9 +44,7 @@ import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; -import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -96,6 +93,37 @@ public class MockData { private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_1 = UUID.randomUUID(); private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_2 = UUID.randomUUID(); + public static final String MOCK_SERVICE_ACCOUNT_1 = "{\n" + + " \"type\" : \"service_account\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + + public static final String MOCK_SERVICE_ACCOUNT_2 = "{\n" + + " \"type\" : \"service_account-2\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + + public static final JsonNode HMAC_SECRET_PAYLOAD_1 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + public static final JsonNode HMAC_SECRET_PAYLOAD_2 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEX", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeX"))); + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); public static List standardWorkspaces() { @@ -555,23 +583,14 @@ public static List actorCatalogFetchEvents() { } public static List workspaceServiceAccounts() { - try { - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", - "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() - .withWorkspaceId(WORKSPACE_ID_1) - .withHmacKey(hmacSecretPayload) - .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") - .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") - .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); - - return Arrays.asList(workspaceServiceAccount); - } catch (IOException e) { - throw new RuntimeException(e); - } + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(WORKSPACE_ID_1) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(MOCK_SERVICE_ACCOUNT_1)); + + return Arrays.asList(workspaceServiceAccount); } private static Map sortMap(Map originalMap) { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java index 4d9c2211ac375..e02b66ce9716d 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java @@ -4,8 +4,11 @@ package io.airbyte.config.persistence; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_1; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; @@ -14,9 +17,11 @@ import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; +import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -24,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -116,4 +122,101 @@ void testDumpConfigsWithSecrets() throws IOException { assertEquals(expected, actual); } + @Test + public void testReadingServiceAccount() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = MockData.MOCK_SERVICE_ACCOUNT_1; + + final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate secretCoordinateJson = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_6b894c2b-71dc-4481-bd9f-572402643cf9", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(Jsons.jsonNode( + Map.of("_secret", secretCoordinateHmac.getFullCoordinate()))).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", secretCoordinateJson.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(secretCoordinateHmac); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(secretCoordinateJson); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)).withHmacKey(HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + + @Test + public void testReadingServiceAccountWithJsonNull() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(Jsons.jsonNode( + Map.of("_secret", secretCoordinateHmac.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(secretCoordinateHmac); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + + @Test + public void testReadingServiceAccountWithHmacNull() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = MockData.MOCK_SERVICE_ACCOUNT_1; + + final SecretCoordinate secretCoordinateJson = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_6b894c2b-71dc-4481-bd9f-572402643cf9", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", secretCoordinateJson.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(secretCoordinateJson); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java index 5cac50b2552e4..0759f38951e2e 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java @@ -4,14 +4,22 @@ package io.airbyte.config.persistence; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_1; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_2; +import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_1; +import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_2; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -24,9 +32,11 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; +import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -212,4 +222,160 @@ private static DestinationConnection injectCoordinateIntoDestination(final Strin return Jsons.clone(DESTINATION_WITH_FULL_CONFIG).withConfiguration(injectCoordinate(coordinate)); } + @Test + public void testWriteWorkspaceServiceAccount() throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID workspaceId = UUID.randomUUID(); + + final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(workspaceId) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + + doThrow(new ConfigNotFoundException(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceId.toString())) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + assertEquals(2, longLivedSecretPersistence.getMap().size()); + + String jsonPayloadInPersistence = null; + String hmacPayloadInPersistence = null; + + SecretCoordinate jsonSecretCoordinate = null; + SecretCoordinate hmacSecretCoordinate = null; + for (Map.Entry entry : longLivedSecretPersistence.getMap().entrySet()) { + if (entry.getKey().getFullCoordinate().contains("json")) { + jsonSecretCoordinate = entry.getKey(); + jsonPayloadInPersistence = entry.getValue(); + } else if (entry.getKey().getFullCoordinate().contains("hmac")) { + hmacSecretCoordinate = entry.getKey(); + hmacPayloadInPersistence = entry.getValue(); + } else { + throw new RuntimeException(""); + } + } + + assertNotNull(jsonPayloadInPersistence); + assertNotNull(hmacPayloadInPersistence); + assertNotNull(jsonSecretCoordinate); + assertNotNull(hmacSecretCoordinate); + + assertEquals(jsonSecretPayload, jsonPayloadInPersistence); + assertEquals(HMAC_SECRET_PAYLOAD_1.toString(), hmacPayloadInPersistence); + + verify(configRepository).writeWorkspaceServiceAccountNoSecrets( + Jsons.clone(workspaceServiceAccount.withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretCoordinate.getFullCoordinate()))))); + } + + @Test + public void testWriteSameStagingConfiguration() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final SecretsRepositoryWriter secretsRepositoryWriter = spy( + new SecretsRepositoryWriter(configRepository, Optional.of(secretPersistence), Optional.of(secretPersistence))); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey( + HMAC_SECRET_PAYLOAD_1) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + + final SecretCoordinate jsonSecretCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate hmacSecretCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final WorkspaceServiceAccount cloned = Jsons.clone(workspaceServiceAccount) + .withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretCoordinate.getFullCoordinate()))); + + doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(jsonSecretCoordinate); + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(hmacSecretCoordinate); + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + ArgumentCaptor coordinates = ArgumentCaptor.forClass(SecretCoordinate.class); + ArgumentCaptor payloads = ArgumentCaptor.forClass(String.class); + + verify(secretPersistence, times(2)).write(coordinates.capture(), payloads.capture()); + List actualCoordinates = coordinates.getAllValues(); + assertEquals(2, actualCoordinates.size()); + assertThat(actualCoordinates, containsInAnyOrder(jsonSecretCoordinate, hmacSecretCoordinate)); + + List actualPayload = payloads.getAllValues(); + assertEquals(2, actualPayload.size()); + assertThat(actualPayload, containsInAnyOrder(jsonSecretPayload, HMAC_SECRET_PAYLOAD_1.toString())); + + verify(secretPersistence).write(hmacSecretCoordinate, HMAC_SECRET_PAYLOAD_1.toString()); + verify(configRepository).writeWorkspaceServiceAccountNoSecrets( + cloned); + } + + @Test + public void testWriteDifferentStagingConfiguration() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final SecretsRepositoryWriter secretsRepositoryWriter = + spy(new SecretsRepositoryWriter(configRepository, Optional.of(secretPersistence), Optional.of(secretPersistence))); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretOldPayload = MOCK_SERVICE_ACCOUNT_1; + final String jsonSecretNewPayload = MOCK_SERVICE_ACCOUNT_2; + + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(workspaceId) + .withHmacKey(HMAC_SECRET_PAYLOAD_2) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretNewPayload)); + + final SecretCoordinate jsonSecretOldCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate hmacSecretOldCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final WorkspaceServiceAccount cloned = Jsons.clone(workspaceServiceAccount) + .withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretOldCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretOldCoordinate.getFullCoordinate()))); + + doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(hmacSecretOldCoordinate); + doReturn(Optional.of(jsonSecretOldPayload)).when(secretPersistence).read(jsonSecretOldCoordinate); + + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + final SecretCoordinate jsonSecretNewCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 2); + + final SecretCoordinate hmacSecretNewCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 2); + + ArgumentCaptor coordinates = ArgumentCaptor.forClass(SecretCoordinate.class); + ArgumentCaptor payloads = ArgumentCaptor.forClass(String.class); + + verify(secretPersistence, times(2)).write(coordinates.capture(), payloads.capture()); + List actualCoordinates = coordinates.getAllValues(); + assertEquals(2, actualCoordinates.size()); + assertThat(actualCoordinates, containsInAnyOrder(jsonSecretNewCoordinate, hmacSecretNewCoordinate)); + + List actualPayload = payloads.getAllValues(); + assertEquals(2, actualPayload.size()); + assertThat(actualPayload, containsInAnyOrder(jsonSecretNewPayload, HMAC_SECRET_PAYLOAD_2.toString())); + + verify(configRepository).writeWorkspaceServiceAccountNoSecrets(Jsons.clone(workspaceServiceAccount).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", jsonSecretNewCoordinate.getFullCoordinate()))).withHmacKey(Jsons.jsonNode( + Map.of("_secret", hmacSecretNewCoordinate.getFullCoordinate())))); + } + } diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key.json b/airbyte-config/persistence/src/test/resources/mock_service_key.json deleted file mode 100644 index 0858792758a12..0000000000000 --- a/airbyte-config/persistence/src/test/resources/mock_service_key.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type" : "service_account", - "project_id" : "random-gcp-project", - "private_key_id" : "123a1234ab1a123ab12345678a1234ab1abc1a12", - "private_key" : "-----BEGIN RSA PRIVATE KEY-----\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\n-----END RSA PRIVATE KEY-----", - "client_email" : "a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com", - "client_id" : "123456789012345678901", - "auth_uri" : "https://blah.blah.com/x/blah1/blah", - "token_uri" : "https://blah.blah.com/blah", - "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", - "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} \ No newline at end of file diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json b/airbyte-config/persistence/src/test/resources/mock_service_key_2.json deleted file mode 100644 index 59006e3a7de5b..0000000000000 --- a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type" : "service_account-2", - "project_id" : "random-gcp-project", - "private_key_id" : "123a1234ab1a123ab12345678a1234ab1abc1a12", - "private_key" : "-----BEGIN RSA PRIVATE KEY-----\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\n-----END RSA PRIVATE KEY-----", - "client_email" : "a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com", - "client_id" : "123456789012345678901", - "auth_uri" : "https://blah.blah.com/x/blah1/blah", - "token_uri" : "https://blah.blah.com/blah", - "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", - "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} \ No newline at end of file