From c6192c3fc3b14a7a171e6d983f37f3aa906dd0ce Mon Sep 17 00:00:00 2001 From: subodh Date: Wed, 13 Apr 2022 04:02:08 +0530 Subject: [PATCH 1/5] implement secret handling for workspace_service_account table --- .../config/persistence/ConfigRepository.java | 12 ++ .../persistence/SecretsRepositoryReader.java | 14 ++ .../persistence/SecretsRepositoryWriter.java | 51 +++++ .../split_secrets/NoOpSecretsHydrator.java | 5 + .../split_secrets/RealSecretsHydrator.java | 5 + .../SecretCoordinateToPayload.java | 13 ++ .../split_secrets/SecretsHelpers.java | 46 ++++- .../split_secrets/SecretsHydrator.java | 9 + .../SecretsRepositoryReaderTest.java | 115 +++++++++++ .../SecretsRepositoryWriterTest.java | 180 ++++++++++++++++++ 10 files changed, 445 insertions(+), 5 deletions(-) create mode 100644 airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 310aff09ff6a1..e5621b1f4de7c 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -38,6 +38,7 @@ import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.State; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import io.airbyte.db.instance.configs.jooq.enums.ActorType; @@ -1030,4 +1031,15 @@ private Condition includeTombstones(final Field tombstoneField, final b } } + public WorkspaceServiceAccount getWorkspaceServiceAccountNoSecrets(final UUID workspaceId) + throws JsonValidationException, IOException, ConfigNotFoundException { + return persistence.getConfig(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceId.toString(), WorkspaceServiceAccount.class); + } + + public void writeWorkspaceServiceAccountNoSecrets(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + persistence.writeConfig(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceServiceAccount.getWorkspaceId().toString(), + workspaceServiceAccount); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java index 525743e559073..22af7bef888a3 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @@ -10,6 +10,7 @@ import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -96,4 +97,17 @@ private void hydrateValuesIfKeyPresent(final String key, final Map> configs, final } } + public void writeServiceAccountJsonCredentials(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + final WorkspaceServiceAccount workspaceServiceAccountForDB = getWorkspaceServiceAccountWithSecretCoordinate(workspaceServiceAccount); + configRepository.writeWorkspaceServiceAccountNoSecrets(workspaceServiceAccountForDB); + } + + private WorkspaceServiceAccount getWorkspaceServiceAccountWithSecretCoordinate(final WorkspaceServiceAccount workspaceServiceAccount) + throws JsonValidationException, IOException { + if (longLivedSecretPersistence.isPresent()) { + final WorkspaceServiceAccount clonedWorkspaceServiceAccount = Jsons.clone(workspaceServiceAccount); + final Optional optionalWorkspaceServiceAccount = getOptionalWorkspaceServiceAccount( + workspaceServiceAccount.getWorkspaceId()); + if (workspaceServiceAccount.getJsonCredential() != null) { + final SecretCoordinateToPayload jsonCredSecretCoordinateToPayload = + SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getJsonCredential().toPrettyString(), + longLivedSecretPersistence.get(), + workspaceServiceAccount.getWorkspaceId(), + UUID::randomUUID, + optionalWorkspaceServiceAccount.map(WorkspaceServiceAccount::getJsonCredential).orElse(null), + "json"); + longLivedSecretPersistence.get().write(jsonCredSecretCoordinateToPayload.secretCoordinate(), jsonCredSecretCoordinateToPayload.payload()); + clonedWorkspaceServiceAccount.setJsonCredential(jsonCredSecretCoordinateToPayload.secretCoordinateForDB()); + } + + if (workspaceServiceAccount.getHmacKey() != null) { + final SecretCoordinateToPayload hmackKeySecretCoordinateToPayload = + SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getHmacKey().toString(), + longLivedSecretPersistence.get(), + workspaceServiceAccount.getWorkspaceId(), + UUID::randomUUID, + optionalWorkspaceServiceAccount.map(WorkspaceServiceAccount::getHmacKey).orElse(null), + "hmac"); + longLivedSecretPersistence.get().write(hmackKeySecretCoordinateToPayload.secretCoordinate(), hmackKeySecretCoordinateToPayload.payload()); + clonedWorkspaceServiceAccount.setHmacKey(hmackKeySecretCoordinateToPayload.secretCoordinateForDB()); + } + return clonedWorkspaceServiceAccount; + } + return workspaceServiceAccount; + } + + public Optional getOptionalWorkspaceServiceAccount(final UUID workspaceId) + throws JsonValidationException, IOException { + try { + return Optional.of(configRepository.getWorkspaceServiceAccountNoSecrets(workspaceId)); + } catch (ConfigNotFoundException e) { + return Optional.empty(); + } + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java index b2185c56093cd..bbad63f48c13e 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java @@ -16,4 +16,9 @@ public JsonNode hydrate(final JsonNode partialConfig) { return partialConfig; } + @Override + public JsonNode simpleHydrate(JsonNode secretCoordinate) { + return secretCoordinate; + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java index cfdffb0f0057d..5ecf135f2ba18 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java @@ -22,4 +22,9 @@ public JsonNode hydrate(final JsonNode partialConfig) { return SecretsHelpers.combineConfig(partialConfig, readOnlySecretPersistence); } + @Override + public JsonNode simpleHydrate(final JsonNode secretCoordinate) { + return SecretsHelpers.simpleHydrate(secretCoordinate, readOnlySecretPersistence); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java new file mode 100644 index 0000000000000..41b9ca584809a --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretCoordinateToPayload.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence.split_secrets; + +import com.fasterxml.jackson.databind.JsonNode; + +public record SecretCoordinateToPayload(SecretCoordinate secretCoordinate, + String payload, + JsonNode secretCoordinateForDB) { + +} diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index 579ec254dd651..0b18be365bd60 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -116,7 +116,7 @@ public static JsonNode combineConfig(final JsonNode partialConfig, final ReadOnl if (config.has(COORDINATE_FIELD)) { final var coordinateNode = config.get(COORDINATE_FIELD); final var coordinate = getCoordinateFromTextNode(coordinateNode); - return getOrThrowSecretValueNode(secretPersistence, coordinate); + return getOrThrowSecretValueNode(secretPersistence, coordinate, true); } // otherwise iterate through all object fields @@ -336,16 +336,18 @@ private static JsonNode getFieldOrEmptyNode(final JsonNode node, final int field * @param secretPersistence storage layer for secrets * @param coordinate reference to a secret in the persistence * @throws RuntimeException when a secret at that coordinate is not available in the persistence - * @return a json text node containing the secret value + * @return a json text node containing the secret value or a JSON */ - private static TextNode getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, final SecretCoordinate coordinate) { + private static JsonNode getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, + final SecretCoordinate coordinate, + final boolean asTextNode) { final var secretValue = secretPersistence.read(coordinate); if (secretValue.isEmpty()) { throw new RuntimeException(String.format("That secret was not found in the store! Coordinate: %s", coordinate.getFullCoordinate())); } - return new TextNode(secretValue.get()); + return asTextNode ? new TextNode(secretValue.get()) : Jsons.deserialize(secretValue.get()); } private static SecretCoordinate getCoordinateFromTextNode(final JsonNode node) { @@ -379,6 +381,15 @@ protected static SecretCoordinate getCoordinate( final UUID workspaceId, final Supplier uuidSupplier, final @Nullable String oldSecretFullCoordinate) { + return getSecretCoordinate("airbyte_workspace_", newSecret, secretReader, workspaceId, uuidSupplier, oldSecretFullCoordinate); + } + + private static SecretCoordinate getSecretCoordinate(final String secretBasePrefix, + final String newSecret, + final ReadOnlySecretPersistence secretReader, + final UUID secretBaseId, + final Supplier uuidSupplier, + final @Nullable String oldSecretFullCoordinate) { String coordinateBase = null; Long version = null; @@ -398,7 +409,7 @@ protected static SecretCoordinate getCoordinate( if (coordinateBase == null) { // IMPORTANT: format of this cannot be changed without introducing migrations for secrets // persistences - coordinateBase = "airbyte_workspace_" + workspaceId + "_secret_" + uuidSupplier.get(); + coordinateBase = secretBasePrefix + secretBaseId + "_secret_" + uuidSupplier.get(); } if (version == null) { @@ -408,4 +419,29 @@ protected static SecretCoordinate getCoordinate( return new SecretCoordinate(coordinateBase, version); } + public static SecretCoordinateToPayload convertServiceAccountCredsToSecret(final String newSecret, + final ReadOnlySecretPersistence secretReader, + final UUID workspaceId, + final Supplier uuidSupplier, + final @Nullable JsonNode oldSecretCoordinate, + final String keyType) { + final String oldSecretFullCoordinate = + (oldSecretCoordinate != null && oldSecretCoordinate.has(COORDINATE_FIELD)) ? oldSecretCoordinate.get(COORDINATE_FIELD).asText() + : null; + final SecretCoordinate coordinateForStagingConfig = getSecretCoordinate("service_account_" + keyType + "_", + newSecret, + secretReader, + workspaceId, + uuidSupplier, + oldSecretFullCoordinate); + return new SecretCoordinateToPayload(coordinateForStagingConfig, newSecret, + Jsons.jsonNode(Map.of(COORDINATE_FIELD, coordinateForStagingConfig.getFullCoordinate()))); + } + + public static JsonNode simpleHydrate(final JsonNode secretCoordinateAsJson, + final ReadOnlySecretPersistence readOnlySecretPersistence) { + final var secretCoordinate = getCoordinateFromTextNode(secretCoordinateAsJson.deepCopy().get(COORDINATE_FIELD)); + return getOrThrowSecretValueNode(readOnlySecretPersistence, secretCoordinate, false); + } + } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java index 3c61c0b45638b..ba06bc66b9cdf 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java @@ -19,4 +19,13 @@ public interface SecretsHydrator { */ JsonNode hydrate(JsonNode partialConfig); + /** + * Takes in the secret coordinate in form of a JSON and fetches the secret from the store without + * any special transformations + * + * @param secretCoordinate The co-ordinate of the secret in the store in JSON format + * @return original secret value + */ + JsonNode simpleHydrate(final JsonNode secretCoordinate); + } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java index 4d9c2211ac375..d1705d6029259 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java @@ -5,25 +5,33 @@ package io.airbyte.config.persistence; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; +import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; +import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -116,4 +124,111 @@ void testDumpConfigsWithSecrets() throws IOException { assertEquals(expected, actual); } + @Test + public void testReadingServiceAccount() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + + final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate secretCoordinateJson = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_6b894c2b-71dc-4481-bd9f-572402643cf9", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(Jsons.jsonNode( + Map.of("_secret", secretCoordinateHmac.getFullCoordinate()))).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", secretCoordinateJson.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(secretCoordinateHmac); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(secretCoordinateJson); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)).withHmacKey(hmacSecretPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + + @Test + public void testReadingServiceAccountWithJsonNull() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + + final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(Jsons.jsonNode( + Map.of("_secret", secretCoordinateHmac.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(secretCoordinateHmac); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withHmacKey(hmacSecretPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + + @Test + public void testReadingServiceAccountWithHmacNull() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final RealSecretsHydrator realSecretsHydrator = new RealSecretsHydrator(secretPersistence); + final SecretsRepositoryReader secretsRepositoryReader = + spy(new SecretsRepositoryReader(configRepository, realSecretsHydrator)); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + + final SecretCoordinate secretCoordinateJson = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_6b894c2b-71dc-4481-bd9f-572402643cf9", 1); + + doReturn(new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", secretCoordinateJson.getFullCoordinate()))) + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(secretCoordinateJson); + + final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); + final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); + assertEquals(expected, actual); + } + + private Map sortMap(Map originalMap) { + return originalMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)); + } + } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java index 5cac50b2552e4..8e01f4e5a2311 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java @@ -4,19 +4,24 @@ package io.airbyte.config.persistence; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.AirbyteConfig; import io.airbyte.config.ConfigSchema; @@ -24,17 +29,21 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.config.persistence.split_secrets.MemorySecretPersistence; import io.airbyte.config.persistence.split_secrets.RealSecretsHydrator; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; +import io.airbyte.config.persistence.split_secrets.SecretPersistence; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -212,4 +221,175 @@ private static DestinationConnection injectCoordinateIntoDestination(final Strin return Jsons.clone(DESTINATION_WITH_FULL_CONFIG).withConfiguration(injectCoordinate(coordinate)); } + @Test + public void testWriteWorkspaceServiceAccount() throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID workspaceId = UUID.randomUUID(); + + final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(workspaceId) + .withHmacKey(hmacSecretPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + + doThrow(new ConfigNotFoundException(ConfigSchema.WORKSPACE_SERVICE_ACCOUNT, workspaceId.toString())) + .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + assertEquals(2, longLivedSecretPersistence.getMap().size()); + + String jsonPayloadInPersistence = null; + String hmacPayloadInPersistence = null; + + SecretCoordinate jsonSecretCoordinate = null; + SecretCoordinate hmacSecretCoordinate = null; + for (Map.Entry entry : longLivedSecretPersistence.getMap().entrySet()) { + if (entry.getKey().getFullCoordinate().contains("json")) { + jsonSecretCoordinate = entry.getKey(); + jsonPayloadInPersistence = entry.getValue(); + } else if (entry.getKey().getFullCoordinate().contains("hmac")) { + hmacSecretCoordinate = entry.getKey(); + hmacPayloadInPersistence = entry.getValue(); + } else { + throw new RuntimeException(""); + } + } + + assertNotNull(jsonPayloadInPersistence); + assertNotNull(hmacPayloadInPersistence); + assertNotNull(jsonSecretCoordinate); + assertNotNull(hmacSecretCoordinate); + + assertEquals(jsonSecretPayload, jsonPayloadInPersistence); + assertEquals(hmacSecretPayload.toString(), hmacPayloadInPersistence); + + verify(configRepository).writeWorkspaceServiceAccountNoSecrets( + Jsons.clone(workspaceServiceAccount.withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretCoordinate.getFullCoordinate()))))); + } + + @Test + public void testWriteSameStagingConfiguration() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final SecretsRepositoryWriter secretsRepositoryWriter = spy( + new SecretsRepositoryWriter(configRepository, Optional.of(secretPersistence), Optional.of(secretPersistence))); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(hmacSecretPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + + final SecretCoordinate jsonSecretCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate hmacSecretCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final WorkspaceServiceAccount cloned = Jsons.clone(workspaceServiceAccount) + .withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretCoordinate.getFullCoordinate()))); + + doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(jsonSecretCoordinate); + doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(hmacSecretCoordinate); + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + ArgumentCaptor coordinates = ArgumentCaptor.forClass(SecretCoordinate.class); + ArgumentCaptor payloads = ArgumentCaptor.forClass(String.class); + + verify(secretPersistence, times(2)).write(coordinates.capture(), payloads.capture()); + List actualCoordinates = coordinates.getAllValues(); + assertEquals(2, actualCoordinates.size()); + assertThat(actualCoordinates, containsInAnyOrder(jsonSecretCoordinate, hmacSecretCoordinate)); + + List actualPayload = payloads.getAllValues(); + assertEquals(2, actualPayload.size()); + assertThat(actualPayload, containsInAnyOrder(jsonSecretPayload, hmacSecretPayload.toString())); + + verify(secretPersistence).write(hmacSecretCoordinate, hmacSecretPayload.toString()); + verify(configRepository).writeWorkspaceServiceAccountNoSecrets( + cloned); + } + + @Test + public void testWriteDifferentStagingConfiguration() throws JsonValidationException, ConfigNotFoundException, IOException { + final ConfigRepository configRepository = mock(ConfigRepository.class); + final SecretPersistence secretPersistence = mock(SecretPersistence.class); + final SecretsRepositoryWriter secretsRepositoryWriter = + spy(new SecretsRepositoryWriter(configRepository, Optional.of(secretPersistence), Optional.of(secretPersistence))); + + final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); + + final String jsonSecretOldPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final String jsonSecretNewPayload = Resources.toString(Resources.getResource("mock_service_key_2.json"), StandardCharsets.UTF_8); + + final JsonNode hmacSecretOldPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + final JsonNode hmacSecretNewPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEX", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeX"))); + + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(workspaceId) + .withHmacKey(hmacSecretNewPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretNewPayload)); + + final SecretCoordinate jsonSecretOldCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final SecretCoordinate hmacSecretOldCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); + + final WorkspaceServiceAccount cloned = Jsons.clone(workspaceServiceAccount) + .withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretOldCoordinate.getFullCoordinate()))) + .withHmacKey(Jsons.jsonNode(Map.of("_secret", hmacSecretOldCoordinate.getFullCoordinate()))); + + doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); + + doReturn(Optional.of(hmacSecretOldPayload.toString())).when(secretPersistence).read(hmacSecretOldCoordinate); + doReturn(Optional.of(jsonSecretOldPayload)).when(secretPersistence).read(jsonSecretOldCoordinate); + + secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); + + final SecretCoordinate jsonSecretNewCoordinate = new SecretCoordinate( + "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 2); + + final SecretCoordinate hmacSecretNewCoordinate = new SecretCoordinate( + "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 2); + + ArgumentCaptor coordinates = ArgumentCaptor.forClass(SecretCoordinate.class); + ArgumentCaptor payloads = ArgumentCaptor.forClass(String.class); + + verify(secretPersistence, times(2)).write(coordinates.capture(), payloads.capture()); + List actualCoordinates = coordinates.getAllValues(); + assertEquals(2, actualCoordinates.size()); + assertThat(actualCoordinates, containsInAnyOrder(jsonSecretNewCoordinate, hmacSecretNewCoordinate)); + + List actualPayload = payloads.getAllValues(); + assertEquals(2, actualPayload.size()); + assertThat(actualPayload, containsInAnyOrder(jsonSecretNewPayload, hmacSecretNewPayload.toString())); + + verify(configRepository).writeWorkspaceServiceAccountNoSecrets(Jsons.clone(workspaceServiceAccount).withJsonCredential(Jsons.jsonNode( + Map.of("_secret", jsonSecretNewCoordinate.getFullCoordinate()))).withHmacKey(Jsons.jsonNode( + Map.of("_secret", hmacSecretNewCoordinate.getFullCoordinate())))); + } + + private Map sortMap(Map originalMap) { + return originalMap.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)); + } + } From 6bc18e5e6ffdb136b55d7d5e86a0a257e4cde34d Mon Sep 17 00:00:00 2001 From: subodh Date: Wed, 13 Apr 2022 12:04:16 +0530 Subject: [PATCH 2/5] add new line to the mock json --- .../persistence/src/test/resources/mock_service_key.json | 2 +- .../persistence/src/test/resources/mock_service_key_2.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key.json b/airbyte-config/persistence/src/test/resources/mock_service_key.json index 0858792758a12..3dee35ffbcb6d 100644 --- a/airbyte-config/persistence/src/test/resources/mock_service_key.json +++ b/airbyte-config/persistence/src/test/resources/mock_service_key.json @@ -9,4 +9,4 @@ "token_uri" : "https://blah.blah.com/blah", "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} \ No newline at end of file +} diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json b/airbyte-config/persistence/src/test/resources/mock_service_key_2.json index 59006e3a7de5b..f95f0b9731e30 100644 --- a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json +++ b/airbyte-config/persistence/src/test/resources/mock_service_key_2.json @@ -9,4 +9,4 @@ "token_uri" : "https://blah.blah.com/blah", "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} \ No newline at end of file +} From 12cfe21d17ee49c18b0ee9770221b7691ae70106 Mon Sep 17 00:00:00 2001 From: subodh Date: Wed, 13 Apr 2022 12:40:46 +0530 Subject: [PATCH 3/5] get rid of file --- .../airbyte/config/persistence/MockData.java | 59 ++++++++++++------- .../SecretsRepositoryReaderTest.java | 6 +- .../SecretsRepositoryWriterTest.java | 12 ++-- .../src/test/resources/mock_service_key.json | 12 ---- .../test/resources/mock_service_key_2.json | 12 ---- 5 files changed, 47 insertions(+), 54 deletions(-) delete mode 100644 airbyte-config/persistence/src/test/resources/mock_service_key.json delete mode 100644 airbyte-config/persistence/src/test/resources/mock_service_key_2.json diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 6a70b7489ee06..6e7531e35ed8c 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ActorCatalog; import io.airbyte.config.ActorCatalogFetchEvent; @@ -45,9 +44,7 @@ import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.SyncMode; -import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.Arrays; import java.util.Collections; @@ -95,6 +92,32 @@ public class MockData { private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_1 = UUID.randomUUID(); private static final UUID ACTOR_CATALOG_FETCH_EVENT_ID_2 = UUID.randomUUID(); + public static final String MOCK_SERVICE_ACCOUNT_1 = "{\n" + + " \"type\" : \"service_account\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + + public static final String MOCK_SERVICE_ACCOUNT_2 = "{\n" + + " \"type\" : \"service_account-2\",\n" + + " \"project_id\" : \"random-gcp-project\",\n" + + " \"private_key_id\" : \"123a1234ab1a123ab12345678a1234ab1abc1a12\",\n" + + " \"private_key\" : \"-----BEGIN RSA PRIVATE KEY-----\\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\\n-----END RSA PRIVATE KEY-----\",\n" + + " \"client_email\" : \"a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com\",\n" + + " \"client_id\" : \"123456789012345678901\",\n" + + " \"auth_uri\" : \"https://blah.blah.com/x/blah1/blah\",\n" + + " \"token_uri\" : \"https://blah.blah.com/blah\",\n" + + " \"auth_provider_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah\",\n" + + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + + "}"; + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); public static List standardWorkspaces() { @@ -538,23 +561,19 @@ public static List actorCatalogFetchEvents() { } public static List workspaceServiceAccounts() { - try { - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", - "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() - .withWorkspaceId(WORKSPACE_ID_1) - .withHmacKey(hmacSecretPayload) - .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") - .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") - .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); - - return Arrays.asList(workspaceServiceAccount); - } catch (IOException e) { - throw new RuntimeException(e); - } + final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; + final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", + "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() + .withWorkspaceId(WORKSPACE_ID_1) + .withHmacKey(hmacSecretPayload) + .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") + .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + + return Arrays.asList(workspaceServiceAccount); } private static Map sortMap(Map originalMap) { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java index d1705d6029259..5a1c0b621e38b 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java @@ -11,7 +11,6 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; @@ -25,7 +24,6 @@ import io.airbyte.config.persistence.split_secrets.SecretsHydrator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -134,7 +132,7 @@ public void testReadingServiceAccount() throws JsonValidationException, ConfigNo final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final String jsonSecretPayload = MockData.MOCK_SERVICE_ACCOUNT_1; final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); @@ -205,7 +203,7 @@ public void testReadingServiceAccountWithHmacNull() throws JsonValidationExcepti final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final String jsonSecretPayload = MockData.MOCK_SERVICE_ACCOUNT_1; final SecretCoordinate secretCoordinateJson = new SecretCoordinate( "service_account_json_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_6b894c2b-71dc-4481-bd9f-572402643cf9", 1); diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java index 8e01f4e5a2311..11bd03670508b 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java @@ -4,6 +4,8 @@ package io.airbyte.config.persistence; +import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_1; +import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_2; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -21,7 +23,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.io.Resources; import io.airbyte.commons.json.Jsons; import io.airbyte.config.AirbyteConfig; import io.airbyte.config.ConfigSchema; @@ -37,7 +38,6 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -225,7 +225,7 @@ private static DestinationConnection injectCoordinateIntoDestination(final Strin public void testWriteWorkspaceServiceAccount() throws JsonValidationException, ConfigNotFoundException, IOException { final UUID workspaceId = UUID.randomUUID(); - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); @@ -281,7 +281,7 @@ public void testWriteSameStagingConfiguration() throws JsonValidationException, final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); - final String jsonSecretPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); + final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); @@ -332,8 +332,8 @@ public void testWriteDifferentStagingConfiguration() throws JsonValidationExcept final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); - final String jsonSecretOldPayload = Resources.toString(Resources.getResource("mock_service_key.json"), StandardCharsets.UTF_8); - final String jsonSecretNewPayload = Resources.toString(Resources.getResource("mock_service_key_2.json"), StandardCharsets.UTF_8); + final String jsonSecretOldPayload = MOCK_SERVICE_ACCOUNT_1; + final String jsonSecretNewPayload = MOCK_SERVICE_ACCOUNT_2; final JsonNode hmacSecretOldPayload = Jsons.jsonNode(sortMap( Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key.json b/airbyte-config/persistence/src/test/resources/mock_service_key.json deleted file mode 100644 index 3dee35ffbcb6d..0000000000000 --- a/airbyte-config/persistence/src/test/resources/mock_service_key.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type" : "service_account", - "project_id" : "random-gcp-project", - "private_key_id" : "123a1234ab1a123ab12345678a1234ab1abc1a12", - "private_key" : "-----BEGIN RSA PRIVATE KEY-----\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\n-----END RSA PRIVATE KEY-----", - "client_email" : "a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com", - "client_id" : "123456789012345678901", - "auth_uri" : "https://blah.blah.com/x/blah1/blah", - "token_uri" : "https://blah.blah.com/blah", - "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", - "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} diff --git a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json b/airbyte-config/persistence/src/test/resources/mock_service_key_2.json deleted file mode 100644 index f95f0b9731e30..0000000000000 --- a/airbyte-config/persistence/src/test/resources/mock_service_key_2.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type" : "service_account-2", - "project_id" : "random-gcp-project", - "private_key_id" : "123a1234ab1a123ab12345678a1234ab1abc1a12", - "private_key" : "-----BEGIN RSA PRIVATE KEY-----\nMIIEoQIBAAKCAQBtkKBs9oe9pFhEWjBls9OrY0PXE/QN6nL4Bfw4+UqcBpTyItXo\n3aBXuVqDIZ377zjbJUcYuc4NzAsLImy7VVT1XrdAkkCKQEMoA9pQgONA/3kD8Xff\nSUGfdup8UJg925paaRhM7u81e3XKGwGyL/qcxpuHtfqimeWWfSPy5AawyOFl+l25\nOqbm8PK4/QVqk4pcorQuISUkrehY0Ji0gVQF+ZeBvg7lvBtjNEl//eysGtcZvk7X\nHqg+EIBqRjVNDsViHj0xeoDFcFgXDeWzxeQ0c7gMsDthfm4SjgaVFdQwsJUeoC6X\nlwUoBbFIVVKW0n+SH+kxLc7mhaGjyRYJLS6tAgMBAAECggEAaowetlf4IR/VBoN+\nVSjPSvg5XMr2pyG7tB597RngyGJOLjpaMx5zc1u4/ZSPghRdAh/6R71I+HnYs3dC\nrdqJyCPXqV+Qi+F6bUtx3p+4X9kQ4hjMLcOboWuPFF1774vDSvCwxQAGd8gb//LL\nb3DhEdzCGvOJTN7EOdhwQSAmsXsfj0qKlmm8vv0HBQDvjYYWhy/UcPry5sAGQ8KU\nnUPTkz/OMS56nBIgKXgZtGRTP1Q7Q9a6oLmlvbDxuKGUByUPNlveZplzyWDO3RUN\nNPt9dwgGk6rZK0umunGr0lq+WOK33Ue1RJy2VIvvV6dt32x20ehfVKND8N8q+wJ3\neJQggQKBgQC//dOX8RwkmIloRzzmbu+qY8o44/F5gtxj8maR+OJhvbpFEID49bBr\nzYqcMKfcgHJr6638CXVGSO66IiKtQcTMJ/Vd8TQVPcNPI1h/RD+wT/nkWX6R/0YH\njwwNmikeUDH2/hLQlRZ8O45hc4frDGRMeHn3MSS2YsBDSl6YL/zHpQKBgQCSF9Ka\nyCZmw5eS63G5/X9SVXbLRPuc6Fus+IbRPttOzSRviUXHaBjwwVEJgIKODx/eVXgD\nA/OvFUmwIn73uZD/XgJrhkwAendaa+yhWKAkO5pO/EdAslxRmgxqTXfRcyslKBbo\ns4YAgeYUgzOaMH4UxY4pJ7H6BLsFlboL+8BcaQKBgDSCM1Cm/M91eH8wnJNZW+r6\nB+CvVueoxqX/MdZSf3fD8CHbdaqhZ3LUcEhvdjl0V9b0Sk1YON7UK5Z0p49DIZPE\nifL7eQcmMTh/rkCAZfrOpMWzRE6hxoFiuiUuOHi17jRjILozTEcF8tbsRgwfA392\no8Tbh/Lp5zOAL4bn+PaRAoGAZ2AgEJJsSe9BRB8CPF+aRoJfKvrHKIJqzHyXuVzH\nBn22uI3kKHQKoeHJG/Ypa6hcHpFP+KJFPrDLkaz3NwfCCFFXWQqQoQ4Hgp43tPvn\nZXwfdqChMrCDDuL4wgfLLxRVhVdWzpapzZYdXopwazzBGqWoMIr8LzRFum/2VCBy\nP3ECgYBGqjuYud6gtrzaQwmMfcA0pSYsii96d2LKwWzjgcMzLxge59PIWXeQJqOb\nh97m3qCkkPzbceD6Id8m/EyrNb04V8Zr0ERlcK/a4nRSHoIWQZY01lDSGhneRKn1\nncBvRqCfz6ajf+zBg3zK0af98IHL0FI2NsNJLPrOBFMcthjx/g==\n-----END RSA PRIVATE KEY-----", - "client_email" : "a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com", - "client_id" : "123456789012345678901", - "auth_uri" : "https://blah.blah.com/x/blah1/blah", - "token_uri" : "https://blah.blah.com/blah", - "auth_provider_x509_cert_url" : "https://www.blah.com/blah/v1/blah", - "client_x509_cert_url" : "https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com" -} From 4e596dc46e8d6eee6f25595ecac7a9c777edde69 Mon Sep 17 00:00:00 2001 From: subodh Date: Tue, 26 Apr 2022 03:25:45 +0530 Subject: [PATCH 4/5] address review comments --- .../persistence/SecretsRepositoryReader.java | 5 +- .../persistence/SecretsRepositoryWriter.java | 5 +- .../split_secrets/NoOpSecretsHydrator.java | 2 +- .../split_secrets/RealSecretsHydrator.java | 4 +- .../split_secrets/SecretsHelpers.java | 47 ++++++++++++++----- .../split_secrets/SecretsHydrator.java | 5 +- .../airbyte/config/persistence/MockData.java | 14 +++--- .../SecretsRepositoryReaderTest.java | 20 ++------ .../SecretsRepositoryWriterTest.java | 38 +++++---------- 9 files changed, 71 insertions(+), 69 deletions(-) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java index 22af7bef888a3..6040734e31544 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @@ -102,10 +102,11 @@ public WorkspaceServiceAccount getWorkspaceServiceAccountWithSecrets(final UUID final WorkspaceServiceAccount workspaceServiceAccount = configRepository.getWorkspaceServiceAccountNoSecrets(workspaceId); final JsonNode jsonCredential = - workspaceServiceAccount.getJsonCredential() != null ? secretsHydrator.simpleHydrate(workspaceServiceAccount.getJsonCredential()) : null; + workspaceServiceAccount.getJsonCredential() != null ? secretsHydrator.hydrateSecretCoordinate(workspaceServiceAccount.getJsonCredential()) + : null; final JsonNode hmacKey = - workspaceServiceAccount.getHmacKey() != null ? secretsHydrator.simpleHydrate(workspaceServiceAccount.getHmacKey()) : null; + workspaceServiceAccount.getHmacKey() != null ? secretsHydrator.hydrateSecretCoordinate(workspaceServiceAccount.getHmacKey()) : null; return Jsons.clone(workspaceServiceAccount).withJsonCredential(jsonCredential).withHmacKey(hmacKey); } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java index 21874f0c88a04..7e2491e5d9bcd 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @@ -278,6 +278,8 @@ private WorkspaceServiceAccount getWorkspaceServiceAccountWithSecretCoordinate(f final WorkspaceServiceAccount clonedWorkspaceServiceAccount = Jsons.clone(workspaceServiceAccount); final Optional optionalWorkspaceServiceAccount = getOptionalWorkspaceServiceAccount( workspaceServiceAccount.getWorkspaceId()); + // Convert the JSON key of Service Account into secret co-oridnate. Ref : + // https://cloud.google.com/iam/docs/service-accounts#key-types if (workspaceServiceAccount.getJsonCredential() != null) { final SecretCoordinateToPayload jsonCredSecretCoordinateToPayload = SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getJsonCredential().toPrettyString(), @@ -289,7 +291,8 @@ private WorkspaceServiceAccount getWorkspaceServiceAccountWithSecretCoordinate(f longLivedSecretPersistence.get().write(jsonCredSecretCoordinateToPayload.secretCoordinate(), jsonCredSecretCoordinateToPayload.payload()); clonedWorkspaceServiceAccount.setJsonCredential(jsonCredSecretCoordinateToPayload.secretCoordinateForDB()); } - + // Convert the HMAC key of Service Account into secret co-oridnate. Ref : + // https://cloud.google.com/storage/docs/authentication/hmackeys if (workspaceServiceAccount.getHmacKey() != null) { final SecretCoordinateToPayload hmackKeySecretCoordinateToPayload = SecretsHelpers.convertServiceAccountCredsToSecret(workspaceServiceAccount.getHmacKey().toString(), diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java index bbad63f48c13e..bb02572bc1bbf 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/NoOpSecretsHydrator.java @@ -17,7 +17,7 @@ public JsonNode hydrate(final JsonNode partialConfig) { } @Override - public JsonNode simpleHydrate(JsonNode secretCoordinate) { + public JsonNode hydrateSecretCoordinate(JsonNode secretCoordinate) { return secretCoordinate; } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java index 5ecf135f2ba18..69e3d07bdd08a 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/RealSecretsHydrator.java @@ -23,8 +23,8 @@ public JsonNode hydrate(final JsonNode partialConfig) { } @Override - public JsonNode simpleHydrate(final JsonNode secretCoordinate) { - return SecretsHelpers.simpleHydrate(secretCoordinate, readOnlySecretPersistence); + public JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinate) { + return SecretsHelpers.hydrateSecretCoordinate(secretCoordinate, readOnlySecretPersistence); } } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index 0b18be365bd60..7cd009f0c0563 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -116,7 +116,7 @@ public static JsonNode combineConfig(final JsonNode partialConfig, final ReadOnl if (config.has(COORDINATE_FIELD)) { final var coordinateNode = config.get(COORDINATE_FIELD); final var coordinate = getCoordinateFromTextNode(coordinateNode); - return getOrThrowSecretValueNode(secretPersistence, coordinate, true); + return new TextNode(getOrThrowSecretValueNode(secretPersistence, coordinate)); } // otherwise iterate through all object fields @@ -338,16 +338,14 @@ private static JsonNode getFieldOrEmptyNode(final JsonNode node, final int field * @throws RuntimeException when a secret at that coordinate is not available in the persistence * @return a json text node containing the secret value or a JSON */ - private static JsonNode getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, - final SecretCoordinate coordinate, - final boolean asTextNode) { + private static String getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, + final SecretCoordinate coordinate) { final var secretValue = secretPersistence.read(coordinate); if (secretValue.isEmpty()) { throw new RuntimeException(String.format("That secret was not found in the store! Coordinate: %s", coordinate.getFullCoordinate())); } - - return asTextNode ? new TextNode(secretValue.get()) : Jsons.deserialize(secretValue.get()); + return secretValue.get(); } private static SecretCoordinate getCoordinateFromTextNode(final JsonNode node) { @@ -419,6 +417,21 @@ private static SecretCoordinate getSecretCoordinate(final String secretBasePrefi return new SecretCoordinate(coordinateBase, version); } + /** + * This method takes in the key (JSON key or HMAC key) of a workspace service account as a secret + * and generates a co-ordinate for the secret so that the secret can be written in secret + * persistence at the generated co-ordinate + * + * @param newSecret The JSON key or HMAC key value + * @param secretReader To read the value from secret persistence for comparison with the new value + * @param workspaceId of the service account + * @param uuidSupplier provided to allow a test case to produce known UUIDs in order for easy * + * fixture creation. + * @param oldSecretCoordinate a nullable full coordinate (base+version) retrieved from the * + * previous config + * @param keyType HMAC ot JSON key + * @return a coordinate (versioned reference to where the secret is stored in the persistence) + */ public static SecretCoordinateToPayload convertServiceAccountCredsToSecret(final String newSecret, final ReadOnlySecretPersistence secretReader, final UUID workspaceId, @@ -434,14 +447,24 @@ public static SecretCoordinateToPayload convertServiceAccountCredsToSecret(final workspaceId, uuidSupplier, oldSecretFullCoordinate); - return new SecretCoordinateToPayload(coordinateForStagingConfig, newSecret, - Jsons.jsonNode(Map.of(COORDINATE_FIELD, coordinateForStagingConfig.getFullCoordinate()))); + return new SecretCoordinateToPayload(coordinateForStagingConfig, + newSecret, + Jsons.jsonNode(Map.of(COORDINATE_FIELD, + coordinateForStagingConfig.getFullCoordinate()))); } - public static JsonNode simpleHydrate(final JsonNode secretCoordinateAsJson, - final ReadOnlySecretPersistence readOnlySecretPersistence) { - final var secretCoordinate = getCoordinateFromTextNode(secretCoordinateAsJson.deepCopy().get(COORDINATE_FIELD)); - return getOrThrowSecretValueNode(readOnlySecretPersistence, secretCoordinate, false); + /** + * Takes in the secret coordinate in form of a JSON and fetches the secret from the store + * + * @param secretCoordinateAsJson The co-ordinate at which we expect the secret value to be present + * in the secret persistence + * @param readOnlySecretPersistence The secret persistence + * @return Original secret value as JsonNode + */ + public static JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinateAsJson, + final ReadOnlySecretPersistence readOnlySecretPersistence) { + final var secretCoordinate = getCoordinateFromTextNode(secretCoordinateAsJson.get(COORDINATE_FIELD)); + return Jsons.deserialize(getOrThrowSecretValueNode(readOnlySecretPersistence, secretCoordinate)); } } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java index ba06bc66b9cdf..808e6a7327a1e 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHydrator.java @@ -20,12 +20,11 @@ public interface SecretsHydrator { JsonNode hydrate(JsonNode partialConfig); /** - * Takes in the secret coordinate in form of a JSON and fetches the secret from the store without - * any special transformations + * Takes in the secret coordinate in form of a JSON and fetches the secret from the store * * @param secretCoordinate The co-ordinate of the secret in the store in JSON format * @return original secret value */ - JsonNode simpleHydrate(final JsonNode secretCoordinate); + JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinate); } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 3611b46c059f1..dcd6c6416e1eb 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -119,6 +119,11 @@ public class MockData { + " \"client_x509_cert_url\" : \"https://www.blah.com/blah/v1/blah/a123/a1e5ac98-7531-48e1-943b-b46636%40random-gcp-project.abc.abcdefghijklmno.com\"\n" + "}"; + public static final JsonNode HMAC_SECRET_PAYLOAD_1 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); + public static final JsonNode HMAC_SECRET_PAYLOAD_2 = Jsons.jsonNode(sortMap( + Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEX", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeX"))); + private static final Instant NOW = Instant.parse("2021-12-15T20:30:40.00Z"); public static List standardWorkspaces() { @@ -578,17 +583,12 @@ public static List actorCatalogFetchEvents() { } public static List workspaceServiceAccounts() { - final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", - "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() .withWorkspaceId(WORKSPACE_ID_1) - .withHmacKey(hmacSecretPayload) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") - .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); + .withJsonCredential(Jsons.deserialize(MOCK_SERVICE_ACCOUNT_1)); return Arrays.asList(workspaceServiceAccount); } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java index 5a1c0b621e38b..e02b66ce9716d 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryReaderTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_1; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -29,7 +30,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -134,8 +134,6 @@ public void testReadingServiceAccount() throws JsonValidationException, ConfigNo final String jsonSecretPayload = MockData.MOCK_SERVICE_ACCOUNT_1; - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); @@ -149,13 +147,13 @@ public void testReadingServiceAccount() throws JsonValidationException, ConfigNo .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); - doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(secretCoordinateHmac); + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(secretCoordinateHmac); doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(secretCoordinateJson); final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) - .withJsonCredential(Jsons.deserialize(jsonSecretPayload)).withHmacKey(hmacSecretPayload) + .withJsonCredential(Jsons.deserialize(jsonSecretPayload)).withHmacKey(HMAC_SECRET_PAYLOAD_1) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); assertEquals(expected, actual); @@ -171,9 +169,6 @@ public void testReadingServiceAccountWithJsonNull() throws JsonValidationExcepti final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - final SecretCoordinate secretCoordinateHmac = new SecretCoordinate( "service_account_hmac_13fb9a84-6bfa-4801-8f5e-ce717677babf_secret_e86e2eab-af9b-42a3-b074-b923b4fa617e", 1); @@ -183,11 +178,11 @@ public void testReadingServiceAccountWithJsonNull() throws JsonValidationExcepti .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636")) .when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); - doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(secretCoordinateHmac); + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(secretCoordinateHmac); final WorkspaceServiceAccount actual = secretsRepositoryReader.getWorkspaceServiceAccountWithSecrets(workspaceId); final WorkspaceServiceAccount expected = new WorkspaceServiceAccount().withWorkspaceId(workspaceId) - .withHmacKey(hmacSecretPayload) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com"); assertEquals(expected, actual); @@ -224,9 +219,4 @@ public void testReadingServiceAccountWithHmacNull() throws JsonValidationExcepti assertEquals(expected, actual); } - private Map sortMap(Map originalMap) { - return originalMap.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)); - } - } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java index 11bd03670508b..0759f38951e2e 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/SecretsRepositoryWriterTest.java @@ -4,6 +4,8 @@ package io.airbyte.config.persistence; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_1; +import static io.airbyte.config.persistence.MockData.HMAC_SECRET_PAYLOAD_2; import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_1; import static io.airbyte.config.persistence.MockData.MOCK_SERVICE_ACCOUNT_2; import static org.hamcrest.MatcherAssert.assertThat; @@ -43,7 +45,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.TreeMap; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -226,12 +227,9 @@ public void testWriteWorkspaceServiceAccount() throws JsonValidationException, C final UUID workspaceId = UUID.randomUUID(); final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() .withWorkspaceId(workspaceId) - .withHmacKey(hmacSecretPayload) + .withHmacKey(HMAC_SECRET_PAYLOAD_1) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); @@ -265,7 +263,7 @@ public void testWriteWorkspaceServiceAccount() throws JsonValidationException, C assertNotNull(hmacSecretCoordinate); assertEquals(jsonSecretPayload, jsonPayloadInPersistence); - assertEquals(hmacSecretPayload.toString(), hmacPayloadInPersistence); + assertEquals(HMAC_SECRET_PAYLOAD_1.toString(), hmacPayloadInPersistence); verify(configRepository).writeWorkspaceServiceAccountNoSecrets( Jsons.clone(workspaceServiceAccount.withJsonCredential(Jsons.jsonNode(Map.of("_secret", jsonSecretCoordinate.getFullCoordinate()))) @@ -282,10 +280,8 @@ public void testWriteSameStagingConfiguration() throws JsonValidationException, final UUID workspaceId = UUID.fromString("13fb9a84-6bfa-4801-8f5e-ce717677babf"); final String jsonSecretPayload = MOCK_SERVICE_ACCOUNT_1; - final JsonNode hmacSecretPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey(hmacSecretPayload) + final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount().withWorkspaceId(workspaceId).withHmacKey( + HMAC_SECRET_PAYLOAD_1) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") .withJsonCredential(Jsons.deserialize(jsonSecretPayload)); @@ -303,7 +299,7 @@ public void testWriteSameStagingConfiguration() throws JsonValidationException, doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); doReturn(Optional.of(jsonSecretPayload)).when(secretPersistence).read(jsonSecretCoordinate); - doReturn(Optional.of(hmacSecretPayload.toString())).when(secretPersistence).read(hmacSecretCoordinate); + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(hmacSecretCoordinate); secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); ArgumentCaptor coordinates = ArgumentCaptor.forClass(SecretCoordinate.class); @@ -316,9 +312,9 @@ public void testWriteSameStagingConfiguration() throws JsonValidationException, List actualPayload = payloads.getAllValues(); assertEquals(2, actualPayload.size()); - assertThat(actualPayload, containsInAnyOrder(jsonSecretPayload, hmacSecretPayload.toString())); + assertThat(actualPayload, containsInAnyOrder(jsonSecretPayload, HMAC_SECRET_PAYLOAD_1.toString())); - verify(secretPersistence).write(hmacSecretCoordinate, hmacSecretPayload.toString()); + verify(secretPersistence).write(hmacSecretCoordinate, HMAC_SECRET_PAYLOAD_1.toString()); verify(configRepository).writeWorkspaceServiceAccountNoSecrets( cloned); } @@ -335,14 +331,9 @@ public void testWriteDifferentStagingConfiguration() throws JsonValidationExcept final String jsonSecretOldPayload = MOCK_SERVICE_ACCOUNT_1; final String jsonSecretNewPayload = MOCK_SERVICE_ACCOUNT_2; - final JsonNode hmacSecretOldPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEF", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeF"))); - final JsonNode hmacSecretNewPayload = Jsons.jsonNode(sortMap( - Map.of("access_id", "ABCD1A1ABCDEFG1ABCDEFGH1ABC12ABCDEF1ABCDE1ABCDE1ABCDE12ABCDEX", "secret", "AB1AbcDEF//ABCDeFGHijKlmNOpqR1ABC1aBCDeX"))); - final WorkspaceServiceAccount workspaceServiceAccount = new WorkspaceServiceAccount() .withWorkspaceId(workspaceId) - .withHmacKey(hmacSecretNewPayload) + .withHmacKey(HMAC_SECRET_PAYLOAD_2) .withServiceAccountId("a1e5ac98-7531-48e1-943b-b46636") .withServiceAccountEmail("a1e5ac98-7531-48e1-943b-b46636@random-gcp-project.abc.abcdefghijklmno.com") .withJsonCredential(Jsons.deserialize(jsonSecretNewPayload)); @@ -359,7 +350,7 @@ public void testWriteDifferentStagingConfiguration() throws JsonValidationExcept doReturn(cloned).when(configRepository).getWorkspaceServiceAccountNoSecrets(workspaceId); - doReturn(Optional.of(hmacSecretOldPayload.toString())).when(secretPersistence).read(hmacSecretOldCoordinate); + doReturn(Optional.of(HMAC_SECRET_PAYLOAD_1.toString())).when(secretPersistence).read(hmacSecretOldCoordinate); doReturn(Optional.of(jsonSecretOldPayload)).when(secretPersistence).read(jsonSecretOldCoordinate); secretsRepositoryWriter.writeServiceAccountJsonCredentials(workspaceServiceAccount); @@ -380,16 +371,11 @@ public void testWriteDifferentStagingConfiguration() throws JsonValidationExcept List actualPayload = payloads.getAllValues(); assertEquals(2, actualPayload.size()); - assertThat(actualPayload, containsInAnyOrder(jsonSecretNewPayload, hmacSecretNewPayload.toString())); + assertThat(actualPayload, containsInAnyOrder(jsonSecretNewPayload, HMAC_SECRET_PAYLOAD_2.toString())); verify(configRepository).writeWorkspaceServiceAccountNoSecrets(Jsons.clone(workspaceServiceAccount).withJsonCredential(Jsons.jsonNode( Map.of("_secret", jsonSecretNewCoordinate.getFullCoordinate()))).withHmacKey(Jsons.jsonNode( Map.of("_secret", hmacSecretNewCoordinate.getFullCoordinate())))); } - private Map sortMap(Map originalMap) { - return originalMap.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)); - } - } From 0bb136d4dc1802458080c329952673368731bbed Mon Sep 17 00:00:00 2001 From: subodh Date: Tue, 26 Apr 2022 18:18:10 +0530 Subject: [PATCH 5/5] update method name and add comment --- .../config/persistence/SecretsRepositoryWriter.java | 5 +++++ .../persistence/split_secrets/SecretsHelpers.java | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java index 7e2491e5d9bcd..03f9487384e1c 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @@ -272,6 +272,11 @@ public void writeServiceAccountJsonCredentials(final WorkspaceServiceAccount wor configRepository.writeWorkspaceServiceAccountNoSecrets(workspaceServiceAccountForDB); } + /** + * This method is to encrypt the secret JSON key and HMAC key of a GCP service account a associated + * with a workspace. If in future we build a similar feature i.e. an AWS account associated with a + * workspace, we will have to build new implementation for it + */ private WorkspaceServiceAccount getWorkspaceServiceAccountWithSecretCoordinate(final WorkspaceServiceAccount workspaceServiceAccount) throws JsonValidationException, IOException { if (longLivedSecretPersistence.isPresent()) { diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index 7cd009f0c0563..e4a3406228ad4 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -116,7 +116,7 @@ public static JsonNode combineConfig(final JsonNode partialConfig, final ReadOnl if (config.has(COORDINATE_FIELD)) { final var coordinateNode = config.get(COORDINATE_FIELD); final var coordinate = getCoordinateFromTextNode(coordinateNode); - return new TextNode(getOrThrowSecretValueNode(secretPersistence, coordinate)); + return new TextNode(getOrThrowSecretValue(secretPersistence, coordinate)); } // otherwise iterate through all object fields @@ -336,10 +336,10 @@ private static JsonNode getFieldOrEmptyNode(final JsonNode node, final int field * @param secretPersistence storage layer for secrets * @param coordinate reference to a secret in the persistence * @throws RuntimeException when a secret at that coordinate is not available in the persistence - * @return a json text node containing the secret value or a JSON + * @return a json string containing the secret value or a JSON */ - private static String getOrThrowSecretValueNode(final ReadOnlySecretPersistence secretPersistence, - final SecretCoordinate coordinate) { + private static String getOrThrowSecretValue(final ReadOnlySecretPersistence secretPersistence, + final SecretCoordinate coordinate) { final var secretValue = secretPersistence.read(coordinate); if (secretValue.isEmpty()) { @@ -464,7 +464,7 @@ public static SecretCoordinateToPayload convertServiceAccountCredsToSecret(final public static JsonNode hydrateSecretCoordinate(final JsonNode secretCoordinateAsJson, final ReadOnlySecretPersistence readOnlySecretPersistence) { final var secretCoordinate = getCoordinateFromTextNode(secretCoordinateAsJson.get(COORDINATE_FIELD)); - return Jsons.deserialize(getOrThrowSecretValueNode(readOnlySecretPersistence, secretCoordinate)); + return Jsons.deserialize(getOrThrowSecretValue(readOnlySecretPersistence, secretCoordinate)); } }