diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java index 70062f294392e..b60dcaa0591ee 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @@ -15,6 +15,7 @@ import com.jayway.jsonpath.spi.json.JsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import com.jayway.jsonpath.spi.mapper.MappingProvider; +import io.airbyte.commons.json.JsonSchemas.FieldNameOrList; import io.airbyte.commons.util.MoreIterators; import java.util.Collections; import java.util.EnumSet; @@ -94,6 +95,20 @@ public static String appendAppendListSplat(final String jsonPath) { return jsonPath + JSON_PATH_LIST_SPLAT; } + /** + * Map path produced by {@link JsonSchemas} to the JSONPath format. + * + * @param jsonSchemaPath - path as described in {@link JsonSchemas} + * @return path as JSONPath + */ + public static String mapJsonSchemaPathToJsonPath(final List jsonSchemaPath) { + String jsonPath = empty(); + for (final FieldNameOrList fieldNameOrList : jsonSchemaPath) { + jsonPath = fieldNameOrList.isList() ? appendAppendListSplat(jsonPath) : appendField(jsonPath, fieldNameOrList.getFieldName()); + } + return jsonPath; + } + /* * This version of the JsonPath Configuration object allows queries to return to the path of values * instead of the values that were found. diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java index b6da1dac351a1..55396a771d077 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @@ -6,19 +6,20 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Preconditions; import io.airbyte.commons.io.IOs; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; +import io.airbyte.commons.util.MoreLists; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.BiConsumer; @@ -95,8 +96,33 @@ public static Path prepareSchemas(final String resourceDir, final Class k } } - public static void traverseJsonSchema(final JsonNode jsonSchemaNode, final BiConsumer consumer) { - traverseJsonSchemaInternal(jsonSchemaNode, JsonPaths.empty(), consumer); + /** + * Traverse a JsonSchema object. The provided consumer will be called at each node with the node and + * the path to the node. + * + * @param jsonSchema - JsonSchema object to traverse + * @param consumer - accepts the current node and the path to that node. + */ + public static void traverseJsonSchema(final JsonNode jsonSchema, final BiConsumer> consumer) { + traverseJsonSchemaInternal(jsonSchema, new ArrayList<>(), consumer); + } + + /** + * Traverse a JsonSchema object. At each node, map a value. + * + * @param jsonSchema - JsonSchema object to traverse + * @param mapper - accepts the current node and the path to that node. whatever is returned will be + * collected and returned by the final collection. + * @param - type of objects being collected + * @return - collection of all items that were collected during the traversal. Returns a { @link + * Collection } because there is no order or uniqueness guarantee so neither List nor Set + * make sense. + */ + public static List traverseJsonSchemaWithCollector(final JsonNode jsonSchema, + final BiFunction, T> mapper) { + // for the sake of code reuse, use the filtered collector method but makes sure the filter always + // returns true. + return traverseJsonSchemaWithFilteredCollector(jsonSchema, (node, path) -> Optional.ofNullable(mapper.apply(node, path))); } /** @@ -111,44 +137,45 @@ public static void traverseJsonSchema(final JsonNode jsonSchemaNode, final BiCon * Collection } because there is no order or uniqueness guarantee so neither List nor Set * make sense. */ - public static Collection traverseJsonSchemaWithCollector(final JsonNode jsonSchema, final BiFunction> mapper) { - final List collectors = new ArrayList<>(); - traverseJsonSchema(jsonSchema, (node, path) -> mapper.apply(node, path).ifPresent(collectors::add)); - return collectors; + public static List traverseJsonSchemaWithFilteredCollector(final JsonNode jsonSchema, + final BiFunction, Optional> mapper) { + final List collector = new ArrayList<>(); + traverseJsonSchema(jsonSchema, (node, path) -> mapper.apply(node, path).ifPresent(collector::add)); + return collector.stream().toList(); // make list unmodifiable } /** * Traverses a JsonSchema object. It returns the path to each node that meet the provided condition. - * The paths are return in JsonPath format + * The paths are return in JsonPath format. The traversal is depth-first search preoorder and values + * are returned in that order. * * @param obj - JsonSchema object to traverse * @param predicate - predicate to determine if the path for a node should be collected. * @return - collection of all paths that were collected during the traversal. */ - public static Set collectJsonPathsThatMeetCondition(final JsonNode obj, final Predicate predicate) { - return new HashSet<>(traverseJsonSchemaWithCollector(obj, (node, path) -> { + public static List> collectPathsThatMeetCondition(final JsonNode obj, final Predicate predicate) { + return traverseJsonSchemaWithFilteredCollector(obj, (node, path) -> { if (predicate.test(node)) { return Optional.of(path); } else { return Optional.empty(); } - })); + }); } /** * Recursive, depth-first implementation of { @link JsonSchemas#traverseJsonSchema(final JsonNode * jsonNode, final BiConsumer> consumer) }. Takes path as argument so that - * the path can be passsed to the consumer. + * the path can be passed to the consumer. * * @param jsonSchemaNode - jsonschema object to traverse. - * @param path - path from the first call of traverseJsonSchema to the current node. * @param consumer - consumer to be called at each node. it accepts the current node and the path to * the node from the root of the object passed at the root level invocation + * */ - // todo (cgardens) - replace with easier to understand traversal logic from SecretsHelper. private static void traverseJsonSchemaInternal(final JsonNode jsonSchemaNode, - final String path, - final BiConsumer consumer) { + final List path, + final BiConsumer> consumer) { if (!jsonSchemaNode.isObject()) { throw new IllegalArgumentException(String.format("json schema nodes should always be object nodes. path: %s actual: %s", path, jsonSchemaNode)); } @@ -162,9 +189,8 @@ private static void traverseJsonSchemaInternal(final JsonNode jsonSchemaNode, switch (nodeType) { // case BOOLEAN_TYPE, NUMBER_TYPE, STRING_TYPE, NULL_TYPE -> do nothing after consumer.accept above. case ARRAY_TYPE -> { - final String newPath = JsonPaths.appendAppendListSplat(path); + final List newPath = MoreLists.add(path, FieldNameOrList.list()); // hit every node. - // log.error("array: " + jsonSchemaNode); traverseJsonSchemaInternal(jsonSchemaNode.get(JSON_SCHEMA_ITEMS_KEY), newPath, consumer); } case OBJECT_TYPE -> { @@ -172,13 +198,11 @@ private static void traverseJsonSchemaInternal(final JsonNode jsonSchemaNode, if (jsonSchemaNode.has(JSON_SCHEMA_PROPERTIES_KEY)) { for (final Iterator> it = jsonSchemaNode.get(JSON_SCHEMA_PROPERTIES_KEY).fields(); it.hasNext();) { final Entry child = it.next(); - final String newPath = JsonPaths.appendField(path, child.getKey()); - // log.error("obj1: " + jsonSchemaNode); + final List newPath = MoreLists.add(path, FieldNameOrList.fieldName(child.getKey())); traverseJsonSchemaInternal(child.getValue(), newPath, consumer); } } else if (comboKeyWordOptional.isPresent()) { for (final JsonNode arrayItem : jsonSchemaNode.get(comboKeyWordOptional.get())) { - // log.error("obj2: " + jsonSchemaNode); traverseJsonSchemaInternal(arrayItem, path, consumer); } } else { @@ -206,8 +230,15 @@ private static Optional getKeywordIfComposite(final JsonNode node) { return Optional.empty(); } - public static List getTypeOrObject(final JsonNode jsonNode) { - final List types = getType(jsonNode); + /** + * Same logic as {@link #getType(JsonNode)} except when no type is found, it defaults to type: + * Object. + * + * @param jsonSchema - JSONSchema object + * @return type of the node. + */ + public static List getTypeOrObject(final JsonNode jsonSchema) { + final List types = getType(jsonSchema); if (types.isEmpty()) { return List.of(OBJECT_TYPE); } else { @@ -215,21 +246,96 @@ public static List getTypeOrObject(final JsonNode jsonNode) { } } - public static List getType(final JsonNode jsonNode) { - if (jsonNode.has(JSON_SCHEMA_TYPE_KEY)) { - if (jsonNode.get(JSON_SCHEMA_TYPE_KEY).isArray()) { - return MoreIterators.toList(jsonNode.get(JSON_SCHEMA_TYPE_KEY).iterator()) + /** + * Get the type of JSONSchema node. Uses JSONSchema types. Only returns the type of the "top-level" + * node. e.g. if more nodes are nested underneath because it is an object or an array, only the top + * level type is returned. + * + * @param jsonSchema - JSONSchema object + * @return type of the node. + */ + public static List getType(final JsonNode jsonSchema) { + if (jsonSchema.has(JSON_SCHEMA_TYPE_KEY)) { + if (jsonSchema.get(JSON_SCHEMA_TYPE_KEY).isArray()) { + return MoreIterators.toList(jsonSchema.get(JSON_SCHEMA_TYPE_KEY).iterator()) .stream() .map(JsonNode::asText) .collect(Collectors.toList()); } else { - return List.of(jsonNode.get(JSON_SCHEMA_TYPE_KEY).asText()); + return List.of(jsonSchema.get(JSON_SCHEMA_TYPE_KEY).asText()); } } - if (jsonNode.has(JSON_SCHEMA_ENUM_KEY)) { + if (jsonSchema.has(JSON_SCHEMA_ENUM_KEY)) { return List.of(STRING_TYPE); } return Collections.emptyList(); } + /** + * Provides a basic scheme for describing the path into a JSON object. Each element in the path is + * either a field name or a list. + * + * This class is helpful in the case where fields can be any UTF-8 string, so the only simple way to + * keep track of the different parts of a path without going crazy with escape characters is to keep + * it in a list with list set aside as a special case. + * + * We prefer using this scheme instead of JSONPath in the tree traversal because, it is easier to + * decompose a path in this scheme than it is in JSONPath. Some callers of the traversal logic want + * to isolate parts of the path easily without the need for complex regex (that would be required if + * we used JSONPath). + */ + public static class FieldNameOrList { + + private final String fieldName; + private final boolean isList; + + public static FieldNameOrList fieldName(final String fieldName) { + return new FieldNameOrList(fieldName); + } + + public static FieldNameOrList list() { + return new FieldNameOrList(null); + } + + private FieldNameOrList(final String fieldName) { + isList = fieldName == null; + this.fieldName = fieldName; + } + + public String getFieldName() { + Preconditions.checkState(!isList, "cannot return field name, is list node"); + return fieldName; + } + + public boolean isList() { + return isList; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof FieldNameOrList)) { + return false; + } + final FieldNameOrList that = (FieldNameOrList) o; + return isList == that.isList && Objects.equals(fieldName, that.fieldName); + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, isList); + } + + @Override + public String toString() { + return "FieldNameOrList{" + + "fieldName='" + fieldName + '\'' + + ", isList=" + isList + + '}'; + } + + } + } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreLists.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreLists.java index c12e5d7df3a72..fa6779f0d64df 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreLists.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreLists.java @@ -48,4 +48,18 @@ public static List concat(final List... lists) { return Stream.of(lists).flatMap(List::stream).toList(); } + /** + * Copies provided list and adds the new item to the copy. + * + * @param list list to copy and add to + * @param toAdd item to add + * @param type of list + * @return new list with contents of provided list and the added item + */ + public static List add(final List list, final T toAdd) { + final ArrayList newList = new ArrayList<>(list); + newList.add(toAdd); + return newList; + } + } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java index b98d919e3cd76..3476bd76f8f78 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java @@ -8,8 +8,11 @@ import static org.mockito.Mockito.mock; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.JsonSchemas.FieldNameOrList; import io.airbyte.commons.resources.MoreResources; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.function.BiConsumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -41,19 +44,24 @@ void testMutateTypeToArrayStandard() { @Test void testTraverse() throws IOException { final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_all_types.json")); - final BiConsumer mock = mock(BiConsumer.class); + final BiConsumer> mock = mock(BiConsumer.class); JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); - inOrder.verify(mock).accept(jsonWithAllTypes, JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name"), "$.name"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("first"), "$.name.first"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("last"), "$.name.last"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), "$.company"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets"), "$.pets"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items"), "$.pets[*]"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("type"), "$.pets[*].type"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("number"), "$.pets[*].number"); + inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name"), List.of(FieldNameOrList.fieldName("name"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("first"), + List.of(FieldNameOrList.fieldName("name"), FieldNameOrList.fieldName("first"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("name").get("properties").get("last"), + List.of(FieldNameOrList.fieldName("name"), FieldNameOrList.fieldName("last"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), List.of(FieldNameOrList.fieldName("company"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets"), List.of(FieldNameOrList.fieldName("pets"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items"), + List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("type"), + List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list(), FieldNameOrList.fieldName("type"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("pets").get("items").get("properties").get("number"), + List.of(FieldNameOrList.fieldName("pets"), FieldNameOrList.list(), FieldNameOrList.fieldName("number"))); inOrder.verifyNoMoreInteractions(); } @@ -68,20 +76,22 @@ void testTraverseComposite(final String compositeKeyword) throws IOException { final String jsonSchemaString = MoreResources.readResource("json_schemas/composite_json_schema.json") .replaceAll("", compositeKeyword); final JsonNode jsonWithAllTypes = Jsons.deserialize(jsonSchemaString); - final BiConsumer mock = mock(BiConsumer.class); + final BiConsumer> mock = mock(BiConsumer.class); JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); - inOrder.verify(mock).accept(jsonWithAllTypes, JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0), JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1), JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("prop1"), "$.prop1"); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2), JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2).get("items"), "$[*]"); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(0), JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1), JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1).get("items"), "$[*]"); + inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0), Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1), Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("prop1"), + List.of(FieldNameOrList.fieldName("prop1"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2), Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(2).get("items"), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(0), Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1), Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(3).get(compositeKeyword).get(1).get("items"), + List.of(FieldNameOrList.list())); inOrder.verifyNoMoreInteractions(); } @@ -89,14 +99,15 @@ void testTraverseComposite(final String compositeKeyword) throws IOException { @Test void testTraverseMultiType() throws IOException { final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_array_type_fields.json")); - final BiConsumer mock = mock(BiConsumer.class); + final BiConsumer> mock = mock(BiConsumer.class); JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); - inOrder.verify(mock).accept(jsonWithAllTypes, JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), "$.company"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), "$[*]"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), "$[*].user"); + inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get("properties").get("company"), List.of(FieldNameOrList.fieldName("company"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), + List.of(FieldNameOrList.list(), FieldNameOrList.fieldName("user"))); inOrder.verifyNoMoreInteractions(); } @@ -105,16 +116,19 @@ void testTraverseMultiType() throws IOException { void testTraverseMultiTypeComposite() throws IOException { final String compositeKeyword = "anyOf"; final JsonNode jsonWithAllTypes = Jsons.deserialize(MoreResources.readResource("json_schemas/json_with_array_type_fields_with_composites.json")); - final BiConsumer mock = mock(BiConsumer.class); + final BiConsumer> mock = mock(BiConsumer.class); JsonSchemas.traverseJsonSchema(jsonWithAllTypes, mock); final InOrder inOrder = Mockito.inOrder(mock); - inOrder.verify(mock).accept(jsonWithAllTypes, JsonPaths.empty()); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0).get("properties").get("company"), "$.company"); - inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("organization"), "$.organization"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), "$[*]"); - inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), "$[*].user"); + inOrder.verify(mock).accept(jsonWithAllTypes, Collections.emptyList()); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(0).get("properties").get("company"), + List.of(FieldNameOrList.fieldName("company"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get(compositeKeyword).get(1).get("properties").get("organization"), + List.of(FieldNameOrList.fieldName("organization"))); + inOrder.verify(mock).accept(jsonWithAllTypes.get("items"), List.of(FieldNameOrList.list())); + inOrder.verify(mock).accept(jsonWithAllTypes.get("items").get("properties").get("user"), + List.of(FieldNameOrList.list(), FieldNameOrList.fieldName("user"))); inOrder.verifyNoMoreInteractions(); } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/MoreListsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/util/MoreListsTest.java index a05db55e00c58..3243f370bc578 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/util/MoreListsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/util/MoreListsTest.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -28,9 +27,26 @@ void testLast() { @Test void testReverse() { - final ArrayList originalList = Lists.newArrayList(1, 2, 3); + final List originalList = List.of(1, 2, 3); assertEquals(List.of(3, 2, 1), MoreLists.reversed(originalList)); assertEquals(List.of(1, 2, 3), originalList); } + @Test + void testConcat() { + final List> lists = List.of(List.of(1, 2, 3), List.of(4, 5, 6), List.of(7, 8, 9)); + final List expected = List.of(1, 2, 3, 4, 5, 6, 7, 8, 9); + final List actual = MoreLists.concat(lists.get(0), lists.get(1), lists.get(2)); + assertEquals(expected, actual); + } + + @Test + void testAdd() { + final List originalList = List.of(1, 2, 3); + + assertEquals(List.of(1, 2, 3, 4), MoreLists.add(originalList, 4)); + // verify original list was not mutated. + assertEquals(List.of(1, 2, 3), originalList); + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java index 88f0c796273dd..f97ce914bb1a4 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/JsonSecretsProcessor.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import lombok.Builder; import lombok.extern.slf4j.Slf4j; @@ -76,11 +77,14 @@ public JsonNode prepareSecretsForOutput(final JsonNode obj, final JsonNode schem * @return json object with all secrets masked. */ public static JsonNode maskAllSecrets(final JsonNode json, final JsonNode schema) { - final Set pathsWithSecrets = JsonSchemas.collectJsonPathsThatMeetCondition( + final Set pathsWithSecrets = JsonSchemas.collectPathsThatMeetCondition( schema, node -> MoreIterators.toList(node.fields()) .stream() - .anyMatch(field -> AIRBYTE_SECRET_FIELD.equals(field.getKey()))); + .anyMatch(field -> AIRBYTE_SECRET_FIELD.equals(field.getKey()))) + .stream() + .map(JsonPaths::mapJsonSchemaPathToJsonPath) + .collect(Collectors.toSet()); JsonNode copy = Jsons.clone(json); for (final String path : pathsWithSecrets) { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java index 4f1d51ea9473c..eef92dcdfd974 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/SecretsHelpers.java @@ -171,12 +171,13 @@ public static SplitSecretConfig splitAndUpdateConfig(final Supplier uuidSu * in an ascending alphabetical order. */ public static List getSortedSecretPaths(final JsonNode spec) { - return JsonSchemas.collectJsonPathsThatMeetCondition( + return JsonSchemas.collectPathsThatMeetCondition( spec, node -> MoreIterators.toList(node.fields()) .stream() .anyMatch(field -> field.getKey().equals(JsonSecretsProcessor.AIRBYTE_SECRET_FIELD))) .stream() + .map(JsonPaths::mapJsonSchemaPathToJsonPath) .sorted() .toList(); }