From 28ac70c7b325c818dc7065337cb806f72fb9838e Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Thu, 16 Jun 2022 19:55:25 +0300 Subject: [PATCH 1/3] Make sure that cursor value doesn't contain unicode null. --- .../source/relationaldb/StateDecoratingIterator.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index 7900aed452c98..122d62ddbb65d 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -43,12 +43,17 @@ public StateDecoratingIterator(final Iterator messageIterator, stateManager.setIsCdc(false); } + private String getCursorCandidate(final AirbyteMessage message) { + String cursorCandidate = message.getRecord().getData().get(cursorField).asText(); + return (cursorCandidate != null ? cursorCandidate.replaceAll("\u0000", "") : null); + } + @Override protected AirbyteMessage computeNext() { if (messageIterator.hasNext()) { final AirbyteMessage message = messageIterator.next(); if (message.getRecord().getData().hasNonNull(cursorField)) { - final String cursorCandidate = message.getRecord().getData().get(cursorField).asText(); + final String cursorCandidate = getCursorCandidate(message); if (IncrementalUtils.compareCursors(maxCursor, cursorCandidate, cursorType) < 0) { maxCursor = cursorCandidate; } From e9ef6dd0f146ec42ee411fbc92e671924a51f75e Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Thu, 16 Jun 2022 20:16:46 +0300 Subject: [PATCH 2/3] Test that cursors doesn't contain unicode null --- .../StateDecoratingIteratorTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index ab521edf87e28..7fb6964d26545 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -41,6 +41,11 @@ class StateDecoratingIteratorTest { .withRecord(new AirbyteRecordMessage() .withData(Jsons.jsonNode(ImmutableMap.of(UUID_FIELD_NAME, "def")))); + private static final AirbyteMessage RECORD_MESSAGE3 = new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withData(Jsons.jsonNode(ImmutableMap.of(UUID_FIELD_NAME, "abc\u0000")))); + private static Iterator messageIterator; private StateManager stateManager; private AirbyteStateMessage stateMessage; @@ -130,4 +135,22 @@ void testEmptyStream() { assertFalse(iterator.hasNext()); } + @Test + void testUnicodeNull() { + messageIterator = MoreIterators.of(RECORD_MESSAGE3); + when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, "abc")).thenReturn(stateMessage); + + final StateDecoratingIterator iterator = new StateDecoratingIterator( + messageIterator, + stateManager, + NAME_NAMESPACE_PAIR, + UUID_FIELD_NAME, + null, + JsonSchemaPrimitive.STRING); + + assertEquals(RECORD_MESSAGE3, iterator.next()); + assertEquals(stateMessage, iterator.next().getState()); + assertFalse(iterator.hasNext()); + } + } From 27e92d9af7267b60d6e0adba006e49af7ab582f1 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Thu, 16 Jun 2022 20:38:11 +0300 Subject: [PATCH 3/3] incr version --- airbyte-integrations/connectors/source-relational-db/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-relational-db/Dockerfile b/airbyte-integrations/connectors/source-relational-db/Dockerfile index 5ae6986dbb993..36baea09eafa1 100644 --- a/airbyte-integrations/connectors/source-relational-db/Dockerfile +++ b/airbyte-integrations/connectors/source-relational-db/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-relational-db COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-relational-db