Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🪟 🎉 Clear user-defined primary key(s) and cursor field(s) if the field is removed #20203

Merged
merged 16 commits into from
Dec 12, 2022
Merged

🪟 🎉 Clear user-defined primary key(s) and cursor field(s) if the field is removed #20203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
73df9bf
first pass
teallarson Dec 7, 2022
f2479f2
remove typecasting, no longer needed
teallarson Dec 7, 2022
4cae0ed
cleanup existing tests
teallarson Dec 7, 2022
262dda5
cleanup the cleanup
teallarson Dec 7, 2022
0770f7e
testing, review response
teallarson Dec 7, 2022
59834c9
Merge branch 'master' into teal/clear-removed-source-defined-fields
teallarson Dec 7, 2022
5f4f27b
more cleanup
teallarson Dec 8, 2022
c919838
cleanup
teallarson Dec 8, 2022
ee9ec04
cleanup with edmundo, fix tests
teallarson Dec 8, 2022
673a398
clean up looping
teallarson Dec 9, 2022
1b4308d
fix ze build, silly undefined config!
teallarson Dec 9, 2022
582f1af
Merge branch 'master' into teal/clear-removed-source-defined-fields
teallarson Dec 9, 2022
782654c
cleanup return types when getting streamsWithBreakingChanges
teallarson Dec 9, 2022
e34f154
cleanup
teallarson Dec 9, 2022
3222b1a
Update airbyte-webapp/src/views/Connection/ConnectionForm/calculateIn…
teallarson Dec 9, 2022
46f245c
Update airbyte-webapp/src/views/Connection/ConnectionForm/calculateIn…
teallarson Dec 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 214 additions & 1 deletion airbyte-webapp/src/views/Connection/ConnectionForm/calculateInitialCatalog.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import { SyncSchema, SyncSchemaStream } from "core/domain/catalog";
import { DestinationSyncMode, StreamDescriptor, SyncMode } from "core/request/AirbyteClient";
import {
DestinationSyncMode,
FieldTransformTransformType,
StreamDescriptor,
StreamTransformTransformType,
SyncMode,
} from "core/request/AirbyteClient";

import calculateInitialCatalog from "./calculateInitialCatalog";

Expand Down Expand Up @@ -33,6 +39,7 @@ describe("calculateInitialCatalog", () => {
streams: [restProps],
} as unknown as SyncSchema,
[],
[],
false
);

Expand All @@ -58,6 +65,7 @@ describe("calculateInitialCatalog", () => {
],
} as unknown as SyncSchema,
[],
[],
false
);

Expand Down Expand Up @@ -107,6 +115,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup, DestinationSyncMode.overwrite],
[],
false
);

Expand Down Expand Up @@ -182,6 +191,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);

Expand Down Expand Up @@ -239,6 +249,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.overwrite],
[],
false
);

Expand Down Expand Up @@ -296,6 +307,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append],
[],
false
);

Expand Down Expand Up @@ -353,6 +365,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append],
[],
false
);

Expand Down Expand Up @@ -384,6 +397,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
true
);

Expand Down Expand Up @@ -444,6 +458,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);

Expand Down Expand Up @@ -473,6 +488,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
true
);
// primary keys
Expand Down Expand Up @@ -509,6 +525,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);

Expand Down Expand Up @@ -536,6 +553,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);

Expand Down Expand Up @@ -563,6 +581,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);

Expand Down Expand Up @@ -590,6 +609,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
false
);
// primary keys
Expand Down Expand Up @@ -642,6 +662,7 @@ describe("calculateInitialCatalog", () => {
],
},
[DestinationSyncMode.append_dedup],
[],
true,
newStreamDescriptors
);
Expand All @@ -654,4 +675,196 @@ describe("calculateInitialCatalog", () => {
expect(calculatedStreams[1].config?.syncMode).toEqual(SyncMode.incremental);
expect(calculatedStreams[1].config?.destinationSyncMode).toEqual(DestinationSyncMode.overwrite);
});

it("should remove the entire primary key if any path from it was removed", () => {
const { config, stream: sourceDefinedStream } = mockSyncSchemaStream;
const values = calculateInitialCatalog(
{
streams: [
{
id: "1",
stream: {
...sourceDefinedStream,
name: "test",
sourceDefinedCursor: true,
defaultCursorField: ["id"],
sourceDefinedPrimaryKey: [],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
primaryKey: [["id"], ["email"]],
},
},
{
id: "2",
stream: {
...sourceDefinedStream,
name: "test-2",
sourceDefinedCursor: true,
defaultCursorField: ["updated_at"],
sourceDefinedPrimaryKey: [],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
primaryKey: [["id"]],
},
},
],
},
[DestinationSyncMode.append_dedup],
[
{
transformType: StreamTransformTransformType.update_stream,
streamDescriptor: { name: "test", namespace: "namespace-test" },
updateStream: [
{
breaking: true,
transformType: FieldTransformTransformType.remove_field,
fieldName: ["id"],
},
],
},
],
true
);
expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared
expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was not affected
});

it("should remove cursor from config if the old cursor field was removed, even if there is a default", () => {
const { config, stream: sourceDefinedStream } = mockSyncSchemaStream;
const values = calculateInitialCatalog(
{
streams: [
{
id: "1",
stream: {
...sourceDefinedStream,
name: "test",
sourceDefinedCursor: false,
defaultCursorField: ["id"],
sourceDefinedPrimaryKey: [],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
cursorField: ["updated_at"],
},
},
{
id: "2",
stream: {
...sourceDefinedStream,
name: "test-2",
sourceDefinedCursor: true,
defaultCursorField: ["updated_at"],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
cursorField: ["updated_at"],
primaryKey: [["id"]],
},
},
],
},
[DestinationSyncMode.append_dedup],
[
{
transformType: StreamTransformTransformType.update_stream,
streamDescriptor: { name: "test", namespace: "namespace-test" },
updateStream: [
{
breaking: true,
transformType: FieldTransformTransformType.remove_field,
fieldName: ["updated_at"],
},
],
},
],
true
);
expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default
expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected
});
it("should clear multiple config fields if multiple fields were removed", () => {
const { config, stream: sourceDefinedStream } = mockSyncSchemaStream;
const values = calculateInitialCatalog(
{
streams: [
{
id: "1",
stream: {
...sourceDefinedStream,
name: "test",
sourceDefinedCursor: false,
defaultCursorField: ["id"],
sourceDefinedPrimaryKey: [],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
cursorField: ["updated_at"],
primaryKey: [["primary_key"], ["another_field"]],
},
},
{
id: "2",
stream: {
...sourceDefinedStream,
name: "test-2",
sourceDefinedCursor: true,
defaultCursorField: ["updated_at"],
sourceDefinedPrimaryKey: [],
supportedSyncModes: [SyncMode.incremental],
},
config: {
...config,
destinationSyncMode: DestinationSyncMode.append_dedup,
syncMode: SyncMode.incremental,
cursorField: ["updated_at"],
primaryKey: [["id"]],
},
},
],
},
[DestinationSyncMode.append_dedup],
[
{
transformType: StreamTransformTransformType.update_stream,
streamDescriptor: { name: "test", namespace: "namespace-test" },
updateStream: [
{
breaking: true,
transformType: FieldTransformTransformType.remove_field,
fieldName: ["updated_at"],
},
{
breaking: true,
transformType: FieldTransformTransformType.remove_field,
fieldName: ["primary_key"],
},
],
},
],
true
);
expect(values.streams[0].config?.primaryKey).toEqual([]); // was entirely cleared and not replaced with default
expect(values.streams[0].config?.cursorField).toEqual([]); // was entirely cleared and not replaced with default

expect(values.streams[1].config?.primaryKey).toEqual([["id"]]); // was unaffected})
expect(values.streams[1].config?.cursorField).toEqual(["updated_at"]); // was unaffected})
});
});
Loading