Skip to content

Commit 7591883

Browse files
andriikorotkovandriikorotkov
authoredDec 12, 2021
Added benchmarks scripts with small instruction (#8529)
* updated source-mongodb-v2 performance * updated code style * fixed remarks * fixed remarks * fixed remarks * updated strict encrypt source mongodb version * updated source mongodb work with empty collections * updated source mongodb timestamp cursor * updated mongodb source perfomance * fix code style * fix code style * updated tests and documentation * updated tests and documentation * updated tests and documentation * added vudangngoc changes * updated code style * updated code style * added benchmarks scripts with small instruction * fixed remarks * updated ci performance test sh script
·
v1.5.0java-cdk-0.28.0
1 parent 70d2f46 commit 7591883

File tree

8 files changed

+426
-1
lines changed

8 files changed

+426
-1
lines changed
 

‎airbyte-integrations/connectors/source-mssql/README.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# MsSQL (SQL Server) Source
2+
3+
## Performance Test
4+
5+
### Use MsSQL script to populate the benchmark database
6+
7+
In order to create a database with a certain number of tables, and a certain number of records in each of them,
8+
you need to follow a few simple steps.
9+
10+
1. Create a new database.
11+
2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes.
12+
3. Execute the script with your changes for the new database.
13+
You can run the script use the MySQL command line client: - **mysql -h hostname -u user database < path/to/script/mssql-script.sql**
14+
After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**.

‎airbyte-integrations/connectors/source-mssql/mssql-script.sql

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
create procedure table_copy(@tablecount int)
2+
as
3+
begin
4+
set nocount on;
5+
6+
DECLARE @v_max_table int;
7+
DECLARE @v_counter_table int;
8+
DECLARE @tnamee VARCHAR(255);
9+
set @v_max_table = @tablecount;
10+
set @v_counter_table = 1;
11+
12+
while @v_counter_table < @v_max_table begin
13+
set @tnamee = concat('SELECT * INTO test_', @v_counter_table, ' FROM test;');
14+
EXEC (@tnamee);
15+
set @v_counter_table = @v_counter_table + 1;
16+
end;
17+
18+
end;
19+
go --
20+
21+
create procedure insert_rows( @allrows int, @insertcount int, @value NVARCHAR(max))
22+
as
23+
begin
24+
set nocount on;
25+
26+
DECLARE @dummyIpsum varchar(255)
27+
DECLARE @fieldText NVARCHAR(max)
28+
set @fieldText = @value
29+
DECLARE @vmax int;
30+
DECLARE @vmaxx int;
31+
DECLARE @vmaxoneinsert int;
32+
DECLARE @counter int;
33+
DECLARE @lastinsertcounter int;
34+
DECLARE @lastinsert int;
35+
DECLARE @fullloop int;
36+
DECLARE @fullloopcounter int;
37+
set @dummyIpsum = '''dummy_ipsum'''
38+
set @vmax = @allrows;
39+
set @vmaxx = @allrows;
40+
set @vmaxoneinsert = @insertcount;
41+
set @counter = 1;
42+
set @lastinsertcounter = 1;
43+
set @lastinsert = 0;
44+
set @fullloop = 0;
45+
set @fullloopcounter = 0;
46+
47+
while @vmaxx <= @vmaxoneinsert begin
48+
set @vmaxoneinsert = @vmaxx;
49+
set @fullloop = @fullloop + 1;
50+
set @vmaxx = @vmaxx + 1;
51+
end;
52+
53+
while @vmax > @vmaxoneinsert begin
54+
set @fullloop = @fullloop + 1;
55+
set @vmax = @vmax - @vmaxoneinsert;
56+
set @lastinsert = @vmax;
57+
end;
58+
59+
DECLARE @insertTable NVARCHAR(MAX)
60+
set @insertTable = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
61+
while @counter < @vmaxoneinsert begin
62+
set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('));
63+
set @counter = @counter + 1;
64+
end;
65+
set @insertTable = CONVERT(NVARCHAR(max), concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'));
66+
67+
while @vmax < 1 begin
68+
set @fullloop = 0
69+
set @vmax = 1
70+
end;
71+
72+
while @fullloopcounter < @fullloop begin
73+
EXEC (@insertTable);
74+
set @fullloopcounter = @fullloopcounter + 1;
75+
end;
76+
77+
DECLARE @insertTableLasted NVARCHAR(max);
78+
set @insertTableLasted = CONVERT(NVARCHAR(max), 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
79+
while @lastinsertcounter < @lastinsert begin
80+
set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), ('));
81+
set @lastinsertcounter = @lastinsertcounter + 1;
82+
end;
83+
84+
set @insertTableLasted = CONVERT(NVARCHAR(max), concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);'));
85+
86+
while @lastinsert > 0 begin
87+
EXEC (@insertTableLasted);
88+
set @lastinsert = 0;
89+
end;
90+
91+
end;
92+
go --
93+
94+
create procedure table_create(@val int)
95+
as
96+
begin
97+
set nocount on;
98+
99+
-- SQLINES LICENSE FOR EVALUATION USE ONLY
100+
create table test
101+
(
102+
id int check (id > 0) not null identity primary key,
103+
varchar1 varchar(255),
104+
varchar2 varchar(255),
105+
varchar3 varchar(255),
106+
varchar4 varchar(255),
107+
varchar5 varchar(255),
108+
longblobfield nvarchar(max),
109+
timestampfield datetime2(0)
110+
);
111+
112+
DECLARE @extraSmallText NVARCHAR(max);
113+
DECLARE @smallText NVARCHAR(max);
114+
DECLARE @regularText NVARCHAR(max);
115+
DECLARE @largeText NVARCHAR(max);
116+
set @extraSmallText = '''test weight 50b - 50b text, 50b text, 50b text'''
117+
set @smallText = CONCAT('''test weight 500b - ', REPLICATE('some text, some text, ', 20), '''')
118+
set @regularText = CONCAT('''test weight 10kb - ', REPLICATE('some text, some text, some text, some text, ', 295), 'some text''')
119+
set @largeText = CONCAT('''test weight 100kb - ', REPLICATE('some text, some text, some text, some text, ', 2225), 'some text''')
120+
-- TODO: change the following @allrows to control the number of records with different sizes
121+
-- number of 50B records
122+
EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @extraSmallText
123+
-- number of 500B records
124+
EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @smallText
125+
-- number of 10KB records
126+
EXEC insert_rows @allrows = 0, @insertcount = 998, @value = @regularText
127+
-- number of 100KB records
128+
EXEC insert_rows @allrows = 0, @insertcount = 98, @value = @largeText
129+
end;
130+
go --
131+
132+
EXEC table_create @val = 0
133+
drop procedure if exists insert_rows;
134+
drop procedure if exists table_create;
135+
136+
-- TODO: change the value to control the number of tables
137+
EXEC table_copy @tablecount = 1;
138+
drop procedure if exists table_copy;
139+
exec sp_rename 'test', 'test_0';

‎airbyte-integrations/connectors/source-mysql/README.md

+14-1
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,17 @@ To run acceptance and custom integration tests:
3636
To run performance tests:
3737
```
3838
./gradlew :airbyte-integrations:connectors:source-mysql:performanceTest
39-
```
39+
```
40+
41+
### Use MySQL script to populate the benchmark database
42+
43+
In order to create a database with a certain number of tables, and a certain number of records in each of them,
44+
you need to follow a few simple steps.
45+
46+
1. Create a new database.
47+
2. Follow the TODOs in **mssql-script.sql** to change the number of tables, and the number of records of different sizes.
48+
3. Execute the script with your changes for the new database.
49+
You can run the script use the MsSQL command line client: - **sqlcmd -S Serverinstance -E -i path/to/script/mssql-script.sql**
50+
After the script finishes its work, you will receive the number of tables specified in the script, with names starting with **test_0** and ending with **test_(the number of tables minus 1)**.
51+
52+

‎airbyte-integrations/connectors/source-mysql/mysql-script.sql

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
delimiter #
2+
create procedure table_copy(in tablecount int)
3+
begin
4+
5+
set @v_max_table = tablecount;
6+
set @v_counter_table = 1;
7+
8+
while @v_counter_table < @v_max_table do
9+
set @tnamee = concat('create table IF NOT EXISTS test_', @v_counter_table, ' SELECT * FROM test;');
10+
PREPARE stmt from @tnamee;
11+
EXECUTE stmt;
12+
DEALLOCATE PREPARE stmt;
13+
set @v_counter_table = @v_counter_table + 1;
14+
end while;
15+
commit;
16+
17+
end #
18+
19+
delimiter ;
20+
21+
delimiter #
22+
create procedure insert_rows(in allrows int, in insertcount int, in value longblob)
23+
begin
24+
25+
set @dummyIpsum = '\'dummy_ipsum\'';
26+
set @fieldText = value;
27+
set @vmax = allrows;
28+
set @vmaxx = allrows;
29+
set @vmaxoneinsert = insertcount;
30+
set @counter = 1;
31+
set @lastinsertcounter = 1;
32+
set @lastinsert = 0;
33+
set @fullloop = 0;
34+
set @fullloopcounter = 0;
35+
36+
while @vmaxx <= @vmaxoneinsert do
37+
set @vmaxoneinsert = @vmaxx;
38+
set @fullloop = @fullloop + 1;
39+
set @vmaxx = @vmaxx + 1;
40+
end while;
41+
commit;
42+
43+
while @vmax > @vmaxoneinsert do
44+
set @fullloop = @fullloop + 1;
45+
set @vmax = @vmax - @vmaxoneinsert;
46+
set @lastinsert = @vmax;
47+
end while;
48+
commit;
49+
50+
set @insertTable = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
51+
while @counter < @vmaxoneinsert do
52+
set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (');
53+
set @counter = @counter + 1;
54+
end while;
55+
commit;
56+
set @insertTable = concat(@insertTable, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);');
57+
58+
while @vmax < 1 do
59+
set @fullloop = 0;
60+
set @vmax = 1;
61+
end while;
62+
commit;
63+
64+
while @fullloopcounter < @fullloop do
65+
PREPARE runinsert from @insertTable;
66+
EXECUTE runinsert;
67+
DEALLOCATE PREPARE runinsert;
68+
set @fullloopcounter = @fullloopcounter + 1;
69+
end while;
70+
commit;
71+
72+
set @insertTableLasted = concat('insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (');
73+
while @lastinsertcounter < @lastinsert do
74+
set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP), (');
75+
set @lastinsertcounter = @lastinsertcounter + 1;
76+
end while;
77+
commit;
78+
set @insertTableLasted = concat(@insertTableLasted, @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @dummyIpsum, ', ', @fieldText, ', CURRENT_TIMESTAMP);');
79+
80+
while @lastinsert > 0 do
81+
PREPARE runinsert from @insertTableLasted;
82+
EXECUTE runinsert;
83+
DEALLOCATE PREPARE runinsert;
84+
set @lastinsert = 0;
85+
end while;
86+
commit;
87+
88+
end #
89+
90+
delimiter ;
91+
92+
delimiter #
93+
create procedure table_create()
94+
begin
95+
96+
create table test
97+
(
98+
id int unsigned not null auto_increment primary key,
99+
varchar1 varchar(255),
100+
varchar2 varchar(255),
101+
varchar3 varchar(255),
102+
varchar4 varchar(255),
103+
varchar5 varchar(255),
104+
longblobfield longblob,
105+
timestampfield timestamp
106+
)
107+
engine=innodb;
108+
109+
set @extraSmallText = '\'test weight 50b - some text, some text, some text\'';
110+
set @smallText = CONCAT('\'test weight 500b - ', REPEAT('some text, some text, ', 20), '\'');
111+
set @regularText = CONCAT('\'test weight 10kb - ', REPEAT('some text, some text, ', 590), '\'');
112+
set @largeText = CONCAT('\'test weight 100kb - ', REPEAT('some text, some text, ', 4450), '\'');
113+
114+
-- TODO: change the following @allrows to control the number of records with different sizes
115+
-- number of 50B records
116+
call insert_rows(0, 5000000, @extraSmallText);
117+
-- number of 500B records
118+
call insert_rows(0, 50000, @smallText);
119+
-- number of 10KB records
120+
call insert_rows(0, 5000, @regularText);
121+
-- number of 100KB records
122+
call insert_rows(0, 50, @largeText);
123+
end #
124+
125+
delimiter ;
126+
127+
call table_create();
128+
drop procedure if exists table_create;
129+
drop procedure if exists insert_rows;
130+
131+
-- TODO: change the value to control the number of tables
132+
call table_copy(1);
133+
drop procedure if exists table_copy;
134+
ALTER TABLE test RENAME test_0;

‎airbyte-integrations/connectors/source-postgres/1-create-copy-tables-procedure.sql

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
create or replace procedure copy_table(tablecount int)
2+
language plpgsql
3+
as $$
4+
declare v_max_table int; v_counter_table int; v_tnamee VARCHAR(255);
5+
begin
6+
v_max_table := tablecount;
7+
v_counter_table := 1;
8+
while v_counter_table < v_max_table loop
9+
EXECUTE format('create table test_%s as (select * from test t)', v_counter_table);
10+
v_counter_table := v_counter_table + 1;
11+
end loop;
12+
commit;
13+
end;$$
14+

‎airbyte-integrations/connectors/source-postgres/2-create-insert-rows-to-table-procedure.sql

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
create or replace procedure insert_rows(allrows int, insertcount int, value text)
2+
language plpgsql
3+
as $$
4+
declare dummyIpsum varchar(255); fieldText text; vmax int; vmaxx int; vmaxoneinsert int; counter int;
5+
declare lastinsertcounter int; lastinsert int; fullloop int; fullloopcounter int; insertTable text; insertTableLasted text;
6+
7+
begin
8+
fieldText := value;
9+
dummyIpsum = '''dummy_ipsum''';
10+
vmax = allrows;
11+
vmaxx = allrows;
12+
vmaxoneinsert = insertcount;
13+
counter = 1;
14+
lastinsertcounter = 1;
15+
lastinsert = 0;
16+
fullloop = 0;
17+
fullloopcounter = 0;
18+
19+
while vmaxx <= vmaxoneinsert loop
20+
vmaxoneinsert := vmaxx;
21+
fullloop := fullloop + 1;
22+
vmaxx := vmaxx + 1;
23+
end loop;
24+
commit;
25+
26+
while vmax > vmaxoneinsert loop
27+
fullloop := fullloop + 1;
28+
vmax := vmax - vmaxoneinsert;
29+
lastinsert := vmax;
30+
end loop;
31+
commit;
32+
33+
insertTable := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (';
34+
while counter < vmaxoneinsert loop
35+
insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), (');
36+
counter := counter + 1;
37+
end loop;
38+
commit;
39+
insertTable := concat(insertTable, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);');
40+
41+
while vmax < 1 loop
42+
fullloop := 0;
43+
vmax := 1;
44+
end loop;
45+
commit;
46+
47+
while fullloopcounter < fullloop loop
48+
EXECUTE insertTable;
49+
fullloopcounter := fullloopcounter + 1;
50+
end loop;
51+
commit;
52+
53+
insertTableLasted := 'insert into test (varchar1, varchar2, varchar3, varchar4, varchar5, longblobfield, timestampfield) values (';
54+
while lastinsertcounter < lastinsert loop
55+
insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP), (');
56+
lastinsertcounter := lastinsertcounter + 1;
57+
end loop;
58+
commit;
59+
insertTableLasted := concat(insertTableLasted, dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', dummyIpsum, ', ', fieldText, ', CURRENT_TIMESTAMP);');
60+
61+
while lastinsert > 0 loop
62+
EXECUTE insertTableLasted;
63+
lastinsert := 0;
64+
end loop;
65+
commit;
66+
end;$$
67+
68+

0 commit comments

Comments
 (0)