From f1ec1a362256429160a320cae6598475df988c7a Mon Sep 17 00:00:00 2001 From: jiangjiang <819421878@qq.com> Date: Wed, 14 Jul 2021 11:40:40 +0800 Subject: [PATCH 01/14] Update sql_mode to more compatible * update load.go/syncer.go with modifying sql_mode, * add sql_mode Integration Test --- loader/loader.go | 34 +++++++++++++++- syncer/syncer.go | 34 +++++++++++++++- tests/sql_mode/conf/diff_config.toml | 56 +++++++++++++++++++++++++++ tests/sql_mode/conf/dm-master.toml | 4 ++ tests/sql_mode/conf/dm-task.yaml | 46 ++++++++++++++++++++++ tests/sql_mode/conf/dm-worker1.toml | 2 + tests/sql_mode/conf/dm-worker2.toml | 2 + tests/sql_mode/conf/source1.yaml | 11 ++++++ tests/sql_mode/conf/source2.yaml | 11 ++++++ tests/sql_mode/data/db1.increment.sql | 32 +++++++++++++++ tests/sql_mode/data/db1.prepare.sql | 11 ++++++ tests/sql_mode/data/db2.increment.sql | 19 +++++++++ tests/sql_mode/data/db2.prepare.sql | 10 +++++ tests/sql_mode/run.sh | 50 ++++++++++++++++++++++++ 14 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 tests/sql_mode/conf/diff_config.toml create mode 100644 tests/sql_mode/conf/dm-master.toml create mode 100644 tests/sql_mode/conf/dm-task.yaml create mode 100644 tests/sql_mode/conf/dm-worker1.toml create mode 100644 tests/sql_mode/conf/dm-worker2.toml create mode 100644 tests/sql_mode/conf/source1.yaml create mode 100644 tests/sql_mode/conf/source2.yaml create mode 100644 tests/sql_mode/data/db1.increment.sql create mode 100644 tests/sql_mode/data/db1.prepare.sql create mode 100644 tests/sql_mode/data/db2.increment.sql create mode 100644 tests/sql_mode/data/db2.prepare.sql create mode 100644 tests/sql_mode/run.sh diff --git a/loader/loader.go b/loader/loader.go index d9c835ed63..cb6e79be00 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -529,7 +529,39 @@ func (l *Loader) Init(ctx context.Context) (err error) { } } if !hasSQLMode { - lcfg.To.Session["sql_mode"] = l.cfg.LoaderConfig.SQLMode + sqlModes := strings.Split(l.cfg.LoaderConfig.SQLMode, ",") + needDisable := []string{ + "NO_ZERO_IN_DATE", + "NO_ZERO_DATE", + "ERROR_FOR_DIVISION_BY_ZERO", + "NO_AUTO_CREATE_USER", + "STRICT_TRANS_TABLES", + "STRICT_ALL_TABLES", + } + for _, disable := range needDisable { + for i, sqlMode := range sqlModes { + if disable == sqlMode { + sqlModes = append(sqlModes[:i], sqlModes[i+1:]...) + break + } + } + } + needEnable := []string{ + "IGNORE_SPACE", + "NO_AUTO_VALUE_ON_ZERO", + "ALLOW_INVALID_DATES", + } + waitEnable := sqlModes + for _, enable := range needEnable { + for _, sqlMode := range sqlModes { + if enable != sqlMode { + waitEnable = append(waitEnable, enable) + break + } + } + } + lcfg.To.Session["sql_mode"] = strings.Join(waitEnable, ",") + l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) } l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize) diff --git a/syncer/syncer.go b/syncer/syncer.go index af3b74c725..f4f760c6b9 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2742,8 +2742,38 @@ func (s *Syncer) createDBs(ctx context.Context) error { if err2 != nil { s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) } else { - s.cfg.To.Session["sql_mode"] = sqlMode - } + sqlModes := strings.Split(sqlMode, ",") + needDisable := []string{ + "NO_ZERO_IN_DATE", + "NO_ZERO_DATE", + "ERROR_FOR_DIVISION_BY_ZERO", + "NO_AUTO_CREATE_USER", + "STRICT_TRANS_TABLES", + "STRICT_ALL_TABLES", + } + for _, disable := range needDisable { + for i, sqlMode := range sqlModes { + if disable == sqlMode { + sqlModes = append(sqlModes[:i], sqlModes[i+1:]...) + break + } + } + } + needEnable := []string{ + "IGNORE_SPACE", + "NO_AUTO_VALUE_ON_ZERO", + "ALLOW_INVALID_DATES", + } + waitEnable := sqlModes + for _, enable := range needEnable { + for _, sqlMode := range sqlModes { + if enable != sqlMode { + waitEnable = append(waitEnable, enable) + break + } + } + } + s.cfg.To.Session["sql_mode"] = strings.Join(waitEnable, ",") } dbCfg = s.cfg.To diff --git a/tests/sql_mode/conf/diff_config.toml b/tests/sql_mode/conf/diff_config.toml new file mode 100644 index 0000000000..29ffdd5b0d --- /dev/null +++ b/tests/sql_mode/conf/diff_config.toml @@ -0,0 +1,56 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 20 + +check-thread-count = 4 + +sample-percent = 100 + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "sql_mode" +tables = ["~.*"] + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "123456" +instance-id = "source-1" + +[[table-config]] +schema = "sql_mode" +table = "t_1" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "sql_mode" +table = "t_1" + +[[source-db]] +host = "127.0.0.1" +port = 3307 +user = "root" +password = "123456" +instance-id = "source-2" + +[[table-config]] +schema = "sql_mode" +table = "t_2" + +[[table-config.source-tables]] +instance-id = "source-2" +schema = "sql_mode" +table = "t_2" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "test" +password = "123456" diff --git a/tests/sql_mode/conf/dm-master.toml b/tests/sql_mode/conf/dm-master.toml new file mode 100644 index 0000000000..7cecf59ad8 --- /dev/null +++ b/tests/sql_mode/conf/dm-master.toml @@ -0,0 +1,4 @@ +# Master Configuration. +master-addr = ":8261" +advertise-addr = "127.0.0.1:8261" +auto-compaction-retention = "3s" diff --git a/tests/sql_mode/conf/dm-task.yaml b/tests/sql_mode/conf/dm-task.yaml new file mode 100644 index 0000000000..dd1acd6924 --- /dev/null +++ b/tests/sql_mode/conf/dm-task.yaml @@ -0,0 +1,46 @@ +--- +name: sql_mode +task-mode: all +is-sharding: false +meta-schema: "dm_meta" +enable-heartbeat: false + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + password: "" + +mysql-instances: + - source-id: "mysql-replica-01" + block-allow-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + + - source-id: "mysql-replica-02" + block-allow-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + +block-allow-list: + instance: + do-dbs: ["sql_mode"] + +mydumpers: + global: + threads: 4 + chunk-filesize: 0 + skip-tz-utc: true + extra-args: "--statement-size=4000" + +loaders: + global: + pool-size: 16 + dir: "./dumped_data" + +syncers: + global: + worker-count: 16 + batch: 100 diff --git a/tests/sql_mode/conf/dm-worker1.toml b/tests/sql_mode/conf/dm-worker1.toml new file mode 100644 index 0000000000..7a72ea72bf --- /dev/null +++ b/tests/sql_mode/conf/dm-worker1.toml @@ -0,0 +1,2 @@ +name = "worker1" +join = "127.0.0.1:8261" diff --git a/tests/sql_mode/conf/dm-worker2.toml b/tests/sql_mode/conf/dm-worker2.toml new file mode 100644 index 0000000000..010e21c73e --- /dev/null +++ b/tests/sql_mode/conf/dm-worker2.toml @@ -0,0 +1,2 @@ +name = "worker2" +join = "127.0.0.1:8261" diff --git a/tests/sql_mode/conf/source1.yaml b/tests/sql_mode/conf/source1.yaml new file mode 100644 index 0000000000..7d67feb8cf --- /dev/null +++ b/tests/sql_mode/conf/source1.yaml @@ -0,0 +1,11 @@ +source-id: mysql-replica-01 +flavor: '' +enable-gtid: false +enable-relay: true +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3306 diff --git a/tests/sql_mode/conf/source2.yaml b/tests/sql_mode/conf/source2.yaml new file mode 100644 index 0000000000..3df6bdf955 --- /dev/null +++ b/tests/sql_mode/conf/source2.yaml @@ -0,0 +1,11 @@ +source-id: mysql-replica-02 +flavor: '' +enable-gtid: false +enable-relay: true +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3307 diff --git a/tests/sql_mode/data/db1.increment.sql b/tests/sql_mode/data/db1.increment.sql new file mode 100644 index 0000000000..0e99bfb77d --- /dev/null +++ b/tests/sql_mode/data/db1.increment.sql @@ -0,0 +1,32 @@ +set @@session.SQL_MODE=''; +use sql_mode; + +-- test sql_mode PIPES_AS_CONCAT +insert into t_1(num) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_1(name) values("a"); + +-- test sql_mode IGNORE_SPACE +create table count (id int not null, primary key(id)); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +insert into t_1(id, name) values (10, 'a'); +insert into t_1(id, name) values (0, 'b'); +insert into t_1(id, name) values (0, 'c'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +insert into t_1(name) values ('\\a'); + +-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES +insert into t_1(dt) values('0000-06-00'); +insert into t_1(dt) values('0000-00-01'); +insert into t_1(dt) values('0000-06-01'); +insert into t_1(dt) values('0000-00-00'); + +-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO +insert into t_1(num) values(4/0); + +-- test sql_mode NO_AUTO_CREATE_USER +drop user if exists 'no_auto_create_user'; +grant select on *.* to 'no_auto_create_user'; \ No newline at end of file diff --git a/tests/sql_mode/data/db1.prepare.sql b/tests/sql_mode/data/db1.prepare.sql new file mode 100644 index 0000000000..cf4843709a --- /dev/null +++ b/tests/sql_mode/data/db1.prepare.sql @@ -0,0 +1,11 @@ +drop database if exists `sql_mode`; +create database `sql_mode`; +use `sql_mode`; +CREATE TABLE `t_1` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(60), + `num` int, + `dt` datetime, + -- `ts` timestamp, + PRIMARY KEY (id) +); \ No newline at end of file diff --git a/tests/sql_mode/data/db2.increment.sql b/tests/sql_mode/data/db2.increment.sql new file mode 100644 index 0000000000..2d9e84d031 --- /dev/null +++ b/tests/sql_mode/data/db2.increment.sql @@ -0,0 +1,19 @@ +set @@session.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; +-- NO_AUTO_CREATE_USER set failed in mysql8.0 +use sql_mode; + +-- test sql_mode PIPES_AS_CONCAT +insert into t_2(name) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_2(name) values("a"); + +-- test sql_mode IGNORE_SPACE +select count (*) as c from t_2; + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +insert into t_2(id, name) values (10, 'a'); +insert into t_2(id, name) values (0, 'b'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +insert into t_2(name) values ('\\a'); \ No newline at end of file diff --git a/tests/sql_mode/data/db2.prepare.sql b/tests/sql_mode/data/db2.prepare.sql new file mode 100644 index 0000000000..05438fea86 --- /dev/null +++ b/tests/sql_mode/data/db2.prepare.sql @@ -0,0 +1,10 @@ +drop database if exists `sql_mode`; +create database `sql_mode`; +use `sql_mode`; +CREATE TABLE `t_2` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(60), + `num` int, + `dt` datetime, + PRIMARY KEY (id) +); \ No newline at end of file diff --git a/tests/sql_mode/run.sh b/tests/sql_mode/run.sh new file mode 100644 index 0000000000..8194bc63c5 --- /dev/null +++ b/tests/sql_mode/run.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -eu + +cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $cur/../_utils/test_prepare +WORK_DIR=$TEST_DIR/$TEST_NAME + +function run() { + run_sql_source1 "SET @@GLOBAL.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'" + # run_sql_source1 "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES'" + run_sql_source2 "SET @@GLOBAL.SQL_MODE=''" + + run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + # operate mysql config to worker + cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml + dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 + cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml + dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 + + # start DM task only + dmctl_start_task $cur/conf/dm-task.yaml + + # use sync_diff_inspector to check full dump loader + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml +} + +cleanup_data $TEST_NAME +# also cleanup dm processes in case of last run failed +cleanup_process $* +run $* +cleanup_process $* + +echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>" From f115e8beae204779c3ff2f25adce57aff1a4474b Mon Sep 17 00:00:00 2001 From: jiangjiang <819421878@qq.com> Date: Wed, 14 Jul 2021 11:54:13 +0800 Subject: [PATCH 02/14] Add some notation --- loader/loader.go | 5 +++++ syncer/syncer.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/loader/loader.go b/loader/loader.go index cb6e79be00..9ba2fcfded 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -529,6 +529,11 @@ func (l *Loader) Init(ctx context.Context) (err error) { } } if !hasSQLMode { + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 sqlModes := strings.Split(l.cfg.LoaderConfig.SQLMode, ",") needDisable := []string{ "NO_ZERO_IN_DATE", diff --git a/syncer/syncer.go b/syncer/syncer.go index f4f760c6b9..0c7ba93b96 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2742,6 +2742,11 @@ func (s *Syncer) createDBs(ctx context.Context) error { if err2 != nil { s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) } else { + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 sqlModes := strings.Split(sqlMode, ",") needDisable := []string{ "NO_ZERO_IN_DATE", From 665a45094576554f267330f43825609840c6a0da Mon Sep 17 00:00:00 2001 From: jiangjiang <819421878@qq.com> Date: Wed, 14 Jul 2021 13:05:54 +0800 Subject: [PATCH 03/14] add sql_mode in other_intergration.txt --- tests/others_integration.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/others_integration.txt b/tests/others_integration.txt index bf043c136d..d111cf7fbf 100644 --- a/tests/others_integration.txt +++ b/tests/others_integration.txt @@ -8,4 +8,5 @@ only_dml adjust_gtid load_task downstream_more_column -expression_filter \ No newline at end of file +expression_filter +sql_mode \ No newline at end of file From b5044325512c3d4b8879b91b5010220c3273c569 Mon Sep 17 00:00:00 2001 From: okJiang Date: Wed, 14 Jul 2021 14:25:13 +0800 Subject: [PATCH 04/14] add omission --- syncer/syncer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/syncer/syncer.go b/syncer/syncer.go index 0c7ba93b96..4dc8e7bbd4 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2779,6 +2779,7 @@ func (s *Syncer) createDBs(ctx context.Context) error { } } s.cfg.To.Session["sql_mode"] = strings.Join(waitEnable, ",") + } } dbCfg = s.cfg.To From aca089c19965900ca9a93c93760ae1b3a8840e21 Mon Sep 17 00:00:00 2001 From: okJiang Date: Wed, 14 Jul 2021 18:09:57 +0800 Subject: [PATCH 05/14] Add AdjustSQLModeCompatible in pkg/utils/db.go --- loader/loader.go | 54 +++++++------------------------ pkg/utils/db.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ syncer/syncer.go | 61 +++++++++-------------------------- 3 files changed, 109 insertions(+), 88 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 9ba2fcfded..5bbfecbaad 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -521,53 +521,23 @@ func (l *Loader) Init(ctx context.Context) (err error) { } lcfg.To.Session["time_zone"] = "+00:00" - hasSQLMode := false for k := range l.cfg.To.Session { if strings.ToLower(k) == "sql_mode" { - hasSQLMode = true + l.logger.Info("your confiured sql_mode will be adjusted to compatible") break } } - if !hasSQLMode { - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 - sqlModes := strings.Split(l.cfg.LoaderConfig.SQLMode, ",") - needDisable := []string{ - "NO_ZERO_IN_DATE", - "NO_ZERO_DATE", - "ERROR_FOR_DIVISION_BY_ZERO", - "NO_AUTO_CREATE_USER", - "STRICT_TRANS_TABLES", - "STRICT_ALL_TABLES", - } - for _, disable := range needDisable { - for i, sqlMode := range sqlModes { - if disable == sqlMode { - sqlModes = append(sqlModes[:i], sqlModes[i+1:]...) - break - } - } - } - needEnable := []string{ - "IGNORE_SPACE", - "NO_AUTO_VALUE_ON_ZERO", - "ALLOW_INVALID_DATES", - } - waitEnable := sqlModes - for _, enable := range needEnable { - for _, sqlMode := range sqlModes { - if enable != sqlMode { - waitEnable = append(waitEnable, enable) - break - } - } - } - lcfg.To.Session["sql_mode"] = strings.Join(waitEnable, ",") - l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) - } + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 + sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) + if err3 != nil { + l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + } + lcfg.To.Session["sql_mode"] = sqlModes + l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize) if err != nil { diff --git a/pkg/utils/db.go b/pkg/utils/db.go index f935c43bc5..c71c8d93b5 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -524,3 +524,85 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid _ = ret.Set(newGset) return ret, nil } + +// AdjustSQLModeCompatible adjust downstream sql mode to compatible +func AdjustSQLModeCompatible(sqlModes string) (string, error) { + needDisable := []string{ + "NO_ZERO_IN_DATE", + "NO_ZERO_DATE", + "ERROR_FOR_DIVISION_BY_ZERO", + "NO_AUTO_CREATE_USER", + "STRICT_TRANS_TABLES", + "STRICT_ALL_TABLES", + } + needEnable := []string{ + "IGNORE_SPACE", + "NO_AUTO_VALUE_ON_ZERO", + "ALLOW_INVALID_DATES", + } + disable := strings.Join(needDisable, ",") + enable := strings.Join(needEnable, ",") + + mode, err := tmysql.GetSQLMode(sqlModes) + if err != nil { + return "", err + } + disableMode, err2 := tmysql.GetSQLMode(disable) + if err2 != nil { + return "", err2 + } + enableMode, err3 := tmysql.GetSQLMode(enable) + if err3 != nil { + return "", err3 + } + mode = (mode &^ disableMode) | enableMode + + return GetSQLModeByStr(mode), nil +} + +// SQLMode2Str is the sql_mode to string represent of sql_mode map. +var SQLMode2Str = map[tmysql.SQLMode]string{ + tmysql.ModeRealAsFloat: "REAL_AS_FLOAT", + tmysql.ModePipesAsConcat: "PIPES_AS_CONCAT", + tmysql.ModeANSIQuotes: "ANSI_QUOTES", + tmysql.ModeIgnoreSpace: "IGNORE_SPACE", + tmysql.ModeNotUsed: "NOT_USED", + tmysql.ModeOnlyFullGroupBy: "ONLY_FULL_GROUP_BY", + tmysql.ModeNoUnsignedSubtraction: "NO_UNSIGNED_SUBTRACTION", + tmysql.ModeNoDirInCreate: "NO_DIR_IN_CREATE", + tmysql.ModePostgreSQL: "POSTGRESQL", + tmysql.ModeOracle: "ORACLE", + tmysql.ModeMsSQL: "MSSQL", + tmysql.ModeDb2: "DB2", + tmysql.ModeMaxdb: "MAXDB", + tmysql.ModeNoKeyOptions: "NO_KEY_OPTIONS", + tmysql.ModeNoTableOptions: "NO_TABLE_OPTIONS", + tmysql.ModeNoFieldOptions: "NO_FIELD_OPTIONS", + tmysql.ModeMySQL323: "MYSQL323", + tmysql.ModeMySQL40: "MYSQL40", + tmysql.ModeANSI: "ANSI", + tmysql.ModeNoAutoValueOnZero: "NO_AUTO_VALUE_ON_ZERO", + tmysql.ModeNoBackslashEscapes: "NO_BACKSLASH_ESCAPES", + tmysql.ModeStrictTransTables: "STRICT_TRANS_TABLES", + tmysql.ModeStrictAllTables: "STRICT_ALL_TABLES", + tmysql.ModeNoZeroInDate: "NO_ZERO_IN_DATE", + tmysql.ModeNoZeroDate: "NO_ZERO_DATE", + tmysql.ModeInvalidDates: "INVALID_DATES", + tmysql.ModeErrorForDivisionByZero: "ERROR_FOR_DIVISION_BY_ZERO", + tmysql.ModeTraditional: "TRADITIONAL", + tmysql.ModeNoAutoCreateUser: "NO_AUTO_CREATE_USER", + tmysql.ModeHighNotPrecedence: "HIGH_NOT_PRECEDENCE", + tmysql.ModeNoEngineSubstitution: "NO_ENGINE_SUBSTITUTION", + tmysql.ModePadCharToFullLength: "PAD_CHAR_TO_FULL_LENGTH", + tmysql.ModeAllowInvalidDates: "ALLOW_INVALID_DATES", +} + +func GetSQLModeByStr(sqlMode tmysql.SQLMode) string { + var sqlModeStr []string + for SQLMode, str := range SQLMode2Str { + if sqlMode&SQLMode != 0 { + sqlModeStr = append(sqlModeStr, str) + } + } + return strings.Join(sqlModeStr, ",") +} diff --git a/syncer/syncer.go b/syncer/syncer.go index 4dc8e7bbd4..e86be88488 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2725,62 +2725,31 @@ func (s *Syncer) createDBs(ctx context.Context) error { return err } - hasSQLMode := false // get sql_mode from upstream db if s.cfg.To.Session == nil { s.cfg.To.Session = make(map[string]string) } else { for k := range s.cfg.To.Session { if strings.ToLower(k) == "sql_mode" { - hasSQLMode = true + s.tctx.L().Info("your confiured sql_mode will be adjusted to compatible") break } } } - if !hasSQLMode { - sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") - if err2 != nil { - s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) - } else { - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 - sqlModes := strings.Split(sqlMode, ",") - needDisable := []string{ - "NO_ZERO_IN_DATE", - "NO_ZERO_DATE", - "ERROR_FOR_DIVISION_BY_ZERO", - "NO_AUTO_CREATE_USER", - "STRICT_TRANS_TABLES", - "STRICT_ALL_TABLES", - } - for _, disable := range needDisable { - for i, sqlMode := range sqlModes { - if disable == sqlMode { - sqlModes = append(sqlModes[:i], sqlModes[i+1:]...) - break - } - } - } - needEnable := []string{ - "IGNORE_SPACE", - "NO_AUTO_VALUE_ON_ZERO", - "ALLOW_INVALID_DATES", - } - waitEnable := sqlModes - for _, enable := range needEnable { - for _, sqlMode := range sqlModes { - if enable != sqlMode { - waitEnable = append(waitEnable, enable) - break - } - } - } - s.cfg.To.Session["sql_mode"] = strings.Join(waitEnable, ",") - } - } + sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") + if err2 != nil { + s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) + } + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 + sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) + if err3 != nil { + s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + } + s.cfg.To.Session["sql_mode"] = sqlModes dbCfg = s.cfg.To dbCfg.RawDBCfg = config.DefaultRawDBConfig(). From 4505b6c069bd5c3f2d5d53a737fc376f0dc86460 Mon Sep 17 00:00:00 2001 From: okJiang Date: Wed, 14 Jul 2021 18:14:32 +0800 Subject: [PATCH 06/14] Modify func name and add comment --- pkg/utils/db.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/utils/db.go b/pkg/utils/db.go index c71c8d93b5..fc23e0563a 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -557,7 +557,7 @@ func AdjustSQLModeCompatible(sqlModes string) (string, error) { } mode = (mode &^ disableMode) | enableMode - return GetSQLModeByStr(mode), nil + return GetSQLModeStrBySQLMode(mode), nil } // SQLMode2Str is the sql_mode to string represent of sql_mode map. @@ -597,7 +597,8 @@ var SQLMode2Str = map[tmysql.SQLMode]string{ tmysql.ModeAllowInvalidDates: "ALLOW_INVALID_DATES", } -func GetSQLModeByStr(sqlMode tmysql.SQLMode) string { +// GetSQLModeByStr get string represent of sql_mode by sql_mode +func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string { var sqlModeStr []string for SQLMode, str := range SQLMode2Str { if sqlMode&SQLMode != 0 { From dc746025a530eed1ec8bcb0d6b567fc89ac482b8 Mon Sep 17 00:00:00 2001 From: okJiang Date: Thu, 15 Jul 2021 10:31:16 +0800 Subject: [PATCH 07/14] Replace SQLMode2Str with tmysql.Str2SQLMode --- pkg/utils/db.go | 43 ++++--------------------------------------- tests/sql_mode/run.sh | 1 - 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/pkg/utils/db.go b/pkg/utils/db.go index fc23e0563a..4224c1f23d 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -555,52 +555,17 @@ func AdjustSQLModeCompatible(sqlModes string) (string, error) { if err3 != nil { return "", err3 } + // About this bit manipulation, details can be seen + // https://github.com/pingcap/dm/pull/1869#discussion_r669771966 mode = (mode &^ disableMode) | enableMode return GetSQLModeStrBySQLMode(mode), nil } -// SQLMode2Str is the sql_mode to string represent of sql_mode map. -var SQLMode2Str = map[tmysql.SQLMode]string{ - tmysql.ModeRealAsFloat: "REAL_AS_FLOAT", - tmysql.ModePipesAsConcat: "PIPES_AS_CONCAT", - tmysql.ModeANSIQuotes: "ANSI_QUOTES", - tmysql.ModeIgnoreSpace: "IGNORE_SPACE", - tmysql.ModeNotUsed: "NOT_USED", - tmysql.ModeOnlyFullGroupBy: "ONLY_FULL_GROUP_BY", - tmysql.ModeNoUnsignedSubtraction: "NO_UNSIGNED_SUBTRACTION", - tmysql.ModeNoDirInCreate: "NO_DIR_IN_CREATE", - tmysql.ModePostgreSQL: "POSTGRESQL", - tmysql.ModeOracle: "ORACLE", - tmysql.ModeMsSQL: "MSSQL", - tmysql.ModeDb2: "DB2", - tmysql.ModeMaxdb: "MAXDB", - tmysql.ModeNoKeyOptions: "NO_KEY_OPTIONS", - tmysql.ModeNoTableOptions: "NO_TABLE_OPTIONS", - tmysql.ModeNoFieldOptions: "NO_FIELD_OPTIONS", - tmysql.ModeMySQL323: "MYSQL323", - tmysql.ModeMySQL40: "MYSQL40", - tmysql.ModeANSI: "ANSI", - tmysql.ModeNoAutoValueOnZero: "NO_AUTO_VALUE_ON_ZERO", - tmysql.ModeNoBackslashEscapes: "NO_BACKSLASH_ESCAPES", - tmysql.ModeStrictTransTables: "STRICT_TRANS_TABLES", - tmysql.ModeStrictAllTables: "STRICT_ALL_TABLES", - tmysql.ModeNoZeroInDate: "NO_ZERO_IN_DATE", - tmysql.ModeNoZeroDate: "NO_ZERO_DATE", - tmysql.ModeInvalidDates: "INVALID_DATES", - tmysql.ModeErrorForDivisionByZero: "ERROR_FOR_DIVISION_BY_ZERO", - tmysql.ModeTraditional: "TRADITIONAL", - tmysql.ModeNoAutoCreateUser: "NO_AUTO_CREATE_USER", - tmysql.ModeHighNotPrecedence: "HIGH_NOT_PRECEDENCE", - tmysql.ModeNoEngineSubstitution: "NO_ENGINE_SUBSTITUTION", - tmysql.ModePadCharToFullLength: "PAD_CHAR_TO_FULL_LENGTH", - tmysql.ModeAllowInvalidDates: "ALLOW_INVALID_DATES", -} - -// GetSQLModeByStr get string represent of sql_mode by sql_mode +// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string { var sqlModeStr []string - for SQLMode, str := range SQLMode2Str { + for str, SQLMode := range tmysql.Str2SQLMode { if sqlMode&SQLMode != 0 { sqlModeStr = append(sqlModeStr, str) } diff --git a/tests/sql_mode/run.sh b/tests/sql_mode/run.sh index 8194bc63c5..92022b1cac 100644 --- a/tests/sql_mode/run.sh +++ b/tests/sql_mode/run.sh @@ -8,7 +8,6 @@ WORK_DIR=$TEST_DIR/$TEST_NAME function run() { run_sql_source1 "SET @@GLOBAL.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'" - # run_sql_source1 "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES'" run_sql_source2 "SET @@GLOBAL.SQL_MODE=''" run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 From 2452d7e44e7ee60ccfbdc33e447f3b83e8545b31 Mon Sep 17 00:00:00 2001 From: okJiang Date: Thu, 15 Jul 2021 14:08:40 +0800 Subject: [PATCH 08/14] revert user configue --- loader/loader.go | 25 ++++++++------ syncer/syncer.go | 33 ++++++++++--------- .../conf/double-source-no-sharding.yaml | 3 ++ 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 5bbfecbaad..40d8b284ec 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -521,22 +521,27 @@ func (l *Loader) Init(ctx context.Context) (err error) { } lcfg.To.Session["time_zone"] = "+00:00" + hasSQLMode := false for k := range l.cfg.To.Session { if strings.ToLower(k) == "sql_mode" { - l.logger.Info("your confiured sql_mode will be adjusted to compatible") + hasSQLMode = true break } } - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 - sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) - if err3 != nil { - l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + + if !hasSQLMode { + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 + sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) + if err3 != nil { + l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + } + lcfg.To.Session["sql_mode"] = sqlModes } - lcfg.To.Session["sql_mode"] = sqlModes + l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize) diff --git a/syncer/syncer.go b/syncer/syncer.go index e86be88488..9f74cf76ec 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2725,31 +2725,34 @@ func (s *Syncer) createDBs(ctx context.Context) error { return err } + hasSQLMode := false // get sql_mode from upstream db if s.cfg.To.Session == nil { s.cfg.To.Session = make(map[string]string) } else { for k := range s.cfg.To.Session { if strings.ToLower(k) == "sql_mode" { - s.tctx.L().Info("your confiured sql_mode will be adjusted to compatible") + hasSQLMode = true break } } } - sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") - if err2 != nil { - s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) - } - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 - sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) - if err3 != nil { - s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) - } - s.cfg.To.Session["sql_mode"] = sqlModes + if !hasSQLMode { + sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") + if err2 != nil { + s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) + } + // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 + // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, + // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. + // This is because the implementation of go-mysql, that you can see + // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 + sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) + if err3 != nil { + s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + } + s.cfg.To.Session["sql_mode"] = sqlModes + } dbCfg = s.cfg.To dbCfg.RawDBCfg = config.DefaultRawDBConfig(). diff --git a/tests/handle_error/conf/double-source-no-sharding.yaml b/tests/handle_error/conf/double-source-no-sharding.yaml index c483994ec3..2da17837c7 100644 --- a/tests/handle_error/conf/double-source-no-sharding.yaml +++ b/tests/handle_error/conf/double-source-no-sharding.yaml @@ -10,6 +10,9 @@ target-database: user: "test" password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" + session: + sql_mode: "STRICT_TRANS_TABLES" + mysql-instances: - source-id: "mysql-replica-01" block-allow-list: "instance" From 7250ee07dd871ce8200acafb152689beeae7e3ec Mon Sep 17 00:00:00 2001 From: okJiang Date: Thu, 15 Jul 2021 15:19:46 +0800 Subject: [PATCH 09/14] Adjust sql order --- pkg/utils/db.go | 4 ++-- tests/sql_mode/data/db2.increment.sql | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/utils/db.go b/pkg/utils/db.go index 4224c1f23d..e11ad99f57 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -525,7 +525,7 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid return ret, nil } -// AdjustSQLModeCompatible adjust downstream sql mode to compatible +// AdjustSQLModeCompatible adjust downstream sql mode to compatible. func AdjustSQLModeCompatible(sqlModes string) (string, error) { needDisable := []string{ "NO_ZERO_IN_DATE", @@ -562,7 +562,7 @@ func AdjustSQLModeCompatible(sqlModes string) (string, error) { return GetSQLModeStrBySQLMode(mode), nil } -// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode +// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode. func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string { var sqlModeStr []string for str, SQLMode := range tmysql.Str2SQLMode { diff --git a/tests/sql_mode/data/db2.increment.sql b/tests/sql_mode/data/db2.increment.sql index 2d9e84d031..d3502b1ef4 100644 --- a/tests/sql_mode/data/db2.increment.sql +++ b/tests/sql_mode/data/db2.increment.sql @@ -1,19 +1,23 @@ -set @@session.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; +set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; -- NO_AUTO_CREATE_USER set failed in mysql8.0 use sql_mode; -- test sql_mode PIPES_AS_CONCAT +set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT'); insert into t_2(name) values('pipes'||'as'||'concat'); -- test sql_mode ANSI_QUOTES insert into t_2(name) values("a"); -- test sql_mode IGNORE_SPACE +set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE'); select count (*) as c from t_2; -- test sql_mode NO_AUTO_VALUE_ON_ZERO +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); insert into t_2(id, name) values (10, 'a'); insert into t_2(id, name) values (0, 'b'); -- test sql_mode NO_BACKSLASH_ESCAPES +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES'); insert into t_2(name) values ('\\a'); \ No newline at end of file From 1cb557bfb2e7aac5af8a49253c503dce7ab1bdec Mon Sep 17 00:00:00 2001 From: okJiang Date: Thu, 15 Jul 2021 17:28:48 +0800 Subject: [PATCH 10/14] Add test data in the batch load phase --- tests/sql_mode/data/db1.increment.sql | 8 +++--- tests/sql_mode/data/db1.prepare.sql | 35 +++++++++++++++++++++++++-- tests/sql_mode/data/db2.increment.sql | 7 +++--- tests/sql_mode/data/db2.prepare.sql | 25 ++++++++++++++++++- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/tests/sql_mode/data/db1.increment.sql b/tests/sql_mode/data/db1.increment.sql index 0e99bfb77d..3ca2eaa30b 100644 --- a/tests/sql_mode/data/db1.increment.sql +++ b/tests/sql_mode/data/db1.increment.sql @@ -1,5 +1,5 @@ set @@session.SQL_MODE=''; -use sql_mode; +use `sql_mode`; -- test sql_mode PIPES_AS_CONCAT insert into t_1(num) values('pipes'||'as'||'concat'); @@ -8,15 +8,15 @@ insert into t_1(num) values('pipes'||'as'||'concat'); insert into t_1(name) values("a"); -- test sql_mode IGNORE_SPACE -create table count (id int not null, primary key(id)); +create table sum (id int not null, primary key(id)); -- test sql_mode NO_AUTO_VALUE_ON_ZERO -insert into t_1(id, name) values (10, 'a'); +insert into t_1(id, name) values (30, 'a'); insert into t_1(id, name) values (0, 'b'); insert into t_1(id, name) values (0, 'c'); -- test sql_mode NO_BACKSLASH_ESCAPES -insert into t_1(name) values ('\\a'); +insert into t_1(name) values ('\\b'); -- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES insert into t_1(dt) values('0000-06-00'); diff --git a/tests/sql_mode/data/db1.prepare.sql b/tests/sql_mode/data/db1.prepare.sql index cf4843709a..f41249dcb8 100644 --- a/tests/sql_mode/data/db1.prepare.sql +++ b/tests/sql_mode/data/db1.prepare.sql @@ -1,3 +1,5 @@ +set @@session.SQL_MODE=''; + drop database if exists `sql_mode`; create database `sql_mode`; use `sql_mode`; @@ -6,6 +8,35 @@ CREATE TABLE `t_1` ( `name` varchar(60), `num` int, `dt` datetime, - -- `ts` timestamp, PRIMARY KEY (id) -); \ No newline at end of file +); + +-- test sql_mode PIPES_AS_CONCAT +insert into t_1(num) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_1(name) values("a"); + +-- test sql_mode IGNORE_SPACE +create table count (id int not null, primary key(id)); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +insert into t_1(id, name) values (10, 'a'); +insert into t_1(id, name) values (0, 'b'); +insert into t_1(id, name) values (0, 'c'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +insert into t_1(name) values ('\\a'); + +-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES +insert into t_1(dt) values('0000-06-00'); +insert into t_1(dt) values('0000-00-01'); +insert into t_1(dt) values('0000-06-01'); +insert into t_1(dt) values('0000-00-00'); + +-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO +insert into t_1(num) values(4/0); + +-- test sql_mode NO_AUTO_CREATE_USER +drop user if exists 'no_auto_create_user'; +grant select on *.* to 'no_auto_create_user'; \ No newline at end of file diff --git a/tests/sql_mode/data/db2.increment.sql b/tests/sql_mode/data/db2.increment.sql index d3502b1ef4..b7b4b9dbc3 100644 --- a/tests/sql_mode/data/db2.increment.sql +++ b/tests/sql_mode/data/db2.increment.sql @@ -1,5 +1,6 @@ set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; -- NO_AUTO_CREATE_USER set failed in mysql8.0 + use sql_mode; -- test sql_mode PIPES_AS_CONCAT @@ -11,12 +12,12 @@ insert into t_2(name) values("a"); -- test sql_mode IGNORE_SPACE set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE'); -select count (*) as c from t_2; +insert into t_2(name) values(concat ('ignore', 'space')); -- test sql_mode NO_AUTO_VALUE_ON_ZERO set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); -insert into t_2(id, name) values (10, 'a'); -insert into t_2(id, name) values (0, 'b'); +insert into t_2(id, name) values (20, 'a'); +replace into t_2(id, name) values (0, 'b'); -- test sql_mode NO_BACKSLASH_ESCAPES set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES'); diff --git a/tests/sql_mode/data/db2.prepare.sql b/tests/sql_mode/data/db2.prepare.sql index 05438fea86..f956c7aaae 100644 --- a/tests/sql_mode/data/db2.prepare.sql +++ b/tests/sql_mode/data/db2.prepare.sql @@ -1,3 +1,6 @@ +set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; +-- NO_AUTO_CREATE_USER set failed in mysql8.0 + drop database if exists `sql_mode`; create database `sql_mode`; use `sql_mode`; @@ -7,4 +10,24 @@ CREATE TABLE `t_2` ( `num` int, `dt` datetime, PRIMARY KEY (id) -); \ No newline at end of file +); + +-- test sql_mode PIPES_AS_CONCAT +set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT'); +insert into t_2(name) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_2(name) values("a"); + +-- test sql_mode IGNORE_SPACE +set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE'); +insert into t_2(name) values(concat ('ignore', 'space')); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); +insert into t_2(id, name) values (10, 'a'); +insert into t_2(id, name) values (0, 'b'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES'); +insert into t_2(name) values ('\\a'); \ No newline at end of file From c0ca579a6be1c6971aae10b9f970b8bf35a109cc Mon Sep 17 00:00:00 2001 From: okJiang Date: Thu, 15 Jul 2021 20:11:54 +0800 Subject: [PATCH 11/14] fix conflict and move comment --- loader/loader.go | 5 ----- pkg/utils/db.go | 5 +++++ syncer/syncer.go | 5 ----- tests/others_integration.txt | 1 + 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 40d8b284ec..54b98f9ac2 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -530,11 +530,6 @@ func (l *Loader) Init(ctx context.Context) (err error) { } if !hasSQLMode { - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) if err3 != nil { l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) diff --git a/pkg/utils/db.go b/pkg/utils/db.go index e11ad99f57..3ee3782d91 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -526,6 +526,11 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid } // AdjustSQLModeCompatible adjust downstream sql mode to compatible. +// When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 +// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, +// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. +// This is because the implementation of go-mysql, that you can see +// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 func AdjustSQLModeCompatible(sqlModes string) (string, error) { needDisable := []string{ "NO_ZERO_IN_DATE", diff --git a/syncer/syncer.go b/syncer/syncer.go index 9f74cf76ec..cf52d3a97d 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2742,11 +2742,6 @@ func (s *Syncer) createDBs(ctx context.Context) error { if err2 != nil { s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) } - // When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 - // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, - // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. - // This is because the implementation of go-mysql, that you can see - // https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) if err3 != nil { s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) diff --git a/tests/others_integration.txt b/tests/others_integration.txt index d111cf7fbf..24fbb92f3b 100644 --- a/tests/others_integration.txt +++ b/tests/others_integration.txt @@ -9,4 +9,5 @@ adjust_gtid load_task downstream_more_column expression_filter +fake_rotate_event sql_mode \ No newline at end of file From c50551a94c60658f17abd0d18a5082a01d26bd38 Mon Sep 17 00:00:00 2001 From: okJiang Date: Fri, 16 Jul 2021 11:37:47 +0800 Subject: [PATCH 12/14] Add log information --- loader/loader.go | 2 +- syncer/syncer.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 54b98f9ac2..284aeb497f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -532,7 +532,7 @@ func (l *Loader) Init(ctx context.Context) (err error) { if !hasSQLMode { sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) if err3 != nil { - l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned \"\"", log.ShortError(err3)) } lcfg.To.Session["sql_mode"] = sqlModes } diff --git a/syncer/syncer.go b/syncer/syncer.go index 4f12e4d361..3c7d9f3ec7 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2754,11 +2754,11 @@ func (s *Syncer) createDBs(ctx context.Context) error { if !hasSQLMode { sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") if err2 != nil { - s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) + s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"\"", log.ShortError(err2)) } sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) if err3 != nil { - s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3)) + s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned \"\"", log.ShortError(err3)) } s.cfg.To.Session["sql_mode"] = sqlModes } From dcbd1256a472084994bbb0161cca741789a11bc5 Mon Sep 17 00:00:00 2001 From: jiangjiang <819421878@qq.com> Date: Mon, 19 Jul 2021 21:22:42 +0800 Subject: [PATCH 13/14] test --- tests/others_integration.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/others_integration.txt b/tests/others_integration.txt index 5a81fa5528..7d6c265782 100644 --- a/tests/others_integration.txt +++ b/tests/others_integration.txt @@ -12,4 +12,4 @@ expression_filter fake_rotate_event metrics case_sensitive -sql_mode \ No newline at end of file +sql_mode From d49933a5c59864e2dbcc310213f25df1aaf21028 Mon Sep 17 00:00:00 2001 From: okJiang Date: Tue, 20 Jul 2021 13:31:09 +0800 Subject: [PATCH 14/14] fix review --- loader/loader.go | 4 ++-- pkg/utils/db.go | 8 ++++---- syncer/syncer.go | 4 ++-- tests/sql_mode/data/db2.increment.sql | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 6fafb47984..f5857b1c43 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -532,12 +532,12 @@ func (l *Loader) Init(ctx context.Context) (err error) { if !hasSQLMode { sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) if err3 != nil { - l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned \"\"", log.ShortError(err3)) + l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will stay the same", log.ShortError(err3)) } lcfg.To.Session["sql_mode"] = sqlModes } - l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) + l.logger.Info("loader's sql_mode is", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize) if err != nil { diff --git a/pkg/utils/db.go b/pkg/utils/db.go index 3ee3782d91..948126c972 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -526,7 +526,7 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid } // AdjustSQLModeCompatible adjust downstream sql mode to compatible. -// When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 +// TODO: When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 // and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, // as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. // This is because the implementation of go-mysql, that you can see @@ -550,15 +550,15 @@ func AdjustSQLModeCompatible(sqlModes string) (string, error) { mode, err := tmysql.GetSQLMode(sqlModes) if err != nil { - return "", err + return sqlModes, err } disableMode, err2 := tmysql.GetSQLMode(disable) if err2 != nil { - return "", err2 + return sqlModes, err2 } enableMode, err3 := tmysql.GetSQLMode(enable) if err3 != nil { - return "", err3 + return sqlModes, err3 } // About this bit manipulation, details can be seen // https://github.com/pingcap/dm/pull/1869#discussion_r669771966 diff --git a/syncer/syncer.go b/syncer/syncer.go index 05f5231bf6..72004ceb3e 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2860,11 +2860,11 @@ func (s *Syncer) createDBs(ctx context.Context) error { if !hasSQLMode { sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") if err2 != nil { - s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"\"", log.ShortError(err2)) + s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"IGNORE_SPACE, NO_AUTO_VALUE_ON_ZERO, ALLOW_INVALID_DATES\"", log.ShortError(err2)) } sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) if err3 != nil { - s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned \"\"", log.ShortError(err3)) + s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned stay the same", log.ShortError(err3)) } s.cfg.To.Session["sql_mode"] = sqlModes } diff --git a/tests/sql_mode/data/db2.increment.sql b/tests/sql_mode/data/db2.increment.sql index b7b4b9dbc3..9301ae86d3 100644 --- a/tests/sql_mode/data/db2.increment.sql +++ b/tests/sql_mode/data/db2.increment.sql @@ -17,7 +17,7 @@ insert into t_2(name) values(concat ('ignore', 'space')); -- test sql_mode NO_AUTO_VALUE_ON_ZERO set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); insert into t_2(id, name) values (20, 'a'); -replace into t_2(id, name) values (0, 'b'); +replace into t_2(id, name) values (0, 'c'); -- test sql_mode NO_BACKSLASH_ESCAPES set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES');