loader, syncer: Update sql_mode to more compatible

loader/loader.go

@@ -521,16 +521,23 @@ func (l *Loader) Init(ctx context.Context) (err error) {
 	}
 	lcfg.To.Session["time_zone"] = "+00:00"
 
-	hasSQLMode := false
 	for k := range l.cfg.To.Session {
 		if strings.ToLower(k) == "sql_mode" {
-			hasSQLMode = true
+			l.logger.Info("your confiured sql_mode will be adjusted to compatible")
 			break
 		}
 	}
-	if !hasSQLMode {
-		lcfg.To.Session["sql_mode"] = l.cfg.LoaderConfig.SQLMode
-	}
+	// When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00
+	// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31,
+	// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'.
+	// This is because the implementation of go-mysql, that you can see
+	// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087
+	sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode)
+	if err3 != nil {
+		l.logger.Warn("cannot adjust sql_mode compatible", log.ShortError(err3))
+	}
+	lcfg.To.Session["sql_mode"] = sqlModes
+	l.logger.Info("loader's sql_mode is ", zap.String("sqlmode", lcfg.To.Session["sql_mode"]))
 
 	l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize)
 	if err != nil {

pkg/utils/db.go

@@ -524,3 +524,86 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid
 	_ = ret.Set(newGset)
 	return ret, nil
 }
+
+// AdjustSQLModeCompatible adjust downstream sql mode to compatible
+func AdjustSQLModeCompatible(sqlModes string) (string, error) {
+	needDisable := []string{
+		"NO_ZERO_IN_DATE",
+		"NO_ZERO_DATE",
+		"ERROR_FOR_DIVISION_BY_ZERO",
+		"NO_AUTO_CREATE_USER",
+		"STRICT_TRANS_TABLES",
+		"STRICT_ALL_TABLES",
+	}
+	needEnable := []string{
+		"IGNORE_SPACE",
+		"NO_AUTO_VALUE_ON_ZERO",
+		"ALLOW_INVALID_DATES",
+	}
+	disable := strings.Join(needDisable, ",")
+	enable := strings.Join(needEnable, ",")
+
+	mode, err := tmysql.GetSQLMode(sqlModes)
+	if err != nil {
+		return "", err
+	}
+	disableMode, err2 := tmysql.GetSQLMode(disable)
+	if err2 != nil {
+		return "", err2
+	}
+	enableMode, err3 := tmysql.GetSQLMode(enable)
+	if err3 != nil {
+		return "", err3
+	}
+	mode = (mode &^ disableMode) | enableMode
+
+	return GetSQLModeStrBySQLMode(mode), nil
+}
+
+// SQLMode2Str is the sql_mode to string represent of sql_mode map.
+var SQLMode2Str = map[tmysql.SQLMode]string{
+	tmysql.ModeRealAsFloat:            "REAL_AS_FLOAT",
+	tmysql.ModePipesAsConcat:          "PIPES_AS_CONCAT",
+	tmysql.ModeANSIQuotes:             "ANSI_QUOTES",
+	tmysql.ModeIgnoreSpace:            "IGNORE_SPACE",
+	tmysql.ModeNotUsed:                "NOT_USED",
+	tmysql.ModeOnlyFullGroupBy:        "ONLY_FULL_GROUP_BY",
+	tmysql.ModeNoUnsignedSubtraction:  "NO_UNSIGNED_SUBTRACTION",
+	tmysql.ModeNoDirInCreate:          "NO_DIR_IN_CREATE",
+	tmysql.ModePostgreSQL:             "POSTGRESQL",
+	tmysql.ModeOracle:                 "ORACLE",
+	tmysql.ModeMsSQL:                  "MSSQL",
+	tmysql.ModeDb2:                    "DB2",
+	tmysql.ModeMaxdb:                  "MAXDB",
+	tmysql.ModeNoKeyOptions:           "NO_KEY_OPTIONS",
+	tmysql.ModeNoTableOptions:         "NO_TABLE_OPTIONS",
+	tmysql.ModeNoFieldOptions:         "NO_FIELD_OPTIONS",
+	tmysql.ModeMySQL323:               "MYSQL323",
+	tmysql.ModeMySQL40:                "MYSQL40",
+	tmysql.ModeANSI:                   "ANSI",
+	tmysql.ModeNoAutoValueOnZero:      "NO_AUTO_VALUE_ON_ZERO",
+	tmysql.ModeNoBackslashEscapes:     "NO_BACKSLASH_ESCAPES",
+	tmysql.ModeStrictTransTables:      "STRICT_TRANS_TABLES",
+	tmysql.ModeStrictAllTables:        "STRICT_ALL_TABLES",
+	tmysql.ModeNoZeroInDate:           "NO_ZERO_IN_DATE",
+	tmysql.ModeNoZeroDate:             "NO_ZERO_DATE",
+	tmysql.ModeInvalidDates:           "INVALID_DATES",
+	tmysql.ModeErrorForDivisionByZero: "ERROR_FOR_DIVISION_BY_ZERO",
+	tmysql.ModeTraditional:            "TRADITIONAL",
+	tmysql.ModeNoAutoCreateUser:       "NO_AUTO_CREATE_USER",
+	tmysql.ModeHighNotPrecedence:      "HIGH_NOT_PRECEDENCE",
+	tmysql.ModeNoEngineSubstitution:   "NO_ENGINE_SUBSTITUTION",
+	tmysql.ModePadCharToFullLength:    "PAD_CHAR_TO_FULL_LENGTH",
+	tmysql.ModeAllowInvalidDates:      "ALLOW_INVALID_DATES",
+}
+
+// GetSQLModeByStr get string represent of sql_mode by sql_mode
+func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string {
+	var sqlModeStr []string
+	for SQLMode, str := range SQLMode2Str {
+		if sqlMode&SQLMode != 0 {
+			sqlModeStr = append(sqlModeStr, str)
+		}
+	}
+	return strings.Join(sqlModeStr, ",")
+}

syncer/syncer.go

@@ -2725,26 +2725,31 @@ func (s *Syncer) createDBs(ctx context.Context) error {
 		return err
 	}
 
-	hasSQLMode := false
 	// get sql_mode from upstream db
 	if s.cfg.To.Session == nil {
 		s.cfg.To.Session = make(map[string]string)
 	} else {
 		for k := range s.cfg.To.Session {
 			if strings.ToLower(k) == "sql_mode" {
-				hasSQLMode = true
+				s.tctx.L().Info("your confiured sql_mode will be adjusted to compatible")
 				break
 			}
 		}
 	}
-	if !hasSQLMode {
-		sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode")
-		if err2 != nil {
-			s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2))
-		} else {
-			s.cfg.To.Session["sql_mode"] = sqlMode
-		}
-	}
+	sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode")
+	if err2 != nil {
+		s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2))
+	}
+	// When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00
+	// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31,
+	// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'.
+	// This is because the implementation of go-mysql, that you can see
+	// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087
+	sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode)
+	if err3 != nil {
+		s.tctx.L().Warn("cannot adjust sql_mode compatible", log.ShortError(err3))
+	}
+	s.cfg.To.Session["sql_mode"] = sqlModes
 
 	dbCfg = s.cfg.To
 	dbCfg.RawDBCfg = config.DefaultRawDBConfig().

tests/others_integration.txt

@@ -8,4 +8,5 @@ only_dml
 adjust_gtid
 load_task
 downstream_more_column
-expression_filter
+expression_filter
+sql_mode

tests/sql_mode/conf/diff_config.toml

@@ -0,0 +1,56 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 20
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "sql_mode"
+tables = ["~.*"]
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+instance-id = "source-1"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_1"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "sql_mode"
+table  = "t_1"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+instance-id = "source-2"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_2"
+
+[[table-config.source-tables]]
+instance-id = "source-2"
+schema = "sql_mode"
+table  = "t_2"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

tests/sql_mode/conf/dm-master.toml

@@ -0,0 +1,4 @@
+# Master Configuration.
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sql_mode/conf/dm-task.yaml

@@ -0,0 +1,46 @@
+---
+name: sql_mode
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+enable-heartbeat: false
+
+target-database:
+  host: "127.0.0.1"
+  port: 4000
+  user: "root"
+  password: ""
+
+mysql-instances:
+  - source-id: "mysql-replica-01"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+  - source-id: "mysql-replica-02"
+    block-allow-list:  "instance"
+    mydumper-config-name: "global"
+    loader-config-name: "global"
+    syncer-config-name: "global"
+
+block-allow-list:
+  instance:
+    do-dbs: ["sql_mode"]
+
+mydumpers:
+  global:
+    threads: 4
+    chunk-filesize: 0
+    skip-tz-utc: true
+    extra-args: "--statement-size=4000"
+
+loaders:
+  global:
+    pool-size: 16
+    dir: "./dumped_data"
+
+syncers:
+  global:
+    worker-count: 16
+    batch: 100

tests/sql_mode/conf/dm-worker1.toml

@@ -0,0 +1,2 @@
+name = "worker1"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/dm-worker2.toml

@@ -0,0 +1,2 @@
+name = "worker2"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/source1.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-01
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+  host: 127.0.0.1
+  user: root
+  password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+  port: 3306

tests/sql_mode/conf/source2.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-02
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+  host: 127.0.0.1
+  user: root
+  password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+  port: 3307

tests/sql_mode/data/db1.increment.sql

@@ -0,0 +1,32 @@
+set @@session.SQL_MODE='';
+use sql_mode;
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_1(num) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_1(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+create table count (id int not null, primary key(id));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_1(id, name) values (10, 'a');
+insert into t_1(id, name) values (0, 'b');
+insert into t_1(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_1(name) values ('\\a');
+
+-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES
+insert into t_1(dt) values('0000-06-00');
+insert into t_1(dt) values('0000-00-01');
+insert into t_1(dt) values('0000-06-01');
+insert into t_1(dt) values('0000-00-00');
+
+-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO
+insert into t_1(num) values(4/0);
+
+-- test sql_mode NO_AUTO_CREATE_USER
+drop user if exists 'no_auto_create_user';
+grant select on *.* to 'no_auto_create_user';

tests/sql_mode/data/db1.prepare.sql

@@ -0,0 +1,11 @@
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_1` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `name` varchar(60),
+  `num` int,
+  `dt` datetime,
+  -- `ts` timestamp,
+  PRIMARY KEY (id)
+);

tests/sql_mode/data/db2.increment.sql

@@ -0,0 +1,19 @@
+set @@session.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT';
+-- NO_AUTO_CREATE_USER set failed in mysql8.0
+use sql_mode;
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_2(name) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_2(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+select count (*) as c from t_2;
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_2(id, name) values (10, 'a');
+insert into t_2(id, name) values (0, 'b');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_2(name) values ('\\a');

tests/sql_mode/data/db2.prepare.sql

@@ -0,0 +1,10 @@
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_2` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `name` varchar(60),
+  `num` int,
+  `dt` datetime,
+  PRIMARY KEY (id)
+);