feat(encrypt): Support both encrypted and unencrypted passwords for t…

…he upstream and downstream database (#633)

cmd/dm-ctl/main.go

@@ -60,6 +60,7 @@ import (
 //
 //Special Commands:
 //  --encrypt encrypt plaintext to ciphertext
+//  --decrypt decrypt ciphertext to plaintext
 //
 //Global Options:
 //  --V prints version and exit
@@ -74,10 +75,12 @@ func helpUsage(cfg *common.Config) {
 	fmt.Println("Special Commands:")
 	f := cfg.FlagSet.Lookup(common.EncryptCmdName)
 	fmt.Println(fmt.Sprintf("  --%s %s", f.Name, f.Usage))
+	f = cfg.FlagSet.Lookup(common.DecryptCmdName)
+	fmt.Println(fmt.Sprintf("  --%s %s", f.Name, f.Usage))
 	fmt.Println()
 	fmt.Println("Global Options:")
 	cfg.FlagSet.VisitAll(func(flag2 *flag.Flag) {
-		if flag2.Name == common.EncryptCmdName {
+		if flag2.Name == common.EncryptCmdName || flag2.Name == common.DecryptCmdName {
 			return
 		}
 		fmt.Println(fmt.Sprintf("  --%s %s", flag2.Name, flag2.Usage))

dm/config/source_config.go

@@ -167,41 +167,31 @@ func (c *SourceConfig) Verify() error {
 		}
 	}
 
-	_, err = c.DecryptPassword()
-	if err != nil {
-		return err
-	}
+	c.DecryptPassword()
 
 	return nil
 }
 
 // DecryptPassword returns a decrypted config replica in config
-func (c *SourceConfig) DecryptPassword() (*SourceConfig, error) {
+func (c *SourceConfig) DecryptPassword() *SourceConfig {
 	clone := c.Clone()
 	var (
 		pswdFrom string
-		err      error
 	)
 	if len(clone.From.Password) > 0 {
-		pswdFrom, err = utils.Decrypt(clone.From.Password)
-		if err != nil {
-			return nil, terror.WithClass(err, terror.ClassDMWorker)
-		}
+		pswdFrom = utils.DecryptOrPlaintext(clone.From.Password)
 	}
 	clone.From.Password = pswdFrom
-	return clone, nil
+	return clone
 }
 
 // GenerateDBConfig creates DBConfig for DB
-func (c *SourceConfig) GenerateDBConfig() (*DBConfig, error) {
+func (c *SourceConfig) GenerateDBConfig() *DBConfig {
 	// decrypt password
-	clone, err := c.DecryptPassword()
-	if err != nil {
-		return nil, err
-	}
+	clone := c.DecryptPassword()
 	from := &clone.From
 	from.RawDBCfg = DefaultRawDBConfig().SetReadTimeout(dbReadTimeout)
-	return from, nil
+	return from
 }
 
 // Adjust flavor and serverid of SourceConfig

dm/config/source_config_test.go

@@ -62,17 +62,14 @@ func (t *testConfig) TestConfig(c *C) {
 	// test decrypt password
 	clone1.From.Password = "1234"
 	clone1.ServerID = 101
-	clone2, err := cfg.DecryptPassword()
-	c.Assert(err, IsNil)
+	clone2 := cfg.DecryptPassword()
 	c.Assert(clone2, DeepEquals, clone1)
 
 	cfg.From.Password = "xxx"
-	_, err = cfg.DecryptPassword()
-	c.Assert(err, NotNil)
+	cfg.DecryptPassword()
 
 	cfg.From.Password = ""
-	clone3, err := cfg.DecryptPassword()
-	c.Assert(err, IsNil)
+	clone3 := cfg.DecryptPassword()
 	c.Assert(clone3, DeepEquals, cfg)
 
 	// test toml and parse again
@@ -167,7 +164,31 @@ func (t *testConfig) TestConfigVerify(c *C) {
 				cfg.From.Password = "not-encrypt"
 				return cfg
 			},
-			"*decode base64 encoded password.*",
+			"",
+		},
+		{
+			func() *SourceConfig {
+				cfg := newConfig()
+				cfg.From.Password = "" // password empty
+				return cfg
+			},
+			"",
+		},
+		{
+			func() *SourceConfig {
+				cfg := newConfig()
+				cfg.From.Password = "123456" // plaintext password
+				return cfg
+			},
+			"",
+		},
+		{
+			func() *SourceConfig {
+				cfg := newConfig()
+				cfg.From.Password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" // encrypt password (123456)
+				return cfg
+			},
+			"",
 		},
 	}
 

dm/config/subtask.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
-	column "github.com/pingcap/tidb-tools/pkg/column-mapping"
+	"github.com/pingcap/tidb-tools/pkg/column-mapping"
 	"github.com/pingcap/tidb-tools/pkg/filter"
 	router "github.com/pingcap/tidb-tools/pkg/table-router"
 	"go.uber.org/zap"
@@ -352,16 +352,10 @@ func (c *SubTaskConfig) DecryptPassword() (*SubTaskConfig, error) {
 		pswdFrom string
 	)
 	if len(clone.To.Password) > 0 {
-		pswdTo, err = utils.Decrypt(clone.To.Password)
-		if err != nil {
-			return nil, terror.WithScope(terror.ErrConfigDecryptDBPassword.Delegate(err, clone.To.Password), terror.ScopeDownstream)
-		}
+		pswdTo = utils.DecryptOrPlaintext(clone.To.Password)
 	}
 	if len(clone.From.Password) > 0 {
-		pswdFrom, err = utils.Decrypt(clone.From.Password)
-		if err != nil {
-			return nil, terror.WithScope(terror.ErrConfigDecryptDBPassword.Delegate(err, clone.From.Password), terror.ScopeUpstream)
-		}
+		pswdFrom = utils.DecryptOrPlaintext(clone.From.Password)
 	}
 	clone.From.Password = pswdFrom
 	clone.To.Password = pswdTo

dm/config/subtask_test.go

@@ -52,9 +52,9 @@ func (t *testConfig) TestSubTask(c *C) {
 
 	cfg.From.Password = "xxx"
 	_, err = cfg.DecryptPassword()
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	err = cfg.Adjust(true)
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	err = cfg.Adjust(false)
 	c.Assert(err, IsNil)
 

dm/ctl/common/config.go

@@ -31,6 +31,8 @@ const (
 
 	// EncryptCmdName is special command
 	EncryptCmdName = "encrypt"
+	// DecryptCmdName is special command
+	DecryptCmdName = "decrypt"
 )
 
 // NewConfig creates a new base config for dmctl.
@@ -47,6 +49,7 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.MasterAddr, "master-addr", "", "master API server addr")
 	fs.StringVar(&cfg.RPCTimeoutStr, "rpc-timeout", defaultRPCTimeout, fmt.Sprintf("rpc timeout, default is %s", defaultRPCTimeout))
 	fs.StringVar(&cfg.encrypt, EncryptCmdName, "", "encrypt plaintext to ciphertext")
+	fs.StringVar(&cfg.decrypt, DecryptCmdName, "", "decrypt ciphertext to plaintext")
 
 	return cfg
 }
@@ -64,6 +67,7 @@ type Config struct {
 
 	printVersion bool
 	encrypt      string // string need to be encrypted
+	decrypt      string // string need to be decrypted
 }
 
 func (c *Config) String() string {
@@ -95,6 +99,15 @@ func (c *Config) Parse(arguments []string) (finish bool, err error) {
 		return true, nil
 	}
 
+	if len(c.decrypt) > 0 {
+		plaintext, err1 := utils.Decrypt(c.decrypt)
+		if err1 != nil {
+			return true, err1
+		}
+		fmt.Println(plaintext)
+		return true, nil
+	}
+
 	// Load config file if specified.
 	if c.ConfigFile != "" {
 		err = c.configFromFile(c.ConfigFile)

dm/master/server.go

@@ -1171,10 +1171,7 @@ func (s *Server) getSourceConfigs(sources []*config.MySQLInstance) (map[string]c
 	for _, source := range sources {
 		if cfg := s.scheduler.GetSourceCfgByID(source.SourceID); cfg != nil {
 			// check the password
-			_, err := cfg.DecryptPassword()
-			if err != nil {
-				return nil, err
-			}
+			cfg.DecryptPassword()
 			cfgs[source.SourceID] = cfg.From
 		}
 	}
@@ -1243,10 +1240,7 @@ func parseAndAdjustSourceConfig(cfg *config.SourceConfig, content string) error
 		return err
 	}
 
-	dbConfig, err := cfg.GenerateDBConfig()
-	if err != nil {
-		return err
-	}
+	dbConfig := cfg.GenerateDBConfig()
 
 	fromDB, err := conn.DefaultDBProvider.Apply(*dbConfig)
 	if err != nil {

dm/master/server_test.go

@@ -363,14 +363,14 @@ func (t *testMaster) TestCheckTask(c *check.C) {
 	c.Assert(resp.Result, check.IsFalse)
 	clearSchedulerEnv(c, cancel, &wg)
 
-	// simulate invalid password returned from scheduler, so cfg.SubTaskConfigs will fail
+	// simulate invalid password returned from scheduler, but config was supported plaintext mysql password, so cfg.SubTaskConfigs will success
 	ctx, cancel = context.WithCancel(context.Background())
 	server.scheduler, _ = testMockScheduler(ctx, &wg, c, sources, workers, "invalid-encrypt-password", t.workerClients)
 	resp, err = server.CheckTask(context.Background(), &pb.CheckTaskRequest{
 		Task: taskConfig,
 	})
 	c.Assert(err, check.IsNil)
-	c.Assert(resp.Result, check.IsFalse)
+	c.Assert(resp.Result, check.IsTrue)
 	clearSchedulerEnv(c, cancel, &wg)
 }
 

dm/portal/api.go

@@ -17,15 +17,16 @@ import (
 
 	// for database
 	_ "github.com/go-sql-driver/mysql"
-	"github.com/pingcap/dm/dm/config"
-	"github.com/pingcap/dm/pkg/log"
-	"github.com/pingcap/dm/pkg/utils"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb-tools/pkg/dbutil"
 	"github.com/pingcap/tidb-tools/pkg/filter"
 	"github.com/unrolled/render"
 	"go.uber.org/zap"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
+
+	"github.com/pingcap/dm/dm/config"
+	"github.com/pingcap/dm/pkg/log"
+	"github.com/pingcap/dm/pkg/utils"
 )
 
 const (
@@ -341,19 +342,7 @@ func (p *Handler) AnalyzeConfig(w http.ResponseWriter, req *http.Request) {
 	log.L().Info("analyze config", zap.String("config name", cfg.Name))
 
 	// decrypt password
-	dePwd, err := utils.Decrypt(cfg.TargetDB.Password)
-	log.L().Error("decrypt password failed", zap.Error(err))
-	if err != nil {
-		p.genJSONResp(w, http.StatusBadRequest, AnalyzeResult{
-			CommonResult: CommonResult{
-				Result: failed,
-				Error:  err.Error(),
-			},
-			Config: DMTaskConfig{},
-		})
-		return
-	}
-	cfg.TargetDB.Password = dePwd
+	cfg.TargetDB.Password = utils.DecryptOrPlaintext(cfg.TargetDB.Password)
 
 	p.genJSONResp(w, http.StatusOK, AnalyzeResult{
 		CommonResult: CommonResult{

dm/worker/relay.go

@@ -82,7 +82,7 @@ type realRelayHolder struct {
 
 // NewRealRelayHolder creates a new RelayHolder
 func NewRealRelayHolder(cfg *config.SourceConfig) RelayHolder {
-	clone, _ := cfg.DecryptPassword()
+	clone := cfg.DecryptPassword()
 	relayCfg := &relay.Config{
 		EnableGTID:  clone.EnableGTID,
 		AutoFixGTID: clone.AutoFixGTID,

dm/worker/worker.go

@@ -673,7 +673,7 @@ func (w *Worker) UpdateRelayConfig(ctx context.Context, content string) error {
 	}
 
 	w.l.Info("update config", zap.Stringer("new config", newCfg))
-	cloneCfg, _ := newCfg.DecryptPassword()
+	cloneCfg := newCfg.DecryptPassword()
 
 	// Update SubTask configure
 	// NOTE: we only update `DB.Config` in SubTaskConfig now

pkg/utils/encrypt.go

@@ -43,3 +43,12 @@ func Decrypt(ciphertextB64 string) (string, error) {
 	}
 	return string(plaintext), nil
 }
+
+// DecryptOrPlaintext tries to decrypt base64 encoded ciphertext to plaintext or return plaintext
+func DecryptOrPlaintext(ciphertextB64 string) string {
+	plaintext, err := Decrypt(ciphertextB64)
+	if err != nil {
+		return ciphertextB64
+	}
+	return plaintext
+}

tests/dm_syncer/conf/source1.toml

@@ -9,5 +9,5 @@ relay-binlog-gtid = ""
 [from]
 host = "127.0.0.1"
 user = "root"
-password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+password = "123456"
 port = 3306

tests/dmctl_basic/conf/source1.toml

@@ -10,5 +10,5 @@ enable-relay = true
 [from]
 host = "127.0.0.1"
 user = "root"
-password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+password = "123456"
 port = 3306

tests/dmctl_command/conf/source1.toml

@@ -9,7 +9,7 @@ relay-binlog-gtid = ""
 [from]
 host = "127.0.0.1"
 user = "root"
-password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+password = "123456"
 port = 3306
 
 [checker]

tests/dmctl_command/run.sh

@@ -6,7 +6,7 @@ cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 source $cur/../_utils/test_prepare
 WORK_DIR=$TEST_DIR/$TEST_NAME
 
-help_cnt=35
+help_cnt=36
 
 function run() {
     # check dmctl alone output