feat: manual cherry merge PR pingcap#633 to release-1.0

cmd/dm-ctl/main.go

@@ -60,6 +60,7 @@ import (
 //
 //Special Commands:
 //  --encrypt encrypt plaintext to ciphertext
+//  --decrypt decrypt ciphertext to plaintext
 //
 //Global Options:
 //  --V prints version and exit
@@ -74,10 +75,12 @@ func helpUsage(cfg *common.Config) {
 	fmt.Println("Special Commands:")
 	f := cfg.FlagSet.Lookup(common.EncryptCmdName)
 	fmt.Println(fmt.Sprintf("  --%s %s", f.Name, f.Usage))
+	f = cfg.FlagSet.Lookup(common.DecryptCmdName)
+	fmt.Println(fmt.Sprintf("  --%s %s", f.Name, f.Usage))
 	fmt.Println()
 	fmt.Println("Global Options:")
 	cfg.FlagSet.VisitAll(func(flag2 *flag.Flag) {
-		if flag2.Name == common.EncryptCmdName {
+		if flag2.Name == common.EncryptCmdName || flag2.Name == common.DecryptCmdName {
 			return
 		}
 		fmt.Println(fmt.Sprintf("  --%s %s", flag2.Name, flag2.Usage))

dm/config/subtask.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
-	column "github.com/pingcap/tidb-tools/pkg/column-mapping"
+	"github.com/pingcap/tidb-tools/pkg/column-mapping"
 	"github.com/pingcap/tidb-tools/pkg/filter"
 	router "github.com/pingcap/tidb-tools/pkg/table-router"
 	"go.uber.org/zap"
@@ -339,16 +339,10 @@ func (c *SubTaskConfig) DecryptPassword() (*SubTaskConfig, error) {
 		pswdFrom string
 	)
 	if len(clone.To.Password) > 0 {
-		pswdTo, err = utils.Decrypt(clone.To.Password)
-		if err != nil {
-			return nil, terror.WithScope(terror.ErrConfigDecryptDBPassword.Delegate(err, clone.To.Password), terror.ScopeDownstream)
-		}
+		pswdTo = utils.DecryptOrPlaintext(clone.To.Password)
 	}
 	if len(clone.From.Password) > 0 {
-		pswdFrom, err = utils.Decrypt(clone.From.Password)
-		if err != nil {
-			return nil, terror.WithScope(terror.ErrConfigDecryptDBPassword.Delegate(err, clone.From.Password), terror.ScopeUpstream)
-		}
+		pswdFrom = utils.DecryptOrPlaintext(clone.From.Password)
 	}
 	clone.From.Password = pswdFrom
 	clone.To.Password = pswdTo

dm/config/subtask_test.go

@@ -50,10 +50,10 @@ func (t *testConfig) TestSubTask(c *C) {
 
 	cfg.From.Password = "xxx"
 	_, err = cfg.DecryptPassword()
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 
 	err = cfg.Adjust(true)
-	c.Assert(err, NotNil)
+	c.Assert(err, IsNil)
 	err = cfg.Adjust(false)
 	c.Assert(err, IsNil)
 

dm/ctl/common/config.go

@@ -31,6 +31,8 @@ const (
 
 	// EncryptCmdName is special command
 	EncryptCmdName = "encrypt"
+	// DecryptCmdName is special command
+	DecryptCmdName = "decrypt"
 )
 
 // NewConfig creates a new base config for dmctl.
@@ -47,6 +49,7 @@ func NewConfig() *Config {
 	fs.StringVar(&cfg.MasterAddr, "master-addr", "", "master API server addr")
 	fs.StringVar(&cfg.RPCTimeoutStr, "rpc-timeout", defaultRPCTimeout, fmt.Sprintf("rpc timeout, default is %s", defaultRPCTimeout))
 	fs.StringVar(&cfg.encrypt, EncryptCmdName, "", "encrypt plaintext to ciphertext")
+	fs.StringVar(&cfg.decrypt, DecryptCmdName, "", "decrypt ciphertext to plaintext")
 
 	return cfg
 }
@@ -64,6 +67,7 @@ type Config struct {
 
 	printVersion bool
 	encrypt      string // string need to be encrypted
+	decrypt      string // string need to be decrypted
 }
 
 func (c *Config) String() string {
@@ -95,6 +99,15 @@ func (c *Config) Parse(arguments []string) (finish bool, err error) {
 		return true, nil
 	}
 
+	if len(c.decrypt) > 0 {
+		plaintext, err1 := utils.Decrypt(c.decrypt)
+		if err1 != nil {
+			return true, err1
+		}
+		fmt.Println(plaintext)
+		return true, nil
+	}
+
 	// Load config file if specified.
 	if c.ConfigFile != "" {
 		err = c.configFromFile(c.ConfigFile)

dm/master/server_test.go

@@ -405,13 +405,13 @@ func (t *testMaster) TestCheckTask(c *check.C) {
 	c.Assert(err, check.IsNil)
 	c.Assert(resp.Result, check.IsFalse)
 
-	// simulate invalid password returned from dm-workers, so cfg.SubTaskConfigs will fail
+	// simulate invalid password returned from scheduler, but config was supported plaintext mysql password, so cfg.SubTaskConfigs will success
 	testMockWorkerConfig(c, server, ctrl, "invalid-encrypt-password", true)
 	resp, err = server.CheckTask(context.Background(), &pb.CheckTaskRequest{
 		Task: taskConfig,
 	})
 	c.Assert(err, check.IsNil)
-	c.Assert(resp.Result, check.IsFalse)
+	c.Assert(resp.Result, check.IsTrue)
 
 	// test query worker config failed
 	testMockWorkerConfig(c, server, ctrl, "", false)

dm/portal/api.go

@@ -17,15 +17,16 @@ import (
 
 	// for database
 	_ "github.com/go-sql-driver/mysql"
-	"github.com/pingcap/dm/dm/config"
-	"github.com/pingcap/dm/pkg/log"
-	"github.com/pingcap/dm/pkg/utils"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb-tools/pkg/dbutil"
 	"github.com/pingcap/tidb-tools/pkg/filter"
 	"github.com/unrolled/render"
 	"go.uber.org/zap"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
+
+	"github.com/pingcap/dm/dm/config"
+	"github.com/pingcap/dm/pkg/log"
+	"github.com/pingcap/dm/pkg/utils"
 )
 
 const (
@@ -341,18 +342,8 @@ func (p *Handler) AnalyzeConfig(w http.ResponseWriter, req *http.Request) {
 	log.L().Info("analyze config", zap.String("config name", cfg.Name))
 
 	// decrypt password
-	dePwd, err := utils.Decrypt(cfg.TargetDB.Password)
-	log.L().Error("decrypt password failed", zap.Error(err))
-	if err != nil {
-		p.genJSONResp(w, http.StatusBadRequest, AnalyzeResult{
-			CommonResult: CommonResult{
-				Result: failed,
-				Error:  err.Error(),
-			},
-			Config: DMTaskConfig{},
-		})
-		return
-	}
+	dePwd := utils.DecryptOrPlaintext(cfg.TargetDB.Password)
+
 	cfg.TargetDB.Password = dePwd
 
 	p.genJSONResp(w, http.StatusOK, AnalyzeResult{

dm/worker/config.go

@@ -236,10 +236,7 @@ func (c *Config) verify() error {
 		}
 	}
 
-	_, err = c.DecryptPassword()
-	if err != nil {
-		return err
-	}
+	c.DecryptPassword()
 
 	return nil
 }
@@ -292,10 +289,8 @@ func (c *Config) adjust() error {
 
 func (c *Config) createFromDB() (*conn.BaseDB, error) {
 	// decrypt password
-	clone, err := c.DecryptPassword()
-	if err != nil {
-		return nil, err
-	}
+	clone := c.DecryptPassword()
+
 	from := clone.From
 	from.RawDBCfg = config.DefaultRawDBConfig().SetReadTimeout(dbReadTimeout)
 	fromDB, err := conn.DefaultDBProvider.Apply(from)
@@ -378,18 +373,14 @@ func (c *Config) Reload() error {
 }
 
 // DecryptPassword returns a decrypted config replica in config
-func (c *Config) DecryptPassword() (*Config, error) {
+func (c *Config) DecryptPassword() *Config {
 	clone := c.Clone()
 	var (
 		pswdFrom string
-		err      error
 	)
 	if len(clone.From.Password) > 0 {
-		pswdFrom, err = utils.Decrypt(clone.From.Password)
-		if err != nil {
-			return nil, terror.WithClass(err, terror.ClassDMWorker)
-		}
+		pswdFrom = utils.DecryptOrPlaintext(clone.From.Password)
 	}
 	clone.From.Password = pswdFrom
-	return clone, nil
+	return clone
 }

dm/worker/config_test.go

@@ -21,7 +21,7 @@ import (
 	"path"
 	"strings"
 
-	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	"github.com/DATA-DOG/go-sqlmock"
 	. "github.com/pingcap/check"
 	"github.com/siddontang/go-mysql/mysql"
 
@@ -64,17 +64,14 @@ func (t *testServer) TestConfig(c *C) {
 	// test decrypt password
 	clone1.From.Password = "1234"
 	clone1.ServerID = 101
-	clone2, err := cfg.DecryptPassword()
-	c.Assert(err, IsNil)
+	clone2 := cfg.DecryptPassword()
 	c.Assert(clone2, DeepEquals, clone1)
 
 	cfg.From.Password = "xxx"
-	_, err = cfg.DecryptPassword()
-	c.Assert(err, NotNil)
+	cfg.DecryptPassword()
 
 	cfg.From.Password = ""
-	clone3, err := cfg.DecryptPassword()
-	c.Assert(err, IsNil)
+	clone3 := cfg.DecryptPassword()
 	c.Assert(clone3, DeepEquals, cfg)
 
 	// test invalid config
@@ -147,6 +144,30 @@ func (t *testServer) TestConfigVerify(c *C) {
 			},
 			"*decode base64 encoded password.*",
 		},
+		{
+			func() *Config {
+				cfg := newConfig()
+				cfg.From.Password = "" // password empty
+				return cfg
+			},
+			"",
+		},
+		{
+			func() *Config {
+				cfg := newConfig()
+				cfg.From.Password = "123456" // plaintext password
+				return cfg
+			},
+			"",
+		},
+		{
+			func() *Config {
+				cfg := newConfig()
+				cfg.From.Password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" // encrypt password (123456)
+				return cfg
+			},
+			"",
+		},
 	}
 
 	for _, tc := range testCases {

dm/worker/relay.go

@@ -81,7 +81,7 @@ type realRelayHolder struct {
 
 // NewRealRelayHolder creates a new RelayHolder
 func NewRealRelayHolder(cfg *Config) RelayHolder {
-	clone, _ := cfg.DecryptPassword()
+	clone := cfg.DecryptPassword()
 	relayCfg := &relay.Config{
 		EnableGTID:  clone.EnableGTID,
 		AutoFixGTID: clone.AutoFixGTID,

dm/worker/worker.go

@@ -610,7 +610,7 @@ func (w *Worker) UpdateRelayConfig(ctx context.Context, content string) error {
 	}
 
 	w.l.Info("update relay config", zap.Stringer("new config", newCfg))
-	cloneCfg, _ := newCfg.DecryptPassword()
+	cloneCfg := newCfg.DecryptPassword()
 
 	// Update SubTask configure
 	// NOTE: we only update `DB.Config` in SubTaskConfig now

pkg/utils/encrypt.go

@@ -43,3 +43,12 @@ func Decrypt(ciphertextB64 string) (string, error) {
 	}
 	return string(plaintext), nil
 }
+
+// DecryptOrPlaintext tries to decrypt base64 encoded ciphertext to plaintext or return plaintext
+func DecryptOrPlaintext(ciphertextB64 string) string {
+	plaintext, err := Decrypt(ciphertextB64)
+	if err != nil {
+		return ciphertextB64
+	}
+	return plaintext
+}

tests/dmctl_basic/conf/dm-worker1.toml

@@ -9,5 +9,5 @@ relay-binlog-gtid = ""
 [from]
 host = "127.0.0.1"
 user = "root"
-password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+password = "123456"
 port = 3306

tests/dmctl_command/conf/dm-worker1.toml

@@ -9,7 +9,7 @@ relay-binlog-gtid = ""
 [from]
 host = "127.0.0.1"
 user = "root"
-password = "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
+password = "123456"
 port = 3306
 
 [checker]

tests/dmctl_command/run.sh

@@ -6,14 +6,16 @@ cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 source $cur/../_utils/test_prepare
 WORK_DIR=$TEST_DIR/$TEST_NAME
 
+help_cnt=35
+
 function run() {
     # check dmctl alone output
     # it should usage for root command
 
     $PWD/bin/dmctl.test DEVEL > $WORK_DIR/help.log
     help_msg=$(cat $WORK_DIR/help.log)
     help_msg_cnt=$(echo "${help_msg}" | wc -l |xargs)
-    if [ "$help_msg_cnt" != 34 ]; then
+    if [ "$help_msg_cnt" != ${help_cnt} ]; then
         echo "dmctl case 1 help failed: $help_msg"
         echo $help_msg_cnt
         exit 1
@@ -24,7 +26,7 @@ function run() {
     $PWD/bin/dmctl.test DEVEL --help > $WORK_DIR/help.log
     help_msg=$(cat $WORK_DIR/help.log)
     help_msg_cnt=$(echo "${help_msg}" | wc -l |xargs)
-    if [ "$help_msg_cnt" != 34 ]; then
+    if [ "$help_msg_cnt" != ${help_cnt} ]; then
         echo "dmctl case 2 help failed: $help_msg"
         exit 1
     fi