From 17fd0f328be8339e2cbbaead0908812912b5fde8 Mon Sep 17 00:00:00 2001 From: DStrand1 Date: Tue, 8 Jun 2021 10:12:17 -0500 Subject: [PATCH 1/6] feat: Port influxd inspect verify-seriesfile --- cmd/influxd/inspect/inspect.go | 2 + .../inspect/verify_seriesfile.go} | 78 ++++++++++++++++++- .../inspect/verify_seriesfile_test.go} | 78 ++++++++----------- 3 files changed, 110 insertions(+), 48 deletions(-) rename cmd/{influx_inspect/verify/seriesfile/verify.go => influxd/inspect/verify_seriesfile.go} (84%) rename cmd/{influx_inspect/verify/seriesfile/verify_test.go => influxd/inspect/verify_seriesfile_test.go} (70%) diff --git a/cmd/influxd/inspect/inspect.go b/cmd/influxd/inspect/inspect.go index b90d3b57111..bf6cd43f35f 100644 --- a/cmd/influxd/inspect/inspect.go +++ b/cmd/influxd/inspect/inspect.go @@ -23,5 +23,7 @@ func NewCommand(v *viper.Viper) (*cobra.Command, error) { base.AddCommand(exportLp) base.AddCommand(NewExportIndexCommand()) + base.AddCommand(NewVerifySeriesfileCommand()) + return base, nil } diff --git a/cmd/influx_inspect/verify/seriesfile/verify.go b/cmd/influxd/inspect/verify_seriesfile.go similarity index 84% rename from cmd/influx_inspect/verify/seriesfile/verify.go rename to cmd/influxd/inspect/verify_seriesfile.go index d4bc860d4b6..63f2a027d78 100644 --- a/cmd/influx_inspect/verify/seriesfile/verify.go +++ b/cmd/influxd/inspect/verify_seriesfile.go @@ -1,4 +1,4 @@ -package seriesfile +package inspect import ( "fmt" @@ -9,10 +9,86 @@ import ( "sort" "sync" + "github.com/influxdata/influxdb/v2/logger" "github.com/influxdata/influxdb/v2/tsdb" + "github.com/spf13/cobra" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) +type args struct { + dir string + db string + seriesFile string + verbose bool + concurrent int +} + +func NewVerifySeriesfileCommand() *cobra.Command { + var arguments args + cmd := &cobra.Command{ + Use: "verify-seriesfile", + Short: "Verifies the integrity of series files.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SetOut(os.Stdout) + + config := logger.NewConfig() + config.Level = zapcore.WarnLevel + if arguments.verbose { + config.Level = zapcore.InfoLevel + } + log, err := config.New(os.Stderr) + if err != nil { + return err + } + + v := NewVerify() + v.Logger = log + v.Concurrent = arguments.concurrent + + if arguments.seriesFile != "" { + _, err := v.VerifySeriesFile(arguments.seriesFile) + return err + } + + if arguments.db != "" { + _, err := v.VerifySeriesFile(filepath.Join(arguments.dir, arguments.db, "_series")) + return err + } + + dbs, err := ioutil.ReadDir(arguments.dir) + if err != nil { + return err + } + + for _, db := range dbs { + if !db.IsDir() { + continue + } + _, err := v.VerifySeriesFile(filepath.Join(arguments.dir, db.Name(), "_series")) + if err != nil { + return err + } + } + return nil + }, + } + + cmd.Flags().StringVar(&arguments.dir, "dir", filepath.Join(os.Getenv("HOME"), ".influxdbv2", "engine", "data"), + "Data Directory.") + cmd.Flags().StringVar(&arguments.db, "db", "", + "Only use this database inside of the data directory.") + cmd.Flags().StringVar(&arguments.seriesFile, "series-file", "", + "Path to a series file. This overrides --db and --dir.") + cmd.Flags().BoolVar(&arguments.verbose, "v", false, + "Verbose output.") + cmd.Flags().IntVar(&arguments.concurrent, "c", runtime.GOMAXPROCS(0), + "How many concurrent workers to run.") + + return cmd +} + // verifyResult contains the result of a Verify... call type verifyResult struct { valid bool diff --git a/cmd/influx_inspect/verify/seriesfile/verify_test.go b/cmd/influxd/inspect/verify_seriesfile_test.go similarity index 70% rename from cmd/influx_inspect/verify/seriesfile/verify_test.go rename to cmd/influxd/inspect/verify_seriesfile_test.go index 97cf20c7c08..ba6071b2562 100644 --- a/cmd/influx_inspect/verify/seriesfile/verify_test.go +++ b/cmd/influxd/inspect/verify_seriesfile_test.go @@ -1,4 +1,4 @@ -package seriesfile_test +package inspect import ( "fmt" @@ -9,33 +9,41 @@ import ( "testing" "time" - "github.com/influxdata/influxdb/v2/cmd/influx_inspect/verify/seriesfile" "github.com/influxdata/influxdb/v2/models" "github.com/influxdata/influxdb/v2/tsdb" + "github.com/stretchr/testify/require" "go.uber.org/zap" ) +func TestVerifies_BasicCobra(t *testing.T) { + test := NewTest(t) + defer os.RemoveAll(test.Path) + + verify := NewVerifySeriesfileCommand() + verify.SetArgs([]string{"--dir", test.Path}) + require.NoError(t, verify.Execute()) +} + func TestVerifies_Valid(t *testing.T) { test := NewTest(t) - defer test.Close() + defer os.RemoveAll(test.Path) - verify := seriesfile.NewVerify() + verify := NewVerify() if testing.Verbose() { verify.Logger, _ = zap.NewDevelopment() } passed, err := verify.VerifySeriesFile(test.Path) - test.AssertNoError(err) - test.Assert(passed) + require.NoError(t, err) + require.True(t, passed) } func TestVerifies_Invalid(t *testing.T) { test := NewTest(t) - defer test.Close() + defer os.RemoveAll(test.Path) + + require.NoError(t, filepath.Walk(test.Path, func(path string, info os.FileInfo, err error) error { + require.NoError(t, err) - test.AssertNoError(filepath.Walk(test.Path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } if info.IsDir() { return nil } @@ -44,25 +52,21 @@ func TestVerifies_Invalid(t *testing.T) { defer test.Restore(path) fh, err := os.OpenFile(path, os.O_RDWR, 0) - test.AssertNoError(err) + require.NoError(t, err) defer fh.Close() - _, err = fh.WriteAt([]byte("BOGUS"), 0) - test.AssertNoError(err) - test.AssertNoError(fh.Close()) + _, err = fh.WriteAt([]byte("foobar"), 0) + require.NoError(t, err) + require.NoError(t, fh.Close()) - passed, err := seriesfile.NewVerify().VerifySeriesFile(test.Path) - test.AssertNoError(err) - test.Assert(!passed) + passed, err := NewVerify().VerifySeriesFile(test.Path) + require.NoError(t, err) + require.False(t, passed) return nil })) } -// -// helpers -// - type Test struct { *testing.T Path string @@ -72,9 +76,7 @@ func NewTest(t *testing.T) *Test { t.Helper() dir, err := ioutil.TempDir("", "verify-seriesfile-") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // create a series file in the directory err = func() error { @@ -135,39 +137,21 @@ func NewTest(t *testing.T) *Test { } } -func (t *Test) Close() { - os.RemoveAll(t.Path) -} - -func (t *Test) AssertNoError(err error) { - t.Helper() - if err != nil { - t.Fatal("unexpected error:", err) - } -} - -func (t *Test) Assert(x bool) { - t.Helper() - if !x { - t.Fatal("unexpected condition") - } -} - // Backup makes a copy of the path for a later Restore. func (t *Test) Backup(path string) { in, err := os.Open(path) - t.AssertNoError(err) + require.NoError(t.T, err) defer in.Close() out, err := os.Create(path + ".backup") - t.AssertNoError(err) + require.NoError(t.T, err) defer out.Close() _, err = io.Copy(out, in) - t.AssertNoError(err) + require.NoError(t.T, err) } // Restore restores the file at the path to the time when Backup was called last. func (t *Test) Restore(path string) { - t.AssertNoError(os.Rename(path+".backup", path)) + require.NoError(t.T, os.Rename(path+".backup", path)) } From d7805a5293ec5a7a77b44993de09d3ea6419ef95 Mon Sep 17 00:00:00 2001 From: DStrand1 Date: Tue, 8 Jun 2021 12:51:51 -0500 Subject: [PATCH 2/6] chore: clean up logic, use zaptest logger --- cmd/influxd/inspect/verify_seriesfile.go | 22 +++++++++++-------- cmd/influxd/inspect/verify_seriesfile_test.go | 7 +++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/influxd/inspect/verify_seriesfile.go b/cmd/influxd/inspect/verify_seriesfile.go index 63f2a027d78..754a2c62c4d 100644 --- a/cmd/influxd/inspect/verify_seriesfile.go +++ b/cmd/influxd/inspect/verify_seriesfile.go @@ -47,13 +47,14 @@ func NewVerifySeriesfileCommand() *cobra.Command { v.Logger = log v.Concurrent = arguments.concurrent + var db string if arguments.seriesFile != "" { - _, err := v.VerifySeriesFile(arguments.seriesFile) - return err + db = arguments.seriesFile + } else if arguments.db != "" { + db = filepath.Join(arguments.dir, arguments.db, "_series") } - - if arguments.db != "" { - _, err := v.VerifySeriesFile(filepath.Join(arguments.dir, arguments.db, "_series")) + if db != "" { + _, err := v.VerifySeriesFile(db) return err } @@ -66,12 +67,15 @@ func NewVerifySeriesfileCommand() *cobra.Command { if !db.IsDir() { continue } - _, err := v.VerifySeriesFile(filepath.Join(arguments.dir, db.Name(), "_series")) - if err != nil { - return err + _, errNew := v.VerifySeriesFile(filepath.Join(arguments.dir, db.Name(), "_series")) + if errNew != nil { + v.Logger.Error("Failed to verify series file", + zap.String("filename", fmt.Sprintf(filepath.Join(arguments.dir, db.Name(), "_series"))), + zap.Error(errNew)) + err = errNew } } - return nil + return err }, } diff --git a/cmd/influxd/inspect/verify_seriesfile_test.go b/cmd/influxd/inspect/verify_seriesfile_test.go index ba6071b2562..0b9c0cacf57 100644 --- a/cmd/influxd/inspect/verify_seriesfile_test.go +++ b/cmd/influxd/inspect/verify_seriesfile_test.go @@ -12,7 +12,7 @@ import ( "github.com/influxdata/influxdb/v2/models" "github.com/influxdata/influxdb/v2/tsdb" "github.com/stretchr/testify/require" - "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) func TestVerifies_BasicCobra(t *testing.T) { @@ -29,9 +29,8 @@ func TestVerifies_Valid(t *testing.T) { defer os.RemoveAll(test.Path) verify := NewVerify() - if testing.Verbose() { - verify.Logger, _ = zap.NewDevelopment() - } + verify.Logger = zaptest.NewLogger(t) + passed, err := verify.VerifySeriesFile(test.Path) require.NoError(t, err) require.True(t, passed) From e90d9834a809d4eb8a912dd7447139204a995a4c Mon Sep 17 00:00:00 2001 From: DStrand1 Date: Tue, 8 Jun 2021 13:40:22 -0500 Subject: [PATCH 3/6] chore: better error handling --- cmd/influxd/inspect/verify_seriesfile.go | 10 ++++++++-- cmd/influxd/inspect/verify_seriesfile_test.go | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/influxd/inspect/verify_seriesfile.go b/cmd/influxd/inspect/verify_seriesfile.go index 754a2c62c4d..ca32725f8d7 100644 --- a/cmd/influxd/inspect/verify_seriesfile.go +++ b/cmd/influxd/inspect/verify_seriesfile.go @@ -63,18 +63,24 @@ func NewVerifySeriesfileCommand() *cobra.Command { return err } + var hasError bool for _, db := range dbs { if !db.IsDir() { continue } - _, errNew := v.VerifySeriesFile(filepath.Join(arguments.dir, db.Name(), "_series")) + filePath := filepath.Join(arguments.dir, db.Name(), "_series") + _, errNew := v.VerifySeriesFile(filePath) if errNew != nil { v.Logger.Error("Failed to verify series file", - zap.String("filename", fmt.Sprintf(filepath.Join(arguments.dir, db.Name(), "_series"))), + zap.String("filename", filePath), zap.Error(errNew)) + hasError = true err = errNew } } + if hasError { + v.Logger.Info("Some files failed verification, see logs for details") + } return err }, } diff --git a/cmd/influxd/inspect/verify_seriesfile_test.go b/cmd/influxd/inspect/verify_seriesfile_test.go index 0b9c0cacf57..151b61a5d99 100644 --- a/cmd/influxd/inspect/verify_seriesfile_test.go +++ b/cmd/influxd/inspect/verify_seriesfile_test.go @@ -58,7 +58,10 @@ func TestVerifies_Invalid(t *testing.T) { require.NoError(t, err) require.NoError(t, fh.Close()) - passed, err := NewVerify().VerifySeriesFile(test.Path) + verify := NewVerify() + verify.Logger = zaptest.NewLogger(t) + + passed, err := verify.VerifySeriesFile(test.Path) require.NoError(t, err) require.False(t, passed) From 38199832f7a6daca4cf389050e9aefc0b06a327f Mon Sep 17 00:00:00 2001 From: DStrand1 Date: Tue, 8 Jun 2021 13:57:38 -0500 Subject: [PATCH 4/6] chore: generic error return statement --- cmd/influxd/inspect/verify_seriesfile.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/influxd/inspect/verify_seriesfile.go b/cmd/influxd/inspect/verify_seriesfile.go index ca32725f8d7..39ee09e2be2 100644 --- a/cmd/influxd/inspect/verify_seriesfile.go +++ b/cmd/influxd/inspect/verify_seriesfile.go @@ -1,6 +1,7 @@ package inspect import ( + "errors" "fmt" "io/ioutil" "os" @@ -69,19 +70,18 @@ func NewVerifySeriesfileCommand() *cobra.Command { continue } filePath := filepath.Join(arguments.dir, db.Name(), "_series") - _, errNew := v.VerifySeriesFile(filePath) - if errNew != nil { + _, err := v.VerifySeriesFile(filePath) + if err != nil { v.Logger.Error("Failed to verify series file", zap.String("filename", filePath), - zap.Error(errNew)) + zap.Error(err)) hasError = true - err = errNew } } if hasError { - v.Logger.Info("Some files failed verification, see logs for details") + return errors.New("some files failed verification, see logs for details") } - return err + return nil }, } From 98a5a1bcd05c07af46fc434ae964a142f146fff5 Mon Sep 17 00:00:00 2001 From: Dane Strandboge Date: Tue, 8 Jun 2021 14:14:02 -0500 Subject: [PATCH 5/6] chore: collapse nil check into if-block Co-authored-by: Daniel Moran --- cmd/influxd/inspect/verify_seriesfile.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/influxd/inspect/verify_seriesfile.go b/cmd/influxd/inspect/verify_seriesfile.go index 39ee09e2be2..66bdd39e652 100644 --- a/cmd/influxd/inspect/verify_seriesfile.go +++ b/cmd/influxd/inspect/verify_seriesfile.go @@ -70,8 +70,7 @@ func NewVerifySeriesfileCommand() *cobra.Command { continue } filePath := filepath.Join(arguments.dir, db.Name(), "_series") - _, err := v.VerifySeriesFile(filePath) - if err != nil { + if _, err := v.VerifySeriesFile(filePath); err != nil { v.Logger.Error("Failed to verify series file", zap.String("filename", filePath), zap.Error(err)) From 01658a7d4626541c22abd13fed9690706d2d162c Mon Sep 17 00:00:00 2001 From: DStrand1 Date: Tue, 8 Jun 2021 14:16:27 -0500 Subject: [PATCH 6/6] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1061fbcb45f..f9858349f88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This release adds an embedded SQLite database for storing metadata required by t 1. [21543](https://github.com/influxdata/influxdb/pull/21543): Updated `influxd` configuration flag `--store` to work with string values `disk` or `memory`. Memory continues to store metadata in-memory for testing; `disk` will persist metadata to disk via bolt and SQLite 1. [21547](https://github.com/influxdata/influxdb/pull/21547): Allow hiding the tooltip independently of the static legend 1. [21584](https://github.com/influxdata/influxdb/pull/21584): Added the `api/v2/backup/metadata` endpoint for backing up both KV and SQL metadata, and the `api/v2/restore/sql` for restoring SQL metadata. +1. [21635](https://github.com/influxdata/influxdb/pull/21635): Port `influxd inspect verify-seriesfile` to 2.x ### Bug Fixes