From 3af328ac6f59f6ced9d9fb8b0d8ffb5d373372f2 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 08:38:29 -0700 Subject: [PATCH 1/7] First pass at construction end conditions --- configuration/configuration.go | 17 ++- go.mod | 2 +- go.sum | 3 + pkg/tester/construction.go | 57 ++++++++ pkg/tester/construction_results.go | 218 +++++++++++++++++++++++++++++ pkg/tester/data.go | 12 +- pkg/tester/data_results.go | 4 +- 7 files changed, 299 insertions(+), 14 deletions(-) create mode 100644 pkg/tester/construction_results.go diff --git a/configuration/configuration.go b/configuration/configuration.go index c26e3ce5..997c3bf8 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -79,9 +79,6 @@ var ( } ) -// TODO: Add support for sophisticated end conditions -// (https://github.com/coinbase/rosetta-cli/issues/66) - // ConstructionConfiguration contains all configurations // to run check:construction. type ConstructionConfiguration struct { @@ -119,12 +116,23 @@ type ConstructionConfiguration struct { // PrefundedAccounts is an array of prefunded accounts // to use while testing. - PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts"` + PrefundedAccounts []*storage.PrefundedAccount `json:"prefunded_accounts,omitempty"` // Workflows are executed by the rosetta-cli to test // certain construction flows. Make sure to define a // "request_funds" and "create_account" workflow. Workflows []*job.Workflow `json:"workflows"` + + // EndConditions is a map of workflow:count that + // indicates how many of each workflow should be performed + // before check:construction should stop. For example, + // {"create_account": 5} indicates that 5 "create_account" + // workflows should be performed before stopping. + EndConditions map[string]int `json:"end_conditions,omitempty"` + + // ResultsOutputFile is the absolute filepath of where to save + // the results of a check:construction run. + ResultsOutputFile string `json:"results_output_file,omitempty"` } // DefaultDataConfiguration returns the default *DataConfiguration @@ -155,7 +163,6 @@ func DefaultConfiguration() *Configuration { // DataEndConditions contains all the conditions for the syncer to stop // when running check:data. -// Only 1 end condition can be populated at once! type DataEndConditions struct { // Index configures the syncer to stop once reaching a particular block height. Index *int64 `json:"index,omitempty"` diff --git a/go.mod b/go.mod index f7f71903..cab4ec88 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901005342-2c16b7a9ff30 + github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901051936-be1d05c3bec0 github.com/fatih/color v1.9.0 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c diff --git a/go.sum b/go.sum index 9a5320e6..4c3880d5 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/coinbase/rosetta-sdk-go v0.3.4 h1:jWKgajozco/T0FNnZb2TqBsmsUoF6ZuCLnUJkEE+vNg= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200824221853-d7b1fe2f9239 h1:eP88DoIkcbMG5zHja3lUh4F2c+9am1T0zk/AChy1Vpo= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200824221853-d7b1fe2f9239/go.mod h1:/dKD5dZJ0bzDXrd/M3XMQmpXRk0bru3s9EMIeyBJfy4= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200825180137-4b6b8c12e59e h1:gPomoE/+wkTz6/eFWdugcOvWlH/FjiKjNgsIrHrrhZ0= @@ -109,6 +110,8 @@ github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831200955-3c1868507a81 h1:6R/X/ github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200831200955-3c1868507a81/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901005342-2c16b7a9ff30 h1:beBdN7OZ0OO/wjADaievjIE0OAuk0qb4zBd2wNn9NfQ= github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901005342-2c16b7a9ff30/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= +github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901051936-be1d05c3bec0 h1:KIjma/2YyrC7S0UJoRO0Zdqneixkbks2uwe1lK5KfqU= +github.com/coinbase/rosetta-sdk-go v0.3.5-0.20200901051936-be1d05c3bec0/go.mod h1:ERvHFmf3+JUQB/6ZysASs8CCEcWJMAhN+y/ioqnBsLQ= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index 9ed2bf2c..e5c36917 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log" + "os" "time" "github.com/coinbase/rosetta-cli/configuration" @@ -51,7 +52,12 @@ type ConstructionTester struct { onlineFetcher *fetcher.Fetcher broadcastStorage *storage.BroadcastStorage blockStorage *storage.BlockStorage + jobStorage *storage.JobStorage coordinator *coordinator.Coordinator + cancel context.CancelFunc + signalReceived *bool + + reachedEndConditions bool } // InitializeConstruction initiates the construction API tester. @@ -61,6 +67,7 @@ func InitializeConstruction( network *types.NetworkIdentifier, onlineFetcher *fetcher.Fetcher, cancel context.CancelFunc, + signalReceived *bool, ) (*ConstructionTester, error) { dataPath, err := utils.CreateCommandPath(config.DataDirectory, constructionCmdName, network) if err != nil { @@ -237,7 +244,10 @@ func InitializeConstruction( coordinator: coordinator, broadcastStorage: broadcastStorage, blockStorage: blockStorage, + jobStorage: jobStorage, onlineFetcher: onlineFetcher, + cancel: cancel, + signalReceived: signalReceived, }, nil } @@ -329,3 +339,50 @@ func (t *ConstructionTester) PerformBroadcasts(ctx context.Context) error { return nil } + +func (t *ConstructionTester) WatchEndConditions( + ctx context.Context, +) error { + endConditions := t.config.Construction.EndConditions + if endConditions == nil { + return nil + } + + tc := time.NewTicker(EndAtTipCheckInterval) + defer tc.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tc.C: + conditionsMet := true + for workflow, minOccurences := range endConditions { + completed, err := t.jobStorage.Completed(ctx, workflow) + if err != nil { + return fmt.Errorf("%w: unable to fetch completed %s", err, workflow) + } + + if len(completed) < minOccurences { + conditionsMet = false + break + } + } + + if conditionsMet { + t.reachedEndConditions = true + t.cancel() + return nil + } + } + } +} + +// HandleErr is called when `check:construction` returns an error. +func (t *ConstructionTester) HandleErr(err error) { + if *t.signalReceived { + color.Red("Check halted") + os.Exit(1) + return + } +} diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go new file mode 100644 index 00000000..f6d7b1e1 --- /dev/null +++ b/pkg/tester/construction_results.go @@ -0,0 +1,218 @@ +package tester + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + + "github.com/coinbase/rosetta-cli/configuration" + + "github.com/coinbase/rosetta-sdk-go/storage" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/fatih/color" + "github.com/olekukonko/tablewriter" +) + +// TODO: add test output (like check data) +type CheckConstructionResults struct { + Error string `json:"error"` + EndConditions map[string]int `json:"end_conditions"` + Stats *CheckConstructionStats `json:"stats"` +} + +func (c *CheckConstructionResults) Print() { + if len(c.Error) > 0 { + fmt.Printf("\n") + color.Red("Error: %s", c.Error) + } + + fmt.Printf("\n") + color.Green("Success: %s", types.PrintStruct(c.EndConditions)) + + fmt.Printf("\n") + c.Stats.Print() + fmt.Printf("\n") +} + +func (c *CheckConstructionResults) Output(path string) { + if len(path) > 0 { + writeErr := utils.SerializeAndWrite(path, c) + if writeErr != nil { + log.Printf("%s: unable to save results\n", writeErr.Error()) + } + } +} + +func ComputeCheckConstructionResults( + cfg *configuration.Configuration, + err error, + counterStorage *storage.CounterStorage, + jobStorage *storage.JobStorage, +) *CheckConstructionResults { + ctx := context.Background() + stats := ComputeCheckConstructionStats(ctx, cfg, counterStorage, jobStorage) + results := &CheckConstructionResults{ + Stats: stats, + } + + if err != nil { + results.Error = err.Error() + + // We never want to populate an end condition + // if there was an error! + return results + } + + results.EndConditions = cfg.Construction.EndConditions + + return results +} + +type CheckConstructionStats struct { + TransactionsConfirmed int64 `json:"transactions_confirmed"` + TransactionsCreated int64 `json:"transactions_created"` + StaleBroadcasts int64 `json:"stale_broadcasts"` + FailedBroadcasts int64 `json:"failed_broadcasts"` + AddressesCreated int64 `json:"addresses_created"` + + WorkflowsCompleted map[string]int64 `json:"workflows_completed"` +} + +func (c *CheckConstructionStats) PrintCounts() { + table := tablewriter.NewWriter(os.Stdout) + table.SetRowLine(true) + table.SetRowSeparator("-") + table.SetHeader([]string{"check:construction Stats", "Description", "Value"}) + table.Append([]string{ + "Addresses Created", + "# of addresses created", + strconv.FormatInt(c.AddressesCreated, 10), + }) + table.Append([]string{ + "Transactions Created", + "# of transactions created", + strconv.FormatInt(c.TransactionsCreated, 10), + }) + table.Append([]string{ + "Stale Broadcasts", + "# of broadcasts missing after stale depth", + strconv.FormatInt(c.StaleBroadcasts, 10), + }) + table.Append([]string{ + "Transactions Confirmed", + "# of transactions seen on-chain", + strconv.FormatInt(c.TransactionsConfirmed, 10), + }) + table.Append([]string{ + "Failed Broadcasts", + "# of transactions that exceeded broadcast limit", + strconv.FormatInt(c.FailedBroadcasts, 10), + }) + + table.Render() +} + +func (c *CheckConstructionStats) PrintWorkflows() { + table := tablewriter.NewWriter(os.Stdout) + table.SetRowLine(true) + table.SetRowSeparator("-") + table.SetHeader([]string{"check:construction Workflows", "Count"}) + for workflow, count := range c.WorkflowsCompleted { + table.Append([]string{ + workflow, + strconv.FormatInt(count, 10), + }) + } + + table.Render() +} + +func (c *CheckConstructionStats) Print() { + c.PrintCounts() + c.PrintWorkflows() +} + +func ComputeCheckConstructionStats( + ctx context.Context, + config *configuration.Configuration, + counters *storage.CounterStorage, + jobs *storage.JobStorage, +) *CheckConstructionStats { + if counters == nil { + return nil + } + + transactionsCreated, err := counters.Get(ctx, storage.TransactionsCreatedCounter) + if err != nil { + log.Printf("%s cannot get transactions created counter\n", err.Error()) + return nil + } + + transactionsConfirmed, err := counters.Get(ctx, storage.TransactionsConfirmedCounter) + if err != nil { + log.Printf("%s cannot get transactions confirmed counter\n", err.Error()) + return nil + } + + staleBroadcasts, err := counters.Get(ctx, storage.StaleBroadcastsCounter) + if err != nil { + log.Printf("%s cannot get stale broadcasts counter\n", err) + return nil + } + + failedBroadcasts, err := counters.Get(ctx, storage.FailedBroadcastsCounter) + if err != nil { + log.Printf("%s cannot get failed broadcasts counter\n", err.Error()) + return nil + } + + addressesCreated, err := counters.Get(ctx, storage.AddressesCreatedCounter) + if err != nil { + log.Printf("%s cannot get addresses created counter\n", err.Error()) + return nil + } + + workflowsCompleted := map[string]int64{} + for _, workflow := range config.Construction.Workflows { + completed, err := jobs.Completed(ctx, workflow.Name) + if err != nil { + log.Printf("%s cannot get completed count for %s\n", err.Error(), workflow.Name) + return nil + } + + workflowsCompleted[workflow.Name] = int64(len(completed)) + } + + return &CheckConstructionStats{ + TransactionsCreated: transactionsCreated.Int64(), + TransactionsConfirmed: transactionsConfirmed.Int64(), + StaleBroadcasts: staleBroadcasts.Int64(), + FailedBroadcasts: failedBroadcasts.Int64(), + AddressesCreated: addressesCreated.Int64(), + WorkflowsCompleted: workflowsCompleted, + } +} + +// ExitConstruction exits check:data, logs the test results to the console, +// and to a provided output path. +func ExitConstruction( + config *configuration.Configuration, + counterStorage *storage.CounterStorage, + jobStorage *storage.JobStorage, + err error, + status int, +) { + results := ComputeCheckConstructionResults( + config, + err, + counterStorage, + jobStorage, + ) + results.Print() + results.Output(config.Data.ResultsOutputFile) + + os.Exit(status) +} diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 8b397f2e..d11afe7e 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -482,7 +482,7 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co } if len(t.endCondition) != 0 { - Exit( + ExitData( t.config, t.counterStorage, t.balanceStorage, @@ -495,19 +495,19 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co fmt.Printf("\n") if t.reconcilerHandler.InactiveFailure == nil { - Exit(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } if t.config.Data.HistoricalBalanceDisabled { color.Yellow( "Can't find the block missing operations automatically, please enable --lookup-balance-by-block", ) - Exit(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } if t.config.Data.InactiveDiscrepencySearchDisabled { color.Yellow("Search for inactive reconciliation discrepency is disabled") - Exit(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") + ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "") } t.FindMissingOps(ctx, err, sigListeners) @@ -531,7 +531,7 @@ func (t *DataTester) FindMissingOps( ) if err != nil { color.Yellow("%s: could not find block with missing ops", err.Error()) - Exit(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") } color.Yellow( @@ -541,7 +541,7 @@ func (t *DataTester) FindMissingOps( badBlock.Hash, ) - Exit(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") + ExitData(t.config, t.counterStorage, t.balanceStorage, originalErr, 1, "", "") } func (t *DataTester) recursiveOpSearch( diff --git a/pkg/tester/data_results.go b/pkg/tester/data_results.go index f4d92bdf..93696b0f 100644 --- a/pkg/tester/data_results.go +++ b/pkg/tester/data_results.go @@ -453,9 +453,9 @@ func ComputeCheckDataResults( return results } -// Exit exits the program, logs the test results to the console, +// ExitData exits check:data, logs the test results to the console, // and to a provided output path. -func Exit( +func ExitData( config *configuration.Configuration, counterStorage *storage.CounterStorage, balanceStorage *storage.BalanceStorage, From 6b8075de98de0ab80e1db35ce4f57abe1b56de6c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 09:00:44 -0700 Subject: [PATCH 2/7] Integrate end conditions into cmd/construction --- cmd/check_construction.go | 46 ++++++++++++++++++------------ cmd/check_data.go | 4 +-- pkg/tester/construction.go | 8 ++++++ pkg/tester/construction_results.go | 4 +-- pkg/tester/data.go | 2 +- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/cmd/check_construction.go b/cmd/check_construction.go index 8d022541..c9adb5e0 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -16,15 +16,14 @@ package cmd import ( "context" + "fmt" "log" - "os" "time" "github.com/coinbase/rosetta-cli/pkg/tester" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/utils" - "github.com/fatih/color" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" ) @@ -70,12 +69,24 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx) if fetchErr != nil { - log.Fatalf("%s: unable to initialize asserter", fetchErr.Err.Error()) + tester.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: unable to initialize asserter", fetchErr.Err), + 1, + ) } _, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - log.Fatalf("%s: unable to confirm network is supported", err.Error()) + tester.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: unable to confirm network is supported", err), + 1, + ) } constructionTester, err := tester.InitializeConstruction( @@ -84,9 +95,16 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { Config.Network, fetcher, cancel, + &SignalReceived, ) if err != nil { - log.Fatalf("%s: unable to initialize construction tester", err.Error()) + tester.ExitConstruction( + Config, + nil, + nil, + fmt.Errorf("%w: unable to initialize construction tester", err), + 1, + ) } defer constructionTester.CloseDatabase(ctx) @@ -108,21 +126,13 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) { return constructionTester.StartConstructor(ctx) }) + g.Go(func() error { + return constructionTester.WatchEndConditions(ctx) + }) + sigListeners := []context.CancelFunc{cancel} go handleSignals(sigListeners) err = g.Wait() - if SignalReceived { - color.Red("Check halted") - os.Exit(1) - return - } - - if err != nil { - color.Red("Check failed: %s", err.Error()) - os.Exit(1) - } - - // Will only hit this once exit conditions are added - color.Green("Check succeeded") + constructionTester.HandleErr(err) } diff --git a/cmd/check_data.go b/cmd/check_data.go index 01de949e..2db97447 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -82,7 +82,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { _, _, fetchErr := fetcher.InitializeAsserter(ctx) if fetchErr != nil { - tester.Exit( + tester.ExitData( Config, nil, nil, @@ -95,7 +95,7 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) { networkStatus, err := utils.CheckNetworkSupported(ctx, Config.Network, fetcher) if err != nil { - tester.Exit( + tester.ExitData( Config, nil, nil, diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index e5c36917..cbbf2b5d 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -53,6 +53,7 @@ type ConstructionTester struct { broadcastStorage *storage.BroadcastStorage blockStorage *storage.BlockStorage jobStorage *storage.JobStorage + counterStorage *storage.CounterStorage coordinator *coordinator.Coordinator cancel context.CancelFunc signalReceived *bool @@ -245,6 +246,7 @@ func InitializeConstruction( broadcastStorage: broadcastStorage, blockStorage: blockStorage, jobStorage: jobStorage, + counterStorage: counterStorage, onlineFetcher: onlineFetcher, cancel: cancel, signalReceived: signalReceived, @@ -385,4 +387,10 @@ func (t *ConstructionTester) HandleErr(err error) { os.Exit(1) return } + + if t.reachedEndConditions { + ExitConstruction(t.config, t.counterStorage, t.jobStorage, nil, 0) + } + + ExitConstruction(t.config, t.counterStorage, t.jobStorage, err, 1) } diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go index f6d7b1e1..39e1dced 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/tester/construction_results.go @@ -16,11 +16,11 @@ import ( "github.com/olekukonko/tablewriter" ) -// TODO: add test output (like check data) type CheckConstructionResults struct { Error string `json:"error"` EndConditions map[string]int `json:"end_conditions"` Stats *CheckConstructionStats `json:"stats"` + // TODO: add test output (like check data) } func (c *CheckConstructionResults) Print() { @@ -212,7 +212,7 @@ func ExitConstruction( jobStorage, ) results.Print() - results.Output(config.Data.ResultsOutputFile) + results.Output(config.Construction.ResultsOutputFile) os.Exit(status) } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index d11afe7e..68c47c82 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -472,7 +472,7 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co return } - if (err == nil || err == context.Canceled) && len(t.endCondition) == 0 && t.config.Data.EndConditions != nil && + if (err == nil || errors.Is(err, context.Canceled)) && len(t.endCondition) == 0 && t.config.Data.EndConditions != nil && t.config.Data.EndConditions.Index != nil { // occurs at syncer end t.endCondition = configuration.IndexEndCondition t.endConditionDetail = fmt.Sprintf( From 799ab75bbb554e542ffaadad0b0fd5bbfa42b97b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 09:10:00 -0700 Subject: [PATCH 3/7] nits --- pkg/tester/construction.go | 7 ++++++- pkg/tester/construction_results.go | 31 +++++++++++++++++++++++++++++- pkg/tester/data.go | 3 ++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index cbbf2b5d..850d8881 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -40,6 +40,8 @@ const ( // constructionCmdName is used as the prefix on the data directory // for all data saved using this command. constructionCmdName = "check-construction" + + endConditionsCheckInterval = 10 * time.Second ) // ConstructionTester coordinates the `check:construction` test. @@ -342,6 +344,9 @@ func (t *ConstructionTester) PerformBroadcasts(ctx context.Context) error { return nil } +// WatchEndConditions cancels check:construction once +// all end conditions are met (provided workflows +// are executed at least minOccurences). func (t *ConstructionTester) WatchEndConditions( ctx context.Context, ) error { @@ -350,7 +355,7 @@ func (t *ConstructionTester) WatchEndConditions( return nil } - tc := time.NewTicker(EndAtTipCheckInterval) + tc := time.NewTicker(endConditionsCheckInterval) defer tc.Stop() for { diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go index 39e1dced..9aa407f7 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/tester/construction_results.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tester import ( @@ -16,6 +30,9 @@ import ( "github.com/olekukonko/tablewriter" ) +// CheckConstructionResults contains any error that +// occurred on a check:construction run and a collection +// of interesting stats. type CheckConstructionResults struct { Error string `json:"error"` EndConditions map[string]int `json:"end_conditions"` @@ -23,6 +40,7 @@ type CheckConstructionResults struct { // TODO: add test output (like check data) } +// Print logs CheckConstructionResults to the console. func (c *CheckConstructionResults) Print() { if len(c.Error) > 0 { fmt.Printf("\n") @@ -37,6 +55,8 @@ func (c *CheckConstructionResults) Print() { fmt.Printf("\n") } +// Output writes CheckConstructionResults to the provided +// path. func (c *CheckConstructionResults) Output(path string) { if len(path) > 0 { writeErr := utils.SerializeAndWrite(path, c) @@ -46,6 +66,8 @@ func (c *CheckConstructionResults) Output(path string) { } } +// ComputeCheckConstructionResults returns a populated +// CheckConstructionResults. func ComputeCheckConstructionResults( cfg *configuration.Configuration, err error, @@ -71,6 +93,8 @@ func ComputeCheckConstructionResults( return results } +// CheckConstructionStats contains interesting stats +// that are tracked while running check:construction. type CheckConstructionStats struct { TransactionsConfirmed int64 `json:"transactions_confirmed"` TransactionsCreated int64 `json:"transactions_created"` @@ -81,6 +105,7 @@ type CheckConstructionStats struct { WorkflowsCompleted map[string]int64 `json:"workflows_completed"` } +// PrintCounts logs counter-related stats to the console. func (c *CheckConstructionStats) PrintCounts() { table := tablewriter.NewWriter(os.Stdout) table.SetRowLine(true) @@ -115,6 +140,7 @@ func (c *CheckConstructionStats) PrintCounts() { table.Render() } +// PrintWorkflows logs workflow counts to the console. func (c *CheckConstructionStats) PrintWorkflows() { table := tablewriter.NewWriter(os.Stdout) table.SetRowLine(true) @@ -130,18 +156,21 @@ func (c *CheckConstructionStats) PrintWorkflows() { table.Render() } +// Print calls PrintCounts and PrintWorkflows. func (c *CheckConstructionStats) Print() { c.PrintCounts() c.PrintWorkflows() } +// ComputeCheckConstructionStats returns a populated +// CheckConstructionStats. func ComputeCheckConstructionStats( ctx context.Context, config *configuration.Configuration, counters *storage.CounterStorage, jobs *storage.JobStorage, ) *CheckConstructionStats { - if counters == nil { + if counters == nil || jobs == nil { return nil } diff --git a/pkg/tester/data.go b/pkg/tester/data.go index 68c47c82..36f24be9 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -472,7 +472,8 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co return } - if (err == nil || errors.Is(err, context.Canceled)) && len(t.endCondition) == 0 && t.config.Data.EndConditions != nil && + if (err == nil || errors.Is(err, context.Canceled)) && + len(t.endCondition) == 0 && t.config.Data.EndConditions != nil && t.config.Data.EndConditions.Index != nil { // occurs at syncer end t.endCondition = configuration.IndexEndCondition t.endConditionDetail = fmt.Sprintf( From b8828fbd2de84466f6b274dc8f1ea2f6591dd975 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 09:25:17 -0700 Subject: [PATCH 4/7] Add end conditions to ethereum example --- examples/configuration/ethereum.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index d6423bc5..f8ad5f26 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -21,7 +21,6 @@ "broadcast_behind_tip": false, "block_broadcast_limit": 0, "rebroadcast_all": false, - "prefunded_accounts": null, "workflows": [ { "name": "request_funds", @@ -149,7 +148,11 @@ } ] } - ] + ], + "end_conditions": { + "create_account": 10, + "transfer": 10 + } }, "data": null } \ No newline at end of file From 3db2b801fd1e4d70c2aaa724c39bcf3f14896ee1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 10:34:25 -0700 Subject: [PATCH 5/7] Validate bitcoin example --- Makefile | 1 + examples/configuration/bitcoin.json | 183 +++++++++++++++++++++++++++ examples/configuration/ethereum.json | 23 +++- pkg/tester/construction_results.go | 6 +- 4 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 examples/configuration/bitcoin.json diff --git a/Makefile b/Makefile index 3b28aaff..08975e22 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ check-format: ! gofmt -s -l . | read; validate-configuration-files: + go run main.go configuration:validate examples/configuration/bitcoin.json; go run main.go configuration:validate examples/configuration/ethereum.json; go run main.go configuration:validate examples/configuration/simple.json; go run main.go configuration:create examples/configuration/default.json; diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json new file mode 100644 index 00000000..2cd5f908 --- /dev/null +++ b/examples/configuration/bitcoin.json @@ -0,0 +1,183 @@ +{ + "network": { + "blockchain": "Bitcoin", + "network": "Testnet3" + }, + "online_url": "http://3.227.211.154:8080", + "data_directory": "bitcoin-data", + "http_timeout": 300, + "retry_elapsed_time": 0, + "sync_concurrency": 128, + "transaction_concurrency": 0, + "tip_delay": 1800, + "disable_memory_limit": false, + "log_configuration": false, + "construction": { + "offline_url": "http://3.227.211.154:8080", + "stale_depth": 0, + "broadcast_limit": 0, + "ignore_broadcast_failures": false, + "clear_broadcasts": false, + "broadcast_behind_tip": false, + "block_broadcast_limit": 0, + "rebroadcast_all": false, + "workflows": [ + { + "name": "request_funds", + "concurrency": 1, + "scenarios": [ + { + "name": "find_address", + "actions": [ + { + "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", + "type": "set_variable", + "output_path": "currency" + }, + { + "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", + "type": "find_balance", + "output_path": "random_address" + } + ] + }, + { + "name": "request", + "actions": [ + { + "input": "{\"address\": {{random_address.account.address}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", + "type": "find_balance", + "output_path": "loaded_address" + } + ] + } + ] + }, + { + "name": "create_account", + "concurrency": 1, + "scenarios": [ + { + "name": "create_account", + "actions": [ + { + "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", + "type": "set_variable", + "output_path": "network" + }, + { + "input": "{\"curve_type\": \"secp256k1\"}", + "type": "generate_key", + "output_path": "key" + }, + { + "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", + "type": "derive", + "output_path": "address" + }, + { + "input": "{\"address\": {{address.address}}, \"keypair\": {{key}}}", + "type": "save_address" + } + ] + } + ] + }, + { + "name": "transfer", + "concurrency": 10, + "scenarios": [ + { + "name": "transfer", + "actions": [ + { + "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", + "type": "set_variable", + "output_path": "transfer.network" + }, + { + "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", + "type": "set_variable", + "output_path": "currency" + }, + { + "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": \"1200\", \"currency\": {{currency}}}}", + "type": "find_balance", + "output_path": "sender" + }, + { + "input": "\"600\"", + "type": "set_variable", + "output_path": "max_fee" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee}}}", + "type": "math", + "output_path": "available_amount" + }, + { + "input": "{\"minimum\": \"600\", \"maximum\": {{available_amount}}}", + "type": "random_number", + "output_path": "recipient_amount" + }, + { + "input": "{\"recipient_amount\":{{recipient_amount}}}", + "type": "print_message" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", + "type": "math", + "output_path": "sender_amount" + }, + { + "input": "{\"not_address\":[{{sender.account.address}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", + "type": "find_balance", + "output_path": "recipient" + }, + { + "input": "\"1\"", + "type": "set_variable", + "output_path": "transfer.confirmation_depth" + }, + { + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{\"address\":{{recipient.account.address}}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", + "type": "set_variable", + "output_path": "transfer.operations" + }, + { + "input": "{{transfer.operations}}", + "type": "print_message" + } + ] + } + ] + } + ], + "end_conditions": { + "create_account": 10, + "transfer": 10 + } + }, + "data": { + "active_reconciliation_concurrency": 0, + "inactive_reconciliation_concurrency": 0, + "inactive_reconciliation_frequency": 0, + "log_blocks": false, + "log_transactions": false, + "log_balance_changes": false, + "log_reconciliations": false, + "ignore_reconciliation_error": false, + "exempt_accounts": "", + "bootstrap_balances": "", + "historical_balance_disabled": true, + "interesting_accounts": "", + "reconciliation_disabled": false, + "inactive_discrepency_search_disabled": false, + "balance_tracking_disabled": false, + "coin_tracking_disabled": false, + "end_conditions": { + "reconciliation_coverage": 0.95 + }, + "results_output_file": "" + } +} diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index f8ad5f26..da7096f9 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -154,5 +154,26 @@ "transfer": 10 } }, - "data": null + "data": { + "active_reconciliation_concurrency": 0, + "inactive_reconciliation_concurrency": 0, + "inactive_reconciliation_frequency": 0, + "log_blocks": false, + "log_transactions": false, + "log_balance_changes": false, + "log_reconciliations": false, + "ignore_reconciliation_error": false, + "exempt_accounts": "", + "bootstrap_balances": "", + "historical_balance_disabled": false, + "interesting_accounts": "", + "reconciliation_disabled": false, + "inactive_discrepency_search_disabled": false, + "balance_tracking_disabled": false, + "coin_tracking_disabled": false, + "end_conditions": { + "reconciliation_coverage": 0.95 + }, + "results_output_file": "" + } } \ No newline at end of file diff --git a/pkg/tester/construction_results.go b/pkg/tester/construction_results.go index 9aa407f7..55ff54b0 100644 --- a/pkg/tester/construction_results.go +++ b/pkg/tester/construction_results.go @@ -45,11 +45,11 @@ func (c *CheckConstructionResults) Print() { if len(c.Error) > 0 { fmt.Printf("\n") color.Red("Error: %s", c.Error) + } else { + fmt.Printf("\n") + color.Green("Success: %s", types.PrintStruct(c.EndConditions)) } - fmt.Printf("\n") - color.Green("Success: %s", types.PrintStruct(c.EndConditions)) - fmt.Printf("\n") c.Stats.Print() fmt.Printf("\n") From e4cdcd80379e31b26b7df4a64792b2daecf68fb1 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 11:14:48 -0700 Subject: [PATCH 6/7] Add change example to bitcoin --- examples/configuration/bitcoin.json | 47 ++++++++++++++++++++++------ examples/configuration/ethereum.json | 2 +- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index 2cd5f908..9669dc9f 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -7,7 +7,7 @@ "data_directory": "bitcoin-data", "http_timeout": 300, "retry_elapsed_time": 0, - "sync_concurrency": 128, + "sync_concurrency": 0, "transaction_concurrency": 0, "tip_delay": 1800, "disable_memory_limit": false, @@ -101,22 +101,37 @@ "output_path": "currency" }, { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": \"1200\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" + "input": "\"600\"", + "type": "set_variable", + "output_path": "dust_amount" }, { "input": "\"600\"", "type": "set_variable", - "output_path": "max_fee" + "output_path": "fee_amount" }, { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee}}}", + "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{fee_amount}}}", + "type": "math", + "output_path": "send_buffer" + }, + { + "input": "\"1800\"", + "type": "set_variable", + "output_path": "reserved_amount" + }, + { + "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", + "type": "find_balance", + "output_path": "sender" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{send_buffer}}}", "type": "math", "output_path": "available_amount" }, { - "input": "{\"minimum\": \"600\", \"maximum\": {{available_amount}}}", + "input": "{\"minimum\": {{dust_amount}}, \"maximum\": {{available_amount}}}", "type": "random_number", "output_path": "recipient_amount" }, @@ -124,6 +139,20 @@ "input": "{\"recipient_amount\":{{recipient_amount}}}", "type": "print_message" }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{recipient_amount}}}", + "type": "math", + "output_path": "change_amount" + }, + { + "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{fee_amount}}}", + "type": "math", + "output_path": "change_amount" + }, + { + "input": "{\"change_amount\":{{change_amount}}}", + "type": "print_message" + }, { "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", "type": "math", @@ -140,7 +169,7 @@ "output_path": "transfer.confirmation_depth" }, { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{\"address\":{{recipient.account.address}}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", + "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{\"address\":{{recipient.account.address}}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{\"address\":{{sender.account.address}}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", "type": "set_variable", "output_path": "transfer.operations" }, @@ -180,4 +209,4 @@ }, "results_output_file": "" } -} +} \ No newline at end of file diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json index da7096f9..65ea4d06 100644 --- a/examples/configuration/ethereum.json +++ b/examples/configuration/ethereum.json @@ -4,7 +4,7 @@ "network": "Ropsten" }, "online_url": "", - "data_directory": "", + "data_directory": "ethereum-data", "http_timeout": 300, "retry_elapsed_time": 0, "sync_concurrency": 0, From b76c02c686419c6160055c752803af7f83b0007f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 1 Sep 2020 11:16:43 -0700 Subject: [PATCH 7/7] Nits --- cmd/root.go | 2 +- examples/configuration/bitcoin.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index bf8d0a20..a8fab93a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.4.2") + fmt.Println("v0.5.0") }, } diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json index 9669dc9f..211c322b 100644 --- a/examples/configuration/bitcoin.json +++ b/examples/configuration/bitcoin.json @@ -3,7 +3,7 @@ "blockchain": "Bitcoin", "network": "Testnet3" }, - "online_url": "http://3.227.211.154:8080", + "online_url": "", "data_directory": "bitcoin-data", "http_timeout": 300, "retry_elapsed_time": 0, @@ -13,7 +13,7 @@ "disable_memory_limit": false, "log_configuration": false, "construction": { - "offline_url": "http://3.227.211.154:8080", + "offline_url": "", "stale_depth": 0, "broadcast_limit": 0, "ignore_broadcast_failures": false, @@ -209,4 +209,4 @@ }, "results_output_file": "" } -} \ No newline at end of file +}