diff --git a/Makefile b/Makefile index 450ab6999c..b0e387e0e8 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \ UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ )) ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... )) -MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 +MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/store/catchpointdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2 default: build diff --git a/cmd/catchpointdump/database.go b/cmd/catchpointdump/database.go index 7706bcd3dd..2205b4c524 100644 --- a/cmd/catchpointdump/database.go +++ b/cmd/catchpointdump/database.go @@ -27,6 +27,7 @@ import ( "github.com/algorand/go-algorand/crypto/merkletrie" "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver" "github.com/algorand/go-algorand/util/db" @@ -103,30 +104,14 @@ func printDbVersion(staging bool, version uint64, outFile *os.File) { } func getVersion(filename string, staging bool) (uint64, error) { - dbAccessor, err := db.MakeAccessor(filename, true, false) - if err != nil || dbAccessor.Handle == nil { - return 0, err - } - defer dbAccessor.Close() - var version uint64 - err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - if staging { - // writing the version of the catchpoint file start only on ver >= CatchpointFileVersionV7. - // in case the catchpoint version does not exists ReadCatchpointStateUint64 returns 0 - cw := sqlitedriver.NewCatchpointSQLReaderWriter(tx) - version, err = cw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion) - return err - } - - versionAsInt32, err := db.GetUserVersion(ctx, tx) - version = uint64(versionAsInt32) - return err - }) - if err != nil { + rdb, err := db.MakeAccessor(filename, true, false) + if err != nil || rdb.Handle == nil { return 0, err } + catchpointDbs := catchpointdb.MakeStore(db.Pair{Rdb: rdb, Wdb: rdb}) + defer catchpointDbs.Close() - return version, nil + return catchpointDbs.GetVersion(context.Background(), staging) } var checkCmd = &cobra.Command{ diff --git a/config/localTemplate.go b/config/localTemplate.go index a9dd313bb8..210e82f196 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -41,7 +41,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28"` // environmental (may be overridden) // When enabled, stores blocks indefinitely, otherwise, only the most recent blocks @@ -510,6 +510,12 @@ type Local struct { // EnableTxnEvalTracer turns on features in the BlockEvaluator which collect data on transactions, exposing them via algod APIs. // It will store txn deltas created during block evaluation, potentially consuming much larger amounts of memory, EnableTxnEvalTracer bool `version[27]:"false"` + + // StorageEngine allows to control which type of storage to use for the ledger. + // Available options are: + // - sqlite (default) + // - pebbledb (experimental, in development) + StorageEngine string `version[28]:"sqlite"` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers diff --git a/config/local_defaults.go b/config/local_defaults.go index 81e4a25870..fdfb77a29b 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 27, + Version: 28, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 15, @@ -119,6 +119,7 @@ var defaultLocal = Local{ RestReadTimeoutSeconds: 15, RestWriteTimeoutSeconds: 120, RunHosted: false, + StorageEngine: "sqlite", SuggestedFeeBlockHistory: 3, SuggestedFeeSlidingWindowSize: 50, TLSCertFile: "", diff --git a/go.mod b/go.mod index efaf373227..b4ce0b3091 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/algorand/oapi-codegen v1.12.0-algorand.0 github.com/algorand/websocket v1.4.6 github.com/aws/aws-sdk-go v1.33.0 + github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 github.com/consensys/gnark-crypto v0.7.0 github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 github.com/dchest/siphash v1.2.1 @@ -21,6 +22,7 @@ require ( github.com/getkin/kin-openapi v0.107.0 github.com/gofrs/flock v0.7.0 github.com/golang/snappy v0.0.4 + github.com/google/go-cmp v0.5.6 github.com/google/go-querystring v1.0.0 github.com/gorilla/mux v1.8.0 github.com/jmoiron/sqlx v1.2.0 @@ -30,7 +32,7 @@ require ( github.com/miekg/dns v1.1.27 github.com/olivere/elastic v6.2.14+incompatible github.com/sirupsen/logrus v1.8.1 - github.com/spf13/cobra v0.0.3 + github.com/spf13/cobra v1.0.0 github.com/stretchr/testify v1.8.1 golang.org/x/crypto v0.1.0 golang.org/x/sys v0.7.0 @@ -41,35 +43,54 @@ require ( require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/cpuguy83/go-md2man v1.0.8 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cockroachdb/errors v1.8.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/redact v1.0.8 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fortytw2/leaktest v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/invopop/yaml v0.1.0 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.11.13 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/russross/blackfriday v1.5.2 // indirect + github.com/prometheus/client_golang v1.12.0 // indirect + github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect + golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/term v0.7.0 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index f173199fbe..304e431521 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,54 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k= github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g= github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 h1:nwYN+GQ7Z5OOfZwqBO1ma7DSlP7S1YrKWICOyjkwqrc= @@ -21,14 +69,52 @@ github.com/algorand/websocket v1.4.6 h1:I0kV4EYwatuUrKtNiwzYYgojgwh6pksDmlqntKG2 github.com/algorand/websocket v1.4.6/go.mod h1:HJmdGzFtnlUQ4nTzZP6WrT29oGYf1t6Ybi64vROcT+M= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY= github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e h1:CHPYEbz71w8DqJ7DRIq+MXyCQsdibK08vdcQTY4ufas= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 h1:Mp4A1rMoAE45WdoVI7a7jwjiIj8eZEXkl3aYbdMJACs= +github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4/go.mod h1:rWEpkT1ud5qGG2m1HqTBRWbLgi+oodC4+BWC46uVPjw= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/gnark-crypto v0.7.0 h1:rwdy8+ssmLYRqKp+ryRRgQJl/rCq2uv+n83cOydm5UE= github.com/consensys/gnark-crypto v0.7.0/go.mod h1:KPSuJzyxkJA8xZ/+CV47tyqkr9MmpZA3PXivK4VPrVg= -github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M= -github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,12 +122,45 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4= github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getkin/kin-openapi v0.107.0 h1:bxhL6QArW7BXQj8NjXfIJQy680NsMKd25nwhvpCXchg= github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= @@ -49,153 +168,667 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g= github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.9.1 h1:GliPYSpzGKlyOhqIbG8nmHBo3i1saKWFOgh41AN3b+Y= github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olivere/elastic v6.2.14+incompatible h1:k+KadwNP/dkXE0/eu+T6otk1+5fe0tEpPyQJ4XVm5i8= github.com/olivere/elastic v6.2.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 h1:q/fZgS8MMadqFFGa8WL4Oyz+TmjiZfi8UrzWhTl8d5w= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009/go.mod h1:O0bY1e/dSoxMYZYTHP0SWKxG5EWLEvKR9/cOjWPPMKU= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -203,5 +836,15 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= pgregory.net/rapid v0.4.8 h1:d+5SGZWUbJPbl3ss6tmPFqnNeQR6VDOFly+eTjwPiEw= pgregory.net/rapid v0.4.8/go.mod h1:Z5PbWqjvWR1I3UGjvboUuan4fe4ZYEYNLNQLExzCoUs= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/installer/config.json.example b/installer/config.json.example index 4ae34a5716..31da4c1ac5 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 27, + "Version": 28, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 15, @@ -98,6 +98,7 @@ "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, + "StorageEngine": "sqlite", "SuggestedFeeBlockHistory": 3, "SuggestedFeeSlidingWindowSize": 50, "TLSCertFile": "", diff --git a/ledger/acctdeltas.go b/ledger/acctdeltas.go index e587a8fcfa..42d5d93b2f 100644 --- a/ledger/acctdeltas.go +++ b/ledger/acctdeltas.go @@ -298,7 +298,7 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope, if len(a.misses) == 0 { return nil } - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } @@ -317,9 +317,9 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope, if delta.oldResource.AcctRef != nil { acctRef = delta.oldResource.AcctRef } else if acctRef, ok = knownAddresses[addr]; !ok { - acctRef, err = arw.LookupAccountRowID(addr) + acctRef, err = ar.LookupAccountRowID(addr) if err != nil { - if err != sql.ErrNoRows { + if err != sql.ErrNoRows && err != trackerdb.ErrNotFound { err = fmt.Errorf("base account cannot be read while processing resource for addr=%s, aidx=%d: %w", addr.String(), aidx, err) return err @@ -330,7 +330,7 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope, continue } } - resDataBuf, err = arw.LookupResourceDataByAddrID(acctRef, aidx) + resDataBuf, err = ar.LookupResourceDataByAddrID(acctRef, aidx) switch err { case nil: if len(resDataBuf) > 0 { @@ -344,6 +344,10 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope, err = fmt.Errorf("empty resource record: addrid=%d, aidx=%d", acctRef, aidx) return err } + case trackerdb.ErrNotFound: + // we don't have that account, just return an empty record. + a.updateOld(missIdx, trackerdb.PersistedResourcesData{AcctRef: acctRef, Aidx: aidx}) + err = nil case sql.ErrNoRows: // we don't have that account, just return an empty record. a.updateOld(missIdx, trackerdb.PersistedResourcesData{AcctRef: acctRef, Aidx: aidx}) @@ -466,7 +470,7 @@ func (a *compactAccountDeltas) accountsLoadOld(tx trackerdb.TransactionScope) (e if len(a.misses) == 0 { return nil } - arw, err := tx.MakeAccountsReaderWriter() + arw, err := tx.MakeAccountsOptimizedReader() if err != nil { return err } @@ -476,29 +480,13 @@ func (a *compactAccountDeltas) accountsLoadOld(tx trackerdb.TransactionScope) (e }() for _, idx := range a.misses { addr := a.deltas[idx].address - ref, acctDataBuf, err := arw.LookupAccountDataByAddress(addr) - switch err { - case nil: - if len(acctDataBuf) > 0 { - persistedAcctData := &trackerdb.PersistedAccountData{Addr: addr, Ref: ref} - err = protocol.Decode(acctDataBuf, &persistedAcctData.AccountData) - if err != nil { - return err - } - a.updateOld(idx, *persistedAcctData) - } else { - // to retain backward compatibility, we will treat this condition as if we don't have the account. - a.updateOld(idx, trackerdb.PersistedAccountData{Addr: addr, Ref: ref}) - } - case sql.ErrNoRows: - // we don't have that account, just return an empty record. - a.updateOld(idx, trackerdb.PersistedAccountData{Addr: addr}) - // Note: the err will be ignored in this case since `err` is being shadowed. - // this behaviour is equivalent to `err = nil` - default: + data, err := arw.LookupAccount(addr) + if err != nil { // unexpected error - let the caller know that we couldn't complete the operation. return err } + // update the account + a.updateOld(idx, data) } return } @@ -607,7 +595,7 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco if len(a.misses) == 0 { return nil } - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } @@ -616,7 +604,7 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco }() for _, idx := range a.misses { addr := a.deltas[idx].address - ref, acctDataBuf, err := arw.LookupOnlineAccountDataByAddress(addr) + ref, acctDataBuf, err := ar.LookupOnlineAccountDataByAddress(addr) switch err { case nil: if len(acctDataBuf) > 0 { @@ -630,6 +618,10 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco // empty data means offline account a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr, Ref: ref}) } + case trackerdb.ErrNotFound: + // we don't have that account, just return an empty record. + a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr}) + // TODO: phase out sql.ErrNoRows case sql.ErrNoRows: // we don't have that account, just return an empty record. a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr}) diff --git a/ledger/acctdeltas_test.go b/ledger/acctdeltas_test.go index 2dc748225b..0342f3fc33 100644 --- a/ledger/acctdeltas_test.go +++ b/ledger/acctdeltas_test.go @@ -52,14 +52,14 @@ import ( ) func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round, accts map[basics.Address]basics.AccountData) { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() require.NoError(t, err) - r, err := arw.AccountsRound() + r, err := ar.AccountsRound() require.NoError(t, err) require.Equal(t, r, rnd) - aor, err := tx.Testing().MakeAccountsOptimizedReader() + aor, err := tx.MakeAccountsOptimizedReader() require.NoError(t, err) var totalOnline, totalOffline, totalNotPart uint64 @@ -83,11 +83,11 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round } } - all, err := arw.Testing().AccountsAllTest() + all, err := ar.Testing().AccountsAllTest() require.NoError(t, err) require.Equal(t, all, accts) - totals, err := arw.AccountsTotals(context.Background(), false) + totals, err := ar.AccountsTotals(context.Background(), false) require.NoError(t, err) require.Equal(t, totalOnline, totals.Online.Money.Raw, "mismatching total online money") require.Equal(t, totalOffline, totals.Offline.Money.Raw) @@ -129,7 +129,7 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round }) for i := 0; i < len(onlineAccounts); i++ { - dbtop, err := arw.AccountsOnlineTop(rnd, 0, uint64(i), proto) + dbtop, err := ar.AccountsOnlineTop(rnd, 0, uint64(i), proto) require.NoError(t, err) require.Equal(t, i, len(dbtop)) @@ -139,7 +139,7 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round } } - top, err := arw.AccountsOnlineTop(rnd, 0, uint64(len(onlineAccounts)+1), proto) + top, err := ar.AccountsOnlineTop(rnd, 0, uint64(len(onlineAccounts)+1), proto) require.NoError(t, err) require.Equal(t, len(top), len(onlineAccounts)) } @@ -149,8 +149,7 @@ func TestAccountDBInit(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { @@ -210,20 +209,21 @@ func TestAccountDBRound(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() + require.NoError(t, err) + aw, err := tx.MakeAccountsWriter() require.NoError(t, err) accts := ledgertesting.RandomAccounts(20, true) tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion) checkAccounts(t, tx, 0, accts) - totals, err := arw.AccountsTotals(context.Background(), false) + totals, err := ar.AccountsTotals(context.Background(), false) require.NoError(t, err) - expectedOnlineRoundParams, endRound, err := arw.AccountsOnlineRoundParams() + expectedOnlineRoundParams, endRound, err := ar.AccountsOnlineRoundParams() require.NoError(t, err) require.Equal(t, 1, len(expectedOnlineRoundParams)) require.Equal(t, 0, int(endRound)) @@ -269,10 +269,10 @@ func TestAccountDBRound(t *testing.T) { err = resourceUpdatesCnt.resourcesLoadOld(tx, knownAddresses) require.NoError(t, err) - err = arw.AccountsPutTotals(totals, false) + err = aw.AccountsPutTotals(totals, false) require.NoError(t, err) onlineRoundParams := ledgercore.OnlineRoundParamsData{RewardsLevel: totals.RewardsLevel, OnlineSupply: totals.Online.Money.Raw, CurrentProtocol: protocol.ConsensusCurrentVersion} - err = arw.AccountsPutOnlineRoundParams([]ledgercore.OnlineRoundParamsData{onlineRoundParams}, basics.Round(i)) + err = aw.AccountsPutOnlineRoundParams([]ledgercore.OnlineRoundParamsData{onlineRoundParams}, basics.Round(i)) require.NoError(t, err) expectedOnlineRoundParams = append(expectedOnlineRoundParams, onlineRoundParams) @@ -289,7 +289,7 @@ func TestAccountDBRound(t *testing.T) { updatedOnlineAccts, err := onlineAccountsNewRound(tx, updatesOnlineCnt, proto, basics.Round(i)) require.NoError(t, err) - err = arw.UpdateAccountsRound(basics.Round(i)) + err = aw.UpdateAccountsRound(basics.Round(i)) require.NoError(t, err) // TODO: calculate exact number of updates? @@ -297,7 +297,7 @@ func TestAccountDBRound(t *testing.T) { require.NotEmpty(t, updatedOnlineAccts) checkAccounts(t, tx, basics.Round(i), accts) - arw.Testing().CheckCreatablesTest(t, i, expectedDbImage) + ar.Testing().CheckCreatablesTest(t, i, expectedDbImage) } // test the accounts totals @@ -307,11 +307,11 @@ func TestAccountDBRound(t *testing.T) { } expectedTotals := ledgertesting.CalculateNewRoundAccountTotals(t, updates, 0, proto, nil, ledgercore.AccountTotals{}) - actualTotals, err := arw.AccountsTotals(context.Background(), false) + actualTotals, err := ar.AccountsTotals(context.Background(), false) require.NoError(t, err) require.Equal(t, expectedTotals, actualTotals) - actualOnlineRoundParams, endRound, err := arw.AccountsOnlineRoundParams() + actualOnlineRoundParams, endRound, err := ar.AccountsOnlineRoundParams() require.NoError(t, err) require.Equal(t, expectedOnlineRoundParams, actualOnlineRoundParams) require.Equal(t, 9, int(endRound)) @@ -321,7 +321,7 @@ func TestAccountDBRound(t *testing.T) { acctCb := func(addr basics.Address, data basics.AccountData) { loaded[addr] = data } - count, err := arw.LoadAllFullAccounts(context.Background(), "accountbase", "resources", acctCb) + count, err := ar.LoadAllFullAccounts(context.Background(), "accountbase", "resources", acctCb) require.NoError(t, err) require.Equal(t, count, len(accts)) require.Equal(t, count, len(loaded)) @@ -366,8 +366,7 @@ func TestAccountDBInMemoryAcct(t *testing.T) { for i, test := range tests { - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { @@ -437,8 +436,7 @@ func TestAccountDBInMemoryAcct(t *testing.T) { func TestAccountStorageWithStateProofID(t *testing.T) { partitiontest.PartitionTest(t) - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { @@ -604,29 +602,21 @@ func benchmarkInitBalances(b testing.TB, numAccounts int, tx trackerdb.Transacti return } -func cleanupTestDb(dbs db.Pair, dbName string, inMemory bool) { - dbs.Close() - if !inMemory { - os.Remove(dbName) - } -} - func benchmarkReadingAllBalances(b *testing.B, inMemory bool) { - dbs, _ := sqlitedriver.DbOpenTrackerTest(b, true) - dbs.SetLogger(logging.TestingLog(b)) + dbs, _ := sqlitedriver.OpenForTesting(b, true) defer dbs.Close() bal := make(map[basics.Address]basics.AccountData) err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { benchmarkInitBalances(b, b.N, tx, protocol.ConsensusCurrentVersion) - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } b.ResetTimer() // read all the balances in the database. var err2 error - bal, err2 = arw.Testing().AccountsAllTest() + bal, err2 = ar.Testing().AccountsAllTest() require.NoError(b, err2) return nil }) @@ -648,9 +638,8 @@ func BenchmarkReadingAllBalancesDisk(b *testing.B) { } func benchmarkReadingRandomBalances(b *testing.B, inMemory bool) { - dbs, fn := sqlitedriver.DbOpenTrackerTest(b, true) - dbs.SetLogger(logging.TestingLog(b)) - defer dbs.CleanupTest(fn, inMemory) + dbs, _ := sqlitedriver.OpenForTesting(b, true) + defer dbs.Close() err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { accounts := benchmarkInitBalances(b, b.N, tx, protocol.ConsensusCurrentVersion) @@ -687,30 +676,6 @@ func BenchmarkReadingRandomBalancesDisk(b *testing.B) { benchmarkReadingRandomBalances(b, false) } -// TestAccountsDbQueriesCreateClose tests to see that we can create the accountsDbQueries and close it. -// it also verify that double-closing it doesn't create an issue. -func TestAccountsDbQueriesCreateClose(t *testing.T) { - partitiontest.PartitionTest(t) - - dbs, _ := storetesting.DbOpenTest(t, true) - storetesting.SetDbLogging(t, dbs) - defer dbs.Close() - - err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - sqlitedriver.AccountsInitTest(t, tx, make(map[basics.Address]basics.AccountData), protocol.ConsensusCurrentVersion) - return nil - }) - require.NoError(t, err) - qs, err := sqlitedriver.AccountsInitDbQueries(dbs.Rdb.Handle) - require.NoError(t, err) - // TODO[store-refactor]: internals are opaque, once we move the the remainder of accountdb we can mvoe this too - // require.NotNil(t, qs.listCreatablesStmt) - qs.Close() - // require.Nil(t, qs.listCreatablesStmt) - qs.Close() - // require.Nil(t, qs.listCreatablesStmt) -} - func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder bool) { proto := config.Consensus[protocol.ConsensusCurrentVersion] genesisInitState, _ := ledgertesting.GenerateInitState(b, protocol.ConsensusCurrentVersion, 100) @@ -771,12 +736,12 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder boo normalizedAccountBalances, err := prepareNormalizedBalancesV6(chunk.Balances, proto) require.NoError(b, err) b.StartTimer() - err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { + err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { cw, err := tx.MakeCatchpointWriter() if err != nil { return err } - err = cw.WriteCatchpointStagingBalances(ctx, normalizedAccountBalances) + _, err = cw.Write(ctx, trackerdb.CatchpointPayload{Accounts: normalizedAccountBalances}) return }) @@ -813,9 +778,8 @@ func TestLookupKeysByPrefix(t *testing.T) { partitiontest.PartitionTest(t) t.Parallel() - dbs, fn := sqlitedriver.DbOpenTrackerTest(t, false) - dbs.SetLogger(logging.TestingLog(t)) - defer dbs.CleanupTest(fn, false) + dbs, _ := sqlitedriver.OpenForTesting(t, false) + defer dbs.Close() err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { // return account data, initialize DB tables from AccountsInitTest @@ -1000,9 +964,8 @@ func TestLookupKeysByPrefix(t *testing.T) { func BenchmarkLookupKeyByPrefix(b *testing.B) { // learn something from BenchmarkWritingRandomBalancesDisk - dbs, fn := sqlitedriver.DbOpenTrackerTest(b, false) - dbs.SetLogger(logging.TestingLog(b)) - defer dbs.CleanupTest(fn, false) + dbs, _ := sqlitedriver.OpenForTesting(b, false) + defer dbs.Close() err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { // return account data, initialize DB tables from AccountsInitTest @@ -1187,11 +1150,8 @@ func TestKVStoreNilBlobConversion(t *testing.T) { // | Section 4: Run migration to see replace nils with empty byte slices | // +---------------------------------------------------------------------+ - trackerDBWrapper := sqlitedriver.CreateTrackerSQLStore(dbs) - err = trackerDBWrapper.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err0 error) { - _, err0 = tx.RunMigrations(ctx, trackerdb.Params{}, log, targetVersion) - return - }) + trackerDBWrapper := sqlitedriver.MakeStore(dbs) + _, err = trackerDBWrapper.RunMigrations(context.Background(), trackerdb.Params{}, log, targetVersion) require.NoError(t, err) // +------------------------------------------------------------------------------------------------+ @@ -1420,8 +1380,7 @@ func TestCompactResourceDeltas(t *testing.T) { func TestLookupAccountAddressFromAddressID(t *testing.T) { partitiontest.PartitionTest(t) - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() addrs := make([]basics.Address, 100) @@ -1449,13 +1408,13 @@ func TestLookupAccountAddressFromAddressID(t *testing.T) { require.NoError(t, err) err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } for addr, addrid := range addrsids { - retAddr, err := arw.LookupAccountAddressFromAddressID(ctx, addrid) + retAddr, err := ar.LookupAccountAddressFromAddressID(ctx, addrid) if err != nil { return err } @@ -1464,7 +1423,7 @@ func TestLookupAccountAddressFromAddressID(t *testing.T) { } } // test fail case: - retAddr, err := arw.LookupAccountAddressFromAddressID(ctx, nil) + retAddr, err := ar.LookupAccountAddressFromAddressID(ctx, nil) if !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("unexpected error : %w", err) @@ -2093,10 +2052,9 @@ func initBoxDatabase(b *testing.B, totalBoxes, boxSize int) (db.Pair, func(), er } proto := config.Consensus[protocol.ConsensusCurrentVersion] - dbs, fn := storetesting.DbOpenTest(b, false) - storetesting.SetDbLogging(b, dbs) + dbs, _ := storetesting.DbOpenTest(b, false) cleanup := func() { - cleanupTestDb(dbs, fn, false) + dbs.Close() } tx, err := dbs.Wdb.Handle.Begin() @@ -2232,20 +2190,20 @@ func TestAccountOnlineQueries(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dbs.SetLogger(logging.TestingLog(t)) + dbs, _ := sqlitedriver.OpenForTesting(t, true) defer dbs.Close() err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - arw, err := tx.MakeAccountsReaderWriter() - if err != nil { - return err - } + ar, err := tx.MakeAccountsReader() + require.NoError(t, err) + + aw, err := tx.MakeAccountsWriter() + require.NoError(t, err) var accts map[basics.Address]basics.AccountData tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion) - totals, err := arw.AccountsTotals(context.Background(), false) + totals, err := ar.AccountsTotals(context.Background(), false) require.NoError(t, err) var baseAccounts lruAccounts @@ -2329,7 +2287,7 @@ func TestAccountOnlineQueries(t *testing.T) { err = updatesOnlineCnt.accountsLoadOld(tx) require.NoError(t, err) - err = arw.AccountsPutTotals(totals, false) + err = aw.AccountsPutTotals(totals, false) require.NoError(t, err) updatedAccts, _, _, err := accountsNewRound(tx, updatesCnt, compactResourcesDeltas{}, nil, nil, proto, rnd) require.NoError(t, err) @@ -2339,7 +2297,7 @@ func TestAccountOnlineQueries(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, updatedOnlineAccts) - err = arw.UpdateAccountsRound(rnd) + err = aw.UpdateAccountsRound(rnd) require.NoError(t, err) return @@ -2367,12 +2325,12 @@ func TestAccountOnlineQueries(t *testing.T) { refoaB3 := round3poads[0].Ref refoaC3 := round3poads[1].Ref - queries, err := tx.Testing().MakeOnlineAccountsOptimizedReader() + queries, err := tx.MakeOnlineAccountsOptimizedReader() require.NoError(t, err) // check round 1 rnd := basics.Round(1) - online, err := arw.AccountsOnlineTop(rnd, 0, 10, proto) + online, err := ar.AccountsOnlineTop(rnd, 0, 10, proto) require.NoError(t, err) require.Equal(t, 2, len(online)) require.NotContains(t, online, addrC) @@ -2411,7 +2369,7 @@ func TestAccountOnlineQueries(t *testing.T) { // check round 2 rnd = basics.Round(2) - online, err = arw.AccountsOnlineTop(rnd, 0, 10, proto) + online, err = ar.AccountsOnlineTop(rnd, 0, 10, proto) require.NoError(t, err) require.Equal(t, 1, len(online)) require.NotContains(t, online, addrA) @@ -2444,7 +2402,7 @@ func TestAccountOnlineQueries(t *testing.T) { // check round 3 rnd = basics.Round(3) - online, err = arw.AccountsOnlineTop(rnd, 0, 10, proto) + online, err = ar.AccountsOnlineTop(rnd, 0, 10, proto) require.NoError(t, err) require.Equal(t, 1, len(online)) require.NotContains(t, online, addrA) @@ -2475,7 +2433,7 @@ func TestAccountOnlineQueries(t *testing.T) { require.Equal(t, dataC3.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos) require.Equal(t, voteIDC, paod.AccountData.VoteID) - paods, err := arw.OnlineAccountsAll(0) + paods, err := ar.OnlineAccountsAll(0) require.NoError(t, err) require.Equal(t, 5, len(paods)) @@ -2512,20 +2470,20 @@ func TestAccountOnlineQueries(t *testing.T) { checkAddrC() checkAddrA() - paods, err = arw.OnlineAccountsAll(3) + paods, err = ar.OnlineAccountsAll(3) require.NoError(t, err) require.Equal(t, 5, len(paods)) checkAddrB() checkAddrC() checkAddrA() - paods, err = arw.OnlineAccountsAll(2) + paods, err = ar.OnlineAccountsAll(2) require.NoError(t, err) require.Equal(t, 3, len(paods)) checkAddrB() checkAddrC() - paods, err = arw.OnlineAccountsAll(1) + paods, err = ar.OnlineAccountsAll(1) require.NoError(t, err) require.Equal(t, 2, len(paods)) checkAddrB() diff --git a/ledger/acctonline.go b/ledger/acctonline.go index ec65592d3e..1e9f55f629 100644 --- a/ledger/acctonline.go +++ b/ledger/acctonline.go @@ -60,7 +60,7 @@ type cachedOnlineAccount struct { // onlineAccounts tracks history of online accounts type onlineAccounts struct { // Connection to the database. - dbs trackerdb.TrackerStore + dbs trackerdb.Store // Prepared SQL statements for fast accounts DB lookups. accountsq trackerdb.OnlineAccountsReader @@ -433,23 +433,23 @@ func (ao *onlineAccounts) commitRound(ctx context.Context, tx trackerdb.Transact return err } - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } - err = arw.OnlineAccountsDelete(dcc.onlineAccountsForgetBefore) + err = aw.OnlineAccountsDelete(dcc.onlineAccountsForgetBefore) if err != nil { return err } - err = arw.AccountsPutOnlineRoundParams(dcc.onlineRoundParams, dcc.oldBase+1) + err = aw.AccountsPutOnlineRoundParams(dcc.onlineRoundParams, dcc.oldBase+1) if err != nil { return err } // delete all entries all older than maxBalLookback (or votersLookback) rounds ago - err = arw.AccountsPruneOnlineRoundParams(dcc.onlineAccountsForgetBefore) + err = aw.AccountsPruneOnlineRoundParams(dcc.onlineAccountsForgetBefore) return } diff --git a/ledger/acctonline_test.go b/ledger/acctonline_test.go index 0bee1f35fc..13d222a62b 100644 --- a/ledger/acctonline_test.go +++ b/ledger/acctonline_test.go @@ -18,7 +18,6 @@ package ledger import ( "context" - "database/sql" "fmt" "sort" "strconv" @@ -80,7 +79,7 @@ func commitSyncPartial(t *testing.T, oa *onlineAccounts, ml *mockLedgerForTracke require.NoError(t, err) } err := ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -92,7 +91,7 @@ func commitSyncPartial(t *testing.T, oa *onlineAccounts, ml *mockLedgerForTracke } } - return arw.UpdateAccountsRound(newBase) + return aw.UpdateAccountsRound(newBase) }) require.NoError(t, err) }() @@ -1316,7 +1315,7 @@ func TestAcctOnlineVotersLongerHistory(t *testing.T) { require.NoError(t, err) _, err = oa.onlineTotalsEx(lowest - 1) - require.ErrorIs(t, err, sql.ErrNoRows) + require.ErrorIs(t, err, trackerdb.ErrNotFound) // ensure the cache size for addrA does not have more entries than maxBalLookback + 1 // +1 comes from the deletion before X without checking account state at X diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go index 9c16e09279..d2850a51a8 100644 --- a/ledger/acctupdates.go +++ b/ledger/acctupdates.go @@ -151,7 +151,7 @@ type modifiedKvValue struct { type accountUpdates struct { // Connection to the database. - dbs trackerdb.TrackerStore + dbs trackerdb.Store // Optimized reader for fast accounts DB lookups. accountsq trackerdb.AccountsReader @@ -1685,12 +1685,12 @@ func (au *accountUpdates) commitRound(ctx context.Context, tx trackerdb.Transact dcc.stats.OldAccountPreloadDuration = time.Duration(time.Now().UnixNano()) - dcc.stats.OldAccountPreloadDuration } - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } - err = arw.AccountsPutTotals(dcc.roundTotals, false) + err = aw.AccountsPutTotals(dcc.roundTotals, false) if err != nil { return err } diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go index 0faaa4de69..c70b969c54 100644 --- a/ledger/acctupdates_test.go +++ b/ledger/acctupdates_test.go @@ -37,8 +37,10 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver" + "github.com/algorand/go-algorand/ledger/store/trackerdb/testdb" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -51,7 +53,8 @@ var testPoolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff var testSinkAddr = basics.Address{0x2c, 0x2a, 0x6c, 0xe9, 0xa9, 0xa7, 0xc2, 0x8c, 0x22, 0x95, 0xfd, 0x32, 0x4f, 0x77, 0xa5, 0x4, 0x8b, 0x42, 0xc2, 0xb7, 0xa8, 0x54, 0x84, 0xb6, 0x80, 0xb1, 0xe1, 0x3d, 0x59, 0x9b, 0xeb, 0x36} type mockLedgerForTracker struct { - dbs trackerdb.TrackerStore + dbs trackerdb.Store + catchpointDbs catchpointdb.Store blocks []blockEntry deltas []ledgercore.StateDelta log logging.Logger @@ -110,8 +113,7 @@ func setupAccts(niter int) []map[basics.Address]basics.AccountData { } func makeMockLedgerForTrackerWithLogger(t testing.TB, inMemory bool, initialBlocksCount int, consensusVersion protocol.ConsensusVersion, accts []map[basics.Address]basics.AccountData, l logging.Logger) *mockLedgerForTracker { - dbs, fileName := sqlitedriver.DbOpenTrackerTest(t, inMemory) - dbs.SetLogger(l) + dbs, fileName := sqlitedriver.OpenForTesting(t, inMemory) blocks := randomInitChain(consensusVersion, initialBlocksCount) deltas := make([]ledgercore.StateDelta, initialBlocksCount) @@ -182,7 +184,7 @@ func (ml *mockLedgerForTracker) fork(t testing.TB) *mockLedgerForTracker { dbs.Rdb.SetLogger(dblogger) dbs.Wdb.SetLogger(dblogger) - newLedgerTracker.dbs = sqlitedriver.CreateTrackerSQLStore(dbs) + newLedgerTracker.dbs = sqlitedriver.MakeStore(dbs) return newLedgerTracker } @@ -253,10 +255,14 @@ func (ml *mockLedgerForTracker) BlockHdr(rnd basics.Round) (bookkeeping.BlockHea return ml.blocks[int(rnd)].block.BlockHeader, nil } -func (ml *mockLedgerForTracker) trackerDB() trackerdb.TrackerStore { +func (ml *mockLedgerForTracker) trackerDB() trackerdb.Store { return ml.dbs } +func (ml *mockLedgerForTracker) catchpointDB() catchpointdb.Store { + return ml.catchpointDbs +} + func (ml *mockLedgerForTracker) blockDB() db.Pair { return db.Pair{} } @@ -297,13 +303,13 @@ func (au *accountUpdates) allBalances(rnd basics.Round) (bals map[basics.Address return } - err = au.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { + err = au.dbs.Snapshot(func(ctx context.Context, tx trackerdb.SnapshotScope) error { var err0 error - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } - bals, err0 = arw.Testing().AccountsAllTest() + bals, err0 = ar.Testing().AccountsAllTest() return err0 }) if err != nil { @@ -1033,64 +1039,69 @@ func TestListCreatables(t *testing.T) { numElementsPerSegement := 25 // set up the database - dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true) - dblogger := logging.TestingLog(t) - dbs.SetLogger(dblogger) + dbs := testdb.OpenForTesting(t) defer dbs.Close() - err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - proto := config.Consensus[protocol.ConsensusCurrentVersion] + proto := config.Consensus[protocol.ConsensusCurrentVersion] + err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { accts := make(map[basics.Address]basics.AccountData) _ = tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion) - require.NoError(t, err) + return + }) + require.NoError(t, err) - au := &accountUpdates{} - au.accountsq, err = tx.Testing().MakeAccountsOptimizedReader() - require.NoError(t, err) + au := &accountUpdates{} + au.accountsq, err = dbs.MakeAccountsOptimizedReader() + require.NoError(t, err) - // ******* All results are obtained from the cache. Empty database ******* - // ******* No deletes ******* - // get random data. Initial batch, no deletes - ctbsList, randomCtbs := randomCreatables(numElementsPerSegement) - expectedDbImage := make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable) - ctbsWithDeletes := randomCreatableSampling(1, ctbsList, randomCtbs, - expectedDbImage, numElementsPerSegement) - // set the cache - au.creatables = ctbsWithDeletes - listAndCompareComb(t, au, expectedDbImage) - - // ******* All results are obtained from the database. Empty cache ******* - // ******* No deletes ******* - // sync with the database - var updates compactAccountDeltas - var resUpdates compactResourcesDeltas + // ******* All results are obtained from the cache. Empty database ******* + // ******* No deletes ******* + // get random data. Initial batch, no deletes + ctbsList, randomCtbs := randomCreatables(numElementsPerSegement) + expectedDbImage := make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable) + ctbsWithDeletes := randomCreatableSampling(1, ctbsList, randomCtbs, + expectedDbImage, numElementsPerSegement) + // set the cache + au.creatables = ctbsWithDeletes + listAndCompareComb(t, au, expectedDbImage) + + // ******* All results are obtained from the database. Empty cache ******* + // ******* No deletes ******* + // sync with the database + var updates compactAccountDeltas + var resUpdates compactResourcesDeltas + err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { _, _, _, err = accountsNewRound(tx, updates, resUpdates, nil, ctbsWithDeletes, proto, basics.Round(1)) require.NoError(t, err) - // nothing left in cache - au.creatables = make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable) - listAndCompareComb(t, au, expectedDbImage) - - // ******* Results are obtained from the database and from the cache ******* - // ******* No deletes in the database. ******* - // ******* Data in the database deleted in the cache ******* - au.creatables = randomCreatableSampling(2, ctbsList, randomCtbs, - expectedDbImage, numElementsPerSegement) - listAndCompareComb(t, au, expectedDbImage) - - // ******* Results are obtained from the database and from the cache ******* - // ******* Deletes are in the database and in the cache ******* - // sync with the database. This has deletes synced to the database. + return + }) + require.NoError(t, err) + + // nothing left in cache + au.creatables = make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable) + listAndCompareComb(t, au, expectedDbImage) + + // ******* Results are obtained from the database and from the cache ******* + // ******* No deletes in the database. ******* + // ******* Data in the database deleted in the cache ******* + au.creatables = randomCreatableSampling(2, ctbsList, randomCtbs, + expectedDbImage, numElementsPerSegement) + listAndCompareComb(t, au, expectedDbImage) + + // ******* Results are obtained from the database and from the cache ******* + // ******* Deletes are in the database and in the cache ******* + // sync with the database. This has deletes synced to the database. + err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { _, _, _, err = accountsNewRound(tx, updates, resUpdates, nil, au.creatables, proto, basics.Round(1)) require.NoError(t, err) - // get new creatables in the cache. There will be deleted in the cache from the previous batch. - au.creatables = randomCreatableSampling(3, ctbsList, randomCtbs, - expectedDbImage, numElementsPerSegement) - listAndCompareComb(t, au, expectedDbImage) - return }) require.NoError(t, err) + // get new creatables in the cache. There will be deleted in the cache from the previous batch. + au.creatables = randomCreatableSampling(3, ctbsList, randomCtbs, + expectedDbImage, numElementsPerSegement) + listAndCompareComb(t, au, expectedDbImage) } func TestBoxNamesByAppIDs(t *testing.T) { @@ -2190,7 +2201,7 @@ func TestAcctUpdatesResources(t *testing.T) { err := au.prepareCommit(dcc) require.NoError(t, err) err = ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -2199,7 +2210,7 @@ func TestAcctUpdatesResources(t *testing.T) { if err != nil { return err } - err = arw.UpdateAccountsRound(newBase) + err = aw.UpdateAccountsRound(newBase) return err }) require.NoError(t, err) @@ -2477,7 +2488,7 @@ func auCommitSync(t *testing.T, rnd basics.Round, au *accountUpdates, ml *mockLe err := au.prepareCommit(dcc) require.NoError(t, err) err = ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -2486,7 +2497,7 @@ func auCommitSync(t *testing.T, rnd basics.Round, au *accountUpdates, ml *mockLe if err != nil { return err } - err = arw.UpdateAccountsRound(newBase) + err = aw.UpdateAccountsRound(newBase) return err }) require.NoError(t, err) diff --git a/ledger/archival_test.go b/ledger/archival_test.go index de483e227c..8dda76b2c2 100644 --- a/ledger/archival_test.go +++ b/ledger/archival_test.go @@ -40,6 +40,7 @@ import ( "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/ledger/store/blockdb" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -76,10 +77,14 @@ func (wl *wrappedLedger) Latest() basics.Round { return wl.l.Latest() } -func (wl *wrappedLedger) trackerDB() trackerdb.TrackerStore { +func (wl *wrappedLedger) trackerDB() trackerdb.Store { return wl.l.trackerDB() } +func (wl *wrappedLedger) catchpointDB() catchpointdb.Store { + return wl.l.catchpointDB() +} + func (wl *wrappedLedger) blockDB() db.Pair { return wl.l.blockDB() } diff --git a/ledger/catchpointtracker.go b/ledger/catchpointtracker.go index a24a0439c4..349eba0466 100644 --- a/ledger/catchpointtracker.go +++ b/ledger/catchpointtracker.go @@ -42,6 +42,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" @@ -110,8 +111,8 @@ type catchpointTracker struct { log logging.Logger // Connection to the database. - dbs trackerdb.TrackerStore - catchpointStore trackerdb.CatchpointReaderWriter + dbs trackerdb.Store + catchpointStore catchpointdb.Store // The last catchpoint label that was written to the database. Should always align with what's in the database. // note that this is the last catchpoint *label* and not the catchpoint file. @@ -227,27 +228,34 @@ func (ct *catchpointTracker) finishFirstStage(ctx context.Context, dbRound basic } } - return ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } + var info catchpointdb.CatchpointFirstStageInfo + err := ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { + info, err = ct.recordFirstStageInfo(ctx, tx, &catchpointGenerationStats, dbRound, totalKVs, totalAccounts, totalChunks, biggestChunkLen, spVerificationHash) + return err + }) + if err != nil { + // failed to record first stage info + return err + } - err = ct.recordFirstStageInfo(ctx, tx, &catchpointGenerationStats, dbRound, totalKVs, totalAccounts, totalChunks, biggestChunkLen, spVerificationHash) + err = ct.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) error { + // save the catchpoint info + err = tx.InsertOrReplaceCatchpointFirstStageInfo(ctx, dbRound, &info) if err != nil { return err } - // Clear the db record. - return crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingFirstStageInfo, 0) + // reset catchpoint staging + return tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateWritingFirstStageInfo, 0) }) + return err } // Possibly finish generating first stage catchpoint db record and data file after // a crash. func (ct *catchpointTracker) finishFirstStageAfterCrash(dbRound basics.Round) error { v, err := ct.catchpointStore.ReadCatchpointStateUint64( - context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo) + context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo) if err != nil { return err } @@ -257,9 +265,9 @@ func (ct *catchpointTracker) finishFirstStageAfterCrash(dbRound basics.Round) er // First, delete the unfinished data file. relCatchpointDataFilePath := filepath.Join( - trackerdb.CatchpointDirName, + catchpointdb.CatchpointDirName, makeCatchpointDataFilePath(dbRound)) - err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath) if err != nil { return err } @@ -276,9 +284,9 @@ func (ct *catchpointTracker) finishCatchpointsAfterCrash(catchpointLookback uint for _, record := range records { // First, delete the unfinished catchpoint file. relCatchpointFilePath := filepath.Join( - trackerdb.CatchpointDirName, - trackerdb.MakeCatchpointFilePath(basics.Round(record.Round))) - err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath) + catchpointdb.CatchpointDirName, + catchpointdb.MakeCatchpointFilePath(basics.Round(record.Round))) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath) if err != nil { return err } @@ -302,7 +310,7 @@ func (ct *catchpointTracker) recoverFromCrash(dbRound basics.Round) error { ctx := context.Background() catchpointLookback, err := ct.catchpointStore.ReadCatchpointStateUint64( - ctx, trackerdb.CatchpointStateCatchpointLookback) + ctx, catchpointdb.CatchpointStateCatchpointLookback) if err != nil { return err } @@ -333,7 +341,7 @@ func (ct *catchpointTracker) recoverFromCrash(dbRound basics.Round) error { func (ct *catchpointTracker) loadFromDisk(l ledgerForTracker, dbRound basics.Round) (err error) { ct.log = l.trackerLog() ct.dbs = l.trackerDB() - ct.catchpointStore, err = l.trackerDB().MakeCatchpointReaderWriter() + ct.catchpointStore = l.catchpointDB() if err != nil { return err } @@ -357,7 +365,7 @@ func (ct *catchpointTracker) loadFromDisk(l ledgerForTracker, dbRound basics.Rou } ct.lastCatchpointLabel, err = ct.catchpointStore.ReadCatchpointStateString( - context.Background(), trackerdb.CatchpointStateLastCatchpoint) + context.Background(), catchpointdb.CatchpointStateLastCatchpoint) if err != nil { return } @@ -512,11 +520,7 @@ func (ct *catchpointTracker) commitRound(ctx context.Context, tx trackerdb.Trans } }() - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -556,34 +560,42 @@ func (ct *catchpointTracker) commitRound(ctx context.Context, tx trackerdb.Trans dcc.stats.MerkleTrieUpdateDuration = now - dcc.stats.MerkleTrieUpdateDuration } - err = arw.UpdateAccountsHashRound(ctx, treeTargetRound) + err = aw.UpdateAccountsHashRound(ctx, treeTargetRound) if err != nil { return err } - if dcc.catchpointFirstStage { - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingFirstStageInfo, 1) - if err != nil { - return err - } - } + return nil +} - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchpointLookback, dcc.catchpointLookback) - if err != nil { - return err - } +func (ct *catchpointTracker) postCommit(ctx context.Context, dcc *deferredCommitContext) { + err := ct.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) { + if dcc.catchpointFirstStage { + // set state as dirty + err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateWritingFirstStageInfo, 1) + if err != nil { + return err + } + } - for _, round := range ct.calculateCatchpointRounds(&dcc.deferredCommitRange) { - err = crw.InsertUnfinishedCatchpoint(ctx, round, dcc.committedRoundDigests[round-dcc.oldBase-1]) + err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchpointLookback, dcc.catchpointLookback) if err != nil { return err } - } - return nil -} + for _, round := range ct.calculateCatchpointRounds(&dcc.deferredCommitRange) { + err = tx.InsertUnfinishedCatchpoint(ctx, round, dcc.committedRoundDigests[round-dcc.oldBase-1]) + if err != nil { + return err + } + } + + return + }) + if err != nil { + ct.log.Errorf("failed to update catchpoint db post commit: %v", err) + } -func (ct *catchpointTracker) postCommit(ctx context.Context, dcc *deferredCommitContext) { if ct.balancesTrie != nil { _, err := ct.balancesTrie.Evict(false) if err != nil { @@ -728,7 +740,7 @@ func repackCatchpoint(ctx context.Context, header CatchpointFileHeader, biggestC // Create a catchpoint (a label and possibly a file with db record) and remove // the unfinished catchpoint record. -func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound basics.Round, round basics.Round, dataInfo trackerdb.CatchpointFirstStageInfo, blockHash crypto.Digest) error { +func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound basics.Round, round basics.Round, dataInfo catchpointdb.CatchpointFirstStageInfo, blockHash crypto.Digest) error { startTime := time.Now() labelMaker := ledgercore.MakeCatchpointLabelMakerCurrent(round, &blockHash, &dataInfo.TrieBalancesHash, dataInfo.Totals, &dataInfo.StateProofVerificationHash) label := ledgercore.MakeLabel(labelMaker) @@ -738,7 +750,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound round, accountsRound, label) err := ct.catchpointStore.WriteCatchpointStateString( - ctx, trackerdb.CatchpointStateLastCatchpoint, label) + ctx, catchpointdb.CatchpointStateLastCatchpoint, label) if err != nil { return err } @@ -751,7 +763,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound return nil } - catchpointDataFilePath := filepath.Join(ct.dbDirectory, trackerdb.CatchpointDirName) + catchpointDataFilePath := filepath.Join(ct.dbDirectory, catchpointdb.CatchpointDirName) catchpointDataFilePath = filepath.Join(catchpointDataFilePath, makeCatchpointDataFilePath(accountsRound)) @@ -778,7 +790,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound } relCatchpointFilePath := - filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(round)) + filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(round)) absCatchpointFilePath := filepath.Join(ct.dbDirectory, relCatchpointFilePath) err = os.MkdirAll(filepath.Dir(absCatchpointFilePath), 0700) @@ -796,18 +808,11 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound return err } - err = ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - - err = ct.recordCatchpointFile(ctx, crw, round, relCatchpointFilePath, fileInfo.Size()) - if err != nil { - return err - } - return crw.DeleteUnfinishedCatchpoint(ctx, round) - }) + err = ct.recordCatchpointFile(ctx, ct.catchpointStore, round, relCatchpointFilePath, fileInfo.Size()) + if err != nil { + return err + } + err = ct.catchpointStore.DeleteUnfinishedCatchpoint(ctx, round) if err != nil { return err } @@ -887,8 +892,8 @@ func (ct *catchpointTracker) pruneFirstStageRecordsData(ctx context.Context, max for _, round := range rounds { relCatchpointDataFilePath := - filepath.Join(trackerdb.CatchpointDirName, makeCatchpointDataFilePath(round)) - err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath) + filepath.Join(catchpointdb.CatchpointDirName, makeCatchpointDataFilePath(round)) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath) if err != nil { return err } @@ -1096,18 +1101,18 @@ func (ct *catchpointTracker) isWritingCatchpointDataFile() bool { // Generates a (first stage) catchpoint data file. // The file is built in the following order: -// - Catchpoint file header (named content.msgpack). The header is generated and appended to the file at the end of the -// second stage of catchpoint generation. -// - State proof verification data chunk (named stateProofVerificationContext.msgpack). -// - Balance and KV chunk (named balances.x.msgpack). -// ... -// - Balance and KV chunk (named balances.x.msgpack). +// - Catchpoint file header (named content.msgpack). The header is generated and appended to the file at the end of the +// second stage of catchpoint generation. +// - State proof verification data chunk (named stateProofVerificationContext.msgpack). +// - Balance and KV chunk (named balances.x.msgpack). +// ... +// - Balance and KV chunk (named balances.x.msgpack). func (ct *catchpointTracker) generateCatchpointData(ctx context.Context, accountsRound basics.Round, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails) (totalKVs, totalAccounts, totalChunks, biggestChunkLen uint64, spVerificationHash crypto.Digest, err error) { ct.log.Debugf("catchpointTracker.generateCatchpointData() writing catchpoint accounts for round %d", accountsRound) startTime := time.Now() - catchpointDataFilePath := filepath.Join(ct.dbDirectory, trackerdb.CatchpointDirName) + catchpointDataFilePath := filepath.Join(ct.dbDirectory, catchpointdb.CatchpointDirName) catchpointDataFilePath = filepath.Join(catchpointDataFilePath, makeCatchpointDataFilePath(accountsRound)) @@ -1199,25 +1204,25 @@ func (ct *catchpointTracker) generateCatchpointData(ctx context.Context, account return catchpointWriter.totalKVs, catchpointWriter.totalAccounts, catchpointWriter.chunkNum, catchpointWriter.biggestChunkLen, spVerificationHash, nil } -func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx trackerdb.TransactionScope, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails, accountsRound basics.Round, totalKVs uint64, totalAccounts uint64, totalChunks uint64, biggestChunkLen uint64, stateProofVerificationHash crypto.Digest) error { - arw, err := tx.MakeAccountsReaderWriter() +func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx trackerdb.TransactionScope, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails, accountsRound basics.Round, totalKVs uint64, totalAccounts uint64, totalChunks uint64, biggestChunkLen uint64, stateProofVerificationHash crypto.Digest) (info catchpointdb.CatchpointFirstStageInfo, err error) { + ar, err := tx.MakeAccountsReader() if err != nil { - return err + return } - accountTotals, err := arw.AccountsTotals(ctx, false) + accountTotals, err := ar.AccountsTotals(ctx, false) if err != nil { - return err + return } mc, err := tx.MakeMerkleCommitter(false) if err != nil { - return err + return info, err } if ct.balancesTrie == nil { trie, trieErr := merkletrie.MakeTrie(mc, trackerdb.TrieMemoryConfig) if trieErr != nil { - return trieErr + return info, trieErr } ct.balancesTrie = trie } else { @@ -1226,15 +1231,10 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke trieBalancesHash, err := ct.balancesTrie.RootHash() if err != nil { - return err - } - - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err + return } - info := trackerdb.CatchpointFirstStageInfo{ + info = catchpointdb.CatchpointFirstStageInfo{ Totals: accountTotals, TotalAccounts: totalAccounts, TotalKVs: totalKVs, @@ -1244,11 +1244,6 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke StateProofVerificationHash: stateProofVerificationHash, } - err = crw.InsertOrReplaceCatchpointFirstStageInfo(ctx, accountsRound, &info) - if err != nil { - return err - } - catchpointGenerationStats.MerkleTrieRootHash = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(trieBalancesHash[:]) catchpointGenerationStats.SPVerificationCtxsHash = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(stateProofVerificationHash[:]) ct.log.EventWithDetails(telemetryspec.Accounts, telemetryspec.CatchpointGenerationEvent, catchpointGenerationStats) @@ -1262,7 +1257,7 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke With("MerkleTrieRootHash", catchpointGenerationStats.MerkleTrieRootHash). With("SPVerificationCtxsHash", catchpointGenerationStats.SPVerificationCtxsHash). Infof("Catchpoint data file was generated") - return nil + return } func makeCatchpointDataFilePath(accountsRound basics.Round) string { @@ -1273,7 +1268,7 @@ func makeCatchpointDataFilePath(accountsRound basics.Round) string { // after a successful insert operation to the database, it would delete up to 2 old entries, as needed. // deleting 2 entries while inserting single entry allow us to adjust the size of the backing storage and have the // database and storage realign. -func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw trackerdb.CatchpointReaderWriter, round basics.Round, relCatchpointFilePath string, fileSize int64) (err error) { +func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw catchpointdb.ReaderWriter, round basics.Round, relCatchpointFilePath string, fileSize int64) (err error) { if ct.catchpointFileHistoryLength != 0 { err = crw.StoreCatchpoint(ctx, round, relCatchpointFilePath, "", fileSize) if err != nil { @@ -1281,7 +1276,7 @@ func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw track return } } else { - err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath) if err != nil { ct.log.Warnf("catchpointTracker.recordCatchpointFile() unable to remove file (%s): %v", relCatchpointFilePath, err) return @@ -1296,7 +1291,7 @@ func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw track return fmt.Errorf("unable to delete catchpoint file, getOldestCatchpointFiles failed : %v", err) } for round, fileToDelete := range filesToDelete { - err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, fileToDelete) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, fileToDelete) if err != nil { return err } @@ -1316,20 +1311,12 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS ledgerGetcatchpointCount.Inc(nil) // TODO: we need to generalize this, check @cce PoC PR, he has something // somewhat broken for some KVs.. - err := ct.dbs.Snapshot(func(ctx context.Context, tx trackerdb.SnapshotScope) (err error) { - cr, err := tx.MakeCatchpointReader() - if err != nil { - return err - } - - dbFileName, _, fileSize, err = cr.GetCatchpoint(ctx, round) - return - }) - ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil) + dbFileName, _, fileSize, err := ct.catchpointStore.GetCatchpoint(context.Background(), round) if err != nil && err != sql.ErrNoRows { // we had some sql error. return nil, fmt.Errorf("catchpointTracker.GetCatchpointStream() unable to lookup catchpoint %d: %v", round, err) } + ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil) if dbFileName != "" { catchpointPath := filepath.Join(ct.dbDirectory, dbFileName) file, openErr := os.OpenFile(catchpointPath, os.O_RDONLY, 0666) @@ -1340,14 +1327,10 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS if os.IsNotExist(openErr) { // the database told us that we have this file.. but we couldn't find it. // delete it from the database. - crw, err2 := ct.dbs.MakeCatchpointReaderWriter() - if err2 != nil { - return nil, err2 - } - err2 = ct.recordCatchpointFile(context.Background(), crw, round, "", 0) - if err2 != nil { - ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to delete missing catchpoint entry: %v", err2) - return nil, err2 + err = ct.recordCatchpointFile(context.Background(), ct.catchpointStore, round, "", 0) + if err != nil { + ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to delete missing catchpoint entry: %v", err) + return nil, err } return nil, ledgercore.ErrNoEntry{} @@ -1358,7 +1341,7 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS // if the database doesn't know about that round, see if we have that file anyway: relCatchpointFilePath := - filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(round)) + filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(round)) absCatchpointFilePath := filepath.Join(ct.dbDirectory, relCatchpointFilePath) file, err := os.OpenFile(absCatchpointFilePath, os.O_RDONLY, 0666) if err == nil && file != nil { @@ -1368,11 +1351,7 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS // we couldn't get the stat, so just return with the file. return &readCloseSizer{ReadCloser: file, size: -1}, nil //nolint:nilerr // intentionally ignoring Stat error } - crw, err := ct.dbs.MakeCatchpointReaderWriter() - if err != nil { - return nil, err - } - err = ct.recordCatchpointFile(context.Background(), crw, round, relCatchpointFilePath, fileInfo.Size()) + err = ct.recordCatchpointFile(context.Background(), ct.catchpointStore, round, relCatchpointFilePath, fileInfo.Size()) if err != nil { ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to save missing catchpoint entry: %v", err) } @@ -1388,11 +1367,17 @@ func (ct *catchpointTracker) catchpointEnabled() bool { // initializeHashes initializes account/resource/kv hashes. // as part of the initialization, it tests if a hash table matches to account base and updates the former. func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb.TransactionScope, rnd basics.Round) error { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } - hashRound, err := arw.AccountsHashRound(ctx) + + aw, err := tx.MakeAccountsWriter() + if err != nil { + return err + } + + hashRound, err := ar.AccountsHashRound(ctx) if err != nil { return err } @@ -1400,7 +1385,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb. if hashRound != rnd { // if the hashed round is different then the base round, something was modified, and the accounts aren't in sync // with the hashes. - err = arw.ResetAccountHashes(ctx) + err = aw.ResetAccountHashes(ctx) if err != nil { return err } @@ -1457,7 +1442,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb. if !added { // we need to translate the "addrid" into actual account address so that // we can report the failure. - addr, lErr := arw.LookupAccountAddressFromAddressID(ctx, acct.AccountRef) + addr, lErr := ar.LookupAccountAddressFromAddressID(ctx, acct.AccountRef) if lErr != nil { ct.log.Warnf("initializeHashes attempted to add duplicate acct hash '%s' to merkle trie for account id %d : %v", hex.EncodeToString(acct.Digest), acct.AccountRef, lErr) } else { @@ -1541,7 +1526,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb. } // we've just updated the merkle trie, update the hashRound to reflect that. - err = arw.UpdateAccountsHashRound(ctx, rnd) + err = aw.UpdateAccountsHashRound(ctx, rnd) if err != nil { return fmt.Errorf("initializeHashes was unable to update the account hash round to %d: %v", rnd, err) } diff --git a/ledger/catchpointtracker_test.go b/ledger/catchpointtracker_test.go index 84a103300f..a1e286b3cf 100644 --- a/ledger/catchpointtracker_test.go +++ b/ledger/catchpointtracker_test.go @@ -40,6 +40,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" @@ -95,7 +96,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) { filesToCreate := 4 temporaryDirectory := t.TempDir() - catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName) err := os.Mkdir(catchpointsDirectory, 0777) require.NoError(t, err) @@ -103,7 +104,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) { // Create the catchpoint files with dummy data for i := 0; i < filesToCreate; i++ { - fileName := filepath.Join(trackerdb.CatchpointDirName, fmt.Sprintf("%d.catchpoint", i)) + fileName := filepath.Join(catchpointdb.CatchpointDirName, fmt.Sprintf("%d.catchpoint", i)) data := []byte{byte(i), byte(i + 1), byte(i + 2)} err = os.WriteFile(filepath.Join(temporaryDirectory, fileName), data, 0666) require.NoError(t, err) @@ -129,7 +130,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) { require.Equal(t, int64(3), len) // File deleted, but record in the database - err = os.Remove(filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName, "2.catchpoint")) + err = os.Remove(filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName, "2.catchpoint")) require.NoError(t, err) reader, err = ct.GetCatchpointStream(basics.Round(2)) require.Equal(t, ledgercore.ErrNoEntry{}, err) @@ -175,7 +176,7 @@ func TestCatchpointsDeleteStored(t *testing.T) { dummyCatchpointFiles := make([]string, dummyCatchpointFilesToCreate) for i := 0; i < dummyCatchpointFilesToCreate; i++ { file := fmt.Sprintf("%s%c%d%c%d%cdummy_catchpoint_file-%d", - trackerdb.CatchpointDirName, os.PathSeparator, + catchpointdb.CatchpointDirName, os.PathSeparator, i/10, os.PathSeparator, i/2, os.PathSeparator, i) @@ -215,7 +216,7 @@ func TestCatchpointsDeleteStoredOnSchemaUpdate(t *testing.T) { return } temporaryDirectroy := t.TempDir() - tempCatchpointDir := filepath.Join(temporaryDirectroy, trackerdb.CatchpointDirName) + tempCatchpointDir := filepath.Join(temporaryDirectroy, catchpointdb.CatchpointDirName) // creating empty catchpoint directories emptyDirPath := path.Join(tempCatchpointDir, "2f", "e1") @@ -252,7 +253,7 @@ func TestCatchpointsDeleteStoredOnSchemaUpdate(t *testing.T) { _, err = trackerDBInitialize(ml, true, ct.dbDirectory) require.NoError(t, err) - emptyDirs, err := trackerdb.GetEmptyDirs(tempCatchpointDir) + emptyDirs, err := catchpointdb.GetEmptyDirs(tempCatchpointDir) require.NoError(t, err) onlyTempDirEmpty := len(emptyDirs) == 0 require.Equal(t, onlyTempDirEmpty, true) @@ -320,7 +321,7 @@ func TestRecordCatchpointFile(t *testing.T) { require.NoError(t, err) require.Equal(t, conf.CatchpointFileHistoryLength, numberOfCatchpointFiles) - emptyDirs, err := trackerdb.GetEmptyDirs(temporaryDirectory) + emptyDirs, err := catchpointdb.GetEmptyDirs(temporaryDirectory) require.NoError(t, err) onlyCatchpointDirEmpty := len(emptyDirs) == 0 || (len(emptyDirs) == 1 && emptyDirs[0] == temporaryDirectory) @@ -335,7 +336,7 @@ func createCatchpoint(t *testing.T, ct *catchpointTracker, accountsRound basics. require.Equal(t, calculateStateProofVerificationHash(t, ml), stateProofVerificationHash) - err = ct.createCatchpoint(context.Background(), accountsRound, round, trackerdb.CatchpointFirstStageInfo{BiggestChunkLen: biggestChunkLen}, crypto.Digest{}) + err = ct.createCatchpoint(context.Background(), accountsRound, round, catchpointdb.CatchpointFirstStageInfo{BiggestChunkLen: biggestChunkLen}, crypto.Digest{}) require.NoError(t, err) } @@ -401,9 +402,7 @@ func writeDummySpVerification(t *testing.T, nextIndexForContext uint64, numberOf e.LastAttestedRound = basics.Round(nextIndexForContext + i) contexts[i] = &e } - writer := tx.MakeSpVerificationCtxReaderWriter() - - return writer.StoreSPContexts(ctx, contexts[:]) + return tx.MakeSpVerificationCtxWriter().StoreSPContexts(ctx, contexts[:]) }) require.NoError(t, err) } @@ -423,7 +422,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) { ct.initialize(cfg, ".") temporaryDirectroy := b.TempDir() - catchpointsDirectory := filepath.Join(temporaryDirectroy, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(temporaryDirectroy, catchpointdb.CatchpointDirName) err := os.Mkdir(catchpointsDirectory, 0777) require.NoError(b, err) @@ -436,7 +435,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) { // at this point, the database was created. We want to fill the accounts data accountsNumber := 6000000 * b.N err = ml.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -457,7 +456,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) { } } - return arw.UpdateAccountsHashRound(ctx, 1) + return aw.UpdateAccountsHashRound(ctx, 1) }) require.NoError(b, err) @@ -1049,7 +1048,7 @@ func TestCatchpointFirstStageInfoPruning(t *testing.T) { defer ct.close() temporaryDirectory := t.TempDir() - catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName) err := os.Mkdir(catchpointsDirectory, 0777) require.NoError(t, err) @@ -1147,7 +1146,7 @@ func TestCatchpointFirstStagePersistence(t *testing.T) { defer ml.Close() tempDirectory := t.TempDir() - catchpointsDirectory := filepath.Join(tempDirectory, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(tempDirectory, catchpointdb.CatchpointDirName) cfg := config.GetDefaultLocal() cfg.CatchpointInterval = 4 @@ -1192,12 +1191,11 @@ func TestCatchpointFirstStagePersistence(t *testing.T) { defer ml2.Close() ml.Close() - cps2, err := ml2.dbs.MakeCatchpointReaderWriter() - require.NoError(t, err) + cps2 := ml2.catchpointDB() // Insert unfinished first stage record. err = cps2.WriteCatchpointStateUint64( - context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo, 1) + context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo, 1) require.NoError(t, err) // Delete the database record. @@ -1221,7 +1219,7 @@ func TestCatchpointFirstStagePersistence(t *testing.T) { // Check that the unfinished first stage record is deleted. v, err := ct2.catchpointStore.ReadCatchpointStateUint64( - context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo) + context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo) require.NoError(t, err) require.Zero(t, v) } @@ -1248,7 +1246,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) { defer ml.Close() tempDirectory := t.TempDir() - catchpointsDirectory := filepath.Join(tempDirectory, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(tempDirectory, catchpointdb.CatchpointDirName) cfg := config.GetDefaultLocal() cfg.CatchpointInterval = 4 @@ -1271,7 +1269,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) { firstStageRound := secondStageRound - basics.Round(protoParams.CatchpointLookback) catchpointDataFilePath := filepath.Join(catchpointsDirectory, makeCatchpointDataFilePath(firstStageRound)) - var firstStageInfo trackerdb.CatchpointFirstStageInfo + var firstStageInfo catchpointdb.CatchpointFirstStageInfo var catchpointData []byte // Add blocks until the first catchpoint round. @@ -1312,7 +1310,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) { // Check that the data file exists. catchpointFilePath := - filepath.Join(catchpointsDirectory, trackerdb.MakeCatchpointFilePath(secondStageRound)) + filepath.Join(catchpointsDirectory, catchpointdb.MakeCatchpointFilePath(secondStageRound)) info, err := os.Stat(catchpointFilePath) require.NoError(t, err) @@ -1331,20 +1329,19 @@ func TestCatchpointSecondStagePersistence(t *testing.T) { err = os.WriteFile(catchpointDataFilePath, catchpointData, 0644) require.NoError(t, err) - cps2, err := ml2.dbs.MakeCatchpointReaderWriter() - require.NoError(t, err) + cw2 := ml2.catchpointDB() // Restore the first stage database record. - err = cps2.InsertOrReplaceCatchpointFirstStageInfo(context.Background(), firstStageRound, &firstStageInfo) + err = cw2.InsertOrReplaceCatchpointFirstStageInfo(context.Background(), firstStageRound, &firstStageInfo) require.NoError(t, err) // Insert unfinished catchpoint record. - err = cps2.InsertUnfinishedCatchpoint( + err = cw2.InsertUnfinishedCatchpoint( context.Background(), secondStageRound, crypto.Digest{}) require.NoError(t, err) // Delete the catchpoint file database record. - err = cps2.StoreCatchpoint( + err = cw2.StoreCatchpoint( context.Background(), secondStageRound, "", "", 0) require.NoError(t, err) @@ -1523,8 +1520,7 @@ func TestCatchpointSecondStageDeletesUnfinishedCatchpointRecordAfterRestart(t *t defer ml2.Close() ml.Close() - cps2, err := ml2.dbs.MakeCatchpointReaderWriter() - require.NoError(t, err) + cps2 := ml2.catchpointDB() // Sanity check: first stage record should be deleted. _, exists, err := cps2.SelectCatchpointFirstStageInfo(context.Background(), firstStageRound) @@ -1811,7 +1807,7 @@ func TestCatchpointLargeAccountCountCatchpointGeneration(t *testing.T) { ct := newCatchpointTracker(t, ml, conf, ".") temporaryDirectory := t.TempDir() - catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName) + catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName) err := os.Mkdir(catchpointsDirectory, 0777) require.NoError(t, err) defer os.RemoveAll(catchpointsDirectory) diff --git a/ledger/catchpointwriter.go b/ledger/catchpointwriter.go index df8b72ba6f..4c22e035b0 100644 --- a/ledger/catchpointwriter.go +++ b/ledger/catchpointwriter.go @@ -108,17 +108,17 @@ func (data catchpointStateProofVerificationContext) ToBeHashed() (protocol.HashI } func makeCatchpointWriter(ctx context.Context, filePath string, tx trackerdb.TransactionScope, maxResourcesPerChunk int) (*catchpointWriter, error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsReader() if err != nil { return nil, err } - totalAccounts, err := arw.TotalAccounts(ctx) + totalAccounts, err := aw.TotalAccounts(ctx) if err != nil { return nil, err } - totalKVs, err := arw.TotalKVs(ctx) + totalKVs, err := aw.TotalKVs(ctx) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (cw *catchpointWriter) Abort() error { } func (cw *catchpointWriter) WriteStateProofVerificationContext() (crypto.Digest, error) { - rawData, err := cw.tx.MakeSpVerificationCtxReaderWriter().GetAllSPContexts(cw.ctx) + rawData, err := cw.tx.MakeSpVerificationCtxReader().GetAllSPContexts(cw.ctx) if err != nil { return crypto.Digest{}, err } diff --git a/ledger/catchpointwriter_test.go b/ledger/catchpointwriter_test.go index db54b4527c..efb3ca86e6 100644 --- a/ledger/catchpointwriter_test.go +++ b/ledger/catchpointwriter_test.go @@ -284,7 +284,7 @@ func TestBasicCatchpointWriter(t *testing.T) { require.Equal(t, uint64(len(accts)), uint64(len(chunk.Balances))) } -func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath string, filepath string, maxResourcesPerChunk int) CatchpointFileHeader { +func testWriteCatchpoint(t *testing.T, rdb trackerdb.Store, datapath string, filepath string, maxResourcesPerChunk int) CatchpointFileHeader { var totalAccounts uint64 var totalChunks uint64 var biggestChunkLen uint64 @@ -300,7 +300,7 @@ func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath stri return err } - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } @@ -318,11 +318,11 @@ func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath stri totalAccounts = writer.totalAccounts totalChunks = writer.chunkNum biggestChunkLen = writer.biggestChunkLen - accountsRnd, err = arw.AccountsRound() + accountsRnd, err = ar.AccountsRound() if err != nil { return } - totals, err = arw.AccountsTotals(ctx, false) + totals, err = ar.AccountsTotals(ctx, false) return }) require.NoError(t, err) @@ -420,12 +420,12 @@ func TestCatchpointReadDatabaseOverflowSingleAccount(t *testing.T) { cw, err := makeCatchpointWriter(context.Background(), catchpointDataFilePath, tx, maxResourcesPerChunk) require.NoError(t, err) - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } - expectedTotalResources, err := arw.TotalResources(ctx) + expectedTotalResources, err := ar.TotalResources(ctx) if err != nil { return err } @@ -506,17 +506,17 @@ func TestCatchpointReadDatabaseOverflowAccounts(t *testing.T) { catchpointDataFilePath := filepath.Join(temporaryDirectory, "15.data") err = ml.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } - expectedTotalAccounts, err := arw.TotalAccounts(ctx) + expectedTotalAccounts, err := ar.TotalAccounts(ctx) if err != nil { return err } - expectedTotalResources, err := arw.TotalResources(ctx) + expectedTotalResources, err := ar.TotalResources(ctx) if err != nil { return err } @@ -602,7 +602,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) { ctx := context.Background() err = l.trackerDBs.TransactionContext(ctx, func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return nil } @@ -618,7 +618,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) { require.NotEmpty(t, h1) // reset hashes - err = arw.ResetAccountHashes(ctx) + err = aw.ResetAccountHashes(ctx) require.NoError(t, err) // rebuild the MT @@ -664,7 +664,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) { require.NoError(t, err) } -func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess trackerdb.TrackerStore, filepath string) *Ledger { +func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess trackerdb.Store, filepath string) *Ledger { // create a ledger. var initState ledgercore.InitState initState.Block.CurrentProtocol = protocol.ConsensusCurrentVersion @@ -686,17 +686,17 @@ func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess tracke err = accessor.BuildMerkleTrie(context.Background(), nil) require.NoError(t, err) - err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) error { + err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { cw, err := tx.MakeCatchpointWriter() if err != nil { return err } - return cw.ApplyCatchpointStagingBalances(ctx, 0, 0) + return cw.Apply(ctx, 0, 0) }) require.NoError(t, err) - balanceTrieStats := func(db trackerdb.TrackerStore) merkletrie.Stats { + balanceTrieStats := func(db trackerdb.Store) merkletrie.Stats { var stats merkletrie.Stats err = db.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { committer, err := tx.MakeMerkleCommitter(false) diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go index 990c977dac..fe8bd0e7c4 100644 --- a/ledger/catchupaccessor.go +++ b/ledger/catchupaccessor.go @@ -34,6 +34,7 @@ import ( "github.com/algorand/go-algorand/ledger/encoded" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/ledger/store/blockdb" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -95,102 +96,30 @@ type CatchpointCatchupAccessor interface { } type stagingWriter interface { - writeBalances(context.Context, []trackerdb.NormalizedAccountBalance) error - writeCreatables(context.Context, []trackerdb.NormalizedAccountBalance) error - writeHashes(context.Context, []trackerdb.NormalizedAccountBalance) error - writeKVs(context.Context, []encoded.KVRecordV6) error - isShared() bool + write(context.Context, trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) } type stagingWriterImpl struct { - wdb trackerdb.TrackerStore + wdb trackerdb.Store } -func (w *stagingWriterImpl) writeBalances(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - return crw.WriteCatchpointStagingBalances(ctx, balances) - }) -} - -func (w *stagingWriterImpl) writeKVs(ctx context.Context, kvrs []encoded.KVRecordV6) error { - return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - - keys := make([][]byte, len(kvrs)) - values := make([][]byte, len(kvrs)) - hashes := make([][]byte, len(kvrs)) - for i := 0; i < len(kvrs); i++ { - keys[i] = kvrs[i].Key - - // Since `encoded.KVRecordV6` is `omitempty` and `omitemptyarray`, - // when we have an instance of `encoded.KVRecordV6` with nil value, - // an empty box is unmarshalled to have `nil` value, - // while this might be mistaken to be a box deletion. - // - // We don't want to mistake this to be a deleted box: - // We are (and should be) during Fast Catchup (FC) - // writing to DB with empty byte string, rather than writing nil. - // - // This matters in sqlite3, - // for sqlite3 differs on writing nil byte slice to table from writing []byte{}: - // - writing nil byte slice is true that `value is NULL` - // - writing []byte{} is false on `value is NULL`. - // - // For the sake of consistency, we convert nil to []byte{}. - // - // Also, from a round by round catchup perspective, - // when we delete a box, in accountsNewRoundImpl method, - // the kv pair with value = nil will be deleted from kvstore table. - // Thus, it seems more consistent and appropriate to write as []byte{}. - - if kvrs[i].Value == nil { - kvrs[i].Value = []byte{} - } - values[i] = kvrs[i].Value - hashes[i] = trackerdb.KvHashBuilderV6(string(keys[i]), values[i]) - } - - return crw.WriteCatchpointStagingKVs(ctx, keys, values, hashes) - }) -} - -func (w *stagingWriterImpl) writeCreatables(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - - return crw.WriteCatchpointStagingCreatable(ctx, balances) - }) -} - -func (w *stagingWriterImpl) writeHashes(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - crw, err := tx.MakeCatchpointReaderWriter() +func (w *stagingWriterImpl) write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) { + var report trackerdb.CatchpointReport + err := w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { + crw, err := tx.MakeCatchpointWriter() if err != nil { return err } - - return crw.WriteCatchpointStagingHashes(ctx, balances) + report, err = crw.Write(ctx, payload) + return err }) -} - -func (w *stagingWriterImpl) isShared() bool { - return w.wdb.IsSharedCacheConnection() + return report, err } // catchpointCatchupAccessorImpl is the concrete implementation of the CatchpointCatchupAccessor interface type catchpointCatchupAccessorImpl struct { ledger *Ledger - catchpointStore trackerdb.CatchpointReaderWriter + catchpointStore catchpointdb.Store stagingWriter stagingWriter @@ -242,10 +171,9 @@ type CatchupAccessorClientLedger interface { // MakeCatchpointCatchupAccessor creates a CatchpointCatchupAccessor given a ledger func MakeCatchpointCatchupAccessor(ledger *Ledger, log logging.Logger) CatchpointCatchupAccessor { - crw, _ := ledger.trackerDB().MakeCatchpointReaderWriter() return &catchpointCatchupAccessorImpl{ ledger: ledger, - catchpointStore: crw, + catchpointStore: ledger.catchpointDB(), stagingWriter: &stagingWriterImpl{wdb: ledger.trackerDB()}, log: log, } @@ -254,9 +182,9 @@ func MakeCatchpointCatchupAccessor(ledger *Ledger, log logging.Logger) Catchpoin // GetState returns the current state of the catchpoint catchup func (c *catchpointCatchupAccessorImpl) GetState(ctx context.Context) (state CatchpointCatchupState, err error) { var istate uint64 - istate, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState) + istate, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState) if err != nil { - return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err) + return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err) } state = CatchpointCatchupState(istate) return @@ -267,18 +195,18 @@ func (c *catchpointCatchupAccessorImpl) SetState(ctx context.Context, state Catc if state < CatchpointCatchupStateInactive || state > catchpointCatchupStateLast { return fmt.Errorf("invalid catchpoint catchup state provided : %d", state) } - err = c.catchpointStore.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, uint64(state)) + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, uint64(state)) if err != nil { - return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err) + return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err) } return } // GetLabel returns the current catchpoint catchup label func (c *catchpointCatchupAccessorImpl) GetLabel(ctx context.Context) (label string, err error) { - label, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel) + label, err = c.catchpointStore.ReadCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel) if err != nil { - return "", fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err) + return "", fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err) } return } @@ -290,9 +218,9 @@ func (c *catchpointCatchupAccessorImpl) SetLabel(ctx context.Context, label stri if err != nil { return } - err = c.catchpointStore.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, label) + err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, label) if err != nil { - return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err) + return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err) } return } @@ -304,38 +232,51 @@ func (c *catchpointCatchupAccessorImpl) ResetStagingBalances(ctx context.Context } start := time.Now() ledgerResetstagingbalancesCount.Inc(nil) - err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { + err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { crw, err := tx.MakeCatchpointWriter() if err != nil { return err } - err = crw.ResetCatchpointStagingBalances(ctx, newCatchup) + err = crw.Reset(ctx, newCatchup) if err != nil { return fmt.Errorf("unable to reset catchpoint catchup balances : %v", err) } - if !newCatchup { - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, 0) + return + }) + if err != nil { + // failed to reset staging balances + ledgerResetstagingbalancesMicros.AddMicrosecondsSince(start, nil) + return + } + + if !newCatchup { + // reset catchup internal durable state + err = c.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) { + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, 0) if err != nil { return err } - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, 0) + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, 0) if err != nil { return err } - err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, "") + err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, "") if err != nil { return err } - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, 0) + + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, 0) if err != nil { - return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err) + return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err) } - } - return - }) + + return + }) + } + ledgerResetstagingbalancesMicros.AddMicrosecondsSince(start, nil) return } @@ -423,33 +364,41 @@ func (c *catchpointCatchupAccessorImpl) processStagingContent(ctx context.Contex // TotalAccounts, TotalAccounts, Catchpoint, BlockHeaderDigest, BalancesRound start := time.Now() ledgerProcessstagingcontentCount.Inc(nil) - err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { - cw, err := tx.MakeCatchpointWriter() - if err != nil { - return err - } - err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion, fileHeader.Version) - if err != nil { - return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup version '%s': %v", trackerdb.CatchpointStateCatchupVersion, err) - } - aw, err := tx.MakeAccountsWriter() + + // update catchup internal durable state + err = c.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) { + err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupVersion, fileHeader.Version) if err != nil { - return err + return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup version '%s': %v", catchpointdb.CatchpointStateCatchupVersion, err) } - - err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, uint64(fileHeader.BlocksRound)) + err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, uint64(fileHeader.BlocksRound)) if err != nil { - return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err) + return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBlockRound, err) } if fileHeader.Version >= CatchpointFileVersionV6 { - err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound, uint64(fileHeader.BlocksRound)) + err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound, uint64(fileHeader.BlocksRound)) if err != nil { - return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupHashRound, err) + return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupHashRound, err) } } - err = aw.AccountsPutTotals(fileHeader.Totals, true) return }) + if err != nil { + // failed to update catchup internal state + ledgerProcessstagingcontentMicros.AddMicrosecondsSince(start, nil) + return err + } + + // put totals on the ledger + err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { + aw, err := tx.MakeAccountsWriter() + if err != nil { + return err + } + + return aw.AccountsPutTotals(fileHeader.Totals, true) + }) + ledgerProcessstagingcontentMicros.AddMicrosecondsSince(start, nil) if err == nil { progress.SeenHeader = true @@ -588,101 +537,49 @@ func (c *catchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte } } - wg := sync.WaitGroup{} - - var errBalances error - var errCreatables error - var errHashes error - var errKVs error - var durBalances time.Duration - var durCreatables time.Duration - var durHashes time.Duration - var durKVs time.Duration - - // start the balances writer - wg.Add(1) - go func() { - defer wg.Done() - writeBalancesStart := time.Now() - errBalances = c.stagingWriter.writeBalances(ctx, normalizedAccountBalances) - durBalances = time.Since(writeBalancesStart) - }() - - // on a in-memory database, wait for the writer to finish before starting the new writer - if c.stagingWriter.isShared() { - wg.Wait() - } - - // starts the creatables writer - wg.Add(1) - go func() { - defer wg.Done() - hasCreatables := false - for _, accBal := range normalizedAccountBalances { - for _, res := range accBal.Resources { - if res.IsOwning() { - hasCreatables = true - break - } - } - } - if hasCreatables { - writeCreatablesStart := time.Now() - errCreatables = c.stagingWriter.writeCreatables(ctx, normalizedAccountBalances) - durCreatables = time.Since(writeCreatablesStart) + // fix empty KV's + for i := range chunkKVs { + // Since `encoded.KVRecordV6` is `omitempty` and `omitemptyarray`, + // when we have an instance of `encoded.KVRecordV6` with nil value, + // an empty box is unmarshalled to have `nil` value, + // while this might be mistaken to be a box deletion. + // + // We don't want to mistake this to be a deleted box: + // We are (and should be) during Fast Catchup (FC) + // writing to DB with empty byte string, rather than writing nil. + // + // This matters in sqlite3, + // for sqlite3 differs on writing nil byte slice to table from writing []byte{}: + // - writing nil byte slice is true that `value is NULL` + // - writing []byte{} is false on `value is NULL`. + // + // For the sake of consistency, we convert nil to []byte{}. + // + // Also, from a round by round catchup perspective, + // when we delete a box, in accountsNewRoundImpl method, + // the kv pair with value = nil will be deleted from kvstore table. + // Thus, it seems more consistent and appropriate to write as []byte{}. + if chunkKVs[i].Value == nil { + chunkKVs[i].Value = []byte{} } - }() - - // on a in-memory database, wait for the writer to finish before starting the new writer - if c.stagingWriter.isShared() { - wg.Wait() } - // start the accounts pending hashes writer - wg.Add(1) - go func() { - defer wg.Done() - writeHashesStart := time.Now() - errHashes = c.stagingWriter.writeHashes(ctx, normalizedAccountBalances) - durHashes = time.Since(writeHashesStart) - }() - - // on a in-memory database, wait for the writer to finish before starting the new writer - if c.stagingWriter.isShared() { - wg.Wait() - } - - // start the kv store writer - wg.Add(1) - go func() { - defer wg.Done() - - writeKVsStart := time.Now() - errKVs = c.stagingWriter.writeKVs(ctx, chunkKVs) - durKVs = time.Since(writeKVsStart) - }() - - wg.Wait() - - if errBalances != nil { - return errBalances - } - if errCreatables != nil { - return errCreatables - } - if errHashes != nil { - return errHashes - } - if errKVs != nil { - return errKVs + // apply catchpoint + var report trackerdb.CatchpointReport + report, err = c.stagingWriter.write(ctx, trackerdb.CatchpointPayload{ + Accounts: normalizedAccountBalances, + KVRecords: chunkKVs, + }) + if err != nil { + return err } - progress.BalancesWriteDuration += durBalances - progress.CreatablesWriteDuration += durCreatables - progress.HashesWriteDuration += durHashes - progress.KVWriteDuration += durKVs - ledgerProcessstagingbalancesMicros.AddMicrosecondsSince(start, nil) + + progress.BalancesWriteDuration += report.BalancesWriteDuration + progress.CreatablesWriteDuration += report.CreatablesWriteDuration + progress.HashesWriteDuration += report.HashesWriteDuration + progress.KVWriteDuration += report.KVWriteDuration progress.ProcessedBytes += uint64(len(bytes)) progress.ProcessedKVs += uint64(len(chunkKVs)) for _, acctBal := range normalizedAccountBalances { @@ -723,23 +620,6 @@ func countHashes(hashes [][]byte) (accountCount, kvCount uint64) { // BuildMerkleTrie would process the catchpointpendinghashes and insert all the items in it into the merkle trie func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error) { dbs := c.ledger.trackerDB() - err = dbs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { - crw, err := tx.MakeCatchpointWriter() - if err != nil { - return err - } - - // creating the index can take a while, so ensure we don't generate false alerts for no good reason. - _, err = tx.ResetTransactionWarnDeadline(ctx, time.Now().Add(120*time.Second)) - if err != nil { - return err - } - - return crw.CreateCatchpointStagingHashesIndex(ctx) - }) - if err != nil { - return - } wg := sync.WaitGroup{} wg.Add(2) @@ -754,7 +634,7 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro defer wg.Done() defer close(writerQueue) - dbErr := dbs.Snapshot(func(transactionCtx context.Context, tx trackerdb.SnapshotScope) (err error) { + dbErr := dbs.Transaction(func(transactionCtx context.Context, tx trackerdb.TransactionScope) (err error) { it := tx.MakeCatchpointPendingHashesIterator(trieRebuildAccountChunkSize) var hashes [][]byte for { @@ -919,9 +799,9 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro // GetCatchupBlockRound returns the latest block round matching the current catchpoint func (c *catchpointCatchupAccessorImpl) GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error) { var iRound uint64 - iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound) + iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound) if err != nil { - return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchpointLookback, err) + return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchpointLookback, err) } return basics.Round(iRound), nil } @@ -935,27 +815,27 @@ func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl var catchpointLabel string var version uint64 - catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel) + catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel) if err != nil { - return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err) + return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err) } - version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion) + version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupVersion) if err != nil { return fmt.Errorf("unable to retrieve catchpoint version: %v", err) } var iRound uint64 - iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound) + iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound) if err != nil { - return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err) + return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBlockRound, err) } blockRound = basics.Round(iRound) start := time.Now() ledgerVerifycatchpointCount.Inc(nil) err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + ar, err := tx.MakeAccountsReader() if err != nil { return err } @@ -976,12 +856,12 @@ func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl return fmt.Errorf("unable to get trie root hash: %v", err) } - totals, err = arw.AccountsTotals(ctx, true) + totals, err = ar.AccountsTotals(ctx, true) if err != nil { return fmt.Errorf("unable to get accounts totals: %v", err) } - rawStateProofVerificationContext, err = tx.MakeSpVerificationCtxReaderWriter().GetAllSPContextsFromCatchpointTbl(ctx) + rawStateProofVerificationContext, err = tx.MakeSpVerificationCtxReader().GetAllSPContextsFromCatchpointTbl(ctx) if err != nil { return fmt.Errorf("unable to get state proof verification data: %v", err) } @@ -1026,18 +906,10 @@ func (c *catchpointCatchupAccessorImpl) StoreBalancesRound(ctx context.Context, balancesRound := blk.Round() - basics.Round(catchpointLookback) start := time.Now() ledgerStorebalancesroundCount.Inc(nil) - err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) { - crw, err := tx.MakeCatchpointWriter() - if err != nil { - return err - } - - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, uint64(balancesRound)) - if err != nil { - return fmt.Errorf("CatchpointCatchupAccessorImpl::StoreBalancesRound: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBalancesRound, err) - } - return - }) + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, uint64(balancesRound)) + if err != nil { + return fmt.Errorf("CatchpointCatchupAccessorImpl::StoreBalancesRound: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBalancesRound, err) + } ledgerStorebalancesroundMicros.AddMicrosecondsSince(start, nil) return } @@ -1126,37 +998,41 @@ func (c *catchpointCatchupAccessorImpl) CompleteCatchup(ctx context.Context) (er func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err error) { start := time.Now() ledgerCatchpointFinishBalsCount.Inc(nil) - err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - crw, err := tx.MakeCatchpointReaderWriter() - if err != nil { - return err - } - arw, err := tx.MakeAccountsReaderWriter() - if err != nil { - return err - } + balancesRound, err := c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound) + if err != nil { + return err + } + + hashRound, err := c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound) + if err != nil { + return err + } - var balancesRound, hashRound uint64 - var totals ledgercore.AccountTotals + err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { + // get current totals - balancesRound, err = crw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound) + ar, err := tx.MakeAccountsReader() if err != nil { return err } - hashRound, err = crw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound) + totals, err := ar.AccountsTotals(ctx, true) if err != nil { return err } - totals, err = arw.AccountsTotals(ctx, true) + // + // reset trackerdb + // + + aw, err := tx.MakeAccountsWriter() if err != nil { return err } if hashRound == 0 { - err = arw.ResetAccountHashes(ctx) + err = aw.ResetAccountHashes(ctx) if err != nil { return err } @@ -1168,7 +1044,7 @@ func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err // it might be necessary to restore it into the latest database version. To do that, one // will need to run the 6->7 migration code manually here or in a similar function to create // onlineaccounts and other V7 tables. - err = arw.AccountsReset(ctx) + err = aw.AccountsReset(ctx) if err != nil { return err } @@ -1188,50 +1064,69 @@ func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err } } - err = crw.ApplyCatchpointStagingBalances(ctx, basics.Round(balancesRound), basics.Round(hashRound)) - if err != nil { - return err - } + // + // apply catchpoint + // - err = arw.AccountsPutTotals(totals, false) + crw, err := tx.MakeCatchpointWriter() if err != nil { return err } - err = crw.ResetCatchpointStagingBalances(ctx, false) + err = crw.Apply(ctx, basics.Round(balancesRound), basics.Round(hashRound)) if err != nil { return err } - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, 0) + err = aw.AccountsPutTotals(totals, false) if err != nil { return err } - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, 0) + err = crw.Reset(ctx, false) if err != nil { return err } - err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, "") - if err != nil { - return err - } + return nil + }) - if hashRound != 0 { - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound, 0) - if err != nil { - return err - } - } + if err != nil { + // apply failed + ledgerCatchpointFinishBalsMicros.AddMicrosecondsSince(start, nil) + return err + } + + // apply succeded + // reset catchup durable state + + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, 0) + if err != nil { + return err + } + + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, 0) + if err != nil { + return err + } - err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, 0) + err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, "") + if err != nil { + return err + } + + if hashRound != 0 { + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound, 0) if err != nil { - return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err) + return err } + } + + err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, 0) + if err != nil { + return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err) + } - return - }) ledgerCatchpointFinishBalsMicros.AddMicrosecondsSince(start, nil) return err } diff --git a/ledger/catchupaccessor_test.go b/ledger/catchupaccessor_test.go index a97377bd3e..b4c249d772 100644 --- a/ledger/catchupaccessor_test.go +++ b/ledger/catchupaccessor_test.go @@ -527,20 +527,8 @@ type testStagingWriter struct { hashes map[[4 + crypto.DigestSize]byte]int } -func (w *testStagingWriter) writeBalances(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - return nil -} - -func (w *testStagingWriter) writeCreatables(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - return nil -} - -func (w *testStagingWriter) writeKVs(ctx context.Context, kvrs []encoded.KVRecordV6) error { - return nil -} - -func (w *testStagingWriter) writeHashes(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error { - for _, bal := range balances { +func (w *testStagingWriter) write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) { + for _, bal := range payload.Accounts { for _, hash := range bal.AccountHashes { var key [4 + crypto.DigestSize]byte require.Len(w.t, hash, 4+crypto.DigestSize) @@ -548,11 +536,7 @@ func (w *testStagingWriter) writeHashes(ctx context.Context, balances []trackerd w.hashes[key] = w.hashes[key] + 1 } } - return nil -} - -func (w *testStagingWriter) isShared() bool { - return false + return trackerdb.CatchpointReport{}, nil } // makeTestCatchpointCatchupAccessor creates a CatchpointCatchupAccessor given a ledger diff --git a/ledger/ledger.go b/ledger/ledger.go index 0c77470873..31a8b04d85 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -37,7 +37,9 @@ import ( "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/ledger/store/blockdb" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver" "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" @@ -51,8 +53,9 @@ type Ledger struct { // Database connections to the DBs storing blocks and tracker state. // We use potentially different databases to avoid SQLite contention // during catchup. - trackerDBs trackerdb.TrackerStore - blockDBs db.Pair + trackerDBs trackerdb.Store + catchpointDbs catchpointdb.Store + blockDBs db.Pair // blockQ is the buffer of added blocks that will be flushed to // persistent storage @@ -144,14 +147,11 @@ func OpenLedger( } }() - l.trackerDBs, l.blockDBs, err = openLedgerDB(dbPathPrefix, dbMem) + l.trackerDBs, l.blockDBs, err = openLedgerDB(dbPathPrefix, dbMem, cfg, log) if err != nil { err = fmt.Errorf("OpenLedger.openLedgerDB %v", err) return nil, err } - l.trackerDBs.SetLogger(log) - l.blockDBs.Rdb.SetLogger(log) - l.blockDBs.Wdb.SetLogger(log) l.setSynchronousMode(context.Background(), l.synchronousMode) @@ -284,12 +284,9 @@ func (l *Ledger) verifyMatchingGenesisHash() (err error) { return } -func openLedgerDB(dbPathPrefix string, dbMem bool) (trackerDBs trackerdb.TrackerStore, blockDBs db.Pair, err error) { +func openLedgerDB(dbPathPrefix string, dbMem bool, cfg config.Local, log logging.Logger) (trackerDBs trackerdb.Store, blockDBs db.Pair, err error) { // Backwards compatibility: we used to store both blocks and tracker // state in a single SQLite db file. - var trackerDBFilename string - var blockDBFilename string - if !dbMem { commonDBFilename := dbPathPrefix + ".sqlite" _, err = os.Stat(commonDBFilename) @@ -302,20 +299,36 @@ func openLedgerDB(dbPathPrefix string, dbMem bool) (trackerDBs trackerdb.Tracker } } - trackerDBFilename = dbPathPrefix + ".tracker.sqlite" - blockDBFilename = dbPathPrefix + ".block.sqlite" - outErr := make(chan error, 2) go func() { var lerr error - trackerDBs, lerr = sqlitedriver.OpenTrackerSQLStore(trackerDBFilename, dbMem) + switch cfg.StorageEngine { + case "pebbledb": + dir := dbPathPrefix + "/tracker" + trackerDBs, lerr = pebbledbdriver.Open(dir, dbMem, config.Consensus[protocol.ConsensusCurrentVersion], log) + // pebbledb needs to be explicitly configured. + // anything else will initialize a sqlite engine. + case "sqlite": + fallthrough + default: + file := dbPathPrefix + ".tracker.sqlite" + trackerDBs, lerr = sqlitedriver.Open(file, dbMem, log) + } + outErr <- lerr }() go func() { var lerr error + blockDBFilename := dbPathPrefix + ".block.sqlite" blockDBs, lerr = db.OpenPair(blockDBFilename, dbMem) - outErr <- lerr + if lerr != nil { + outErr <- lerr + return + } + blockDBs.Rdb.SetLogger(log) + blockDBs.Wdb.SetLogger(log) + outErr <- nil }() err = <-outErr @@ -805,10 +818,15 @@ func (l *Ledger) GetCatchpointStream(round basics.Round) (ReadCloseSizer, error) } // ledgerForTracker methods -func (l *Ledger) trackerDB() trackerdb.TrackerStore { +func (l *Ledger) trackerDB() trackerdb.Store { return l.trackerDBs } +// ledgerForTracker methods +func (l *Ledger) catchpointDB() catchpointdb.Store { + return l.catchpointDbs +} + // ledgerForTracker methods func (l *Ledger) blockDB() db.Pair { return l.blockDBs diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go index e9a413678c..5165fcfa06 100644 --- a/ledger/ledger_test.go +++ b/ledger/ledger_test.go @@ -43,6 +43,7 @@ import ( "github.com/algorand/go-algorand/data/transactions/verify" "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" ledgertesting "github.com/algorand/go-algorand/ledger/testing" "github.com/algorand/go-algorand/logging" @@ -2277,7 +2278,7 @@ func TestLedgerReloadTxTailHistoryAccess(t *testing.T) { // reset tables and re-init again, similary to the catchpount apply code // since the ledger has only genesis accounts, this recreates them - err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) error { + err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { arw, err := tx.MakeAccountsWriter() if err != nil { return err @@ -2296,7 +2297,7 @@ func TestLedgerReloadTxTailHistoryAccess(t *testing.T) { DbPathPrefix: l.catchpoint.dbDirectory, BlockDb: l.blockDBs, } - _, err0 = tx.Testing().RunMigrations(ctx, tp, l.log, preReleaseDBVersion /*target database version*/) + _, err0 = tx.RunMigrations(ctx, tp, l.log, preReleaseDBVersion /*target database version*/) if err0 != nil { return err0 } @@ -2439,7 +2440,7 @@ func TestLedgerMigrateV6ShrinkDeltas(t *testing.T) { cfg.MaxAcctLookback = proto.MaxBalLookback log := logging.TestingLog(t) log.SetLevel(logging.Info) // prevent spamming with ledger.AddValidatedBlock debug message - trackerDB, blockDB, err := openLedgerDB(dbName, inMem) + trackerDB, blockDB, err := openLedgerDB(dbName, inMem, cfg, log) require.NoError(t, err) defer func() { trackerDB.Close() @@ -3301,7 +3302,7 @@ func TestLedgerCatchpointSPVerificationTracker(t *testing.T) { catchpointAccessor, accessorProgress := initializeTestCatchupAccessor(t, l, uint64(len(initkeys))) - relCatchpointFilePath := filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(basics.Round(cfg.CatchpointInterval))) + relCatchpointFilePath := filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(basics.Round(cfg.CatchpointInterval))) catchpointData := readCatchpointFile(t, relCatchpointFilePath) err = catchpointAccessor.ProcessStagingBalances(context.Background(), catchpointData[1].headerName, catchpointData[1].data, &accessorProgress) diff --git a/ledger/spverificationtracker.go b/ledger/spverificationtracker.go index d98974897d..cca86fcca9 100644 --- a/ledger/spverificationtracker.go +++ b/ledger/spverificationtracker.go @@ -134,7 +134,7 @@ func (spt *spVerificationTracker) commitRound(ctx context.Context, tx trackerdb. } if dcc.spVerification.lastDeleteIndex >= 0 { - err = tx.MakeSpVerificationCtxReaderWriter().DeleteOldSPContexts(ctx, dcc.spVerification.earliestLastAttestedRound) + err = tx.MakeSpVerificationCtxWriter().DeleteOldSPContexts(ctx, dcc.spVerification.earliestLastAttestedRound) } return err @@ -146,7 +146,7 @@ func commitSPContexts(ctx context.Context, tx trackerdb.TransactionScope, commit ptrToCtxs[i] = &commitData[i].verificationContext } - return tx.MakeSpVerificationCtxReaderWriter().StoreSPContexts(ctx, ptrToCtxs) + return tx.MakeSpVerificationCtxWriter().StoreSPContexts(ctx, ptrToCtxs) } func (spt *spVerificationTracker) postCommit(_ context.Context, dcc *deferredCommitContext) { diff --git a/ledger/spverificationtracker_test.go b/ledger/spverificationtracker_test.go index e0f073fe4d..a88c05f0ed 100644 --- a/ledger/spverificationtracker_test.go +++ b/ledger/spverificationtracker_test.go @@ -424,7 +424,7 @@ func TestStateProofVerificationTracker_LookupVerificationContext(t *testing.T) { _, err := spt.LookupVerificationContext(basics.Round(0)) a.ErrorIs(err, errSPVerificationContextNotFound) - a.ErrorContains(err, "no rows") + a.ErrorContains(err, "not found") finalLastAttestedRound := basics.Round(defaultStateProofInterval + contextToAdd*defaultStateProofInterval) _, err = spt.LookupVerificationContext(finalLastAttestedRound + basics.Round(defaultStateProofInterval)) diff --git a/ledger/store/catchpointdb/catchpoint.go b/ledger/store/catchpointdb/catchpoint.go new file mode 100644 index 0000000000..a361e81f7d --- /dev/null +++ b/ledger/store/catchpointdb/catchpoint.go @@ -0,0 +1,157 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchpointdb + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" +) + +// CatchpointDirName represents the directory name in which all the catchpoints files are stored +var CatchpointDirName = "catchpoints" + +// CatchpointState is used to store catchpoint related variables into the catchpointstate table. +// +//msgp:ignore CatchpointState +type CatchpointState string + +const ( + // CatchpointStateLastCatchpoint is written by a node once a catchpoint label is created for a round + CatchpointStateLastCatchpoint = CatchpointState("lastCatchpoint") + // CatchpointStateWritingFirstStageInfo state variable is set to 1 if catchpoint's first stage is unfinished, + // and is 0 otherwise. Used to clear / restart the first stage after a crash. + // This key is set in the same db transaction as the account updates, so the + // unfinished first stage corresponds to the current db round. + CatchpointStateWritingFirstStageInfo = CatchpointState("writingFirstStageInfo") + // CatchpointStateWritingCatchpoint if there is an unfinished catchpoint, this state variable is set to + // the catchpoint's round. Otherwise, it is set to 0. + // DEPRECATED. + CatchpointStateWritingCatchpoint = CatchpointState("writingCatchpoint") + // CatchpointStateCatchupState is the state of the catchup process. The variable is stored only during the catchpoint catchup process, and removed afterward. + CatchpointStateCatchupState = CatchpointState("catchpointCatchupState") + // CatchpointStateCatchupLabel is the label to which the currently catchpoint catchup process is trying to catchup to. + CatchpointStateCatchupLabel = CatchpointState("catchpointCatchupLabel") + // CatchpointStateCatchupBlockRound is the block round that is associated with the current running catchpoint catchup. + CatchpointStateCatchupBlockRound = CatchpointState("catchpointCatchupBlockRound") + // CatchpointStateCatchupBalancesRound is the balance round that is associated with the current running catchpoint catchup. Typically it would be + // equal to CatchpointStateCatchupBlockRound - 320. + CatchpointStateCatchupBalancesRound = CatchpointState("catchpointCatchupBalancesRound") + // CatchpointStateCatchupHashRound is the round that is associated with the hash of the merkle trie. Normally, it's identical to CatchpointStateCatchupBalancesRound, + // however, it could differ when we catchup from a catchpoint that was created using a different version : in this case, + // we set it to zero in order to reset the merkle trie. This would force the merkle trie to be re-build on startup ( if needed ). + CatchpointStateCatchupHashRound = CatchpointState("catchpointCatchupHashRound") + // CatchpointStateCatchpointLookback is the number of rounds we keep catchpoints for + CatchpointStateCatchpointLookback = CatchpointState("catchpointLookback") + // CatchpointStateCatchupVersion is the catchpoint version which the currently catchpoint catchup process is trying to catchup to. + CatchpointStateCatchupVersion = CatchpointState("catchpointCatchupVersion") +) + +// UnfinishedCatchpointRecord represents a stored record of an unfinished catchpoint. +type UnfinishedCatchpointRecord struct { + Round basics.Round + BlockHash crypto.Digest +} + +// CatchpointFirstStageInfo For the `catchpointfirststageinfo` table. +type CatchpointFirstStageInfo struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Totals ledgercore.AccountTotals `codec:"accountTotals"` + TrieBalancesHash crypto.Digest `codec:"trieBalancesHash"` + // Total number of accounts in the catchpoint data file. Only set when catchpoint + // data files are generated. + TotalAccounts uint64 `codec:"accountsCount"` + + // Total number of accounts in the catchpoint data file. Only set when catchpoint + // data files are generated. + TotalKVs uint64 `codec:"kvsCount"` + + // Total number of chunks in the catchpoint data file. Only set when catchpoint + // data files are generated. + TotalChunks uint64 `codec:"chunksCount"` + // BiggestChunkLen is the size in the bytes of the largest chunk, used when re-packing. + BiggestChunkLen uint64 `codec:"biggestChunk"` + + // StateProofVerificationHash is the hash of the state proof verification data contained in the catchpoint data file. + StateProofVerificationHash crypto.Digest `codec:"spVerificationHash"` +} + +// MakeCatchpointFilePath builds the path of a catchpoint file. +func MakeCatchpointFilePath(round basics.Round) string { + irnd := int64(round) / 256 + outStr := "" + for irnd > 0 { + outStr = filepath.Join(outStr, fmt.Sprintf("%02x", irnd%256)) + irnd = irnd / 256 + } + outStr = filepath.Join(outStr, strconv.FormatInt(int64(round), 10)+".catchpoint") + return outStr +} + +// RemoveSingleCatchpointFileFromDisk removes a single catchpoint file from the disk. this function does not leave empty directories +func RemoveSingleCatchpointFileFromDisk(dbDirectory, fileToDelete string) (err error) { + absCatchpointFileName := filepath.Join(dbDirectory, fileToDelete) + err = os.Remove(absCatchpointFileName) + if err == nil || os.IsNotExist(err) { + // it's ok if the file doesn't exist. + err = nil + } else { + // we can't delete the file, abort - + return fmt.Errorf("unable to delete old catchpoint file '%s' : %v", absCatchpointFileName, err) + } + splitedDirName := strings.Split(fileToDelete, string(os.PathSeparator)) + + var subDirectoriesToScan []string + //build a list of all the subdirs + currentSubDir := "" + for _, element := range splitedDirName { + currentSubDir = filepath.Join(currentSubDir, element) + subDirectoriesToScan = append(subDirectoriesToScan, currentSubDir) + } + + // iterating over the list of directories. starting from the sub dirs and moving up. + // skipping the file itself. + for i := len(subDirectoriesToScan) - 2; i >= 0; i-- { + absSubdir := filepath.Join(dbDirectory, subDirectoriesToScan[i]) + if _, err := os.Stat(absSubdir); os.IsNotExist(err) { + continue + } + + isEmpty, err := isDirEmpty(absSubdir) + if err != nil { + return fmt.Errorf("unable to read old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err) + } + if isEmpty { + err = os.Remove(absSubdir) + if err != nil { + if os.IsNotExist(err) { + continue + } + return fmt.Errorf("unable to delete old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err) + } + } + } + + return nil +} diff --git a/ledger/store/catchpointdb/catchpointdb.go b/ledger/store/catchpointdb/catchpointdb.go new file mode 100644 index 0000000000..4e639001a3 --- /dev/null +++ b/ledger/store/catchpointdb/catchpointdb.go @@ -0,0 +1,182 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchpointdb + +import ( + "context" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" +) + +type Store interface { + ReaderWriter + // init + RunMigrations(ctx context.Context, targetVersion int32) (err error) + // batch support + Batch(fn BatchFn) (err error) + BatchContext(ctx context.Context, fn BatchFn) (err error) + // root methods + DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) + // cleanup + Close() +} + +// BatchFn is the callback lambda used in `Batch`. +type BatchFn func(ctx context.Context, tx BatchScope) error + +// BatchScope is an atomic write-only scope to the store. +type BatchScope interface { + Writer +} + +// Writer is the write interface for the catchpoint store +type Writer interface { + MetadataWriter + GeneratorWriter + CatchupWriter +} + +// GeneratorWriter all writes needs into the catchpoint store during catchpoint generation +type GeneratorWriter interface { + CatchpointDB_catchpointstate_Writer + CatchpointDB_unfinishedcatchpoints_Writer + CatchpointDB_catchpointfirststageinfo_Writer +} + +// CatchupWriter all catchpoint processing writes needed during catchpoint catchup +type CatchupWriter interface { + CatchpointDB_catchpointstate_Writer +} + +// Reader is the read interface for the catchpoint store +type Reader interface { + GetVersion(ctx context.Context, staging bool) (uint64, error) + + MetadataReader + GeneratorReader + CatchupReader +} + +// GeneratorReader all read needs from the catchpoint store during catchpoint generation +type GeneratorReader interface { + CatchpointDB_catchpointstate_Reader + CatchpointDB_unfinishedcatchpoints_Reader + CatchpointDB_catchpointfirststageinfo_Reader +} + +// CatchupReader all catchpoint processing reads needed during catchpoint catchup +type CatchupReader interface { + CatchpointDB_catchpointstate_Reader +} + +// ReaderWriter combines the Reader and Writer +type ReaderWriter interface { + Reader + Writer +} + +/// ------------- +/// catchpoints metadata register + +type MetadataWriter interface { + StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) +} + +type MetadataReader interface { + GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) + GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) +} + +///------------------------ +/// catchup runtime (durable) +/// could this be part of apply? + +// used during: +// - mainly in catchpoint application, with a couple uses in catchpoint generation +// - used in catchpoint tracker during commit round +// - used to signal that the db state is ready for being read by the catchpoint generator +// - CatchpointStateWritingFirstStageInfo, catchpoint is unfinished (dirty flag) +// - CatchpointStateCatchpointLookback, handle protocol changes during crash recovery +// +// decision here: +// we could simplify the code, and restart cathcpoint from a clean state after a crash +// and not need to resume where we left of + +// During Generation: +// - CatchpointStateWritingFirstStageInfo (write) +// - CatchpointStateCatchpointLookback (write) +// - CatchpointStateLastCatchpoint (write) +// During Apply: +// - CatchpointStateCatchupVersion .. version (write) +// - CatchpointStateCatchupState .. state (write) +// - CatchpointStateCatchupBlockRound .. block round (write) +// - CatchpointStateCatchupHashRound .. hash round (write) +// - CatchpointStateCatchupBalancesRound .. balances round (write) +// - CatchpointStateCatchupLabel .. label (write) +type CatchpointDB_catchpointstate_Writer interface { + WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error) + WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error) +} + +// During Generation: +// - CatchpointStateWritingFirstStageInfo (read) +// - CatchpointStateCatchpointLookback (read) +// - CatchpointStateLastCatchpoint (read) +// During Apply: +// - CatchpointStateCatchupVersion .. version (read) +// - CatchpointStateCatchupState .. state (read) +// - CatchpointStateCatchupBlockRound .. block round (read) +// - CatchpointStateCatchupHashRound .. hash round (read) +// - CatchpointStateCatchupBalancesRound .. balances round (read) +// - CatchpointStateCatchupLabel .. label (read) +// CatchpointDump?: +// - CatchpointStateCatchupVersion .. version (read) +type CatchpointDB_catchpointstate_Reader interface { + ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error) + ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error) +} + +// During Generation: +// - InsertUnfinishedCatchpoint +// - DeleteUnfinishedCatchpoint +type CatchpointDB_unfinishedcatchpoints_Writer interface { + InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error + DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error +} + +// During Generation: +// - SelectUnfinishedCatchpoints +type CatchpointDB_unfinishedcatchpoints_Reader interface { + SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error) +} + +// During Generation: +// - InsertOrReplaceCatchpointFirstStageInfo +// - DeleteOldCatchpointFirstStageInfo +type CatchpointDB_catchpointfirststageinfo_Writer interface { + InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error + DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error +} + +// During Generation: +// - SelectCatchpointFirstStageInfo +// - SelectOldCatchpointFirstStageInfoRounds +type CatchpointDB_catchpointfirststageinfo_Reader interface { + SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool /*exists*/, error) + SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) +} diff --git a/ledger/store/catchpointdb/catchpointdb_impl.go b/ledger/store/catchpointdb/catchpointdb_impl.go new file mode 100644 index 0000000000..a47cc79b22 --- /dev/null +++ b/ledger/store/catchpointdb/catchpointdb_impl.go @@ -0,0 +1,101 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchpointdb + +import ( + "context" + "database/sql" + + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/util/db" +) + +var LatestSchemaVersion int32 = 1 + +type catchpointStore struct { + pair db.Pair + Reader + Writer +} + +func Open(dbFilename string, dbMem bool, log logging.Logger) (Store, error) { + pair, err := db.OpenPair(dbFilename, dbMem) + if err != nil { + return nil, err + } + pair.Rdb.SetLogger(log) + pair.Wdb.SetLogger(log) + return MakeStore(pair), nil +} + +func MakeStore(pair db.Pair) Store { + return &catchpointStore{pair, makeReader(pair.Rdb.Handle), makeWriter(pair.Wdb.Handle)} +} + +// RunMigrations implements Store +func (*catchpointStore) RunMigrations(ctx context.Context, targetVersion int32) (err error) { + panic("unimplemented") +} + +// Batch implements Store +func (store *catchpointStore) Batch(fn BatchFn) (err error) { + return store.BatchContext(context.Background(), fn) +} + +// BatchContext implements Store +func (store *catchpointStore) BatchContext(ctx context.Context, fn BatchFn) (err error) { + return store.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + return fn(ctx, &batchScope{makeWriter(tx)}) + }) +} + +// DeleteStoredCatchpoints iterates over the storedcatchpoints table and deletes all the files stored on disk. +// once all the files have been deleted, it would go ahead and remove the entries from the table. +func (store *catchpointStore) DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) { + catchpointsFilesChunkSize := 50 + for { + fileNames, err := store.GetOldestCatchpointFiles(ctx, catchpointsFilesChunkSize, 0) + if err != nil { + return err + } + if len(fileNames) == 0 { + break + } + + for round, fileName := range fileNames { + err = RemoveSingleCatchpointFileFromDisk(dbDirectory, fileName) + if err != nil { + return err + } + // clear the entry from the database + err = store.StoreCatchpoint(ctx, round, "", "", 0) + if err != nil { + return err + } + } + } + return nil +} + +// Close implements Store +func (store *catchpointStore) Close() { + store.pair.Close() +} + +type batchScope struct { + Writer +} diff --git a/ledger/store/trackerdb/sqlitedriver/catchpoint_test.go b/ledger/store/catchpointdb/catchpointdb_test.go similarity index 64% rename from ledger/store/trackerdb/sqlitedriver/catchpoint_test.go rename to ledger/store/catchpointdb/catchpointdb_test.go index e2c57c0f01..de5e222040 100644 --- a/ledger/store/trackerdb/sqlitedriver/catchpoint_test.go +++ b/ledger/store/catchpointdb/catchpointdb_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package sqlitedriver +package catchpointdb import ( "context" @@ -24,7 +24,6 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" storetesting "github.com/algorand/go-algorand/ledger/store/testing" - "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) @@ -33,47 +32,46 @@ import ( func TestCatchpointFirstStageInfoTable(t *testing.T) { partitiontest.PartitionTest(t) - dbs, _ := storetesting.DbOpenTest(t, true) + pair, _ := storetesting.DbOpenTest(t, true) + dbs := MakeStore(pair) defer dbs.Close() ctx := context.Background() - err := accountsCreateCatchpointFirstStageInfoTable(ctx, dbs.Wdb.Handle) + err := dbs.RunMigrations(ctx, LatestSchemaVersion) require.NoError(t, err) - crw := NewCatchpointSQLReaderWriter(dbs.Wdb.Handle) - for _, round := range []basics.Round{4, 6, 8} { - info := trackerdb.CatchpointFirstStageInfo{ + info := CatchpointFirstStageInfo{ TotalAccounts: uint64(round) * 10, } - err = crw.InsertOrReplaceCatchpointFirstStageInfo(ctx, round, &info) + err = dbs.InsertOrReplaceCatchpointFirstStageInfo(ctx, round, &info) require.NoError(t, err) } for _, round := range []basics.Round{4, 6, 8} { - info, exists, err := crw.SelectCatchpointFirstStageInfo(ctx, round) + info, exists, err := dbs.SelectCatchpointFirstStageInfo(ctx, round) require.NoError(t, err) require.True(t, exists) - infoExpected := trackerdb.CatchpointFirstStageInfo{ + infoExpected := CatchpointFirstStageInfo{ TotalAccounts: uint64(round) * 10, } require.Equal(t, infoExpected, info) } - _, exists, err := crw.SelectCatchpointFirstStageInfo(ctx, 7) + _, exists, err := dbs.SelectCatchpointFirstStageInfo(ctx, 7) require.NoError(t, err) require.False(t, exists) - rounds, err := crw.SelectOldCatchpointFirstStageInfoRounds(ctx, 6) + rounds, err := dbs.SelectOldCatchpointFirstStageInfoRounds(ctx, 6) require.NoError(t, err) require.Equal(t, []basics.Round{4, 6}, rounds) - err = crw.DeleteOldCatchpointFirstStageInfo(ctx, 6) + err = dbs.DeleteOldCatchpointFirstStageInfo(ctx, 6) require.NoError(t, err) - rounds, err = crw.SelectOldCatchpointFirstStageInfoRounds(ctx, 9) + rounds, err = dbs.SelectOldCatchpointFirstStageInfoRounds(ctx, 9) require.NoError(t, err) require.Equal(t, []basics.Round{8}, rounds) } @@ -81,28 +79,26 @@ func TestCatchpointFirstStageInfoTable(t *testing.T) { func TestUnfinishedCatchpointsTable(t *testing.T) { partitiontest.PartitionTest(t) - dbs, _ := storetesting.DbOpenTest(t, true) + pair, _ := storetesting.DbOpenTest(t, true) + dbs := MakeStore(pair) defer dbs.Close() - cts := NewCatchpointSQLReaderWriter(dbs.Wdb.Handle) - - err := accountsCreateUnfinishedCatchpointsTable( - context.Background(), dbs.Wdb.Handle) + err := dbs.RunMigrations(context.Background(), LatestSchemaVersion) require.NoError(t, err) var d3 crypto.Digest rand.Read(d3[:]) - err = cts.InsertUnfinishedCatchpoint(context.Background(), 3, d3) + err = dbs.InsertUnfinishedCatchpoint(context.Background(), 3, d3) require.NoError(t, err) var d5 crypto.Digest rand.Read(d5[:]) - err = cts.InsertUnfinishedCatchpoint(context.Background(), 5, d5) + err = dbs.InsertUnfinishedCatchpoint(context.Background(), 5, d5) require.NoError(t, err) - ret, err := cts.SelectUnfinishedCatchpoints(context.Background()) + ret, err := dbs.SelectUnfinishedCatchpoints(context.Background()) require.NoError(t, err) - expected := []trackerdb.UnfinishedCatchpointRecord{ + expected := []UnfinishedCatchpointRecord{ { Round: 3, BlockHash: d3, @@ -114,12 +110,12 @@ func TestUnfinishedCatchpointsTable(t *testing.T) { } require.Equal(t, expected, ret) - err = cts.DeleteUnfinishedCatchpoint(context.Background(), 3) + err = dbs.DeleteUnfinishedCatchpoint(context.Background(), 3) require.NoError(t, err) - ret, err = cts.SelectUnfinishedCatchpoints(context.Background()) + ret, err = dbs.SelectUnfinishedCatchpoints(context.Background()) require.NoError(t, err) - expected = []trackerdb.UnfinishedCatchpointRecord{ + expected = []UnfinishedCatchpointRecord{ { Round: 5, BlockHash: d5, diff --git a/ledger/store/catchpointdb/msgp_gen.go b/ledger/store/catchpointdb/msgp_gen.go new file mode 100644 index 0000000000..9ddeabc697 --- /dev/null +++ b/ledger/store/catchpointdb/msgp_gen.go @@ -0,0 +1,261 @@ +package catchpointdb + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// CatchpointFirstStageInfo +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// + +// MarshalMsg implements msgp.Marshaler +func (z *CatchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 8 bits */ + if (*z).Totals.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).TotalAccounts == 0 { + zb0001Len-- + zb0001Mask |= 0x4 + } + if (*z).BiggestChunkLen == 0 { + zb0001Len-- + zb0001Mask |= 0x8 + } + if (*z).TotalChunks == 0 { + zb0001Len-- + zb0001Mask |= 0x10 + } + if (*z).TotalKVs == 0 { + zb0001Len-- + zb0001Mask |= 0x20 + } + if (*z).StateProofVerificationHash.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x40 + } + if (*z).TrieBalancesHash.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "accountTotals" + o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + o = (*z).Totals.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "accountsCount" + o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalAccounts) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "biggestChunk" + o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b) + o = msgp.AppendUint64(o, (*z).BiggestChunkLen) + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // string "chunksCount" + o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalChunks) + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // string "kvsCount" + o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, (*z).TotalKVs) + } + if (zb0001Mask & 0x40) == 0 { // if not empty + // string "spVerificationHash" + o = append(o, 0xb2, 0x73, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68) + o = (*z).StateProofVerificationHash.MarshalMsg(o) + } + if (zb0001Mask & 0x80) == 0 { // if not empty + // string "trieBalancesHash" + o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68) + o = (*z).TrieBalancesHash.MarshalMsg(o) + } + } + return +} + +func (_ *CatchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*CatchpointFirstStageInfo) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CatchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Totals.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Totals") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalAccounts") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalKVs") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "TotalChunks") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProofVerificationHash") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = CatchpointFirstStageInfo{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "accountTotals": + bts, err = (*z).Totals.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + case "trieBalancesHash": + bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TrieBalancesHash") + return + } + case "accountsCount": + (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalAccounts") + return + } + case "kvsCount": + (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalKVs") + return + } + case "chunksCount": + (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalChunks") + return + } + case "biggestChunk": + (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BiggestChunkLen") + return + } + case "spVerificationHash": + bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProofVerificationHash") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *CatchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*CatchpointFirstStageInfo) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CatchpointFirstStageInfo) Msgsize() (s int) { + s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size + 19 + (*z).StateProofVerificationHash.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *CatchpointFirstStageInfo) MsgIsZero() bool { + return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) && ((*z).StateProofVerificationHash.MsgIsZero()) +} diff --git a/ledger/store/catchpointdb/msgp_gen_test.go b/ledger/store/catchpointdb/msgp_gen_test.go new file mode 100644 index 0000000000..1b416cdd8e --- /dev/null +++ b/ledger/store/catchpointdb/msgp_gen_test.go @@ -0,0 +1,75 @@ +//go:build !skip_msgp_testing +// +build !skip_msgp_testing + +package catchpointdb + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "testing" + + "github.com/algorand/msgp/msgp" + + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestMarshalUnmarshalCatchpointFirstStageInfo(t *testing.T) { + partitiontest.PartitionTest(t) + v := CatchpointFirstStageInfo{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingCatchpointFirstStageInfo(t *testing.T) { + protocol.RunEncodingTest(t, &CatchpointFirstStageInfo{}) +} + +func BenchmarkMarshalMsgCatchpointFirstStageInfo(b *testing.B) { + v := CatchpointFirstStageInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCatchpointFirstStageInfo(b *testing.B) { + v := CatchpointFirstStageInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCatchpointFirstStageInfo(b *testing.B) { + v := CatchpointFirstStageInfo{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/ledger/store/catchpointdb/reader.go b/ledger/store/catchpointdb/reader.go new file mode 100644 index 0000000000..82c60a14ca --- /dev/null +++ b/ledger/store/catchpointdb/reader.go @@ -0,0 +1,227 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchpointdb + +import ( + "context" + "database/sql" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +type reader struct { + q db.Queryable +} + +func makeReader(q db.Queryable) Reader { + return &reader{q} +} + +// Version implements Reader +func (r *reader) GetVersion(ctx context.Context, staging bool) (version uint64, err error) { + if staging { + // writing the version of the catchpoint file start only on ver >= CatchpointFileVersionV7. + // in case the catchpoint version does not exists ReadCatchpointStateUint64 returns 0 + version, err = r.ReadCatchpointStateUint64(context.Background(), CatchpointStateCatchupVersion) + if err != nil { + return 0, err + } + return version, err + } + + versionAsInt32, err := db.GetUserVersion(ctx, r.q) + if err != nil { + return 0, err + } + version = uint64(versionAsInt32) + + return +} + +// GetCatchpoint implements Reader +func (r *reader) GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) { + err = r.q.QueryRowContext(ctx, "SELECT filename, catchpoint, filesize FROM storedcatchpoints WHERE round=?", int64(round)).Scan(&fileName, &catchpoint, &fileSize) + return +} + +// GetOldestCatchpointFiles implements Reader +func (r *reader) GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) { + err = db.Retry(func() (err error) { + query := "SELECT round, filename FROM storedcatchpoints WHERE pinned = 0 and round <= COALESCE((SELECT round FROM storedcatchpoints WHERE pinned = 0 ORDER BY round DESC LIMIT ?, 1),0) ORDER BY round ASC LIMIT ?" + rows, err := r.q.QueryContext(ctx, query, filesToKeep, fileCount) + if err != nil { + return err + } + defer rows.Close() + + fileNames = make(map[basics.Round]string) + for rows.Next() { + var fileName string + var round basics.Round + err = rows.Scan(&round, &fileName) + if err != nil { + return err + } + fileNames[round] = fileName + } + + return rows.Err() + }) + if err != nil { + fileNames = nil + } + return +} + +// ReadCatchpointStateString implements Reader +func (r *reader) ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error) { + err = db.Retry(func() (err error) { + query := "SELECT strval FROM catchpointstate WHERE id=?" + var v sql.NullString + err = r.q.QueryRowContext(ctx, query, stateName).Scan(&v) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return err + } + + if v.Valid { + val = v.String + } + return nil + }) + return val, err +} + +// ReadCatchpointStateUint64 implements Reader +func (r *reader) ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error) { + err = db.Retry(func() (err error) { + query := "SELECT intval FROM catchpointstate WHERE id=?" + var v sql.NullInt64 + err = r.q.QueryRowContext(ctx, query, stateName).Scan(&v) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return err + } + if v.Valid { + val = uint64(v.Int64) + } + return nil + }) + return val, err +} + +// SelectUnfinishedCatchpoints implements Reader +func (r *reader) SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error) { + var res []UnfinishedCatchpointRecord + + f := func() error { + query := "SELECT round, blockhash FROM unfinishedcatchpoints ORDER BY round" + rows, err := r.q.QueryContext(ctx, query) + if err != nil { + return err + } + + // Clear `res` in case this function is repeated. + res = res[:0] + for rows.Next() { + var record UnfinishedCatchpointRecord + var blockHash []byte + err = rows.Scan(&record.Round, &blockHash) + if err != nil { + return err + } + copy(record.BlockHash[:], blockHash) + res = append(res, record) + } + + return nil + } + err := db.Retry(f) + if err != nil { + return nil, err + } + + return res, nil +} + +// SelectCatchpointFirstStageInfo implements Reader +func (r *reader) SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool, error) { + var data []byte + f := func() error { + query := "SELECT info FROM catchpointfirststageinfo WHERE round=?" + err := r.q.QueryRowContext(ctx, query, round).Scan(&data) + if err == sql.ErrNoRows { + data = nil + return nil + } + return err + } + err := db.Retry(f) + if err != nil { + return CatchpointFirstStageInfo{}, false, err + } + + if data == nil { + return CatchpointFirstStageInfo{}, false, nil + } + + var res CatchpointFirstStageInfo + err = protocol.Decode(data, &res) + if err != nil { + return CatchpointFirstStageInfo{}, false, err + } + + return res, true, nil +} + +// SelectOldCatchpointFirstStageInfoRounds implements Reader +func (r *reader) SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) { + var res []basics.Round + + f := func() error { + query := "SELECT round FROM catchpointfirststageinfo WHERE round <= ?" + rows, err := r.q.QueryContext(ctx, query, maxRound) + if err != nil { + return err + } + + // Clear `res` in case this function is repeated. + res = res[:0] + for rows.Next() { + var r basics.Round + err = rows.Scan(&r) + if err != nil { + return err + } + res = append(res, r) + } + + return nil + } + err := db.Retry(f) + if err != nil { + return nil, err + } + + return res, nil +} diff --git a/ledger/store/trackerdb/utils.go b/ledger/store/catchpointdb/utils.go similarity index 98% rename from ledger/store/trackerdb/utils.go rename to ledger/store/catchpointdb/utils.go index 0d9e8b4d8a..2a665c2ab0 100644 --- a/ledger/store/trackerdb/utils.go +++ b/ledger/store/catchpointdb/utils.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package trackerdb +package catchpointdb import ( "io" diff --git a/ledger/store/catchpointdb/writer.go b/ledger/store/catchpointdb/writer.go new file mode 100644 index 0000000000..3cd405704e --- /dev/null +++ b/ledger/store/catchpointdb/writer.go @@ -0,0 +1,127 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchpointdb + +import ( + "context" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +type writer struct { + e db.Executable +} + +func makeWriter(e db.Executable) Writer { + return &writer{e} +} + +// StoreCatchpoint implements Writer +func (w *writer) StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) { + err = db.Retry(func() (err error) { + query := "DELETE FROM storedcatchpoints WHERE round=?" + _, err = w.e.ExecContext(ctx, query, round) + if err != nil || (fileName == "" && catchpoint == "" && fileSize == 0) { + return err + } + + query = "INSERT INTO storedcatchpoints(round, filename, catchpoint, filesize, pinned) VALUES(?, ?, ?, ?, 0)" + _, err = w.e.ExecContext(ctx, query, round, fileName, catchpoint, fileSize) + return err + }) + return +} + +// WriteCatchpointStateString implements Writer +func (w *writer) WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error) { + err = db.Retry(func() (err error) { + if setValue == "" { + return deleteCatchpointStateImpl(ctx, w.e, stateName) + } + + // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case. + query := "INSERT OR REPLACE INTO catchpointstate(id, strval) VALUES(?, ?)" + _, err = w.e.ExecContext(ctx, query, stateName, setValue) + return err + }) + return err +} + +// WriteCatchpointStateUint64 implements Writer +func (w *writer) WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error) { + err = db.Retry(func() (err error) { + if setValue == 0 { + return deleteCatchpointStateImpl(ctx, w.e, stateName) + } + + // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case. + query := "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)" + _, err = w.e.ExecContext(ctx, query, stateName, setValue) + return err + }) + return err +} + +// DeleteUnfinishedCatchpoint implements Writer +func (w *writer) DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error { + f := func() error { + query := "DELETE FROM unfinishedcatchpoints WHERE round = ?" + _, err := w.e.ExecContext(ctx, query, round) + return err + } + return db.Retry(f) +} + +// InsertUnfinishedCatchpoint implements Writer +func (w *writer) InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error { + f := func() error { + query := "INSERT INTO unfinishedcatchpoints(round, blockhash) VALUES(?, ?)" + _, err := w.e.ExecContext(ctx, query, round, blockHash[:]) + return err + } + return db.Retry(f) +} + +// DeleteOldCatchpointFirstStageInfo implements Writer +func (w *writer) DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error { + f := func() error { + query := "DELETE FROM catchpointfirststageinfo WHERE round <= ?" + _, err := w.e.ExecContext(ctx, query, maxRoundToDelete) + return err + } + return db.Retry(f) +} + +// InsertOrReplaceCatchpointFirstStageInfo implements Writer +func (w *writer) InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error { + infoSerialized := protocol.Encode(info) + f := func() error { + query := "INSERT OR REPLACE INTO catchpointfirststageinfo(round, info) VALUES(?, ?)" + _, err := w.e.ExecContext(ctx, query, round, infoSerialized) + return err + } + return db.Retry(f) +} + +func deleteCatchpointStateImpl(ctx context.Context, e db.Executable, stateName CatchpointState) error { + query := "DELETE FROM catchpointstate WHERE id=?" + _, err := e.ExecContext(ctx, query, stateName) + return err +} diff --git a/ledger/store/merkle_committer.go b/ledger/store/merkle_committer.go deleted file mode 100644 index bc7502dac6..0000000000 --- a/ledger/store/merkle_committer.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (C) 2019-2023 Algorand, Inc. -// This file is part of go-algorand -// -// go-algorand is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// go-algorand is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with go-algorand. If not, see . - -package store - -import "database/sql" - -// MerkleCommitter allows storing and loading merkletrie pages from a sqlite database. -// -//msgp:ignore MerkleCommitter -type MerkleCommitter struct { - tx *sql.Tx - deleteStmt *sql.Stmt - insertStmt *sql.Stmt - selectStmt *sql.Stmt -} - -// MakeMerkleCommitter creates a MerkleCommitter object that implements the merkletrie.Committer interface allowing storing and loading -// merkletrie pages from a sqlite database. -func MakeMerkleCommitter(tx *sql.Tx, staging bool) (mc *MerkleCommitter, err error) { - mc = &MerkleCommitter{tx: tx} - accountHashesTable := "accounthashes" - if staging { - accountHashesTable = "catchpointaccounthashes" - } - mc.deleteStmt, err = tx.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?") - if err != nil { - return nil, err - } - mc.insertStmt, err = tx.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)") - if err != nil { - return nil, err - } - mc.selectStmt, err = tx.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?") - if err != nil { - return nil, err - } - return mc, nil -} - -// StorePage is the merkletrie.Committer interface implementation, stores a single page in a sqlite database table. -func (mc *MerkleCommitter) StorePage(page uint64, content []byte) error { - if len(content) == 0 { - _, err := mc.deleteStmt.Exec(page) - return err - } - _, err := mc.insertStmt.Exec(page, content) - return err -} - -// LoadPage is the merkletrie.Committer interface implementation, load a single page from a sqlite database table. -func (mc *MerkleCommitter) LoadPage(page uint64) (content []byte, err error) { - err = mc.selectStmt.QueryRow(page).Scan(&content) - if err == sql.ErrNoRows { - content = nil - err = nil - return - } else if err != nil { - return nil, err - } - return content, nil -} diff --git a/ledger/store/testing/helpers.go b/ledger/store/testing/helpers.go index 34ba3e3ffb..b9e41ee286 100644 --- a/ledger/store/testing/helpers.go +++ b/ledger/store/testing/helpers.go @@ -29,7 +29,7 @@ import ( // DbOpenTest opens a db file for testing purposes. func DbOpenTest(t testing.TB, inMemory bool) (db.Pair, string) { - fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) + fn := fmt.Sprintf("%s/%s.%d", t.TempDir(), strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) dbs, err := db.OpenPair(fn, inMemory) require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory) return dbs, fn diff --git a/ledger/store/trackerdb/catchpoint.go b/ledger/store/trackerdb/catchpoint.go index dfbacbd86f..1af7ad1e5d 100644 --- a/ledger/store/trackerdb/catchpoint.go +++ b/ledger/store/trackerdb/catchpoint.go @@ -17,16 +17,8 @@ package trackerdb import ( - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/crypto/merkletrie" "github.com/algorand/go-algorand/data/basics" - "github.com/algorand/go-algorand/ledger/ledgercore" ) // TrieMemoryConfig is the memory configuration setup used for the merkle trie. @@ -45,51 +37,6 @@ var MerkleCommitterNodesPerPage = int64(116) // value was calibrated using BenchmarkCalibrateCacheNodeSize var TrieCachedNodesCount = 9000 -// CatchpointDirName represents the directory name in which all the catchpoints files are stored -var CatchpointDirName = "catchpoints" - -// CatchpointState is used to store catchpoint related variables into the catchpointstate table. -// -//msgp:ignore CatchpointState -type CatchpointState string - -const ( - // CatchpointStateLastCatchpoint is written by a node once a catchpoint label is created for a round - CatchpointStateLastCatchpoint = CatchpointState("lastCatchpoint") - // CatchpointStateWritingFirstStageInfo state variable is set to 1 if catchpoint's first stage is unfinished, - // and is 0 otherwise. Used to clear / restart the first stage after a crash. - // This key is set in the same db transaction as the account updates, so the - // unfinished first stage corresponds to the current db round. - CatchpointStateWritingFirstStageInfo = CatchpointState("writingFirstStageInfo") - // CatchpointStateWritingCatchpoint if there is an unfinished catchpoint, this state variable is set to - // the catchpoint's round. Otherwise, it is set to 0. - // DEPRECATED. - CatchpointStateWritingCatchpoint = CatchpointState("writingCatchpoint") - // CatchpointStateCatchupState is the state of the catchup process. The variable is stored only during the catchpoint catchup process, and removed afterward. - CatchpointStateCatchupState = CatchpointState("catchpointCatchupState") - // CatchpointStateCatchupLabel is the label to which the currently catchpoint catchup process is trying to catchup to. - CatchpointStateCatchupLabel = CatchpointState("catchpointCatchupLabel") - // CatchpointStateCatchupBlockRound is the block round that is associated with the current running catchpoint catchup. - CatchpointStateCatchupBlockRound = CatchpointState("catchpointCatchupBlockRound") - // CatchpointStateCatchupBalancesRound is the balance round that is associated with the current running catchpoint catchup. Typically it would be - // equal to CatchpointStateCatchupBlockRound - 320. - CatchpointStateCatchupBalancesRound = CatchpointState("catchpointCatchupBalancesRound") - // CatchpointStateCatchupHashRound is the round that is associated with the hash of the merkle trie. Normally, it's identical to CatchpointStateCatchupBalancesRound, - // however, it could differ when we catchup from a catchpoint that was created using a different version : in this case, - // we set it to zero in order to reset the merkle trie. This would force the merkle trie to be re-build on startup ( if needed ). - CatchpointStateCatchupHashRound = CatchpointState("catchpointCatchupHashRound") - // CatchpointStateCatchpointLookback is the number of rounds we keep catchpoints for - CatchpointStateCatchpointLookback = CatchpointState("catchpointLookback") - // CatchpointStateCatchupVersion is the catchpoint version which the currently catchpoint catchup process is trying to catchup to. - CatchpointStateCatchupVersion = CatchpointState("catchpointCatchupVersion") -) - -// UnfinishedCatchpointRecord represents a stored record of an unfinished catchpoint. -type UnfinishedCatchpointRecord struct { - Round basics.Round - BlockHash crypto.Digest -} - // NormalizedAccountBalance is a staging area for a catchpoint file account information before it's being added to the catchpoint staging tables. type NormalizedAccountBalance struct { // The public key address to which the account belongs. @@ -110,86 +57,3 @@ type NormalizedAccountBalance struct { // partial balance indicates that the original account balance was split into multiple parts in catchpoint creation time PartialBalance bool } - -// CatchpointFirstStageInfo For the `catchpointfirststageinfo` table. -type CatchpointFirstStageInfo struct { - _struct struct{} `codec:",omitempty,omitemptyarray"` - - Totals ledgercore.AccountTotals `codec:"accountTotals"` - TrieBalancesHash crypto.Digest `codec:"trieBalancesHash"` - // Total number of accounts in the catchpoint data file. Only set when catchpoint - // data files are generated. - TotalAccounts uint64 `codec:"accountsCount"` - - // Total number of accounts in the catchpoint data file. Only set when catchpoint - // data files are generated. - TotalKVs uint64 `codec:"kvsCount"` - - // Total number of chunks in the catchpoint data file. Only set when catchpoint - // data files are generated. - TotalChunks uint64 `codec:"chunksCount"` - // BiggestChunkLen is the size in the bytes of the largest chunk, used when re-packing. - BiggestChunkLen uint64 `codec:"biggestChunk"` - - // StateProofVerificationHash is the hash of the state proof verification data contained in the catchpoint data file. - StateProofVerificationHash crypto.Digest `codec:"spVerificationHash"` -} - -// MakeCatchpointFilePath builds the path of a catchpoint file. -func MakeCatchpointFilePath(round basics.Round) string { - irnd := int64(round) / 256 - outStr := "" - for irnd > 0 { - outStr = filepath.Join(outStr, fmt.Sprintf("%02x", irnd%256)) - irnd = irnd / 256 - } - outStr = filepath.Join(outStr, strconv.FormatInt(int64(round), 10)+".catchpoint") - return outStr -} - -// RemoveSingleCatchpointFileFromDisk removes a single catchpoint file from the disk. this function does not leave empty directories -func RemoveSingleCatchpointFileFromDisk(dbDirectory, fileToDelete string) (err error) { - absCatchpointFileName := filepath.Join(dbDirectory, fileToDelete) - err = os.Remove(absCatchpointFileName) - if err == nil || os.IsNotExist(err) { - // it's ok if the file doesn't exist. - err = nil - } else { - // we can't delete the file, abort - - return fmt.Errorf("unable to delete old catchpoint file '%s' : %v", absCatchpointFileName, err) - } - splitedDirName := strings.Split(fileToDelete, string(os.PathSeparator)) - - var subDirectoriesToScan []string - //build a list of all the subdirs - currentSubDir := "" - for _, element := range splitedDirName { - currentSubDir = filepath.Join(currentSubDir, element) - subDirectoriesToScan = append(subDirectoriesToScan, currentSubDir) - } - - // iterating over the list of directories. starting from the sub dirs and moving up. - // skipping the file itself. - for i := len(subDirectoriesToScan) - 2; i >= 0; i-- { - absSubdir := filepath.Join(dbDirectory, subDirectoriesToScan[i]) - if _, err := os.Stat(absSubdir); os.IsNotExist(err) { - continue - } - - isEmpty, err := isDirEmpty(absSubdir) - if err != nil { - return fmt.Errorf("unable to read old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err) - } - if isEmpty { - err = os.Remove(absSubdir) - if err != nil { - if os.IsNotExist(err) { - continue - } - return fmt.Errorf("unable to delete old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err) - } - } - } - - return nil -} diff --git a/ledger/store/trackerdb/dualdriver/accounts_reader.go b/ledger/store/trackerdb/dualdriver/accounts_reader.go new file mode 100644 index 0000000000..cda1a0687c --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/accounts_reader.go @@ -0,0 +1,209 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/google/go-cmp/cmp" +) + +type accountsReader struct { + primary trackerdb.AccountsReader + secondary trackerdb.AccountsReader +} + +// Close implements trackerdb.AccountsReader +func (ar *accountsReader) Close() { + ar.primary.Close() + ar.secondary.Close() +} + +// ListCreatables implements trackerdb.AccountsReader +func (ar *accountsReader) ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) { + resultsP, dbRoundP, errP := ar.primary.ListCreatables(maxIdx, maxResults, ctype) + resultsS, dbRoundS, errS := ar.secondary.ListCreatables(maxIdx, maxResults, ctype) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(resultsP, resultsS) { + err = ErrInconsistentResult + return + } + if dbRoundP != dbRoundS { + err = ErrInconsistentResult + return + } + // return primary results + return resultsP, dbRoundP, nil +} + +// LookupAccount implements trackerdb.AccountsReader +func (ar *accountsReader) LookupAccount(addr basics.Address) (data trackerdb.PersistedAccountData, err error) { + dataP, errP := ar.primary.LookupAccount(addr) + dataS, errS := ar.secondary.LookupAccount(addr) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + ref, err := coalesceAccountRefs(dataP.Ref, dataS.Ref) + if err != nil { + return + } + // update ref in results + // Note: this is safe because the refs are engine specific + dataP.Ref = ref + dataS.Ref = ref + // check results match + if dataP != dataS { + err = ErrInconsistentResult + return + } + // return primary results + return dataP, nil +} + +// LookupAllResources implements trackerdb.AccountsReader +func (ar *accountsReader) LookupAllResources(addr basics.Address) (data []trackerdb.PersistedResourcesData, rnd basics.Round, err error) { + dataP, rndP, errP := ar.primary.LookupAllResources(addr) + dataS, rndS, errS := ar.secondary.LookupAllResources(addr) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + if len(dataP) != len(dataS) { + err = ErrInconsistentResult + return + } + var ref trackerdb.AccountRef + for i := range dataP { + ref, err = coalesceAccountRefs(dataP[i].AcctRef, dataS[i].AcctRef) + if err != nil { + return data, rnd, err + } + // update ref in results + dataP[i].AcctRef = ref + dataS[i].AcctRef = ref + } + // check results match + if !cmp.Equal(dataP, dataS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + if rndP != rndS { + err = ErrInconsistentResult + return + } + // return primary results + return dataP, rndP, nil +} + +// LookupCreator implements trackerdb.AccountsReader +func (ar *accountsReader) LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) { + addrP, okP, dbRoundP, errP := ar.primary.LookupCreator(cidx, ctype) + addrS, okS, dbRoundS, errS := ar.secondary.LookupCreator(cidx, ctype) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if addrP != addrS { + err = ErrInconsistentResult + return + } + if okP != okS { + err = ErrInconsistentResult + return + } + if dbRoundP != dbRoundS { + err = ErrInconsistentResult + return + } + // return primary results + return addrP, okP, dbRoundP, nil +} + +// LookupKeyValue implements trackerdb.AccountsReader +func (ar *accountsReader) LookupKeyValue(key string) (pv trackerdb.PersistedKVData, err error) { + pvP, errP := ar.primary.LookupKeyValue(key) + pvS, errS := ar.secondary.LookupKeyValue(key) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(pvP, pvS) { + err = ErrInconsistentResult + return + } + // return primary results + return pvP, nil +} + +// LookupKeysByPrefix implements trackerdb.AccountsReader +func (ar *accountsReader) LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) { + roundP, errP := ar.primary.LookupKeysByPrefix(prefix, maxKeyNum, results, resultCount) + roundS, errS := ar.secondary.LookupKeysByPrefix(prefix, maxKeyNum, results, resultCount) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if roundP != roundS { + err = ErrInconsistentResult + return + } + // return primary results + return roundP, nil +} + +// LookupResources implements trackerdb.AccountsReader +func (ar *accountsReader) LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data trackerdb.PersistedResourcesData, err error) { + dataP, errP := ar.primary.LookupResources(addr, aidx, ctype) + dataS, errS := ar.secondary.LookupResources(addr, aidx, ctype) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + ref, err := coalesceAccountRefs(dataP.AcctRef, dataS.AcctRef) + if err != nil { + return + } + // update ref in results + // Note: this is safe because the refs are engine specific + dataP.AcctRef = ref + dataS.AcctRef = ref + // check results match + if !cmp.Equal(dataP, dataS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + // return primary results + return dataP, nil +} diff --git a/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go b/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go new file mode 100644 index 0000000000..6a67c0db53 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go @@ -0,0 +1,356 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/google/go-cmp/cmp" +) + +type accountsReaderExt struct { + primary trackerdb.AccountsReaderExt + secondary trackerdb.AccountsReaderExt +} + +// AccountsHashRound implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) { + hashrndP, errP := ar.primary.AccountsHashRound(ctx) + hashrndS, errS := ar.secondary.AccountsHashRound(ctx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if hashrndP != hashrndS { + err = ErrInconsistentResult + return + } + // return primary results + return hashrndP, nil +} + +// AccountsOnlineRoundParams implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error) { + onlineRoundParamsDataP, endRoundP, errP := ar.primary.AccountsOnlineRoundParams() + onlineRoundParamsDataS, endRoundS, errS := ar.secondary.AccountsOnlineRoundParams() + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(onlineRoundParamsDataP, onlineRoundParamsDataS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + if endRoundP != endRoundS { + err = ErrInconsistentResult + return + } + // return primary results + return onlineRoundParamsDataP, endRoundP, nil +} + +// AccountsOnlineTop implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (onlineAccounts map[basics.Address]*ledgercore.OnlineAccount, err error) { + onlineAccountsP, errP := ar.primary.AccountsOnlineTop(rnd, offset, n, proto) + onlineAccountsS, errS := ar.secondary.AccountsOnlineTop(rnd, offset, n, proto) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(onlineAccountsP, onlineAccountsS) { + err = ErrInconsistentResult + return + } + // return primary results + return onlineAccountsP, nil +} + +// AccountsRound implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) AccountsRound() (rnd basics.Round, err error) { + rndP, errP := ar.primary.AccountsRound() + rndS, errS := ar.secondary.AccountsRound() + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if rndP != rndS { + err = ErrInconsistentResult + return + } + // return primary results + return rndP, nil +} + +// AccountsTotals implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error) { + totalsP, errP := ar.primary.AccountsTotals(ctx, catchpointStaging) + totalsS, errS := ar.secondary.AccountsTotals(ctx, catchpointStaging) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if totalsP != totalsS { + err = ErrInconsistentResult + return + } + // return primary results + return totalsP, nil +} + +// LoadAllFullAccounts implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error) { + countP, errP := ar.primary.LoadAllFullAccounts(ctx, balancesTable, resourcesTable, acctCb) + countS, errS := ar.secondary.LoadAllFullAccounts(ctx, balancesTable, resourcesTable, acctCb) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if countP != countS { + err = ErrInconsistentResult + return + } + // return primary results + return countP, nil +} + +// LoadTxTail implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*trackerdb.TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error) { + roundDataP, roundHashP, baseRoundP, errP := ar.primary.LoadTxTail(ctx, dbRound) + roundDataS, roundHashS, baseRoundS, errS := ar.secondary.LoadTxTail(ctx, dbRound) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(roundDataP, roundDataS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + if !cmp.Equal(roundHashP, roundHashS) { + err = ErrInconsistentResult + return + } + if baseRoundP != baseRoundS { + err = ErrInconsistentResult + return + } + // return primary results + return roundDataP, roundHashP, baseRoundP, nil +} + +// LookupAccountAddressFromAddressID implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LookupAccountAddressFromAddressID(ctx context.Context, ref trackerdb.AccountRef) (address basics.Address, err error) { + addressP, errP := ar.primary.LookupAccountAddressFromAddressID(ctx, ref) + addressS, errS := ar.secondary.LookupAccountAddressFromAddressID(ctx, ref) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if addressP != addressS { + err = ErrInconsistentResult + return + } + // return primary results + return addressP, nil +} + +// LookupAccountRowID implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LookupAccountRowID(addr basics.Address) (ref trackerdb.AccountRef, err error) { + // Note: we do not check the refs since they are internal to the engines and wont match + refP, errP := ar.primary.LookupAccountRowID(addr) + refS, errS := ar.secondary.LookupAccountRowID(addr) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + return accountRef{refP, refS}, nil +} + +// LookupOnlineAccountDataByAddress implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) { + // Note: we do not check the refs since they are internal to the engines and wont match + refP, dataP, errP := ar.primary.LookupOnlineAccountDataByAddress(addr) + refS, dataS, errS := ar.secondary.LookupOnlineAccountDataByAddress(addr) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(dataP, dataS) { + err = ErrInconsistentResult + return + } + // return primary results + return onlineAccountRef{refP, refS}, dataP, nil +} + +// LookupResourceDataByAddrID implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) LookupResourceDataByAddrID(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) { + if accRef == nil { + return data, trackerdb.ErrNotFound + } + // parse ref + xRef := accRef.(accountRef) + // lookup + dataP, errP := ar.primary.LookupResourceDataByAddrID(xRef.primary, aidx) + dataS, errS := ar.secondary.LookupResourceDataByAddrID(xRef.secondary, aidx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(dataP, dataS) { + err = ErrInconsistentResult + return + } + // return primary results + return dataP, nil +} + +// OnlineAccountsAll implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) OnlineAccountsAll(maxAccounts uint64) (accounts []trackerdb.PersistedOnlineAccountData, err error) { + accountsP, errP := ar.primary.OnlineAccountsAll(maxAccounts) + accountsS, errS := ar.secondary.OnlineAccountsAll(maxAccounts) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + if len(accountsP) != len(accountsS) { + err = ErrInconsistentResult + return + } + var ref trackerdb.OnlineAccountRef + for i := range accountsP { + ref, err = coalesceOnlineAccountRefs(accountsP[i].Ref, accountsS[i].Ref) + if err != nil { + return accounts, err + } + // update ref in results + accountsP[i].Ref = ref + accountsS[i].Ref = ref + } + // check results match + if !cmp.Equal(accountsP, accountsS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + // return primary results + return accountsP, nil +} + +// ExpiredOnlineAccountsForRound implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) ExpiredOnlineAccountsForRound(rnd basics.Round, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (expAccounts map[basics.Address]*ledgercore.OnlineAccountData, err error) { + expAccountsP, errP := ar.primary.ExpiredOnlineAccountsForRound(rnd, voteRnd, proto, rewardsLevel) + expAccountsS, errS := ar.secondary.ExpiredOnlineAccountsForRound(rnd, voteRnd, proto, rewardsLevel) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if !cmp.Equal(expAccountsP, expAccountsS) { + err = ErrInconsistentResult + return + } + // return primary results + return expAccountsP, nil +} + +// Testing implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) Testing() trackerdb.AccountsReaderTestExt { + // TODO + return nil +} + +// TotalAccounts implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) TotalAccounts(ctx context.Context) (total uint64, err error) { + totalP, errP := ar.primary.TotalAccounts(ctx) + totalS, errS := ar.secondary.TotalAccounts(ctx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if totalP != totalS { + err = ErrInconsistentResult + return + } + // return primary results + return totalP, nil +} + +// TotalKVs implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) TotalKVs(ctx context.Context) (total uint64, err error) { + totalP, errP := ar.primary.TotalKVs(ctx) + totalS, errS := ar.secondary.TotalKVs(ctx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if totalP != totalS { + err = ErrInconsistentResult + return + } + // return primary results + return totalP, nil +} + +// TotalResources implements trackerdb.AccountsReaderExt +func (ar *accountsReaderExt) TotalResources(ctx context.Context) (total uint64, err error) { + totalP, errP := ar.primary.TotalResources(ctx) + totalS, errS := ar.secondary.TotalResources(ctx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if totalP != totalS { + err = ErrInconsistentResult + return + } + // return primary results + return totalP, nil +} diff --git a/ledger/store/trackerdb/dualdriver/accounts_writer.go b/ledger/store/trackerdb/dualdriver/accounts_writer.go new file mode 100644 index 0000000000..318ccd36c0 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/accounts_writer.go @@ -0,0 +1,170 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type accountsWriter struct { + primary trackerdb.AccountsWriter + secondary trackerdb.AccountsWriter +} + +// Close implements trackerdb.AccountsWriter +func (aw *accountsWriter) Close() { + aw.primary.Close() + aw.secondary.Close() +} + +// DeleteAccount implements trackerdb.AccountsWriter +func (aw *accountsWriter) DeleteAccount(accRef trackerdb.AccountRef) (rowsAffected int64, err error) { + // parse ref + xRef := accRef.(accountRef) + // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines + rowsAffectedP, errP := aw.primary.DeleteAccount(xRef.primary) + _, errS := aw.secondary.DeleteAccount(xRef.secondary) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + return rowsAffectedP, nil +} + +// DeleteCreatable implements trackerdb.AccountsWriter +func (aw *accountsWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { + // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines + rowsAffectedP, errP := aw.primary.DeleteCreatable(cidx, ctype) + _, errS := aw.secondary.DeleteCreatable(cidx, ctype) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + return rowsAffectedP, nil +} + +// DeleteKvPair implements trackerdb.AccountsWriter +func (aw *accountsWriter) DeleteKvPair(key string) error { + errP := aw.primary.DeleteKvPair(key) + errS := aw.secondary.DeleteKvPair(key) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// DeleteResource implements trackerdb.AccountsWriter +func (aw *accountsWriter) DeleteResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (rowsAffected int64, err error) { + // parse ref + xRef := accRef.(accountRef) + // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines + rowsAffectedP, errP := aw.primary.DeleteResource(xRef.primary, aidx) + _, errS := aw.secondary.DeleteResource(xRef.secondary, aidx) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + return rowsAffectedP, nil +} + +// InsertAccount implements trackerdb.AccountsWriter +func (aw *accountsWriter) InsertAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseAccountData) (ref trackerdb.AccountRef, err error) { + // Note: we do not check the refs since they are internal to the engines and wont match + refP, errP := aw.primary.InsertAccount(addr, normBalance, data) + refS, errS := aw.secondary.InsertAccount(addr, normBalance, data) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // return ref + return accountRef{refP, refS}, nil +} + +// InsertCreatable implements trackerdb.AccountsWriter +func (aw *accountsWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (ref trackerdb.CreatableRef, err error) { + // Note: we do not check the refs since they are internal to the engines and wont match + refP, errP := aw.primary.InsertCreatable(cidx, ctype, creator) + refS, errS := aw.secondary.InsertCreatable(cidx, ctype, creator) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // return ref + return creatableRef{refP, refS}, nil +} + +// InsertResource implements trackerdb.AccountsWriter +func (aw *accountsWriter) InsertResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (ref trackerdb.ResourceRef, err error) { + // parse ref + xRef := accRef.(accountRef) + // Note: we do not check the refs since they are internal to the engines and wont match + refP, errP := aw.primary.InsertResource(xRef.primary, aidx, data) + refS, errS := aw.secondary.InsertResource(xRef.secondary, aidx, data) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // return ref + return resourceRef{refP, refS}, nil +} + +// UpdateAccount implements trackerdb.AccountsWriter +func (aw *accountsWriter) UpdateAccount(accRef trackerdb.AccountRef, normBalance uint64, data trackerdb.BaseAccountData) (rowsAffected int64, err error) { + // parse ref + xRef := accRef.(accountRef) + // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines + rowsAffectedP, errP := aw.primary.UpdateAccount(xRef.primary, normBalance, data) + _, errS := aw.secondary.UpdateAccount(xRef.secondary, normBalance, data) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + return rowsAffectedP, nil +} + +// UpdateResource implements trackerdb.AccountsWriter +func (aw *accountsWriter) UpdateResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (rowsAffected int64, err error) { + // parse ref + xRef := accRef.(accountRef) + // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines + rowsAffectedP, errP := aw.primary.UpdateResource(xRef.primary, aidx, data) + _, errS := aw.secondary.UpdateResource(xRef.secondary, aidx, data) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + return rowsAffectedP, nil +} + +// UpsertKvPair implements trackerdb.AccountsWriter +func (aw *accountsWriter) UpsertKvPair(key string, value []byte) error { + errP := aw.primary.UpsertKvPair(key, value) + errS := aw.secondary.UpsertKvPair(key, value) + // coalesce errors + return coalesceErrors(errP, errS) +} diff --git a/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go b/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go new file mode 100644 index 0000000000..27b74381f0 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go @@ -0,0 +1,101 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type accountsWriterExt struct { + primary trackerdb.AccountsWriterExt + secondary trackerdb.AccountsWriterExt +} + +// AccountsPruneOnlineRoundParams implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error { + errP := aw.primary.AccountsPruneOnlineRoundParams(deleteBeforeRound) + errS := aw.secondary.AccountsPruneOnlineRoundParams(deleteBeforeRound) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// AccountsPutOnlineRoundParams implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error { + errP := aw.primary.AccountsPutOnlineRoundParams(onlineRoundParamsData, startRound) + errS := aw.secondary.AccountsPutOnlineRoundParams(onlineRoundParamsData, startRound) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// AccountsPutTotals implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) AccountsPutTotals(totals ledgercore.AccountTotals, catchpointStaging bool) error { + errP := aw.primary.AccountsPutTotals(totals, catchpointStaging) + errS := aw.secondary.AccountsPutTotals(totals, catchpointStaging) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// AccountsReset implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) AccountsReset(ctx context.Context) error { + errP := aw.primary.AccountsReset(ctx) + errS := aw.secondary.AccountsReset(ctx) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// OnlineAccountsDelete implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) OnlineAccountsDelete(forgetBefore basics.Round) (err error) { + errP := aw.primary.OnlineAccountsDelete(forgetBefore) + errS := aw.secondary.OnlineAccountsDelete(forgetBefore) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// ResetAccountHashes implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) ResetAccountHashes(ctx context.Context) (err error) { + errP := aw.primary.ResetAccountHashes(ctx) + errS := aw.secondary.ResetAccountHashes(ctx) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// TxtailNewRound implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error { + errP := aw.primary.TxtailNewRound(ctx, baseRound, roundData, forgetBeforeRound) + errS := aw.secondary.TxtailNewRound(ctx, baseRound, roundData, forgetBeforeRound) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// UpdateAccountsHashRound implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error) { + errP := aw.primary.UpdateAccountsHashRound(ctx, hashRound) + errS := aw.secondary.UpdateAccountsHashRound(ctx, hashRound) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// UpdateAccountsRound implements trackerdb.AccountsWriterExt +func (aw *accountsWriterExt) UpdateAccountsRound(rnd basics.Round) (err error) { + errP := aw.primary.UpdateAccountsRound(rnd) + errS := aw.secondary.UpdateAccountsRound(rnd) + // coalesce errors + return coalesceErrors(errP, errS) +} diff --git a/ledger/store/trackerdb/dualdriver/dualdriver.go b/ledger/store/trackerdb/dualdriver/dualdriver.go new file mode 100644 index 0000000000..64692a9d7e --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/dualdriver.go @@ -0,0 +1,498 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + "errors" + "reflect" + "time" + + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/util/db" + "github.com/google/go-cmp/cmp" +) + +// ErrInconsistentResult is returned when the two stores return different results. +var ErrInconsistentResult = errors.New("inconsistent results between store engines") + +var allowAllUnexported = cmp.Exporter(func(f reflect.Type) bool { return true }) + +type trackerStore struct { + primary trackerdb.Store + secondary trackerdb.Store + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +// MakeStore creates a dual tracker store that verifies that both stores return the same results. +func MakeStore(primary trackerdb.Store, secondary trackerdb.Store) trackerdb.Store { + return &trackerStore{primary, secondary, &reader{primary, secondary}, &writer{primary, secondary}, &catchpoint{primary, secondary}} +} + +func (s *trackerStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) { + errP := s.primary.SetSynchronousMode(ctx, mode, fullfsync) + errS := s.secondary.SetSynchronousMode(ctx, mode, fullfsync) + return coalesceErrors(errP, errS) +} + +func (s *trackerStore) IsSharedCacheConnection() bool { + // Note: this is not something to check for being equal but rather keep the most conservative answer. + return s.primary.IsSharedCacheConnection() || s.secondary.IsSharedCacheConnection() +} + +func (s *trackerStore) Batch(fn trackerdb.BatchFn) (err error) { + return s.BatchContext(context.Background(), fn) +} + +func (s *trackerStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) { + handle, err := s.BeginBatch(ctx) + if err != nil { + return err + } + defer handle.Close() + + err = fn(ctx, handle) + if err != nil { + return err + } + + return handle.Commit() +} + +func (s *trackerStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) { + primary, err := s.primary.BeginBatch(ctx) + if err != nil { + return nil, err + } + secondary, err := s.secondary.BeginBatch(ctx) + if err != nil { + return nil, err + } + return &batch{primary, secondary, &writer{primary, secondary}}, nil +} + +func (s *trackerStore) Snapshot(fn trackerdb.SnapshotFn) (err error) { + return s.SnapshotContext(context.Background(), fn) +} + +func (s *trackerStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) { + handle, err := s.BeginSnapshot(ctx) + if err != nil { + return err + } + defer handle.Close() + + err = fn(ctx, handle) + if err != nil { + return err + } + return nil +} + +func (s *trackerStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) { + primary, err := s.primary.BeginSnapshot(ctx) + if err != nil { + return nil, err + } + secondary, err := s.secondary.BeginSnapshot(ctx) + if err != nil { + return nil, err + } + return &snapshot{primary, secondary, &reader{primary, secondary}}, nil +} + +func (s *trackerStore) Transaction(fn trackerdb.TransactionFn) (err error) { + return s.TransactionContext(context.Background(), fn) +} + +func (s *trackerStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) error { + handle, err := s.BeginTransaction(ctx) + if err != nil { + return err + } + defer handle.Close() + + err = fn(ctx, handle) + if err != nil { + return err + } + + return handle.Commit() +} + +func (s *trackerStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) { + primary, err := s.primary.BeginTransaction(ctx) + if err != nil { + return nil, err + } + secondary, err := s.secondary.BeginTransaction(ctx) + if err != nil { + return nil, err + } + return &transaction{primary, secondary, &reader{primary, secondary}, &writer{primary, secondary}, &catchpoint{primary, secondary}}, nil +} + +// RunMigrations implements trackerdb.Transaction +func (s *trackerStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + paramsP, errP := s.primary.RunMigrations(ctx, params, log, targetVersion) + paramsS, errS := s.secondary.RunMigrations(ctx, params, log, targetVersion) + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results + if paramsP != paramsS { + err = ErrInconsistentResult + return + } + // return primary result + return paramsP, nil +} + +func (s *trackerStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) { + // ignore the stats + // Note: this is a SQL specific operation, so the are unlikely to match + stats, errP := s.primary.Vacuum(ctx) + _, errS := s.secondary.Vacuum(ctx) + err = coalesceErrors(errP, errS) + return +} + +func (s *trackerStore) ResetToV6Test(ctx context.Context) error { + // TODO + return nil +} + +func (s *trackerStore) Close() { + s.primary.Close() + s.secondary.Close() +} + +type reader struct { + primary trackerdb.Reader + secondary trackerdb.Reader +} + +// MakeAccountsOptimizedReader implements trackerdb.Reader +func (r *reader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) { + primary, errP := r.primary.MakeAccountsOptimizedReader() + secondary, errS := r.secondary.MakeAccountsOptimizedReader() + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &accountsReader{primary, secondary}, nil +} + +// MakeAccountsReader implements trackerdb.Reader +func (r *reader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) { + primary, errP := r.primary.MakeAccountsReader() + secondary, errS := r.secondary.MakeAccountsReader() + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &accountsReaderExt{primary, secondary}, nil +} + +// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader +func (r *reader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) { + primary, errP := r.primary.MakeOnlineAccountsOptimizedReader() + secondary, errS := r.secondary.MakeOnlineAccountsOptimizedReader() + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &onlineAccountsReader{primary, secondary}, nil +} + +// MakeSpVerificationCtxReader implements trackerdb.Reader +func (r *reader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader { + primary := r.primary.MakeSpVerificationCtxReader() + secondary := r.secondary.MakeSpVerificationCtxReader() + return &stateproofReader{primary, secondary} +} + +type writer struct { + primary trackerdb.Writer + secondary trackerdb.Writer +} + +// MakeAccountsOptimizedWriter implements trackerdb.Writer +func (w *writer) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) { + primary, errP := w.primary.MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables) + secondary, errS := w.secondary.MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables) + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &accountsWriter{primary, secondary}, nil +} + +// MakeAccountsWriter implements trackerdb.Writer +func (w *writer) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) { + primary, errP := w.primary.MakeAccountsWriter() + secondary, errS := w.secondary.MakeAccountsWriter() + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &accountsWriterExt{primary, secondary}, nil +} + +// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer +func (w *writer) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) { + primary, errP := w.primary.MakeOnlineAccountsOptimizedWriter(hasAccounts) + secondary, errS := w.secondary.MakeOnlineAccountsOptimizedWriter(hasAccounts) + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + return &onlineAccountsWriter{primary, secondary}, nil +} + +// MakeSpVerificationCtxWriter implements trackerdb.Writer +func (w *writer) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter { + primary := w.primary.MakeSpVerificationCtxWriter() + secondary := w.secondary.MakeSpVerificationCtxWriter() + return &stateproofWriter{primary, secondary} +} + +// Testing implements trackerdb.Writer +func (w *writer) Testing() trackerdb.WriterTestExt { + primary := w.primary.Testing() + secondary := w.secondary.Testing() + return &writerForTesting{primary, secondary} +} + +type catchpoint struct { + primary trackerdb.Catchpoint + secondary trackerdb.Catchpoint +} + +// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint +func (*catchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter { + // TODO: catchpoint + return nil +} + +// MakeCatchpointWriter implements trackerdb.Catchpoint +func (*catchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) { + // TODO: catchpoint + return nil, nil +} + +// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint +func (*catchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter { + // TODO: catchpoint + return nil +} + +// MakeKVsIter implements trackerdb.Catchpoint +func (*catchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) { + // TODO: catchpoint + return nil, nil +} + +// MakeMerkleCommitter implements trackerdb.Catchpoint +func (*catchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) { + // TODO: catchpoint + return nil, nil +} + +// MakeOrderedAccountsIter implements trackerdb.Catchpoint +func (*catchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter { + // TODO: catchpoint + return nil +} + +type batch struct { + primary trackerdb.Batch + secondary trackerdb.Batch + trackerdb.Writer +} + +// ResetTransactionWarnDeadline implements trackerdb.Batch +func (b *batch) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + _, _ = b.primary.ResetTransactionWarnDeadline(ctx, deadline) + _, _ = b.secondary.ResetTransactionWarnDeadline(ctx, deadline) + // ignore results, this is very engine specific + return +} + +// Commit implements trackerdb.Batch +func (b *batch) Commit() error { + errP := b.primary.Commit() + errS := b.secondary.Commit() + // errors are unlikely to match between engines + return coalesceErrors(errP, errS) +} + +// Close implements trackerdb.Batch +func (b *batch) Close() error { + errP := b.primary.Close() + errS := b.secondary.Close() + // errors are unlikely to match between engines + return coalesceErrors(errP, errS) +} + +type transaction struct { + primary trackerdb.Transaction + secondary trackerdb.Transaction + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +// ResetTransactionWarnDeadline implements trackerdb.Transaction +func (tx *transaction) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + _, _ = tx.primary.ResetTransactionWarnDeadline(ctx, deadline) + _, _ = tx.secondary.ResetTransactionWarnDeadline(ctx, deadline) + // ignore results, this is very engine specific + return +} + +// RunMigrations implements trackerdb.Transaction +func (tx *transaction) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + paramsP, errP := tx.primary.RunMigrations(ctx, params, log, targetVersion) + paramsS, errS := tx.secondary.RunMigrations(ctx, params, log, targetVersion) + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results + if paramsP != paramsS { + err = ErrInconsistentResult + return + } + // return primary result + return paramsP, nil +} + +// Commit implements trackerdb.Transaction +func (tx *transaction) Commit() error { + errP := tx.primary.Commit() + errS := tx.secondary.Commit() + // errors are unlikely to match between engines + return coalesceErrors(errP, errS) +} + +// Close implements trackerdb.Transaction +func (tx *transaction) Close() error { + errP := tx.primary.Close() + errS := tx.secondary.Close() + // errors are unlikely to match between engines + return coalesceErrors(errP, errS) +} + +type snapshot struct { + primary trackerdb.Snapshot + secondary trackerdb.Snapshot + trackerdb.Reader +} + +// Close implements trackerdb.Snapshot +func (s *snapshot) Close() error { + errP := s.primary.Close() + errS := s.secondary.Close() + // errors are unlikely to match between engines + return coalesceErrors(errP, errS) +} + +// +// refs +// + +type accountRef struct { + primary trackerdb.AccountRef + secondary trackerdb.AccountRef +} + +// AccountRefMarker implements trackerdb.AccountRef +func (accountRef) AccountRefMarker() {} + +func coalesceAccountRefs(primary, secondary trackerdb.AccountRef) (trackerdb.AccountRef, error) { + if primary != nil && secondary != nil { + return accountRef{primary, secondary}, nil + } else if primary == nil && secondary == nil { + // all good, ref is nil + return nil, nil + } else { + // ref mismatch + return nil, ErrInconsistentResult + } +} + +type onlineAccountRef struct { + primary trackerdb.OnlineAccountRef + secondary trackerdb.OnlineAccountRef +} + +// OnlineAccountRefMarker implements trackerdb.OnlineAccountRef +func (onlineAccountRef) OnlineAccountRefMarker() {} + +func coalesceOnlineAccountRefs(primary, secondary trackerdb.OnlineAccountRef) (trackerdb.OnlineAccountRef, error) { + if primary != nil && secondary != nil { + return onlineAccountRef{primary, secondary}, nil + } else if primary == nil && secondary == nil { + // all good, ref is nil + return nil, nil + } else { + // ref mismatch + return nil, ErrInconsistentResult + } +} + +type resourceRef struct { + primary trackerdb.ResourceRef + secondary trackerdb.ResourceRef +} + +// ResourceRefMarker implements trackerdb.ResourceRef +func (resourceRef) ResourceRefMarker() {} + +type creatableRef struct { + primary trackerdb.CreatableRef + secondary trackerdb.CreatableRef +} + +// CreatableRefMarker implements trackerdb.CreatableRef +func (creatableRef) CreatableRefMarker() {} + +// +// helpers +// + +func coalesceErrors(errP error, errS error) error { + if errP == nil && errS != nil { + logging.Base().Error("secondary engine error, ", errS) + return ErrInconsistentResult + } + if errP != nil && errS == nil { + logging.Base().Error("primary engine error, ", errP) + return ErrInconsistentResult + } + // happy case (no errors) + if errP == nil && errS == nil { + return nil + } + // happy case (both errored) + return errP +} diff --git a/ledger/store/trackerdb/dualdriver/online_accounts_reader.go b/ledger/store/trackerdb/dualdriver/online_accounts_reader.go new file mode 100644 index 0000000000..f933639911 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/online_accounts_reader.go @@ -0,0 +1,116 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/google/go-cmp/cmp" +) + +type onlineAccountsReader struct { + primary trackerdb.OnlineAccountsReader + secondary trackerdb.OnlineAccountsReader +} + +// Close implements trackerdb.OnlineAccountsReader +func (oar *onlineAccountsReader) Close() { + oar.primary.Close() + oar.secondary.Close() +} + +// LookupOnline implements trackerdb.OnlineAccountsReader +func (oar *onlineAccountsReader) LookupOnline(addr basics.Address, rnd basics.Round) (data trackerdb.PersistedOnlineAccountData, err error) { + dataP, errP := oar.primary.LookupOnline(addr, rnd) + dataS, errS := oar.secondary.LookupOnline(addr, rnd) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + ref, err := coalesceOnlineAccountRefs(dataP.Ref, dataS.Ref) + if err != nil { + return + } + // update ref in results + // Note: this is safe because the refs are engine specific + dataP.Ref = ref + dataS.Ref = ref + // check results match + if dataP != dataS { + err = ErrInconsistentResult + return + } + // return primary results + return dataP, nil +} + +// LookupOnlineHistory implements trackerdb.OnlineAccountsReader +func (oar *onlineAccountsReader) LookupOnlineHistory(addr basics.Address) (result []trackerdb.PersistedOnlineAccountData, rnd basics.Round, err error) { + resultP, rndP, errP := oar.primary.LookupOnlineHistory(addr) + resultS, rndS, errS := oar.secondary.LookupOnlineHistory(addr) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // coalesce refs + if len(resultP) != len(resultS) { + err = ErrInconsistentResult + return + } + var ref trackerdb.OnlineAccountRef + for i := range resultP { + ref, err = coalesceOnlineAccountRefs(resultP[i].Ref, resultS[i].Ref) + if err != nil { + return result, rnd, err + } + // update ref in results + resultP[i].Ref = ref + resultS[i].Ref = ref + } + // check results match + if !cmp.Equal(resultP, resultS, allowAllUnexported) { + err = ErrInconsistentResult + return + } + if rndP != rndS { + err = ErrInconsistentResult + return + } + // return primary results + return resultP, rndP, nil +} + +// LookupOnlineTotalsHistory implements trackerdb.OnlineAccountsReader +func (oar *onlineAccountsReader) LookupOnlineTotalsHistory(round basics.Round) (algos basics.MicroAlgos, err error) { + algosP, errP := oar.primary.LookupOnlineTotalsHistory(round) + algosS, errS := oar.secondary.LookupOnlineTotalsHistory(round) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if algosP != algosS { + err = ErrInconsistentResult + return + } + // return primary results + return algosP, nil +} diff --git a/ledger/store/trackerdb/dualdriver/online_accounts_writer.go b/ledger/store/trackerdb/dualdriver/online_accounts_writer.go new file mode 100644 index 0000000000..823ed54eee --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/online_accounts_writer.go @@ -0,0 +1,47 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type onlineAccountsWriter struct { + primary trackerdb.OnlineAccountsWriter + secondary trackerdb.OnlineAccountsWriter +} + +// Close implements trackerdb.OnlineAccountsWriter +func (oaw *onlineAccountsWriter) Close() { + oaw.primary.Close() + oaw.secondary.Close() +} + +// InsertOnlineAccount implements trackerdb.OnlineAccountsWriter +func (oaw *onlineAccountsWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (ref trackerdb.OnlineAccountRef, err error) { + // Note: we do not check the refs since they are internal to the engines and wont match + refP, errP := oaw.primary.InsertOnlineAccount(addr, normBalance, data, updRound, voteLastValid) + refS, errS := oaw.secondary.InsertOnlineAccount(addr, normBalance, data, updRound, voteLastValid) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // return ref + return onlineAccountRef{refP, refS}, nil +} diff --git a/ledger/store/trackerdb/dualdriver/stateproof_reader.go b/ledger/store/trackerdb/dualdriver/stateproof_reader.go new file mode 100644 index 0000000000..1ed61b4ebc --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/stateproof_reader.go @@ -0,0 +1,85 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/google/go-cmp/cmp" +) + +type stateproofReader struct { + primary trackerdb.SpVerificationCtxReader + secondary trackerdb.SpVerificationCtxReader +} + +// LookupSPContext implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) LookupSPContext(stateProofLastAttestedRound basics.Round) (*ledgercore.StateProofVerificationContext, error) { + vcP, errP := r.primary.LookupSPContext(stateProofLastAttestedRound) + vcS, errS := r.secondary.LookupSPContext(stateProofLastAttestedRound) + // coalesce errors + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + // check results match + if !cmp.Equal(vcP, vcS, allowAllUnexported) { + err = ErrInconsistentResult + return nil, err + } + // return primary results + return vcP, nil +} + +// GetAllSPContexts implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) GetAllSPContexts(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) { + resultsP, errP := r.primary.GetAllSPContexts(ctx) + resultsS, errS := r.secondary.GetAllSPContexts(ctx) + // coalesce errors + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + // check results match + if !cmp.Equal(resultsP, resultsS, allowAllUnexported) { + err = ErrInconsistentResult + return nil, err + } + // return primary results + return resultsP, nil +} + +// GetAllSPContextsFromCatchpointTbl implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) GetAllSPContextsFromCatchpointTbl(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) { + resultsP, errP := r.primary.GetAllSPContextsFromCatchpointTbl(ctx) + resultsS, errS := r.secondary.GetAllSPContextsFromCatchpointTbl(ctx) + // coalesce errors + err := coalesceErrors(errP, errS) + if err != nil { + return nil, err + } + // check results match + if !cmp.Equal(resultsP, resultsS, allowAllUnexported) { + err = ErrInconsistentResult + return nil, err + } + // return primary results + return resultsP, nil +} diff --git a/ledger/store/trackerdb/dualdriver/stateproof_writer.go b/ledger/store/trackerdb/dualdriver/stateproof_writer.go new file mode 100644 index 0000000000..208ce3cbd4 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/stateproof_writer.go @@ -0,0 +1,53 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type stateproofWriter struct { + primary trackerdb.SpVerificationCtxWriter + secondary trackerdb.SpVerificationCtxWriter +} + +// StoreSPContexts implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) StoreSPContexts(ctx context.Context, verificationContext []*ledgercore.StateProofVerificationContext) error { + errP := w.primary.StoreSPContexts(ctx, verificationContext) + errS := w.secondary.StoreSPContexts(ctx, verificationContext) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// StoreSPContextsToCatchpointTbl implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) StoreSPContextsToCatchpointTbl(ctx context.Context, verificationContexts []ledgercore.StateProofVerificationContext) error { + errP := w.primary.StoreSPContextsToCatchpointTbl(ctx, verificationContexts) + errS := w.secondary.StoreSPContextsToCatchpointTbl(ctx, verificationContexts) + // coalesce errors + return coalesceErrors(errP, errS) +} + +// DeleteOldSPContexts implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) DeleteOldSPContexts(ctx context.Context, earliestLastAttestedRound basics.Round) error { + errP := w.primary.DeleteOldSPContexts(ctx, earliestLastAttestedRound) + errS := w.secondary.DeleteOldSPContexts(ctx, earliestLastAttestedRound) + // coalesce errors + return coalesceErrors(errP, errS) +} diff --git a/ledger/store/trackerdb/dualdriver/transaction_for_testing.go b/ledger/store/trackerdb/dualdriver/transaction_for_testing.go new file mode 100644 index 0000000000..18ba888a52 --- /dev/null +++ b/ledger/store/trackerdb/dualdriver/transaction_for_testing.go @@ -0,0 +1,68 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dualdriver + +import ( + "context" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +type writerForTesting struct { + primary trackerdb.WriterTestExt + secondary trackerdb.WriterTestExt +} + +// AccountsInitLightTest implements trackerdb.WriterTestExt +func (tx *writerForTesting) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabse bool, err error) { + newDatabaseP, errP := tx.primary.AccountsInitLightTest(tb, initAccounts, proto) + newDatabaseS, errS := tx.secondary.AccountsInitLightTest(tb, initAccounts, proto) + // coalesce errors + err = coalesceErrors(errP, errS) + if err != nil { + return + } + // check results match + if newDatabaseP != newDatabaseS { + err = ErrInconsistentResult + return + } + // return primary results + return newDatabaseP, nil +} + +// AccountsInitTest implements trackerdb.WriterTestExt +func (tx *writerForTesting) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { + newDatabaseP := tx.primary.AccountsInitTest(tb, initAccounts, proto) + tx.secondary.AccountsInitTest(tb, initAccounts, proto) + // return primary results + return newDatabaseP +} + +// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt +func (*writerForTesting) AccountsUpdateSchemaTest(ctx context.Context) (err error) { + panic("unimplemented") +} + +// ModifyAcctBaseTest implements trackerdb.WriterTestExt +func (*writerForTesting) ModifyAcctBaseTest() error { + panic("unimplemented") +} diff --git a/ledger/store/trackerdb/generickv/accounts_ext_reader.go b/ledger/store/trackerdb/generickv/accounts_ext_reader.go new file mode 100644 index 0000000000..553d143a45 --- /dev/null +++ b/ledger/store/trackerdb/generickv/accounts_ext_reader.go @@ -0,0 +1,521 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +func (r *accountsReader) AccountsRound() (rnd basics.Round, err error) { + // SQL at time of impl: + // + // "SELECT rnd FROM acctrounds WHERE id='acctbase'" + + // read round entry + value, closer, err := r.kvr.Get(roundKey()) + if err != nil { + return + } + defer closer.Close() + + // parse the bytes into a u64 + rnd = basics.Round(binary.BigEndian.Uint64(value)) + + return +} + +func (r *accountsReader) AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error) { + // read round entry + value, closer, err := r.kvr.Get(totalsKey(catchpointStaging)) + if err != nil { + return + } + defer closer.Close() + + err = protocol.Decode(value, &totals) + if err != nil { + return + } + + return +} + +func (r *accountsReader) AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) { + // TODO: catchpoint + return +} + +func (r *accountsReader) LookupAccountAddressFromAddressID(ctx context.Context, ref trackerdb.AccountRef) (address basics.Address, err error) { + // TODO: catchpoint + return +} + +func (r *accountsReader) LookupAccountRowID(addr basics.Address) (ref trackerdb.AccountRef, err error) { + // Note: [perf] this is not a very cheap operation since we have to pull up the entire record + acc, err := r.LookupAccount(addr) + if err != nil { + return + } + + if acc.Ref == nil { + return nil, trackerdb.ErrNotFound + } + + return acc.Ref, nil +} + +func (r *accountsReader) LookupResourceDataByAddrID(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) { + // TODO: this can probably get removed in favor of LookupResources + // the only issue here is that the only caller of this is not doing anything with the ctype + // so we might have to change the signature of LookupResources to skip the ctype, which might be reasonable + if accRef == nil { + return data, trackerdb.ErrNotFound + } + xref := accRef.(accountRef) + + value, closer, err := r.kvr.Get(resourceKey(xref.addr, aidx)) + if err != nil { + return + } + defer closer.Close() + + return value, nil +} + +func (r *accountsReader) TotalResources(ctx context.Context) (total uint64, err error) { + // TODO: catchpoint + return +} + +func (r *accountsReader) TotalAccounts(ctx context.Context) (total uint64, err error) { + // TODO: catchpoint + return +} + +func (r *accountsReader) TotalKVs(ctx context.Context) (total uint64, err error) { + // TODO: catchpoint + return +} + +// TODO: this replicates some functionality from LookupOnlineHistory, implemented for onlineAccountsReader +func (r *accountsReader) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) { + low := onlineAccountOnlyPartialKey(addr) + high := onlineAccountOnlyPartialKey(addr) + high[len(high)-1]++ + iter := r.kvr.NewIter(low, high, true) + defer iter.Close() + + if iter.Next() { + // key is -- = + key := iter.Key() + + rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1 + u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + + addrOffset := len(kvPrefixOnlineAccount) + 1 + var addr basics.Address + copy(addr[:], key[addrOffset:addrOffset+32]) + + data, err = iter.Value() + if err != nil { + return + } + + var oa trackerdb.BaseOnlineAccountData + err = protocol.Decode(data, &oa) + if err != nil { + return + } + + ref = onlineAccountRef{ + addr: addr, + round: basics.Round(u64Rnd), + normBalance: oa.NormalizedOnlineBalance(r.proto), + } + } else { + err = trackerdb.ErrNotFound + return + } + + return +} + +// AccountsOnlineTop returns the top n online accounts starting at position offset +// (that is, the top offset'th account through the top offset+n-1'th account). +// +// The accounts are sorted by their normalized balance and address. The normalized +// balance has to do with the reward parts of online account balances. See the +// normalization procedure in AccountData.NormalizedOnlineBalance(). +// +// Note that this does not check if the accounts have a vote key valid for any +// particular round (past, present, or future). +func (r *accountsReader) AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (data map[basics.Address]*ledgercore.OnlineAccount, err error) { + // The SQL before the impl + // SELECT + // address, normalizedonlinebalance, data, max(updround) FROM onlineaccounts + // WHERE updround <= ? + // GROUP BY address HAVING normalizedonlinebalance > 0 + // ORDER BY normalizedonlinebalance DESC, address + // DESC LIMIT ? + // OFFSET ? + + // initialize return map + data = make(map[basics.Address]*ledgercore.OnlineAccount) + + // prepare iter over online accounts (by balance) + low := []byte(kvPrefixOnlineAccountBalance) + low = append(low, "-"...) + high := onlineAccountBalanceOnlyPartialKey(rnd) + high[len(high)-1]++ + // reverse order iterator to get high-to-low + iter := r.kvr.NewIter(low, high, true) + defer iter.Close() + + var value []byte + + // first, drop the results from 0 to the offset + for i := uint64(0); i < offset; i++ { + iter.Next() + } + + // add the other results to the map + for i := uint64(0); i < n; i++ { + // if no more results, return early + if !iter.Next() { + return + } + + // key is --- = + key := iter.Key() + + // TODO: make this cleaner + // get the offset where the address starts + offset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1 + + // extract address + var addr basics.Address + copy(addr[:], key[offset:]) + + // skip if already in map + if _, ok := data[addr]; ok { + continue + } + + value, err = iter.Value() + if err != nil { + return + } + + oa := trackerdb.BaseOnlineAccountData{} + err = protocol.Decode(value, &oa) + if err != nil { + return + } + // load the data as a ledgercore OnlineAccount + data[addr] = &ledgercore.OnlineAccount{ + Address: addr, + MicroAlgos: oa.MicroAlgos, + RewardsBase: oa.RewardsBase, + NormalizedOnlineBalance: oa.NormalizedOnlineBalance(proto), + VoteFirstValid: oa.VoteFirstValid, + VoteLastValid: oa.VoteLastValid, + StateProofID: oa.StateProofID, + } + } + return +} + +func (r *accountsReader) AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error) { + // The SQL at the time of writing: + // + // SELECT rnd, data FROM onlineroundparamstail ORDER BY rnd ASC + + start := []byte(kvOnlineAccountRoundParams + "-") + end := []byte(kvOnlineAccountRoundParams + ".") + iter := r.kvr.NewIter(start, end, false) + defer iter.Close() + + var value []byte + + for iter.Next() { + // read the key + // schema: - + key := iter.Key() + + // extract the round from the key + rndOffset := len(kvOnlineAccountRoundParams) + 1 + u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + // assign current item round as endRound + endRound = basics.Round(u64Rnd) + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return nil, endRound, err + } + + // decode the param + roundParams := ledgercore.OnlineRoundParamsData{} + err = protocol.Decode(value, &roundParams) + if err != nil { + return nil, endRound, err + } + + // add the params to the return list + onlineRoundParamsData = append(onlineRoundParamsData, roundParams) + } + + return +} + +// OnlineAccountsAll returns all online accounts up to a provided maximum +// the returned list of PersistedOnlineAccountData includes all of the available +// data for each included account in ascending order of account and round +// (example [account-1-round-1, account1-round-2, ..., account2-round-1, ...]) +func (r *accountsReader) OnlineAccountsAll(maxAccounts uint64) ([]trackerdb.PersistedOnlineAccountData, error) { + // The SQL at the time of impl: + // + // SELECT rowid, address, updround, data + // FROM onlineaccounts + // ORDER BY address, updround ASC + // + // Note: the SQL implementation does not seem to load the current db round to the resulting objects + + // read the current db round + var round basics.Round + round, err := r.AccountsRound() + if err != nil { + return nil, err + } + + low := []byte(kvPrefixOnlineAccount + "-") + high := []byte(kvPrefixOnlineAccount + ".") + iter := r.kvr.NewIter(low, high, false) + defer iter.Close() + + result := make([]trackerdb.PersistedOnlineAccountData, 0, maxAccounts) + + var value []byte + var updround uint64 + + // keep track of the most recently seen account so we can tally up the total number seen + lastAddr := basics.Address{} + seen := uint64(0) + + for iter.Next() { + pitem := trackerdb.PersistedOnlineAccountData{Round: round} + + // schema: -- + key := iter.Key() + + addrOffset := len(kvPrefixOnlineAccount) + 1 + var addr basics.Address + copy(addr[:], key[addrOffset:addrOffset+32]) + // extract updround, it's the last section after the "-" + + rndOffset := addrOffset + 32 + 1 + updround = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + + // load addr, round and data into the persisted item + pitem.Addr = addr + pitem.UpdRound = basics.Round(updround) + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return nil, err + } + // decode raw value + err = protocol.Decode(value, &pitem.AccountData) + if err != nil { + return nil, err + } + // set ref + normBalance := pitem.AccountData.NormalizedOnlineBalance(r.proto) + pitem.Ref = onlineAccountRef{addr, normBalance, pitem.UpdRound} + // if maxAccounts is supplied, potentially stop reading data if we've collected enough + if maxAccounts > 0 { + // we have encountered a new address + if !bytes.Equal(addr[:], lastAddr[:]) { + copy(lastAddr[:], addr[:]) + seen++ + } + // this newest account seen is beyond the maxAccounts requested, meaning we've seen all the data we need + if seen > maxAccounts { + break + } + } + + // append entry to accum + result = append(result, pitem) + } + + return result, nil +} + +// ExpiredOnlineAccountsForRound implements trackerdb.AccountsReaderExt +func (r *accountsReader) ExpiredOnlineAccountsForRound(rnd basics.Round, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (data map[basics.Address]*ledgercore.OnlineAccountData, err error) { + // The SQL at the time of writing: + // + // SELECT address, data, max(updround) + // FROM onlineaccounts + // WHERE updround <= ? <---- ? = rnd + // GROUP BY address + // HAVING votelastvalid < ? and votelastvalid > 0 <---- ? = voteRnd + // ORDER BY address + + // initialize return map + data = make(map[basics.Address]*ledgercore.OnlineAccountData) + expired := make(map[basics.Address]struct{}) + + // prepare iter over online accounts (by balance) + low := []byte(kvPrefixOnlineAccountBalance) + low = append(low, "-"...) + high := onlineAccountBalanceOnlyPartialKey(rnd) + high[len(high)-1]++ + // reverse order iterator to get high-to-low + iter := r.kvr.NewIter(low, high, true) + defer iter.Close() + + var value []byte + + // add the other results to the map + for iter.Next() { + // key is --- = + key := iter.Key() + + // get the addrOffset where the address starts + addrOffset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1 + + // extract address + var addr basics.Address + copy(addr[:], key[addrOffset:]) + + // skip if already in map + // we keep only the one with `max(updround)` + // the reverse iter makes us hit the max first + if _, ok := data[addr]; ok { + continue + } + // when the a ccount is expired we do not add it to the data + // but we might have an older version that is not expired show up + // this would be wrong, so we skip those accounts if the latest version is expired + if _, ok := expired[addr]; ok { + continue + } + + value, err = iter.Value() + if err != nil { + return + } + + oa := trackerdb.BaseOnlineAccountData{} + err = protocol.Decode(value, &oa) + if err != nil { + return + } + + // filter by vote expiration + // sql: HAVING votelastvalid < ? and votelastvalid > 0 + // Note: we might have to add an extra index during insert if this doing this in memory becomes a perf issue + if !(oa.VoteLastValid < voteRnd && oa.VoteLastValid > 0) { + expired[addr] = struct{}{} + continue + } + + // load the data as a ledgercore OnlineAccount + oadata := oa.GetOnlineAccountData(proto, rewardsLevel) + data[addr] = &oadata + } + + return +} + +func (r *accountsReader) LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*trackerdb.TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error) { + // The SQL at the time of writing: + // + // "SELECT rnd, data FROM txtail ORDER BY rnd DESC" + + start := []byte(kvTxTail + "-") + end := []byte(kvTxTail + ".") + iter := r.kvr.NewIter(start, end, true) + defer iter.Close() + + var value []byte + + expectedRound := dbRound + for iter.Next() { + // read the key + key := iter.Key() + + // extract the txTail round from the key + rndOffset := len(kvTxTail) + 1 + u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + round := basics.Round(u64Rnd) + // check that we are on the right round + if round != expectedRound { + return nil, nil, 0, fmt.Errorf("txtail table contain unexpected round %d; round %d was expected", round, expectedRound) + } + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return nil, nil, 0, err + } + + // decode the TxTail + tail := &trackerdb.TxTailRound{} + err = protocol.Decode(value, tail) + if err != nil { + return nil, nil, 0, err + } + + // add the tail + roundData = append(roundData, tail) + // add the hash + roundHash = append(roundHash, crypto.Hash(value)) + + // step the round down (we expect the "previous" round next..) + expectedRound-- + } + + // reverse the array ordering in-place so that it would be incremental order. + for i := 0; i < len(roundData)/2; i++ { + roundData[i], roundData[len(roundData)-i-1] = roundData[len(roundData)-i-1], roundData[i] + roundHash[i], roundHash[len(roundHash)-i-1] = roundHash[len(roundHash)-i-1], roundHash[i] + } + return roundData, roundHash, expectedRound + 1, nil +} + +func (r *accountsReader) LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error) { + // TODO: catchpoint CLI + return +} + +func (r *accountsReader) Testing() trackerdb.AccountsReaderTestExt { + // TODO: this can wait + return nil +} diff --git a/ledger/store/trackerdb/generickv/accounts_ext_writer.go b/ledger/store/trackerdb/generickv/accounts_ext_writer.go new file mode 100644 index 0000000000..bdb05b9138 --- /dev/null +++ b/ledger/store/trackerdb/generickv/accounts_ext_writer.go @@ -0,0 +1,265 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + "encoding/binary" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +func (w *accountsWriter) AccountsReset(ctx context.Context) error { + // TODO: catchpoint + return nil +} + +func (w *accountsWriter) ResetAccountHashes(ctx context.Context) (err error) { + // TODO: catchpoint + return +} + +func (w *accountsWriter) TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error { + // The SQL at the time fo writing: + // + // for i, data := range roundData: + // the inserted rnd value is baseRound + i + // + // INSERT INTO txtail(rnd, data) VALUES(?, ?) + // + // then it also cleans up everything before `forgetBeforeRound`: + // + // DELETE FROM txtail WHERE rnd < ? + + // insert the new txTail's + for i, data := range roundData { + rnd := basics.Round(int(baseRound) + i) + err := w.kvw.Set(txTailKey(rnd), data) + if err != nil { + return err + } + } + + // delete old ones + start := []byte(kvTxTail + "-") + end := txTailKey(forgetBeforeRound) + err := w.kvw.DeleteRange(start, end) + if err != nil { + return err + } + + return nil +} + +func (w *accountsWriter) UpdateAccountsRound(rnd basics.Round) (err error) { + // The SQL at the time of writing: + // + // UPDATE acctrounds SET rnd=? WHERE id='acctbase' AND rnd "-".join(kvPrefixOnlineAccount, addr, round) + // - and the `onlineAccountBalanceKey(round, normBalance, addr) -> "-".join(kvPrefixOnlineAccountBalance, round, normBalance, addr) + + // 1. read from the `onlineAccountBalanceKey` range since we can the addr's that will need to be deleted + start := []byte(kvPrefixOnlineAccountBalance + "-") + end := []byte(kvPrefixOnlineAccountBalance + "-") + end = append(end, bigEndianUint64(uint64(forgetBefore))...) + iter := w.kvr.NewIter(start, end, true) + defer iter.Close() + + toDeletePrimaryIndex := make([]struct { + basics.Address + basics.Round + }, 0) + + toDeleteSecondaryIndex := make([][]byte, 0) + + var prevAddr basics.Address + + for iter.Next() { + // read the key + // schema: --- + key := iter.Key() + + // extract the round from the key (offset: 1) + rndOffset := len(kvPrefixOnlineAccountBalance) + 1 + u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + round := basics.Round(u64Rnd) + + // get the offset where the address starts + addrOffset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1 + var addr basics.Address + copy(addr[:], key[addrOffset:addrOffset+32]) + + if addr != prevAddr { + // new address + // if the first (latest) entry is + // - offline then delete all + // - online then safe to delete all previous except this first (latest) + + // reset the state + prevAddr = addr + + // delete on voting empty + var oad trackerdb.BaseOnlineAccountData + var data []byte + data, err = iter.Value() + if err != nil { + return err + } + err = protocol.Decode(data, &oad) + if err != nil { + return err + } + if oad.IsVotingEmpty() { + // delete this and all subsequent + toDeletePrimaryIndex = append(toDeletePrimaryIndex, struct { + basics.Address + basics.Round + }{addr, round}) + toDeleteSecondaryIndex = append(toDeleteSecondaryIndex, key) + } + + // restart the loop + // if there are some subsequent entries, they will deleted on the next iteration + // if no subsequent entries, the loop will reset the state and the latest entry does not get deleted + continue + } + + // mark the item for deletion + toDeletePrimaryIndex = append(toDeletePrimaryIndex, struct { + basics.Address + basics.Round + }{addr, round}) + toDeleteSecondaryIndex = append(toDeleteSecondaryIndex, key) + } + + // 2. delete the individual addr+round entries + for _, item := range toDeletePrimaryIndex { + // TODO: [perf] we might be able to optimize this with a SingleDelete call + err = w.kvw.Delete(onlineAccountKey(item.Address, item.Round)) + if err != nil { + return + } + } + + // 3. delete the range from `onlineAccountBalanceKey` + for _, key := range toDeleteSecondaryIndex { + // TODO: [perf] we might be able to optimize this with a SingleDelete call + err = w.kvw.Delete(key) + if err != nil { + return + } + } + + return +} + +func (w *accountsWriter) AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error { + // The SQL at the time of impl: + // + // for i, data := range onlineRoundParamsData { + // the inserted rnd value is startRound + i + // + // INSERT INTO onlineroundparamstail (rnd, data) VALUES (?, ?) + // + + // insert the round params + for i := range onlineRoundParamsData { + rnd := basics.Round(int(startRound) + i) + raw := protocol.Encode(&onlineRoundParamsData[i]) + err := w.kvw.Set(onlineAccountRoundParamsKey(rnd), raw) + if err != nil { + return err + } + } + + return nil +} + +func (w *accountsWriter) AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error { + // The SQL at the time of impl: + // + // DELETE FROM onlineroundparamstail WHERE rnd. + +package generickv + +import ( + "encoding/binary" + "fmt" + "io" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +// KvRead is a low level KV db interface for reading. +type KvRead interface { + Get(key []byte) ([]byte, io.Closer, error) + NewIter(low, high []byte, reverse bool) KvIter +} + +// KvIter is a low level KV iterator. +type KvIter interface { + Next() bool + Key() []byte + KeySlice() Slice + Value() ([]byte, error) + ValueSlice() (Slice, error) + Valid() bool + Close() +} + +// Slice is a low level slice used during the KV iterator. +type Slice interface { + Data() []byte + Free() + Size() int + Exists() bool +} + +type accountsReader struct { + kvr KvRead + proto config.ConsensusParams +} + +// MakeAccountsReader returns a kv db agnostic AccountsReader. +func MakeAccountsReader(kvr KvRead, proto config.ConsensusParams) *accountsReader { + return &accountsReader{kvr, proto} +} + +func (r *accountsReader) LookupAccount(addr basics.Address) (data trackerdb.PersistedAccountData, err error) { + // SQL impl at time of writing: + // + // SELECT + // accountbase.rowid, + // acctrounds.rnd, + // accountbase.data + // FROM acctrounds + // LEFT JOIN accountbase ON address=? + // WHERE id='acctbase' + + data.Addr = addr + + // read the current db round + data.Round, err = r.AccountsRound() + if err != nil { + return + } + + value, closer, err := r.kvr.Get(accountKey(addr)) + if err == trackerdb.ErrNotFound { + // Note: the SQL implementation returns a data value and no error even when the account does not exist. + return data, nil + } else if err != nil { + return + } + defer closer.Close() + + err = protocol.Decode(value, &data.AccountData) + if err != nil { + return + } + + data.Ref = accountRef{addr} + + return +} + +func (r *accountsReader) LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data trackerdb.PersistedResourcesData, err error) { + data.Aidx = aidx + + // read the current db round + data.Round, err = r.AccountsRound() + if err != nil { + return + } + + value, closer, err := r.kvr.Get(resourceKey(addr, aidx)) + if err == trackerdb.ErrNotFound { + // Note: the SQL implementation returns a data value and no error even when the account does not exist. + data.Data = trackerdb.MakeResourcesData(0) + return data, nil + } else if err != nil { + err = fmt.Errorf("unable to query resource data for address %v aidx %v ctype %v : %w", addr, aidx, ctype, err) + return + } + defer closer.Close() + + err = protocol.Decode(value, &data.Data) + if err != nil { + return + } + + // Note: the ctype is not filtered during the query, but rather asserted to be what the caller expected + if ctype == basics.AssetCreatable && !data.Data.IsAsset() { + err = fmt.Errorf("lookupResources asked for an asset but got %v", data.Data) + } + if ctype == basics.AppCreatable && !data.Data.IsApp() { + err = fmt.Errorf("lookupResources asked for an app but got %v", data.Data) + } + + data.AcctRef = accountRef{addr} + + return +} + +func (r *accountsReader) LookupAllResources(addr basics.Address) (data []trackerdb.PersistedResourcesData, rnd basics.Round, err error) { + low := resourceAddrOnlyPartialKey(addr) + high := resourceAddrOnlyPartialKey(addr) + high[len(high)-1]++ + + iter := r.kvr.NewIter(low, high, false) + defer iter.Close() + + var value []byte + var aidx uint64 + + // read the current db round + rnd, err = r.AccountsRound() + if err != nil { + return + } + + for iter.Next() { + pitem := trackerdb.PersistedResourcesData{AcctRef: accountRef{addr}, Round: rnd} + + // read the key to parse the aidx + // key is -- = + key := iter.Key() + + aidxOffset := len(kvPrefixResource) + 1 + 32 + 1 + aidx = binary.BigEndian.Uint64(key[aidxOffset : aidxOffset+8]) + pitem.Aidx = basics.CreatableIndex(aidx) + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return + } + // decode raw value + err = protocol.Decode(value, &pitem.Data) + if err != nil { + return + } + // append entry to accum + data = append(data, pitem) + } + + return +} + +func (r *accountsReader) LookupKeyValue(key string) (pv trackerdb.PersistedKVData, err error) { + // read the current db round + pv.Round, err = r.AccountsRound() + if err != nil { + return + } + + value, closer, err := r.kvr.Get(appKvKey(key)) + if err == trackerdb.ErrNotFound { + // Note: the SQL implementation returns a data value and no error even when the account does not exist. + return pv, nil + } else if err != nil { + return + } + defer closer.Close() + + pv.Value = value + + return +} + +// TODO: lifted from sql.go, we might want to refactor it +func keyPrefixIntervalPreprocessing(prefix []byte) ([]byte, []byte) { + if prefix == nil { + prefix = []byte{} + } + prefixIncr := make([]byte, len(prefix)) + copy(prefixIncr, prefix) + for i := len(prefix) - 1; i >= 0; i-- { + currentByteIncr := int(prefix[i]) + 1 + if currentByteIncr > 0xFF { + prefixIncr = prefixIncr[:len(prefixIncr)-1] + continue + } + prefixIncr[i] = byte(currentByteIncr) + return prefix, prefixIncr + } + return prefix, nil +} + +func (r *accountsReader) LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) { + // SQL at time of writing: + // + // SELECT acctrounds.rnd, kvstore.key + // FROM acctrounds LEFT JOIN kvstore ON kvstore.key >= ? AND kvstore.key < ? + // WHERE id='acctbase' + + // read the current db round + round, err = r.AccountsRound() + if err != nil { + return + } + + start, end := keyPrefixIntervalPreprocessing([]byte(prefix)) + + iter := r.kvr.NewIter(start, end, false) + defer iter.Close() + + var value []byte + + for iter.Next() { + // end iteration if we reached max results + if resultCount == maxKeyNum { + return + } + + // read the key + key := string(iter.Key()) + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return + } + + // mark if the key has data on the result map + results[key] = len(value) > 0 + + // inc results in range + resultCount++ + } + + return +} + +func (r *accountsReader) ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) { + // The old SQL impl: + // + // SELECT + // acctrounds.rnd, + // assetcreators.asset, + // assetcreators.creator + // FROM acctrounds + // LEFT JOIN assetcreators ON assetcreators.asset <= ? AND assetcreators.ctype = ? + // WHERE acctrounds.id='acctbase' + // ORDER BY assetcreators.asset desc + // LIMIT ? + + // read the current db round + dbRound, err = r.AccountsRound() + if err != nil { + return + } + + start := []byte(fmt.Sprintf("%s-", kvPrefixCreatorIndex)) + end := creatableKey(basics.CreatableIndex(uint64(maxIdx) + 1)) + + // assets are returned in descending order of cidx + iter := r.kvr.NewIter(start, end, true) + defer iter.Close() + + var value []byte + var resultCount uint64 + var cidx uint64 + + for iter.Next() { + // end iteration if we reached max results + if resultCount == maxResults { + return + } + + // read the key + // schema: - + key := iter.Key() + + // extract cidx + cidxOffset := len(kvPrefixCreatorIndex) + 1 + cidx = binary.BigEndian.Uint64(key[cidxOffset : cidxOffset+8]) + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return + } + + // decode the raw value + var entry creatableEntry + err = protocol.Decode(value, &entry) + if err != nil { + return + } + + // TODO: the ctype as part of key makes this filterable during the iter + if entry.Ctype != ctype { + continue + } + + // create the "creatable" struct + cl := basics.CreatableLocator{Type: ctype, Index: basics.CreatableIndex(cidx)} + copy(cl.Creator[:], entry.CreatorAddr) + + // add it to the the results + results = append(results, cl) + + // inc results in range + resultCount++ + } + + // read the current db round + dbRound, err = r.AccountsRound() + if err != nil { + return + } + + return +} + +func (r *accountsReader) LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) { + // The old SQL impl: + // + // SELECT + // acctrounds.rnd, + // assetcreators.creator + // FROM acctrounds + // LEFT JOIN assetcreators ON asset = ? AND ctype = ? + // WHERE id='acctbase' + + // read the current db round + dbRound, err = r.AccountsRound() + if err != nil { + return + } + + value, closer, err := r.kvr.Get(creatableKey(cidx)) + if err == trackerdb.ErrNotFound { + // the record does not exist + // clean up the error and just return ok=false + err = nil + ok = false + return + } else if err != nil { + return + } + defer closer.Close() + + // decode the raw value + var entry creatableEntry + err = protocol.Decode(value, &entry) + if err != nil { + return + } + + // assert that the ctype is the one expected + if entry.Ctype != ctype { + ok = false + return + } + + // copy the addr to the return + copy(addr[:], entry.CreatorAddr) + + // mark result as ok + ok = true + + return +} + +func (r *accountsReader) Close() { + +} diff --git a/ledger/store/trackerdb/generickv/accounts_writer.go b/ledger/store/trackerdb/generickv/accounts_writer.go new file mode 100644 index 0000000000..b2f808a436 --- /dev/null +++ b/ledger/store/trackerdb/generickv/accounts_writer.go @@ -0,0 +1,188 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +// KvWrite is a low level KV db interface for writing. +type KvWrite interface { + Set(key, value []byte) error + Delete(key []byte) error + DeleteRange(start, end []byte) error +} + +type accountsWriter struct { + kvw KvWrite + kvr KvRead +} + +type accountRef struct { + addr basics.Address +} + +func (ref accountRef) AccountRefMarker() {} + +type resourceRef struct { + addr basics.Address + aidx basics.CreatableIndex +} + +func (ref resourceRef) ResourceRefMarker() {} + +type creatableRef struct { + cidx basics.CreatableIndex +} + +func (ref creatableRef) CreatableRefMarker() {} + +// MakeAccountsWriter returns a kv db agnostic AccountsWriter. +// TODO: we should discuss what is the best approach for this `kvr KvRead`. +// the problem is that `OnlineAccountsDelete` requires reading and writing. +// the cleanest approach is to move that method to a separate interface, and have it only be available on 'transactions'. +// although a snapshot+batch should be able to support it too since its all reads, then writes. +func MakeAccountsWriter(kvw KvWrite, kvr KvRead) *accountsWriter { + return &accountsWriter{kvw, kvr} +} + +func (w *accountsWriter) InsertAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseAccountData) (ref trackerdb.AccountRef, err error) { + // write account entry + raw := protocol.Encode(&data) + err = w.kvw.Set(accountKey(addr), raw) + if err != nil { + return nil, err + } + + return accountRef{addr}, nil +} + +func (w *accountsWriter) DeleteAccount(ref trackerdb.AccountRef) (rowsAffected int64, err error) { + xref := ref.(accountRef) + + // delete account entry + err = w.kvw.Delete(accountKey(xref.addr)) + if err != nil { + return 0, err + } + + return 1, nil +} + +func (w *accountsWriter) UpdateAccount(ref trackerdb.AccountRef, normBalance uint64, data trackerdb.BaseAccountData) (rowsAffected int64, err error) { + xref := ref.(accountRef) + + // overwrite account entry + raw := protocol.Encode(&data) + err = w.kvw.Set(accountKey(xref.addr), raw) + if err != nil { + return 0, err + } + + return 1, nil +} + +func (w *accountsWriter) InsertResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (ref trackerdb.ResourceRef, err error) { + xref := acctRef.(accountRef) + + // write resource entry + raw := protocol.Encode(&data) + err = w.kvw.Set(resourceKey(xref.addr, aidx), raw) + if err != nil { + return nil, err + } + + return resourceRef{xref.addr, aidx}, nil +} + +func (w *accountsWriter) DeleteResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex) (rowsAffected int64, err error) { + xref := acctRef.(accountRef) + + // delete resource entry + err = w.kvw.Delete(resourceKey(xref.addr, aidx)) + if err != nil { + return 0, err + } + + return 1, nil +} + +func (w *accountsWriter) UpdateResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (rowsAffected int64, err error) { + xref := acctRef.(accountRef) + + // update resource entry + raw := protocol.Encode(&data) + err = w.kvw.Set(resourceKey(xref.addr, aidx), raw) + if err != nil { + return 0, err + } + + return 1, nil +} + +func (w *accountsWriter) UpsertKvPair(key string, value []byte) error { + // upsert kv entry + err := w.kvw.Set(appKvKey(key), value) + if err != nil { + return err + } + + return nil +} + +func (w *accountsWriter) DeleteKvPair(key string) error { + // delete kv entry + err := w.kvw.Delete(appKvKey(key)) + if err != nil { + return err + } + + return nil +} + +type creatableEntry struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + Ctype basics.CreatableType + CreatorAddr []byte +} + +func (w *accountsWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (ref trackerdb.CreatableRef, err error) { + // insert creatable entry + raw := protocol.Encode(&creatableEntry{Ctype: ctype, CreatorAddr: creator}) + err = w.kvw.Set(creatableKey(cidx), raw) + if err != nil { + return + } + + return creatableRef{cidx}, nil +} + +func (w *accountsWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) { + // delete creatable entry + err = w.kvw.Delete(creatableKey(cidx)) + if err != nil { + return 0, err + } + + return 1, nil +} + +func (w *accountsWriter) Close() { + +} diff --git a/ledger/store/trackerdb/generickv/catchpoint.go b/ledger/store/trackerdb/generickv/catchpoint.go new file mode 100644 index 0000000000..18b4e209fa --- /dev/null +++ b/ledger/store/trackerdb/generickv/catchpoint.go @@ -0,0 +1,60 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type catchpoint struct{} + +// MakeCatchpoint returns a trackerdb.Catchpoint for a KV +func MakeCatchpoint() trackerdb.Catchpoint { + return &catchpoint{} +} + +// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint +func (*catchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter { + panic("unimplemented") +} + +// MakeCatchpointWriter implements trackerdb.Catchpoint +func (*catchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) { + panic("unimplemented") +} + +// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint +func (*catchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter { + panic("unimplemented") +} + +// MakeKVsIter implements trackerdb.Catchpoint +func (*catchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) { + panic("unimplemented") +} + +// MakeMerkleCommitter implements trackerdb.Catchpoint +func (*catchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) { + panic("unimplemented") +} + +// MakeOrderedAccountsIter implements trackerdb.Catchpoint +func (*catchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter { + panic("unimplemented") +} diff --git a/ledger/store/trackerdb/generickv/init_accounts.go b/ledger/store/trackerdb/generickv/init_accounts.go new file mode 100644 index 0000000000..af587af105 --- /dev/null +++ b/ledger/store/trackerdb/generickv/init_accounts.go @@ -0,0 +1,60 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" + "github.com/stretchr/testify/require" +) + +type dbForInit interface { + trackerdb.Store + KvRead + KvWrite +} + +// AccountsInitTest initializes the database for testing with the given accounts. +func AccountsInitTest(tb testing.TB, db dbForInit, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { + params := trackerdb.Params{ + InitAccounts: initAccounts, + InitProto: proto, + } + _, err := RunMigrations(context.Background(), db, params, trackerdb.AccountDBVersion) + require.NoError(tb, err) + return true +} + +// AccountsInitLightTest initializes the database for testing with the given accounts. +// +// This is duplicate due to a specific legacy test in accdeltas_test.go. +// TODO: remove the need for this. +func AccountsInitLightTest(tb testing.TB, db dbForInit, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { + params := trackerdb.Params{ + InitAccounts: initAccounts, + // TODO: how do we get the correct version from the proto arg? + InitProto: protocol.ConsensusCurrentVersion, + } + _, err = RunMigrations(context.Background(), db, params, trackerdb.AccountDBVersion) + require.NoError(tb, err) + return true, nil +} diff --git a/ledger/store/trackerdb/generickv/migrations.go b/ledger/store/trackerdb/generickv/migrations.go new file mode 100644 index 0000000000..9c8e942e39 --- /dev/null +++ b/ledger/store/trackerdb/generickv/migrations.go @@ -0,0 +1,225 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + "encoding/binary" + "fmt" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +func getSchemaVersion(ctx context.Context, kvr KvRead) (int32, error) { + // read version entry + value, closer, err := kvr.Get(schemaVersionKey()) + if err == trackerdb.ErrNotFound { + // ignore the error, return version 0 + return 0, nil + } else if err != nil { + return 0, err + } + defer closer.Close() + + // parse the bytes into a i32 + version := int32(binary.BigEndian.Uint32(value)) + + return version, nil +} + +func setSchemaVersion(ctx context.Context, kvw KvWrite, version int32) error { + // write version entry + raw := bigEndianUint32(uint32(version)) + err := kvw.Set(schemaVersionKey(), raw) + if err != nil { + return err + } + + return nil +} + +type dbForMigrations interface { + trackerdb.Store + KvRead + KvWrite +} + +// RunMigrations runs the migrations on the store up to the target version. +func RunMigrations(ctx context.Context, db dbForMigrations, params trackerdb.Params, targetVersion int32) (mgr trackerdb.InitParams, err error) { + + dbVersion, err := getSchemaVersion(ctx, db) + if err != nil { + return + } + + mgr.SchemaVersion = dbVersion + mgr.VacuumOnStartup = false + + migrator := &migrator{ + currentVersion: dbVersion, + targetVersion: targetVersion, + params: params, + db: db, + } + + err = migrator.Migrate(ctx) + if err != nil { + return + } + + mgr.SchemaVersion = migrator.currentVersion + + return mgr, nil +} + +type migrator struct { + currentVersion int32 + targetVersion int32 + params trackerdb.Params + db dbForMigrations +} + +func (m *migrator) Migrate(ctx context.Context) error { + // we cannot rollback + if m.currentVersion > m.targetVersion { + return nil + } + // upgrade the db one version at at time + for m.currentVersion < m.targetVersion { + // run next version upgrade + switch m.currentVersion { + case 0: // initial version + err := m.initialVersion(ctx) + if err != nil { + return err + } + default: + // any other version we do nothing + return nil + } + } + return nil +} + +func (m *migrator) setVersion(ctx context.Context, version int32) error { + // update crrent version in the db + err := setSchemaVersion(ctx, m.db, version) + if err != nil { + return err + } + // update current version in the migrator + m.currentVersion = version + return nil +} + +func (m *migrator) initialVersion(ctx context.Context) error { + proto := config.Consensus[m.params.InitProto] + + // TODO: make this a batch scope + err := m.db.TransactionContext(ctx, func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { + aow, err := tx.MakeAccountsOptimizedWriter(true, false, false, false) + if err != nil { + return err + } + + oaow, err := tx.MakeOnlineAccountsOptimizedWriter(true) + if err != nil { + return err + } + + aw, err := tx.MakeAccountsWriter() + if err != nil { + return err + } + + updRound := basics.Round(0) + + // mark the db as round 0 + err = aw.UpdateAccountsRound(updRound) + if err != nil { + return err + } + + var ot basics.OverflowTracker + var totals ledgercore.AccountTotals + + // insert initial accounts + for addr, account := range m.params.InitAccounts { + // build a trackerdb.BaseAccountData to pass to the DB + var bad trackerdb.BaseAccountData + bad.SetAccountData(&account) + // insert the account + _, err = aow.InsertAccount(addr, account.NormalizedOnlineBalance(proto), bad) + if err != nil { + return err + } + + // build a ledgercore.AccountData to track the totals + ad := ledgercore.ToAccountData(account) + // track the totals + totals.AddAccount(proto, ad, &ot) + + // insert online account (if online) + if bad.Status == basics.Online { + var baseOnlineAD trackerdb.BaseOnlineAccountData + baseOnlineAD.BaseVotingData = bad.BaseVotingData + baseOnlineAD.MicroAlgos = bad.MicroAlgos + baseOnlineAD.RewardsBase = bad.RewardsBase + + _, err = oaow.InsertOnlineAccount(addr, account.NormalizedOnlineBalance(proto), baseOnlineAD, uint64(updRound), uint64(baseOnlineAD.VoteLastValid)) + if err != nil { + return err + } + } + } + + // make sure we didn't overflow + if ot.Overflowed { + return fmt.Errorf("overflow computing totals") + } + + // insert the totals + err = aw.AccountsPutTotals(totals, false) + if err != nil { + return err + } + + // insert online params + params := []ledgercore.OnlineRoundParamsData{ + { + OnlineSupply: totals.Online.Money.Raw, + RewardsLevel: totals.RewardsLevel, + CurrentProtocol: m.params.InitProto, + }, + } + err = aw.AccountsPutOnlineRoundParams(params, basics.Round(0)) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + // KV store starts at version 10 + return m.setVersion(ctx, 10) +} diff --git a/ledger/store/trackerdb/generickv/msgp_gen.go b/ledger/store/trackerdb/generickv/msgp_gen.go new file mode 100644 index 0000000000..fd825ca99d --- /dev/null +++ b/ledger/store/trackerdb/generickv/msgp_gen.go @@ -0,0 +1,146 @@ +package generickv + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "github.com/algorand/msgp/msgp" +) + +// The following msgp objects are implemented in this file: +// creatableEntry +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// + +// MarshalMsg implements msgp.Marshaler +func (z *creatableEntry) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if len((*z).CreatorAddr) == 0 { + zb0001Len-- + zb0001Mask |= 0x1 + } + if (*z).Ctype.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not empty + // string "CreatorAddr" + o = append(o, 0xab, 0x43, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x41, 0x64, 0x64, 0x72) + o = msgp.AppendBytes(o, (*z).CreatorAddr) + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "Ctype" + o = append(o, 0xa5, 0x43, 0x74, 0x79, 0x70, 0x65) + o = (*z).Ctype.MarshalMsg(o) + } + } + return +} + +func (_ *creatableEntry) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*creatableEntry) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *creatableEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Ctype.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Ctype") + return + } + } + if zb0001 > 0 { + zb0001-- + (*z).CreatorAddr, bts, err = msgp.ReadBytesBytes(bts, (*z).CreatorAddr) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "CreatorAddr") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = creatableEntry{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "Ctype": + bts, err = (*z).Ctype.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Ctype") + return + } + case "CreatorAddr": + (*z).CreatorAddr, bts, err = msgp.ReadBytesBytes(bts, (*z).CreatorAddr) + if err != nil { + err = msgp.WrapError(err, "CreatorAddr") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *creatableEntry) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*creatableEntry) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *creatableEntry) Msgsize() (s int) { + s = 1 + 6 + (*z).Ctype.Msgsize() + 12 + msgp.BytesPrefixSize + len((*z).CreatorAddr) + return +} + +// MsgIsZero returns whether this is a zero value +func (z *creatableEntry) MsgIsZero() bool { + return ((*z).Ctype.MsgIsZero()) && (len((*z).CreatorAddr) == 0) +} diff --git a/ledger/store/trackerdb/generickv/msgp_gen_test.go b/ledger/store/trackerdb/generickv/msgp_gen_test.go new file mode 100644 index 0000000000..88d79eec21 --- /dev/null +++ b/ledger/store/trackerdb/generickv/msgp_gen_test.go @@ -0,0 +1,75 @@ +//go:build !skip_msgp_testing +// +build !skip_msgp_testing + +package generickv + +// Code generated by github.com/algorand/msgp DO NOT EDIT. + +import ( + "testing" + + "github.com/algorand/msgp/msgp" + + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestMarshalUnmarshalcreatableEntry(t *testing.T) { + partitiontest.PartitionTest(t) + v := creatableEntry{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingcreatableEntry(t *testing.T) { + protocol.RunEncodingTest(t, &creatableEntry{}) +} + +func BenchmarkMarshalMsgcreatableEntry(b *testing.B) { + v := creatableEntry{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgcreatableEntry(b *testing.B) { + v := creatableEntry{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalcreatableEntry(b *testing.B) { + v := creatableEntry{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/ledger/store/trackerdb/generickv/onlineaccounts_reader.go b/ledger/store/trackerdb/generickv/onlineaccounts_reader.go new file mode 100644 index 0000000000..029b0317c2 --- /dev/null +++ b/ledger/store/trackerdb/generickv/onlineaccounts_reader.go @@ -0,0 +1,165 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "encoding/binary" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +// LookupOnline pulls the Online Account data for a given account+round +func (r *accountsReader) LookupOnline(addr basics.Address, rnd basics.Round) (data trackerdb.PersistedOnlineAccountData, err error) { + // SQL at the time of writing this: + // + // SELECT + // onlineaccounts.rowid, onlineaccounts.updround, + // acctrounds.rnd, + // onlineaccounts.data + // FROM acctrounds + // LEFT JOIN onlineaccounts ON address=? AND updround <= ? + // WHERE id='acctbase' + // ORDER BY updround DESC LIMIT 1 + + // read the current db round + data.Round, err = r.AccountsRound() + if err != nil { + return + } + + // read latest account up to `rnd`` + low := onlineAccountOnlyPartialKey(addr) + high := onlineAccountKey(addr, rnd) + // inc the last byte to make it inclusive + high[len(high)-1]++ + iter := r.kvr.NewIter(low, high, true) + defer iter.Close() + + var value []byte + var updRound uint64 + + if iter.Next() { + // schema: -- + key := iter.Key() + + // extract updround, its the last section after the "-" + rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1 + updRound = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + if err != nil { + return + } + data.Addr = addr + data.UpdRound = basics.Round(updRound) + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return + } + + // parse the value + err = protocol.Decode(value, &data.AccountData) + if err != nil { + return + } + + normBalance := data.AccountData.NormalizedOnlineBalance(r.proto) + data.Ref = onlineAccountRef{addr, normBalance, rnd} + + // we have the record, we can leave + return + } + + // nothing was found + // Note: the SQL implementation returns a data value and no error even when the account does not exist. + return data, nil +} + +// LookupOnlineTotalsHistory pulls the total Online Algos on a given round +func (r *accountsReader) LookupOnlineTotalsHistory(round basics.Round) (basics.MicroAlgos, error) { + // SQL at the time of writing this: + // + // SELECT data FROM onlineroundparamstail WHERE rnd=? + + value, closer, err := r.kvr.Get(onlineAccountRoundParamsKey(round)) + if err != nil { + return basics.MicroAlgos{}, err + } + defer closer.Close() + data := ledgercore.OnlineRoundParamsData{} + err = protocol.Decode(value, &data) + if err != nil { + return basics.MicroAlgos{}, err + } + return basics.MicroAlgos{Raw: data.OnlineSupply}, nil +} + +func (r *accountsReader) LookupOnlineHistory(addr basics.Address) (result []trackerdb.PersistedOnlineAccountData, rnd basics.Round, err error) { + low := onlineAccountOnlyPartialKey(addr) + high := onlineAccountOnlyPartialKey(addr) + high[len(high)-1]++ + iter := r.kvr.NewIter(low, high, false) + defer iter.Close() + + var value []byte + var updround uint64 + + // read the current db round + rnd, err = r.AccountsRound() + if err != nil { + return + } + + for iter.Next() { + pitem := trackerdb.PersistedOnlineAccountData{} + + // schema: -- + key := iter.Key() + // extract updround, its the last section after the "-" + rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1 + updround = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8]) + if err != nil { + return + } + pitem.Addr = addr + pitem.UpdRound = basics.Round(updround) + // Note: for compatibility with the SQL impl, this is not included on each item + // pitem.Round = rnd + + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return + } + // decode raw value + err = protocol.Decode(value, &pitem.AccountData) + if err != nil { + return + } + + // set the ref + pitem.Ref = onlineAccountRef{addr, pitem.AccountData.NormalizedOnlineBalance(r.proto), pitem.UpdRound} + + // append entry to accum + result = append(result, pitem) + } + + return +} diff --git a/ledger/store/trackerdb/generickv/onlineaccounts_writer.go b/ledger/store/trackerdb/generickv/onlineaccounts_writer.go new file mode 100644 index 0000000000..cca847c608 --- /dev/null +++ b/ledger/store/trackerdb/generickv/onlineaccounts_writer.go @@ -0,0 +1,64 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +type onlineAccountsWriter struct { + kvw KvWrite +} + +type onlineAccountRef struct { + addr basics.Address + normBalance uint64 + round basics.Round +} + +func (ref onlineAccountRef) OnlineAccountRefMarker() {} + +// MakeOnlineAccountsWriter constructs an kv agnostic OnlineAccountsWriter +func MakeOnlineAccountsWriter(kvw KvWrite) trackerdb.OnlineAccountsWriter { + return &onlineAccountsWriter{kvw} +} + +func (w *onlineAccountsWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (ref trackerdb.OnlineAccountRef, err error) { + raw := protocol.Encode(&data) + rnd := basics.Round(updRound) + + // write to the online account key + err = w.kvw.Set(onlineAccountKey(addr, rnd), raw) + if err != nil { + return nil, err + } + + // write to the secondary account balance key + // TODO: this is not the most efficient use of space, but its a tradeoff with a second lookup per object. + // this impacts `AccountsOnlineTop`, and some experiments will be needed to determine if we do extra lookups or duplicate the values. + err = w.kvw.Set(onlineAccountBalanceKey(updRound, normBalance, addr), raw) + if err != nil { + return nil, err + } + + return onlineAccountRef{addr, normBalance, rnd}, nil +} + +func (w *onlineAccountsWriter) Close() { +} diff --git a/ledger/store/trackerdb/generickv/reader.go b/ledger/store/trackerdb/generickv/reader.go new file mode 100644 index 0000000000..88761c6013 --- /dev/null +++ b/ledger/store/trackerdb/generickv/reader.go @@ -0,0 +1,52 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb" +) + +type reader struct { + proto config.ConsensusParams + KvRead +} + +// MakeReader returns a trackerdb.Reader for a KV +func MakeReader(kvr KvRead, proto config.ConsensusParams) trackerdb.Reader { + return &reader{proto, kvr} +} + +// MakeAccountsOptimizedReader implements trackerdb.Reader +func (r *reader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) { + return MakeAccountsReader(r, r.proto), nil +} + +// MakeAccountsReader implements trackerdb.Reader +func (r *reader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) { + return MakeAccountsReader(r, r.proto), nil +} + +// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader +func (r *reader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) { + return MakeAccountsReader(r, r.proto), nil +} + +// MakeSpVerificationCtxReader implements trackerdb.Reader +func (r *reader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader { + return MakeStateproofReader(r) +} diff --git a/ledger/store/trackerdb/generickv/schema.go b/ledger/store/trackerdb/generickv/schema.go new file mode 100644 index 0000000000..8afb41e9e2 --- /dev/null +++ b/ledger/store/trackerdb/generickv/schema.go @@ -0,0 +1,171 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "encoding/binary" + + "github.com/algorand/go-algorand/data/basics" +) + +const ( + kvPrefixAccount = "account" + kvPrefixResource = "resource" + kvPrefixAppKv = "appkv" + kvPrefixCreatorIndex = "creator" + kvPrefixOnlineAccount = "online_account_base" + kvPrefixOnlineAccountBalance = "online_account_balance" + kvRoundKey = "global_round" + kvSchemaVersionKey = "global_schema_version" + kvTotalsKey = "global_total" + kvTxTail = "txtail" + kvOnlineAccountRoundParams = "online_account_round_params" + kvPrefixStateproof = "stateproofs" +) + +// return the big-endian binary encoding of a uint64 +func bigEndianUint64(v uint64) []byte { + ret := make([]byte, 8) + binary.BigEndian.PutUint64(ret, v) + return ret +} + +// return the big-endian binary encoding of a uint32 +func bigEndianUint32(v uint32) []byte { + ret := make([]byte, 4) + binary.BigEndian.PutUint32(ret, v) + return ret +} + +// accountKey: 4-byte prefix + 32-byte address +func accountKey(address basics.Address) []byte { + ret := []byte(kvPrefixAccount) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + return ret +} + +// resourceKey: 4-byte prefix + 32-byte address + 8-byte big-endian uint64 +func resourceKey(address basics.Address, aidx basics.CreatableIndex) []byte { + ret := []byte(kvPrefixResource) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(aidx))...) + return ret +} + +func resourceAddrOnlyPartialKey(address basics.Address) []byte { + ret := []byte(kvPrefixResource) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + ret = append(ret, "-"...) + return ret +} + +func appKvKey(key string) []byte { + ret := []byte(kvPrefixAppKv) + ret = append(ret, "-"...) + ret = append(ret, key...) + return ret +} + +func creatableKey(cidx basics.CreatableIndex) []byte { + ret := []byte(kvPrefixCreatorIndex) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(cidx))...) + return ret +} + +func onlineAccountKey(address basics.Address, round basics.Round) []byte { + ret := []byte(kvPrefixOnlineAccount) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(round))...) + return ret +} + +func onlineAccountOnlyPartialKey(address basics.Address) []byte { + ret := []byte(kvPrefixOnlineAccount) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + ret = append(ret, "-"...) + return ret +} + +// TODO: use basics.Round +func onlineAccountBalanceKey(round uint64, normBalance uint64, address basics.Address) []byte { + ret := []byte(kvPrefixOnlineAccountBalance) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(round)...) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(normBalance)...) + ret = append(ret, "-"...) + ret = append(ret, address[:]...) + return ret +} + +func onlineAccountBalanceOnlyPartialKey(round basics.Round) []byte { + ret := []byte(kvPrefixOnlineAccountBalance) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(round))...) + ret = append(ret, "-"...) + return ret +} + +func roundKey() []byte { + ret := []byte(kvRoundKey) + return ret +} + +func schemaVersionKey() []byte { + ret := []byte(kvSchemaVersionKey) + return ret +} + +func totalsKey(catchpointStaging bool) []byte { + ret := []byte(kvTotalsKey) + ret = append(ret, "-"...) + if catchpointStaging { + ret = append(ret, "staging"...) + } else { + ret = append(ret, "live"...) + } + return ret +} + +func txTailKey(rnd basics.Round) []byte { + ret := []byte(kvTxTail) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(rnd))...) + return ret +} + +func onlineAccountRoundParamsKey(rnd basics.Round) []byte { + ret := []byte(kvOnlineAccountRoundParams) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(rnd))...) + return ret +} + +func stateproofKey(rnd basics.Round) []byte { + ret := []byte(kvPrefixStateproof) + ret = append(ret, "-"...) + ret = append(ret, bigEndianUint64(uint64(rnd))...) + return ret +} diff --git a/ledger/store/trackerdb/generickv/stateproof_reader.go b/ledger/store/trackerdb/generickv/stateproof_reader.go new file mode 100644 index 0000000000..0c3ed62b58 --- /dev/null +++ b/ledger/store/trackerdb/generickv/stateproof_reader.go @@ -0,0 +1,104 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +type stateproofReader struct { + kvr KvRead +} + +// MakeStateproofReader returns a trackerdb.SpVerificationCtxReader for a KV +func MakeStateproofReader(kvr KvRead) trackerdb.SpVerificationCtxReader { + return &stateproofReader{kvr} +} + +// LookupSPContext implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) LookupSPContext(stateProofLastAttestedRound basics.Round) (*ledgercore.StateProofVerificationContext, error) { + // SQL at the time of writing: + // + // SELECT + // verificationcontext + // FROM stateproofverification + // WHERE lastattestedround=? + + value, closer, err := r.kvr.Get(stateproofKey(stateProofLastAttestedRound)) + if err != nil { + return nil, err + } + defer closer.Close() + + var vc ledgercore.StateProofVerificationContext + err = protocol.Decode(value, &vc) + if err != nil { + return nil, err + } + + return &vc, nil +} + +// GetAllSPContexts implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) GetAllSPContexts(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) { + // SQL at the time of writing: + // + // SELECT + // verificationContext + // FROM stateProofVerification + // ORDER BY lastattestedround + + low := []byte(kvPrefixStateproof + "-") + high := []byte(kvPrefixStateproof + ".") + iter := r.kvr.NewIter(low, high, false) + defer iter.Close() + + results := make([]ledgercore.StateProofVerificationContext, 0) + + var value []byte + var err error + for iter.Next() { + // get value for current item in the iterator + value, err = iter.Value() + if err != nil { + return nil, err + } + + // decode the value + vc := ledgercore.StateProofVerificationContext{} + err = protocol.Decode(value, &vc) + if err != nil { + return nil, err + } + + // add the item to the results + results = append(results, vc) + } + + return results, nil +} + +// GetAllSPContextsFromCatchpointTbl implements trackerdb.SpVerificationCtxReader +func (r *stateproofReader) GetAllSPContextsFromCatchpointTbl(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) { + // TODO: catchpoint + return nil, nil +} diff --git a/ledger/store/trackerdb/generickv/stateproof_writer.go b/ledger/store/trackerdb/generickv/stateproof_writer.go new file mode 100644 index 0000000000..42565d9edc --- /dev/null +++ b/ledger/store/trackerdb/generickv/stateproof_writer.go @@ -0,0 +1,76 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" +) + +type stateproofWriter struct { + kvw KvWrite +} + +// MakeStateproofWriter returns a trackerdb.SpVerificationCtxWriter for a KV +func MakeStateproofWriter(kvw KvWrite) trackerdb.SpVerificationCtxWriter { + return &stateproofWriter{kvw} +} + +// StoreSPContexts implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) StoreSPContexts(ctx context.Context, verificationContext []*ledgercore.StateProofVerificationContext) error { + // SQL at the time of writing: + // + // INSERT INTO stateProofVerification + // (lastattestedround, verificationContext) + // VALUES + // (?, ?) + + for i := range verificationContext { + // write stateproof entry + vc := verificationContext[i] + raw := protocol.Encode(vc) + err := w.kvw.Set(stateproofKey(vc.LastAttestedRound), raw) + if err != nil { + return err + } + } + + return nil +} + +// DeleteOldSPContexts implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) DeleteOldSPContexts(ctx context.Context, earliestLastAttestedRound basics.Round) error { + // SQL at the time of writing: + // + // DELETE FROM stateproofverification + // WHERE lastattestedround < ? + + start := []byte(kvPrefixStateproof + "-") + end := stateproofKey(earliestLastAttestedRound) + + return w.kvw.DeleteRange(start, end) +} + +// StoreSPContextsToCatchpointTbl implements trackerdb.SpVerificationCtxWriter +func (w *stateproofWriter) StoreSPContextsToCatchpointTbl(ctx context.Context, verificationContexts []ledgercore.StateProofVerificationContext) error { + // TODO: catchpoint + return nil +} diff --git a/ledger/store/trackerdb/generickv/writer.go b/ledger/store/trackerdb/generickv/writer.go new file mode 100644 index 0000000000..c5d8dcfc0f --- /dev/null +++ b/ledger/store/trackerdb/generickv/writer.go @@ -0,0 +1,91 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package generickv + +import ( + "context" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" + "testing" +) + +type writer struct { + // TODO: the need for the store here is quite broken + // this is due to exposing RunMigrations and AccountsInit on the writers + // the internals of this methods completly ignore the "writer" and recreate a transaction + store trackerdb.Store + KvWrite + KvRead +} + +// MakeWriter returns a trackerdb.Writer for a KV +func MakeWriter(store trackerdb.Store, kvw KvWrite, kvr KvRead) trackerdb.Writer { + return &writer{store, kvw, kvr} +} + +// MakeAccountsOptimizedWriter implements trackerdb.Writer +func (w *writer) MakeAccountsOptimizedWriter(hasAccounts bool, hasResources bool, hasKvPairs bool, hasCreatables bool) (trackerdb.AccountsWriter, error) { + return MakeAccountsWriter(w, w), nil +} + +// MakeAccountsWriter implements trackerdb.Writer +func (w *writer) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) { + return MakeAccountsWriter(w, w), nil +} + +// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer +func (w *writer) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) { + return MakeOnlineAccountsWriter(w), nil +} + +// MakeSpVerificationCtxWriter implements trackerdb.Writer +func (w *writer) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter { + return MakeStateproofWriter(w) +} + +// Testing implements trackerdb.Writer +func (w *writer) Testing() trackerdb.WriterTestExt { + return &writerForTesting{w.store, w, w} +} + +type writerForTesting struct { + trackerdb.Store + KvWrite + KvRead +} + +// AccountsInitLightTest implements trackerdb.WriterTestExt +func (w *writerForTesting) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { + panic("unimplemented") +} + +// AccountsInitTest implements trackerdb.WriterTestExt +func (w *writerForTesting) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { + return AccountsInitTest(tb, w, initAccounts, proto) +} + +// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt +func (w *writerForTesting) AccountsUpdateSchemaTest(ctx context.Context) (err error) { + panic("unimplemented") +} + +// ModifyAcctBaseTest implements trackerdb.WriterTestExt +func (w *writerForTesting) ModifyAcctBaseTest() error { + panic("unimplemented") +} diff --git a/ledger/store/trackerdb/interface.go b/ledger/store/trackerdb/interface.go index 9e9fbb1a10..b532fa238a 100644 --- a/ledger/store/trackerdb/interface.go +++ b/ledger/store/trackerdb/interface.go @@ -18,6 +18,7 @@ package trackerdb import ( "context" + "errors" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -26,6 +27,9 @@ import ( "github.com/algorand/go-algorand/ledger/ledgercore" ) +// ErrNotFound is returned when a record is not found. +var ErrNotFound = errors.New("trackerdb: not found") + // AccountRef is an opaque ref to an account in the db. type AccountRef interface { AccountRefMarker() @@ -68,15 +72,16 @@ type AccountsWriter interface { // AccountsWriterExt is the write interface used inside transactions and batch operations. type AccountsWriterExt interface { - AccountsReset(ctx context.Context) error - ResetAccountHashes(ctx context.Context) (err error) - TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error + CatchpointApplicationWriter + UpdateAccountsRound(rnd basics.Round) (err error) - UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error) AccountsPutTotals(totals ledgercore.AccountTotals, catchpointStaging bool) error + OnlineAccountsDelete(forgetBefore basics.Round) (err error) AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error + + TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error } // AccountsReader is the "optimized" read interface for: @@ -100,24 +105,38 @@ type AccountsReader interface { // AccountsReaderExt is the read interface for: // - accounts, resources, app kvs, creatables type AccountsReaderExt interface { + CatchpointCreationReader + + AccountsRound() (rnd basics.Round, err error) AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error) - AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) - LookupAccountAddressFromAddressID(ctx context.Context, ref AccountRef) (address basics.Address, err error) - LookupAccountDataByAddress(basics.Address) (ref AccountRef, data []byte, err error) + LookupAccountRowID(basics.Address) (ref AccountRef, err error) LookupResourceDataByAddrID(accountRef AccountRef, aidx basics.CreatableIndex) (data []byte, err error) - TotalResources(ctx context.Context) (total uint64, err error) - TotalAccounts(ctx context.Context) (total uint64, err error) - TotalKVs(ctx context.Context) (total uint64, err error) - AccountsRound() (rnd basics.Round, err error) LookupOnlineAccountDataByAddress(addr basics.Address) (ref OnlineAccountRef, data []byte, err error) + AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (map[basics.Address]*ledgercore.OnlineAccount, error) AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error) ExpiredOnlineAccountsForRound(rnd, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (map[basics.Address]*ledgercore.OnlineAccountData, error) OnlineAccountsAll(maxAccounts uint64) ([]PersistedOnlineAccountData, error) + LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error) + // testing + Testing() AccountsReaderTestExt +} + +type CatchpointApplicationWriter interface { + AccountsReset(ctx context.Context) error + ResetAccountHashes(ctx context.Context) (err error) + UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error) +} + +type CatchpointCreationReader interface { + AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) + LookupAccountAddressFromAddressID(ctx context.Context, ref AccountRef) (address basics.Address, err error) + TotalResources(ctx context.Context) (total uint64, err error) + TotalAccounts(ctx context.Context) (total uint64, err error) + TotalKVs(ctx context.Context) (total uint64, err error) LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error) - Testing() TestAccountsReaderExt } // AccountsReaderWriter is AccountsReader+AccountsWriter @@ -146,52 +165,6 @@ type OnlineAccountsReader interface { Close() } -// CatchpointWriter is the write interface for: -// - catchpoints -type CatchpointWriter interface { - CreateCatchpointStagingHashesIndex(ctx context.Context) (err error) - - StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) - - WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error) - WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error) - - WriteCatchpointStagingBalances(ctx context.Context, bals []NormalizedAccountBalance) error - WriteCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error - WriteCatchpointStagingCreatable(ctx context.Context, bals []NormalizedAccountBalance) error - WriteCatchpointStagingHashes(ctx context.Context, bals []NormalizedAccountBalance) error - - ApplyCatchpointStagingBalances(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error) - ResetCatchpointStagingBalances(ctx context.Context, newCatchup bool) (err error) - - InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error - DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error - DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error - InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error - - DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) -} - -// CatchpointReader is the read interface for: -// - catchpoints -type CatchpointReader interface { - GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) - GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) - - ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error) - ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error) - - SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error) - SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool /*exists*/, error) - SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) -} - -// CatchpointReaderWriter is CatchpointReader+CatchpointWriter -type CatchpointReaderWriter interface { - CatchpointReader - CatchpointWriter -} - // MerkleCommitter allows storing and loading merkletrie pages from a sqlite database. type MerkleCommitter interface { StorePage(page uint64, content []byte) error diff --git a/ledger/store/trackerdb/msgp_gen.go b/ledger/store/trackerdb/msgp_gen.go index e6fa865d65..41f41d0a89 100644 --- a/ledger/store/trackerdb/msgp_gen.go +++ b/ledger/store/trackerdb/msgp_gen.go @@ -35,14 +35,6 @@ import ( // |-----> (*) Msgsize // |-----> (*) MsgIsZero // -// CatchpointFirstStageInfo -// |-----> (*) MarshalMsg -// |-----> (*) CanMarshalMsg -// |-----> (*) UnmarshalMsg -// |-----> (*) CanUnmarshalMsg -// |-----> (*) Msgsize -// |-----> (*) MsgIsZero -// // ResourceFlags // |-----> MarshalMsg // |-----> CanMarshalMsg @@ -1130,250 +1122,6 @@ func (z *BaseVotingData) MsgIsZero() bool { return ((*z).VoteID.MsgIsZero()) && ((*z).SelectionID.MsgIsZero()) && ((*z).VoteFirstValid.MsgIsZero()) && ((*z).VoteLastValid.MsgIsZero()) && ((*z).VoteKeyDilution == 0) && ((*z).StateProofID.MsgIsZero()) } -// MarshalMsg implements msgp.Marshaler -func (z *CatchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) { - o = msgp.Require(b, z.Msgsize()) - // omitempty: check for empty values - zb0001Len := uint32(7) - var zb0001Mask uint8 /* 8 bits */ - if (*z).Totals.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x2 - } - if (*z).TotalAccounts == 0 { - zb0001Len-- - zb0001Mask |= 0x4 - } - if (*z).BiggestChunkLen == 0 { - zb0001Len-- - zb0001Mask |= 0x8 - } - if (*z).TotalChunks == 0 { - zb0001Len-- - zb0001Mask |= 0x10 - } - if (*z).TotalKVs == 0 { - zb0001Len-- - zb0001Mask |= 0x20 - } - if (*z).StateProofVerificationHash.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x40 - } - if (*z).TrieBalancesHash.MsgIsZero() { - zb0001Len-- - zb0001Mask |= 0x80 - } - // variable map header, size zb0001Len - o = append(o, 0x80|uint8(zb0001Len)) - if zb0001Len != 0 { - if (zb0001Mask & 0x2) == 0 { // if not empty - // string "accountTotals" - o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) - o = (*z).Totals.MarshalMsg(o) - } - if (zb0001Mask & 0x4) == 0 { // if not empty - // string "accountsCount" - o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalAccounts) - } - if (zb0001Mask & 0x8) == 0 { // if not empty - // string "biggestChunk" - o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b) - o = msgp.AppendUint64(o, (*z).BiggestChunkLen) - } - if (zb0001Mask & 0x10) == 0 { // if not empty - // string "chunksCount" - o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalChunks) - } - if (zb0001Mask & 0x20) == 0 { // if not empty - // string "kvsCount" - o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74) - o = msgp.AppendUint64(o, (*z).TotalKVs) - } - if (zb0001Mask & 0x40) == 0 { // if not empty - // string "spVerificationHash" - o = append(o, 0xb2, 0x73, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68) - o = (*z).StateProofVerificationHash.MarshalMsg(o) - } - if (zb0001Mask & 0x80) == 0 { // if not empty - // string "trieBalancesHash" - o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68) - o = (*z).TrieBalancesHash.MarshalMsg(o) - } - } - return -} - -func (_ *CatchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool { - _, ok := (z).(*CatchpointFirstStageInfo) - return ok -} - -// UnmarshalMsg implements msgp.Unmarshaler -func (z *CatchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { - var field []byte - _ = field - var zb0001 int - var zb0002 bool - zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) - if _, ok := err.(msgp.TypeError); ok { - zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).Totals.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "Totals") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalAccounts") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalKVs") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "TotalChunks") - return - } - } - if zb0001 > 0 { - zb0001-- - (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen") - return - } - } - if zb0001 > 0 { - zb0001-- - bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "struct-from-array", "StateProofVerificationHash") - return - } - } - if zb0001 > 0 { - err = msgp.ErrTooManyArrayFields(zb0001) - if err != nil { - err = msgp.WrapError(err, "struct-from-array") - return - } - } - } else { - if err != nil { - err = msgp.WrapError(err) - return - } - if zb0002 { - (*z) = CatchpointFirstStageInfo{} - } - for zb0001 > 0 { - zb0001-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err) - return - } - switch string(field) { - case "accountTotals": - bts, err = (*z).Totals.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "Totals") - return - } - case "trieBalancesHash": - bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "TrieBalancesHash") - return - } - case "accountsCount": - (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalAccounts") - return - } - case "kvsCount": - (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalKVs") - return - } - case "chunksCount": - (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "TotalChunks") - return - } - case "biggestChunk": - (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "BiggestChunkLen") - return - } - case "spVerificationHash": - bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts) - if err != nil { - err = msgp.WrapError(err, "StateProofVerificationHash") - return - } - default: - err = msgp.ErrNoField(string(field)) - if err != nil { - err = msgp.WrapError(err) - return - } - } - } - } - o = bts - return -} - -func (_ *CatchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool { - _, ok := (z).(*CatchpointFirstStageInfo) - return ok -} - -// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message -func (z *CatchpointFirstStageInfo) Msgsize() (s int) { - s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size + 19 + (*z).StateProofVerificationHash.Msgsize() - return -} - -// MsgIsZero returns whether this is a zero value -func (z *CatchpointFirstStageInfo) MsgIsZero() bool { - return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) && ((*z).StateProofVerificationHash.MsgIsZero()) -} - // MarshalMsg implements msgp.Marshaler func (z ResourceFlags) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) diff --git a/ledger/store/trackerdb/msgp_gen_test.go b/ledger/store/trackerdb/msgp_gen_test.go index be8a232c32..1d1be174f5 100644 --- a/ledger/store/trackerdb/msgp_gen_test.go +++ b/ledger/store/trackerdb/msgp_gen_test.go @@ -194,66 +194,6 @@ func BenchmarkUnmarshalBaseVotingData(b *testing.B) { } } -func TestMarshalUnmarshalCatchpointFirstStageInfo(t *testing.T) { - partitiontest.PartitionTest(t) - v := CatchpointFirstStageInfo{} - bts := v.MarshalMsg(nil) - left, err := v.UnmarshalMsg(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) - } - - left, err = msgp.Skip(bts) - if err != nil { - t.Fatal(err) - } - if len(left) > 0 { - t.Errorf("%d bytes left over after Skip(): %q", len(left), left) - } -} - -func TestRandomizedEncodingCatchpointFirstStageInfo(t *testing.T) { - protocol.RunEncodingTest(t, &CatchpointFirstStageInfo{}) -} - -func BenchmarkMarshalMsgCatchpointFirstStageInfo(b *testing.B) { - v := CatchpointFirstStageInfo{} - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - v.MarshalMsg(nil) - } -} - -func BenchmarkAppendMsgCatchpointFirstStageInfo(b *testing.B) { - v := CatchpointFirstStageInfo{} - bts := make([]byte, 0, v.Msgsize()) - bts = v.MarshalMsg(bts[0:0]) - b.SetBytes(int64(len(bts))) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - bts = v.MarshalMsg(bts[0:0]) - } -} - -func BenchmarkUnmarshalCatchpointFirstStageInfo(b *testing.B) { - v := CatchpointFirstStageInfo{} - bts := v.MarshalMsg(nil) - b.ReportAllocs() - b.SetBytes(int64(len(bts))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := v.UnmarshalMsg(bts) - if err != nil { - b.Fatal(err) - } - } -} - func TestMarshalUnmarshalResourcesData(t *testing.T) { partitiontest.PartitionTest(t) v := ResourcesData{} diff --git a/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go b/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go new file mode 100644 index 0000000000..619cfb3e5f --- /dev/null +++ b/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go @@ -0,0 +1,494 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package pebbledbdriver + +import ( + "context" + "io" + "runtime" + "time" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/ledger/store/trackerdb/generickv" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/util/db" + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/bloom" + "github.com/cockroachdb/pebble/vfs" +) + +const ( + // minCache is the minimum amount of memory in megabytes to allocate to pebble + // read and write caching, split half and half. + minCache = 16 + + // minHandles is the minimum number of files handles to allocate to the open + // database files. + minHandles = 16 +) + +type trackerStore struct { + kvs kvstore + proto config.ConsensusParams + // use the generickv implementations + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +// Open opens a Pebble db database +func Open(dbdir string, inMem bool, proto config.ConsensusParams, log logging.Logger) (trackerdb.Store, error) { + cache := 100 + handles := 64 + + // Ensure we have some minimal caching and file guarantees + if cache < minCache { + cache = minCache + } + if handles < minHandles { + handles = minHandles + } + + // The max memtable size is limited by the uint32 offsets stored in + // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. + // Taken from https://github.com/cockroachdb/pebble/blob/master/open.go#L38 + maxMemTableSize := 4<<30 - 1 // Capped by 4 GB + + memTableLimit := 2 + memTableSize := cache * 1024 * 1024 / 2 / memTableLimit + if memTableSize > maxMemTableSize { + memTableSize = maxMemTableSize + } + + // configure pebbledb + opts := &pebble.Options{ + // logging + Logger: log, + + // Pebble has a single combined cache area and the write + // buffers are taken from this too. Assign all available + // memory allowance for cache. + Cache: pebble.NewCache(int64(cache * 1024 * 1024)), + MaxOpenFiles: handles, + + // The size of memory table(as well as the write buffer). + // Note, there may have more than two memory tables in the system. + MemTableSize: memTableSize, + + // MemTableStopWritesThreshold places a hard limit on the size + // of the existent MemTables(including the frozen one). + // Note, this must be the number of tables not the size of all memtables + // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 + // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. + MemTableStopWritesThreshold: memTableLimit, + + // The default compaction concurrency(1 thread), + // Here use all available CPUs for faster compaction. + MaxConcurrentCompactions: func() int { return runtime.NumCPU() }, + + // Per-level options. Options for at least one level must be specified. The + // options for the last level are used for all subsequent levels. + Levels: []pebble.LevelOptions{ + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, + }, + } + // Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130 + // for more details. + opts.Experimental.ReadSamplingMultiplier = -1 + + if inMem { + opts.FS = vfs.NewMem() + } + db, err := pebble.Open(dbdir+".pebbledb", opts) + if err != nil { + return nil, err + } + // no fsync + wo := &pebble.WriteOptions{Sync: false} + kvs := kvstore{Pdb: db, wo: wo} + var store trackerdb.Store + store = &trackerStore{ + kvs, + proto, + generickv.MakeReader(&kvs, proto), + generickv.MakeWriter(store, &kvs, &kvs), + generickv.MakeCatchpoint(), + } + return store, nil +} + +// IsSharedCacheConnection implements trackerdb.Store +func (s *trackerStore) IsSharedCacheConnection() bool { + return false +} + +// SetSynchronousMode implements trackerdb.Store +func (s *trackerStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) { + // TODO + return nil +} + +// RunMigrations implements trackerdb.Store +func (s *trackerStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + // create a anonym struct that impls the interface for the migration runner + db := struct { + *trackerStore + *kvstore + }{s, &s.kvs} + return generickv.RunMigrations(ctx, db, params, targetVersion) +} + +// Batch implements trackerdb.Store +func (s *trackerStore) Batch(fn trackerdb.BatchFn) (err error) { + return s.BatchContext(context.Background(), fn) +} + +// BatchContext implements trackerdb.Store +func (s *trackerStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) { + handle, err := s.BeginBatch(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the batch + err = fn(ctx, handle) + if err != nil { + return + } + + // commit the batch + err = handle.Commit() + if err != nil { + return + } + + return err +} + +// BeginBatch implements trackerdb.Store +func (s *trackerStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) { + scope := batchScope{store: s, wb: s.kvs.Pdb.NewBatch(), wo: s.kvs.wo, db: s.kvs.Pdb} + + return &struct { + batchScope + trackerdb.Writer + }{scope, generickv.MakeWriter(s, &scope, &s.kvs)}, nil +} + +// Snapshot implements trackerdb.Store +func (s *trackerStore) Snapshot(fn trackerdb.SnapshotFn) (err error) { + return s.SnapshotContext(context.Background(), fn) +} + +// SnapshotContext implements trackerdb.Store +func (s *trackerStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) { + handle, err := s.BeginSnapshot(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the snapshot + err = fn(ctx, handle) + if err != nil { + return + } + + return err +} + +// BeginSnapshot implements trackerdb.Store +func (s *trackerStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) { + scope := snapshotScope{db: s.kvs.Pdb, snap: s.kvs.Pdb.NewSnapshot()} + return &struct { + snapshotScope + trackerdb.Reader + }{scope, generickv.MakeReader(&scope, s.proto)}, nil +} + +// Transaction implements trackerdb.Store +func (s *trackerStore) Transaction(fn trackerdb.TransactionFn) (err error) { + return s.TransactionContext(context.Background(), fn) +} + +// TransactionContext implements trackerdb.Store +func (s *trackerStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) { + handle, err := s.BeginTransaction(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the transaction + err = fn(ctx, handle) + if err != nil { + return + } + + // commit the transaction + err = handle.Commit() + if err != nil { + return + } + + return err +} + +// BeginTransaction implements trackerdb.Store +func (s *trackerStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) { + scope := transactionScope{ + store: s, + db: s.kvs.Pdb, + wo: s.kvs.wo, + snap: s.kvs.Pdb.NewSnapshot(), + wb: s.kvs.Pdb.NewBatch(), + } + + return &struct { + transactionScope + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint + }{scope, generickv.MakeReader(&scope, s.proto), generickv.MakeWriter(s, &scope, &scope), generickv.MakeCatchpoint()}, nil +} + +// Vacuum implements trackerdb.Store +func (s *trackerStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) { + // TODO + return db.VacuumStats{}, nil +} + +// ResetToV6Test implements trackerdb.Store +func (s *trackerStore) ResetToV6Test(ctx context.Context) error { + // TODO + return nil +} + +// Close implements trackerdb.Store +func (s *trackerStore) Close() { + s.kvs.Pdb.Close() +} + +// +// generic impls +// + +func mapPebbleErrors(err error) error { + switch err { + case pebble.ErrNotFound: + return trackerdb.ErrNotFound + default: + return err + } +} + +type kvstore struct { + Pdb *pebble.DB + wo *pebble.WriteOptions +} + +func (s *kvstore) Set(key, value []byte) error { + return s.Pdb.Set(key, value, s.wo) +} + +func (s *kvstore) Get(key []byte) (value []byte, closer io.Closer, err error) { + value, closer, err = s.Pdb.Get(key) + err = mapPebbleErrors(err) + return +} + +func (s *kvstore) NewIter(low, high []byte, reverse bool) generickv.KvIter { + opts := pebble.IterOptions{LowerBound: low, UpperBound: high} + return newIter(s.Pdb.NewIter(&opts), reverse) +} + +func (s *kvstore) Delete(key []byte) error { + return s.Pdb.Delete(key, s.wo) +} + +func (s *kvstore) DeleteRange(start, end []byte) error { + return s.Pdb.DeleteRange(start, end, s.wo) +} + +type batchScope struct { + // Hack: we should tray to impl without this field + store *trackerStore + db *pebble.DB + wo *pebble.WriteOptions + wb *pebble.Batch +} + +func (bs batchScope) Set(key, value []byte) error { + return bs.wb.Set(key, value, bs.wo) +} + +func (bs batchScope) Delete(key []byte) error { + return bs.wb.Delete(key, bs.wo) +} + +func (bs batchScope) DeleteRange(start, end []byte) error { + return bs.wb.DeleteRange(start, end, bs.wo) +} + +func (bs batchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + // noop + return time.Now(), nil +} + +func (bs batchScope) Commit() error { + return bs.wb.Commit(bs.wo) +} + +func (bs batchScope) Close() error { + return bs.wb.Close() +} + +type snapshotScope struct { + db *pebble.DB + snap *pebble.Snapshot +} + +func (ss snapshotScope) Get(key []byte) (value []byte, closer io.Closer, err error) { + value, closer, err = ss.snap.Get(key) + err = mapPebbleErrors(err) + return +} + +func (ss snapshotScope) NewIter(low, high []byte, reverse bool) generickv.KvIter { + opts := pebble.IterOptions{LowerBound: low, UpperBound: high} + return newIter(ss.snap.NewIter(&opts), reverse) +} + +func (ss snapshotScope) Close() error { + return ss.snap.Close() +} + +type transactionScope struct { + // Hack: we should tray to impl without this field + store *trackerStore + db *pebble.DB + wo *pebble.WriteOptions + snap *pebble.Snapshot + wb *pebble.Batch +} + +func (txs transactionScope) Set(key, value []byte) error { + return txs.wb.Set(key, value, txs.wo) +} + +func (txs transactionScope) Get(key []byte) (value []byte, closer io.Closer, err error) { + value, closer, err = txs.snap.Get(key) + err = mapPebbleErrors(err) + return +} + +func (txs transactionScope) NewIter(low, high []byte, reverse bool) generickv.KvIter { + opts := pebble.IterOptions{LowerBound: low, UpperBound: high} + return newIter(txs.snap.NewIter(&opts), reverse) +} + +func (txs transactionScope) Delete(key []byte) error { + return txs.wb.Delete(key, txs.wo) +} + +func (txs transactionScope) DeleteRange(start, end []byte) error { + return txs.wb.DeleteRange(start, end, txs.wo) +} + +func (txs *transactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + // create a anonym struct that impls the interface for the migration runner + db := struct { + *trackerStore + *kvstore + }{txs.store, &txs.store.kvs} + return generickv.RunMigrations(ctx, db, params, targetVersion) +} + +func (txs transactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + // noop + return time.Now(), nil +} + +func (txs transactionScope) Commit() error { + return txs.wb.Commit(txs.wo) +} + +func (txs transactionScope) Close() error { + txs.snap.Close() // ignore error + return txs.wb.Close() +} + +type pebbleIter struct { + iter *pebble.Iterator + reverse bool + firstCall bool +} + +func newIter(iter *pebble.Iterator, reverse bool) *pebbleIter { + return &pebbleIter{iter, reverse, true} +} + +func (i *pebbleIter) Next() bool { + if i.firstCall { + i.firstCall = false + if i.reverse { + return i.iter.Last() + } + return i.iter.First() + } + if i.reverse { + return i.iter.Prev() + } + return i.iter.Next() +} +func (i *pebbleIter) Valid() bool { return i.iter.Valid() } +func (i *pebbleIter) Close() { i.iter.Close() } + +func (i *pebbleIter) Key() []byte { + k := i.iter.Key() + ret := make([]byte, len(k)) + copy(ret, k) + return ret +} + +func (i *pebbleIter) Value() ([]byte, error) { + v := i.iter.Value() + ret := make([]byte, len(v)) + copy(ret, v) + return ret, nil +} + +// KeySlice is a zero copy slice only valid until iter.Next() or iter.Close() is called. +func (i *pebbleIter) KeySlice() generickv.Slice { return pebbleSlice(i.iter.Key()) } + +// ValueSlice is a zero copy slice only valid until iter.Next() or iter.Close() is called. +func (i *pebbleIter) ValueSlice() (generickv.Slice, error) { return pebbleSlice(i.iter.Value()), nil } + +type pebbleSlice []byte + +func (s pebbleSlice) Data() []byte { return s } +func (s pebbleSlice) Free() {} +func (s pebbleSlice) Size() int { return len(s) } +func (s pebbleSlice) Exists() bool { return s != nil } diff --git a/ledger/store/trackerdb/pebbledbdriver/testing.go b/ledger/store/trackerdb/pebbledbdriver/testing.go new file mode 100644 index 0000000000..2724eb01fb --- /dev/null +++ b/ledger/store/trackerdb/pebbledbdriver/testing.go @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package pebbledbdriver + +import ( + "fmt" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/stretchr/testify/require" +) + +// OpenForTesting opens a sqlite db file for testing purposes. +func OpenForTesting(t testing.TB, inMemory bool) trackerdb.Store { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + + // create a tmp dir for the db, the testing runtime will clean it up automatically + dir := fmt.Sprintf("%s/db", t.TempDir()) + + db, err := Open(dir, inMemory, proto, logging.TestingLog(t)) + require.NoErrorf(t, err, "Dir : %s\nInMemory: %v", dir, inMemory) + + return db +} diff --git a/ledger/store/trackerdb/sqlitedriver/accountsV2.go b/ledger/store/trackerdb/sqlitedriver/accountsV2.go index a551bc4975..51e4da54f8 100644 --- a/ledger/store/trackerdb/sqlitedriver/accountsV2.go +++ b/ledger/store/trackerdb/sqlitedriver/accountsV2.go @@ -56,8 +56,13 @@ func NewAccountsSQLReaderWriter(e db.Executable) *accountsV2ReaderWriter { } } +// NewAccountsSQLReader creates an SQL reader+writer +func NewAccountsSQLReader(q db.Queryable) *accountsV2Reader { + return &accountsV2Reader{q: q, preparedStatements: make(map[string]*sql.Stmt)} +} + // Testing returns this reader, exposed as an interface with test functions -func (r *accountsV2Reader) Testing() trackerdb.TestAccountsReaderExt { +func (r *accountsV2Reader) Testing() trackerdb.AccountsReaderTestExt { return r } @@ -433,21 +438,6 @@ func (r *accountsV2Reader) LookupAccountAddressFromAddressID(ctx context.Context return } -func (r *accountsV2Reader) LookupAccountDataByAddress(addr basics.Address) (ref trackerdb.AccountRef, data []byte, err error) { - // optimize this query for repeated usage - selectStmt, err := r.getOrPrepare("SELECT rowid, data FROM accountbase WHERE address=?") - if err != nil { - return - } - - var rowid int64 - err = selectStmt.QueryRow(addr[:]).Scan(&rowid, &data) - if err != nil { - return - } - return sqlRowRef{rowid}, data, err -} - // LookupOnlineAccountDataByAddress looks up online account data by address. func (r *accountsV2Reader) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) { // optimize this query for repeated usage @@ -458,7 +448,10 @@ func (r *accountsV2Reader) LookupOnlineAccountDataByAddress(addr basics.Address) var rowid int64 err = selectStmt.QueryRow(addr[:]).Scan(&rowid, &data) - if err != nil { + if err == sql.ErrNoRows { + err = trackerdb.ErrNotFound + return + } else if err != nil { return } return sqlRowRef{rowid}, data, err @@ -474,7 +467,10 @@ func (r *accountsV2Reader) LookupAccountRowID(addr basics.Address) (ref trackerd var rowid int64 err = addrRowidStmt.QueryRow(addr[:]).Scan(&rowid) - if err != nil { + if err == sql.ErrNoRows { + err = trackerdb.ErrNotFound + return + } else if err != nil { return } return sqlRowRef{rowid}, err @@ -483,7 +479,7 @@ func (r *accountsV2Reader) LookupAccountRowID(addr basics.Address) (ref trackerd // LookupResourceDataByAddrID looks up the resource data by account rowid + resource aidx. func (r *accountsV2Reader) LookupResourceDataByAddrID(accountRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) { if accountRef == nil { - return data, sql.ErrNoRows + return data, trackerdb.ErrNotFound } addrid := accountRef.(sqlRowRef).rowid // optimize this query for repeated usage @@ -493,7 +489,10 @@ func (r *accountsV2Reader) LookupResourceDataByAddrID(accountRef trackerdb.Accou } err = selectStmt.QueryRow(addrid, aidx).Scan(&data) - if err != nil { + if err == sql.ErrNoRows { + err = trackerdb.ErrNotFound + return + } else if err != nil { return } return data, err diff --git a/ledger/store/trackerdb/sqlitedriver/catchpoint.go b/ledger/store/trackerdb/sqlitedriver/catchpoint.go index 388749858d..1a4009e0f0 100644 --- a/ledger/store/trackerdb/sqlitedriver/catchpoint.go +++ b/ledger/store/trackerdb/sqlitedriver/catchpoint.go @@ -21,290 +21,127 @@ import ( "database/sql" "errors" "fmt" + "sync" "time" - "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger/store/trackerdb" - "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/db" "github.com/mattn/go-sqlite3" ) -type catchpointReader struct { - q db.Queryable -} - type catchpointWriter struct { - e db.Executable -} - -type catchpointReaderWriter struct { - catchpointReader - catchpointWriter -} - -// NewCatchpointSQLReaderWriter creates a Catchpoint SQL reader+writer -func NewCatchpointSQLReaderWriter(e db.Executable) *catchpointReaderWriter { - return &catchpointReaderWriter{ - catchpointReader{q: e}, - catchpointWriter{e: e}, - } -} - -func (cr *catchpointReader) GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) { - err = cr.q.QueryRowContext(ctx, "SELECT filename, catchpoint, filesize FROM storedcatchpoints WHERE round=?", int64(round)).Scan(&fileName, &catchpoint, &fileSize) - return -} - -func (cr *catchpointReader) GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) { - err = db.Retry(func() (err error) { - query := "SELECT round, filename FROM storedcatchpoints WHERE pinned = 0 and round <= COALESCE((SELECT round FROM storedcatchpoints WHERE pinned = 0 ORDER BY round DESC LIMIT ?, 1),0) ORDER BY round ASC LIMIT ?" - rows, err := cr.q.QueryContext(ctx, query, filesToKeep, fileCount) - if err != nil { - return err - } - defer rows.Close() - - fileNames = make(map[basics.Round]string) - for rows.Next() { - var fileName string - var round basics.Round - err = rows.Scan(&round, &fileName) - if err != nil { - return err - } - fileNames[round] = fileName - } - - return rows.Err() - }) - if err != nil { - fileNames = nil - } - return + e db.Executable + isShared bool } -func (cr *catchpointReader) ReadCatchpointStateUint64(ctx context.Context, stateName trackerdb.CatchpointState) (val uint64, err error) { - err = db.Retry(func() (err error) { - query := "SELECT intval FROM catchpointstate WHERE id=?" - var v sql.NullInt64 - err = cr.q.QueryRowContext(ctx, query, stateName).Scan(&v) - if err == sql.ErrNoRows { - return nil - } - if err != nil { - return err - } - if v.Valid { - val = uint64(v.Int64) - } - return nil - }) - return val, err +// MakeCatchpointApplier creates a Catchpoint SQL reader+writer +func MakeCatchpointApplier(e db.Executable, isShared bool) trackerdb.CatchpointApply { + return &catchpointWriter{e, isShared} } -func (cr *catchpointReader) ReadCatchpointStateString(ctx context.Context, stateName trackerdb.CatchpointState) (val string, err error) { - err = db.Retry(func() (err error) { - query := "SELECT strval FROM catchpointstate WHERE id=?" - var v sql.NullString - err = cr.q.QueryRowContext(ctx, query, stateName).Scan(&v) - if err == sql.ErrNoRows { - return nil - } - if err != nil { - return err - } - - if v.Valid { - val = v.String - } - return nil - }) - return val, err -} - -func (cr *catchpointReader) SelectUnfinishedCatchpoints(ctx context.Context) ([]trackerdb.UnfinishedCatchpointRecord, error) { - var res []trackerdb.UnfinishedCatchpointRecord - - f := func() error { - query := "SELECT round, blockhash FROM unfinishedcatchpoints ORDER BY round" - rows, err := cr.q.QueryContext(ctx, query) - if err != nil { - return err - } - - // Clear `res` in case this function is repeated. - res = res[:0] - for rows.Next() { - var record trackerdb.UnfinishedCatchpointRecord - var blockHash []byte - err = rows.Scan(&record.Round, &blockHash) - if err != nil { - return err +// Write implements trackerdb.CatchpointApply +func (c *catchpointWriter) Write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) { + wg := sync.WaitGroup{} + + var report trackerdb.CatchpointReport + + var errBalances error + var errCreatables error + var errHashes error + var errKVs error + + // start the balances writer + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + errBalances = c.writeCatchpointStagingBalances(ctx, payload.Accounts) + report.BalancesWriteDuration = time.Since(start) + }() + + // on a in-memory database, wait for the writer to finish before starting the new writer + if c.isShared { + wg.Wait() + } + + // starts the creatables writer + wg.Add(1) + go func() { + defer wg.Done() + hasCreatables := false + for _, accBal := range payload.Accounts { + for _, res := range accBal.Resources { + if res.IsOwning() { + hasCreatables = true + break + } } - copy(record.BlockHash[:], blockHash) - res = append(res, record) } - - return nil - } - err := db.Retry(f) - if err != nil { - return nil, err - } - - return res, nil -} - -func (cr *catchpointReader) SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (trackerdb.CatchpointFirstStageInfo, bool /*exists*/, error) { - var data []byte - f := func() error { - query := "SELECT info FROM catchpointfirststageinfo WHERE round=?" - err := cr.q.QueryRowContext(ctx, query, round).Scan(&data) - if err == sql.ErrNoRows { - data = nil - return nil + if hasCreatables { + start := time.Now() + errCreatables = c.writeCatchpointStagingCreatable(ctx, payload.Accounts) + report.CreatablesWriteDuration = time.Since(start) } - return err - } - err := db.Retry(f) - if err != nil { - return trackerdb.CatchpointFirstStageInfo{}, false, err - } - - if data == nil { - return trackerdb.CatchpointFirstStageInfo{}, false, nil - } - - var res trackerdb.CatchpointFirstStageInfo - err = protocol.Decode(data, &res) - if err != nil { - return trackerdb.CatchpointFirstStageInfo{}, false, err - } - - return res, true, nil -} - -func (cr *catchpointReader) SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) { - var res []basics.Round - - f := func() error { - query := "SELECT round FROM catchpointfirststageinfo WHERE round <= ?" - rows, err := cr.q.QueryContext(ctx, query, maxRound) - if err != nil { - return err + }() + + // on a in-memory database, wait for the writer to finish before starting the new writer + if c.isShared { + wg.Wait() + } + + // start the accounts pending hashes writer + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + errHashes = c.writeCatchpointStagingHashes(ctx, payload.Accounts) + report.HashesWriteDuration = time.Since(start) + }() + + // on a in-memory database, wait for the writer to finish before starting the new writer + if c.isShared { + wg.Wait() + } + + // start the kv store writer + wg.Add(1) + go func() { + defer wg.Done() + + start := time.Now() + keys := make([][]byte, len(payload.KVRecords)) + values := make([][]byte, len(payload.KVRecords)) + hashes := make([][]byte, len(payload.KVRecords)) + for i := 0; i < len(payload.KVRecords); i++ { + keys[i] = payload.KVRecords[i].Key + values[i] = payload.KVRecords[i].Value + hashes[i] = trackerdb.KvHashBuilderV6(string(keys[i]), values[i]) } + errKVs = c.writeCatchpointStagingKVs(ctx, keys, values, hashes) + report.KVWriteDuration = time.Since(start) + }() - // Clear `res` in case this function is repeated. - res = res[:0] - for rows.Next() { - var r basics.Round - err = rows.Scan(&r) - if err != nil { - return err - } - res = append(res, r) - } + wg.Wait() - return nil + if errBalances != nil { + return report, errBalances } - err := db.Retry(f) - if err != nil { - return nil, err + if errCreatables != nil { + return report, errCreatables } - - return res, nil -} - -func (cw *catchpointWriter) StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) { - err = db.Retry(func() (err error) { - query := "DELETE FROM storedcatchpoints WHERE round=?" - _, err = cw.e.ExecContext(ctx, query, round) - if err != nil || (fileName == "" && catchpoint == "" && fileSize == 0) { - return err - } - - query = "INSERT INTO storedcatchpoints(round, filename, catchpoint, filesize, pinned) VALUES(?, ?, ?, ?, 0)" - _, err = cw.e.ExecContext(ctx, query, round, fileName, catchpoint, fileSize) - return err - }) - return -} - -func (cw *catchpointWriter) WriteCatchpointStateUint64(ctx context.Context, stateName trackerdb.CatchpointState, setValue uint64) (err error) { - err = db.Retry(func() (err error) { - if setValue == 0 { - return deleteCatchpointStateImpl(ctx, cw.e, stateName) - } - - // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case. - query := "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)" - _, err = cw.e.ExecContext(ctx, query, stateName, setValue) - return err - }) - return err -} - -func (cw *catchpointWriter) WriteCatchpointStateString(ctx context.Context, stateName trackerdb.CatchpointState, setValue string) (err error) { - err = db.Retry(func() (err error) { - if setValue == "" { - return deleteCatchpointStateImpl(ctx, cw.e, stateName) - } - - // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case. - query := "INSERT OR REPLACE INTO catchpointstate(id, strval) VALUES(?, ?)" - _, err = cw.e.ExecContext(ctx, query, stateName, setValue) - return err - }) - return err -} - -func (cw *catchpointWriter) InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error { - f := func() error { - query := "INSERT INTO unfinishedcatchpoints(round, blockhash) VALUES(?, ?)" - _, err := cw.e.ExecContext(ctx, query, round, blockHash[:]) - return err + if errHashes != nil { + return report, errHashes } - return db.Retry(f) -} - -func (cw *catchpointWriter) DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error { - f := func() error { - query := "DELETE FROM unfinishedcatchpoints WHERE round = ?" - _, err := cw.e.ExecContext(ctx, query, round) - return err - } - return db.Retry(f) -} - -func deleteCatchpointStateImpl(ctx context.Context, e db.Executable, stateName trackerdb.CatchpointState) error { - query := "DELETE FROM catchpointstate WHERE id=?" - _, err := e.ExecContext(ctx, query, stateName) - return err -} - -func (cw *catchpointWriter) InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *trackerdb.CatchpointFirstStageInfo) error { - infoSerialized := protocol.Encode(info) - f := func() error { - query := "INSERT OR REPLACE INTO catchpointfirststageinfo(round, info) VALUES(?, ?)" - _, err := cw.e.ExecContext(ctx, query, round, infoSerialized) - return err + if errKVs != nil { + return report, errKVs } - return db.Retry(f) -} -func (cw *catchpointWriter) DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error { - f := func() error { - query := "DELETE FROM catchpointfirststageinfo WHERE round <= ?" - _, err := cw.e.ExecContext(ctx, query, maxRoundToDelete) - return err - } - return db.Retry(f) + return report, nil } // WriteCatchpointStagingBalances inserts all the account balances in the provided array into the catchpoint balance staging table catchpointbalances. -func (cw *catchpointWriter) WriteCatchpointStagingBalances(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { +func (cw *catchpointWriter) writeCatchpointStagingBalances(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { selectAcctStmt, err := cw.e.PrepareContext(ctx, "SELECT rowid FROM catchpointbalances WHERE address = ?") if err != nil { return err @@ -371,7 +208,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingBalances(ctx context.Context, } // WriteCatchpointStagingHashes inserts all the account hashes in the provided array into the catchpoint pending hashes table catchpointpendinghashes. -func (cw *catchpointWriter) WriteCatchpointStagingHashes(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { +func (cw *catchpointWriter) writeCatchpointStagingHashes(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { insertStmt, err := cw.e.PrepareContext(ctx, "INSERT INTO catchpointpendinghashes(data) VALUES(?)") if err != nil { return err @@ -399,7 +236,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingHashes(ctx context.Context, ba // WriteCatchpointStagingCreatable inserts all the creatables in the provided array into the catchpoint asset creator staging table catchpointassetcreators. // note that we cannot insert the resources here : in order to insert the resources, we need the rowid of the accountbase entry. This is being inserted by // writeCatchpointStagingBalances via a separate go-routine. -func (cw *catchpointWriter) WriteCatchpointStagingCreatable(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { +func (cw *catchpointWriter) writeCatchpointStagingCreatable(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error { var insertCreatorsStmt *sql.Stmt var err error insertCreatorsStmt, err = cw.e.PrepareContext(ctx, "INSERT INTO catchpointassetcreators(asset, creator, ctype) VALUES(?, ?, ?)") @@ -433,7 +270,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingCreatable(ctx context.Context, // WriteCatchpointStagingKVs inserts all the KVs in the provided array into the // catchpoint kvstore staging table catchpointkvstore, and their hashes to the pending -func (cw *catchpointWriter) WriteCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error { +func (cw *catchpointWriter) writeCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error { insertKV, err := cw.e.PrepareContext(ctx, "INSERT INTO catchpointkvstore(key, value) VALUES(?, ?)") if err != nil { return err @@ -460,7 +297,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingKVs(ctx context.Context, keys return nil } -func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context, newCatchup bool) (err error) { +func (cw *catchpointWriter) Reset(ctx context.Context, newCatchup bool) (err error) { s := []string{ "DROP TABLE IF EXISTS catchpointbalances", "DROP TABLE IF EXISTS catchpointassetcreators", @@ -494,6 +331,7 @@ func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context, createNormalizedOnlineBalanceIndex(idxnameBalances, "catchpointbalances"), // should this be removed ? createUniqueAddressBalanceIndex(idxnameAddress, "catchpointbalances"), + "CREATE INDEX IF NOT EXISTS catchpointpendinghashesidx ON catchpointpendinghashes(data)", ) } @@ -509,7 +347,7 @@ func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context, // ApplyCatchpointStagingBalances switches the staged catchpoint catchup tables onto the actual // tables and update the correct balance round. This is the final step in switching onto the new catchpoint round. -func (cw *catchpointWriter) ApplyCatchpointStagingBalances(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error) { +func (cw *catchpointWriter) Apply(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error) { stmts := []string{ "DROP TABLE IF EXISTS accountbase", "DROP TABLE IF EXISTS assetcreators", @@ -545,40 +383,3 @@ func (cw *catchpointWriter) ApplyCatchpointStagingBalances(ctx context.Context, return } - -// CreateCatchpointStagingHashesIndex creates an index on catchpointpendinghashes to allow faster scanning according to the hash order -func (cw *catchpointWriter) CreateCatchpointStagingHashesIndex(ctx context.Context) (err error) { - _, err = cw.e.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS catchpointpendinghashesidx ON catchpointpendinghashes(data)") - if err != nil { - return - } - return -} - -// DeleteStoredCatchpoints iterates over the storedcatchpoints table and deletes all the files stored on disk. -// once all the files have been deleted, it would go ahead and remove the entries from the table. -func (crw *catchpointReaderWriter) DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) { - catchpointsFilesChunkSize := 50 - for { - fileNames, err := crw.GetOldestCatchpointFiles(ctx, catchpointsFilesChunkSize, 0) - if err != nil { - return err - } - if len(fileNames) == 0 { - break - } - - for round, fileName := range fileNames { - err = trackerdb.RemoveSingleCatchpointFileFromDisk(dbDirectory, fileName) - if err != nil { - return err - } - // clear the entry from the database - err = crw.StoreCatchpoint(ctx, round, "", "", 0) - if err != nil { - return err - } - } - } - return nil -} diff --git a/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go b/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go index 32d2614e1d..611c67c877 100644 --- a/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go +++ b/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go @@ -19,27 +19,29 @@ package sqlitedriver import ( "context" "database/sql" + + "github.com/algorand/go-algorand/util/db" ) // catchpointPendingHashesIterator allows us to iterate over the hashes in the catchpointpendinghashes table in their order. type catchpointPendingHashesIterator struct { hashCount int - tx *sql.Tx + q db.Queryable rows *sql.Rows } // MakeCatchpointPendingHashesIterator create a pending hashes iterator that retrieves the hashes in the catchpointpendinghashes table. -func MakeCatchpointPendingHashesIterator(hashCount int, tx *sql.Tx) *catchpointPendingHashesIterator { +func MakeCatchpointPendingHashesIterator(hashCount int, q db.Queryable) *catchpointPendingHashesIterator { return &catchpointPendingHashesIterator{ hashCount: hashCount, - tx: tx, + q: q, } } // Next returns an array containing the hashes, returning HashCount hashes at a time. func (iterator *catchpointPendingHashesIterator) Next(ctx context.Context) (hashes [][]byte, err error) { if iterator.rows == nil { - iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT data FROM catchpointpendinghashes ORDER BY data") + iterator.rows, err = iterator.q.QueryContext(ctx, "SELECT data FROM catchpointpendinghashes ORDER BY data") if err != nil { return } diff --git a/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go b/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go index f48a4f82c7..6dc3b67226 100644 --- a/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go +++ b/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go @@ -23,12 +23,13 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger/encoded" "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/msgp/msgp" ) // encodedAccountsBatchIter allows us to iterate over the accounts data stored in the accountbase table. type encodedAccountsBatchIter struct { - tx *sql.Tx + q db.Queryable accountsRows *sql.Rows resourcesRows *sql.Rows nextBaseRow pendingBaseRow @@ -45,21 +46,21 @@ type catchpointAccountResourceCounter struct { } // MakeEncodedAccoutsBatchIter creates an empty accounts batch iterator. -func MakeEncodedAccoutsBatchIter(tx *sql.Tx) *encodedAccountsBatchIter { - return &encodedAccountsBatchIter{tx: tx} +func MakeEncodedAccoutsBatchIter(q db.Queryable) *encodedAccountsBatchIter { + return &encodedAccountsBatchIter{q: q} } // Next returns an array containing the account data, in the same way it appear in the database // returning accountCount accounts data at a time. func (iterator *encodedAccountsBatchIter) Next(ctx context.Context, accountCount int, resourceCount int) (bals []encoded.BalanceRecordV6, numAccountsProcessed uint64, err error) { if iterator.accountsRows == nil { - iterator.accountsRows, err = iterator.tx.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid") + iterator.accountsRows, err = iterator.q.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid") if err != nil { return } } if iterator.resourcesRows == nil { - iterator.resourcesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx") + iterator.resourcesRows, err = iterator.q.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx") if err != nil { return } diff --git a/ledger/store/trackerdb/sqlitedriver/kvsIter.go b/ledger/store/trackerdb/sqlitedriver/kvsIter.go index 9cd1c9a7a1..3beb6ae0d6 100644 --- a/ledger/store/trackerdb/sqlitedriver/kvsIter.go +++ b/ledger/store/trackerdb/sqlitedriver/kvsIter.go @@ -19,22 +19,24 @@ package sqlitedriver import ( "context" "database/sql" + + "github.com/algorand/go-algorand/util/db" ) type kvsIter struct { - tx *sql.Tx + q db.Queryable rows *sql.Rows } // MakeKVsIter creates a KV iterator. -func MakeKVsIter(ctx context.Context, tx *sql.Tx) (*kvsIter, error) { - rows, err := tx.QueryContext(ctx, "SELECT key, value FROM kvstore") +func MakeKVsIter(ctx context.Context, q db.Queryable) (*kvsIter, error) { + rows, err := q.QueryContext(ctx, "SELECT key, value FROM kvstore") if err != nil { return nil, err } return &kvsIter{ - tx: tx, + q: q, rows: rows, }, nil } diff --git a/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go b/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go index 41589345fe..052dfdac93 100644 --- a/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go +++ b/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go @@ -16,11 +16,15 @@ package sqlitedriver -import "database/sql" +import ( + "database/sql" + + "github.com/algorand/go-algorand/util/db" +) //msgp:ignore MerkleCommitter type merkleCommitter struct { - tx *sql.Tx + e db.Executable deleteStmt *sql.Stmt insertStmt *sql.Stmt selectStmt *sql.Stmt @@ -28,21 +32,21 @@ type merkleCommitter struct { // MakeMerkleCommitter creates a MerkleCommitter object that implements the merkletrie.Committer interface allowing storing and loading // merkletrie pages from a sqlite database. -func MakeMerkleCommitter(tx *sql.Tx, staging bool) (mc *merkleCommitter, err error) { - mc = &merkleCommitter{tx: tx} +func MakeMerkleCommitter(e db.Executable, staging bool) (mc *merkleCommitter, err error) { + mc = &merkleCommitter{e: e} accountHashesTable := "accounthashes" if staging { accountHashesTable = "catchpointaccounthashes" } - mc.deleteStmt, err = tx.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?") + mc.deleteStmt, err = e.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?") if err != nil { return nil, err } - mc.insertStmt, err = tx.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)") + mc.insertStmt, err = e.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)") if err != nil { return nil, err } - mc.selectStmt, err = tx.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?") + mc.selectStmt, err = e.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?") if err != nil { return nil, err } diff --git a/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go b/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go index cf927042c5..2752001131 100644 --- a/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go +++ b/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go @@ -26,6 +26,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" ) // orderedAccountsIter allows us to iterate over the accounts addresses in the order of the account hashes. @@ -34,7 +35,7 @@ type orderedAccountsIter struct { accountBaseRows *sql.Rows hashesRows *sql.Rows resourcesRows *sql.Rows - tx *sql.Tx + e db.Executable pendingBaseRow pendingBaseRow pendingResourceRow pendingResourceRow accountCount int @@ -84,9 +85,9 @@ type pendingResourceRow struct { // MakeOrderedAccountsIter creates an ordered account iterator. Note that due to implementation reasons, // only a single iterator can be active at a time. -func MakeOrderedAccountsIter(tx *sql.Tx, accountCount int) *orderedAccountsIter { +func MakeOrderedAccountsIter(e db.Executable, accountCount int) *orderedAccountsIter { return &orderedAccountsIter{ - tx: tx, + e: e, accountCount: accountCount, step: oaiStepStartup, } @@ -104,7 +105,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb if iterator.step == oaiStepDeleteOldOrderingTable { // although we're going to delete this table anyway when completing the iterator execution, we'll try to // clean up any intermediate table. - _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") + _, err = iterator.e.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") if err != nil { return } @@ -113,7 +114,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb } if iterator.step == oaiStepCreateOrderingTable { // create the temporary table - _, err = iterator.tx.ExecContext(ctx, "CREATE TABLE accountsiteratorhashes(addrid INTEGER, hash blob)") + _, err = iterator.e.ExecContext(ctx, "CREATE TABLE accountsiteratorhashes(addrid INTEGER, hash blob)") if err != nil { return } @@ -122,17 +123,17 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb } if iterator.step == oaiStepQueryAccounts { // iterate over the existing accounts - iterator.accountBaseRows, err = iterator.tx.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid") + iterator.accountBaseRows, err = iterator.e.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid") if err != nil { return } // iterate over the existing resources - iterator.resourcesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx") + iterator.resourcesRows, err = iterator.e.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx") if err != nil { return } // prepare the insert statement into the temporary table - iterator.insertStmt, err = iterator.tx.PrepareContext(ctx, "INSERT INTO accountsiteratorhashes(addrid, hash) VALUES(?, ?)") + iterator.insertStmt, err = iterator.e.PrepareContext(ctx, "INSERT INTO accountsiteratorhashes(addrid, hash) VALUES(?, ?)") if err != nil { return } @@ -200,7 +201,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb if iterator.step == oaiStepCreateOrderingAccountIndex { // create an index. It shown that even when we're making a single select statement in step 5, it would be better to have this index vs. not having it at all. // note that this index is using the rowid of the accountsiteratorhashes table. - _, err = iterator.tx.ExecContext(ctx, "CREATE INDEX accountsiteratorhashesidx ON accountsiteratorhashes(hash)") + _, err = iterator.e.ExecContext(ctx, "CREATE INDEX accountsiteratorhashesidx ON accountsiteratorhashes(hash)") if err != nil { iterator.Close(ctx) return @@ -210,7 +211,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb } if iterator.step == oaiStepSelectFromOrderedTable { // select the data from the ordered table - iterator.hashesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, hash FROM accountsiteratorhashes ORDER BY hash") + iterator.hashesRows, err = iterator.e.QueryContext(ctx, "SELECT addrid, hash FROM accountsiteratorhashes ORDER BY hash") if err != nil { iterator.Close(ctx) @@ -272,7 +273,7 @@ func (iterator *orderedAccountsIter) Close(ctx context.Context) (err error) { iterator.insertStmt.Close() iterator.insertStmt = nil } - _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") + _, err = iterator.e.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes") return } diff --git a/ledger/store/trackerdb/sqlitedriver/schema.go b/ledger/store/trackerdb/sqlitedriver/schema.go index abc53675eb..ea6f577868 100644 --- a/ledger/store/trackerdb/sqlitedriver/schema.go +++ b/ledger/store/trackerdb/sqlitedriver/schema.go @@ -181,9 +181,9 @@ var accountsResetExprs = []string{ // // accountsInit returns nil if either it has initialized the database // correctly, or if the database has already been initialized. -func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { +func accountsInit(e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { for _, tableCreate := range accountsSchema { - _, err = tx.Exec(tableCreate) + _, err = e.Exec(tableCreate) if err != nil { return } @@ -191,11 +191,11 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData // Run creatables migration if it hasn't run yet var creatableMigrated bool - err = tx.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated) + err = e.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated) if err == sql.ErrNoRows { // Run migration for _, migrateCmd := range creatablesMigration { - _, err = tx.Exec(migrateCmd) + _, err = e.Exec(migrateCmd) if err != nil { return } @@ -204,13 +204,13 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData return } - _, err = tx.Exec("INSERT INTO acctrounds (id, rnd) VALUES ('acctbase', 0)") + _, err = e.Exec("INSERT INTO acctrounds (id, rnd) VALUES ('acctbase', 0)") if err == nil { var ot basics.OverflowTracker var totals ledgercore.AccountTotals for addr, data := range initAccounts { - _, err = tx.Exec("INSERT INTO accountbase (address, data) VALUES (?, ?)", + _, err = e.Exec("INSERT INTO accountbase (address, data) VALUES (?, ?)", addr[:], protocol.Encode(&data)) //nolint:gosec // Encode does not hold on to reference if err != nil { return true, err @@ -224,7 +224,7 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData return true, fmt.Errorf("overflow computing totals") } - arw := NewAccountsSQLReaderWriter(tx) + arw := NewAccountsSQLReaderWriter(e) err = arw.AccountsPutTotals(totals, false) if err != nil { return true, err @@ -245,9 +245,9 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData // accountsAddNormalizedBalance adds the normalizedonlinebalance column // to the accountbase table. -func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) error { +func accountsAddNormalizedBalance(e db.Executable, proto config.ConsensusParams) error { var exists bool - err := tx.QueryRow("SELECT 1 FROM pragma_table_info('accountbase') WHERE name='normalizedonlinebalance'").Scan(&exists) + err := e.QueryRow("SELECT 1 FROM pragma_table_info('accountbase') WHERE name='normalizedonlinebalance'").Scan(&exists) if err == nil { // Already exists. return nil @@ -257,13 +257,13 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro } for _, stmt := range createOnlineAccountIndex { - _, err = tx.Exec(stmt) + _, err = e.Exec(stmt) if err != nil { return err } } - rows, err := tx.Query("SELECT address, data FROM accountbase") + rows, err := e.Query("SELECT address, data FROM accountbase") if err != nil { return err } @@ -285,7 +285,7 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro normBalance := data.NormalizedOnlineBalance(proto) if normBalance > 0 { - _, err = tx.Exec("UPDATE accountbase SET normalizedonlinebalance=? WHERE address=?", normBalance, addrbuf) + _, err = e.Exec("UPDATE accountbase SET normalizedonlinebalance=? WHERE address=?", normBalance, addrbuf) if err != nil { return err } @@ -296,9 +296,9 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro } // accountsCreateResourceTable creates the resource table in the database. -func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error { +func accountsCreateResourceTable(ctx context.Context, e db.Executable) error { var exists bool - err := tx.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('resources') WHERE name='addrid'").Scan(&exists) + err := e.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('resources') WHERE name='addrid'").Scan(&exists) if err == nil { // Already exists. return nil @@ -307,7 +307,7 @@ func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error { return err } for _, stmt := range createResourcesTable { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return err } @@ -315,9 +315,9 @@ func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error { return nil } -func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error { +func accountsCreateOnlineAccountsTable(ctx context.Context, e db.Executable) error { var exists bool - err := tx.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('onlineaccounts') WHERE name='address'").Scan(&exists) + err := e.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('onlineaccounts') WHERE name='address'").Scan(&exists) if err == nil { // Already exists. return nil @@ -326,7 +326,7 @@ func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error { return err } for _, stmt := range createOnlineAccountsTable { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return err } @@ -335,9 +335,9 @@ func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error { } // accountsCreateBoxTable creates the KVStore table for box-storage in the database. -func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error { +func accountsCreateBoxTable(ctx context.Context, e db.Executable) error { var exists bool - err := tx.QueryRow("SELECT 1 FROM pragma_table_info('kvstore') WHERE name='key'").Scan(&exists) + err := e.QueryRow("SELECT 1 FROM pragma_table_info('kvstore') WHERE name='key'").Scan(&exists) if err == nil { // already exists return nil @@ -346,7 +346,7 @@ func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error { return err } for _, stmt := range createBoxTable { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return err } @@ -355,14 +355,14 @@ func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error { } // performKVStoreNullBlobConversion scans keys with null blob value, and convert the value to `[]byte{}`. -func performKVStoreNullBlobConversion(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "UPDATE kvstore SET value = '' WHERE value is NULL") +func performKVStoreNullBlobConversion(ctx context.Context, e db.Executable) error { + _, err := e.ExecContext(ctx, "UPDATE kvstore SET value = '' WHERE value is NULL") return err } -func accountsCreateTxTailTable(ctx context.Context, tx *sql.Tx) (err error) { +func accountsCreateTxTailTable(ctx context.Context, e db.Executable) (err error) { for _, stmt := range createTxTailTable { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return } @@ -370,9 +370,9 @@ func accountsCreateTxTailTable(ctx context.Context, tx *sql.Tx) (err error) { return nil } -func accountsCreateOnlineRoundParamsTable(ctx context.Context, tx *sql.Tx) (err error) { +func accountsCreateOnlineRoundParamsTable(ctx context.Context, e db.Executable) (err error) { for _, stmt := range createOnlineRoundParamsTable { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return } @@ -396,7 +396,7 @@ func createStateProofVerificationTable(ctx context.Context, e db.Executable) err } // performResourceTableMigration migrate the database to use the resources table. -func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(processed, total uint64)) (err error) { +func performResourceTableMigration(ctx context.Context, e db.Executable, log func(processed, total uint64)) (err error) { now := time.Now().UnixNano() idxnameBalances := fmt.Sprintf("onlineaccountbals_idx_%d", now) idxnameAddress := fmt.Sprintf("accountbase_address_idx_%d", now) @@ -418,7 +418,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro } for _, stmt := range createNewAcctBase { - _, err = tx.ExecContext(ctx, stmt) + _, err = e.ExecContext(ctx, stmt) if err != nil { return err } @@ -426,26 +426,26 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro var insertNewAcctBase *sql.Stmt var insertResources *sql.Stmt var insertNewAcctBaseNormBal *sql.Stmt - insertNewAcctBase, err = tx.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data) VALUES(?, ?)") + insertNewAcctBase, err = e.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data) VALUES(?, ?)") if err != nil { return err } defer insertNewAcctBase.Close() - insertNewAcctBaseNormBal, err = tx.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data, normalizedonlinebalance) VALUES(?, ?, ?)") + insertNewAcctBaseNormBal, err = e.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data, normalizedonlinebalance) VALUES(?, ?, ?)") if err != nil { return err } defer insertNewAcctBaseNormBal.Close() - insertResources, err = tx.PrepareContext(ctx, "INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") + insertResources, err = e.PrepareContext(ctx, "INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") if err != nil { return err } defer insertResources.Close() var rows *sql.Rows - rows, err = tx.QueryContext(ctx, "SELECT address, data, normalizedonlinebalance FROM accountbase ORDER BY address") + rows, err = e.QueryContext(ctx, "SELECT address, data, normalizedonlinebalance FROM accountbase ORDER BY address") if err != nil { return err } @@ -457,7 +457,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro var processedAccounts uint64 var totalBaseAccounts uint64 - arw := NewAccountsSQLReaderWriter(tx) + arw := NewAccountsSQLReaderWriter(e) totalBaseAccounts, err = arw.TotalAccounts(ctx) if err != nil { return err @@ -524,7 +524,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro } for _, stmt := range applyNewAcctBase { - _, err = tx.Exec(stmt) + _, err = e.Exec(stmt) if err != nil { return err } @@ -532,12 +532,12 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro return nil } -func performTxTailTableMigration(ctx context.Context, tx *sql.Tx, blockDb db.Accessor) (err error) { - if tx == nil { +func performTxTailTableMigration(ctx context.Context, e db.Executable, blockDb db.Accessor) (err error) { + if e == nil { return nil } - arw := NewAccountsSQLReaderWriter(tx) + arw := NewAccountsSQLReaderWriter(e) dbRound, err := arw.AccountsRound() if err != nil { return fmt.Errorf("latest block number cannot be retrieved : %w", err) @@ -596,8 +596,8 @@ func performTxTailTableMigration(ctx context.Context, tx *sql.Tx, blockDb db.Acc return err } -func performOnlineRoundParamsTailMigration(ctx context.Context, tx *sql.Tx, blockDb db.Accessor, newDatabase bool, initProto protocol.ConsensusVersion) (err error) { - arw := NewAccountsSQLReaderWriter(tx) +func performOnlineRoundParamsTailMigration(ctx context.Context, e db.Executable, blockDb db.Accessor, newDatabase bool, initProto protocol.ConsensusVersion) (err error) { + arw := NewAccountsSQLReaderWriter(e) totals, err := arw.AccountsTotals(ctx, false) if err != nil { return err @@ -632,24 +632,24 @@ func performOnlineRoundParamsTailMigration(ctx context.Context, tx *sql.Tx, bloc return arw.AccountsPutOnlineRoundParams(onlineRoundParams, rnd) } -func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progress func(processed, total uint64), log logging.Logger) (err error) { +func performOnlineAccountsTableMigration(ctx context.Context, e db.Executable, progress func(processed, total uint64), log logging.Logger) (err error) { var insertOnlineAcct *sql.Stmt - insertOnlineAcct, err = tx.PrepareContext(ctx, "INSERT INTO onlineaccounts(address, data, normalizedonlinebalance, updround, votelastvalid) VALUES(?, ?, ?, ?, ?)") + insertOnlineAcct, err = e.PrepareContext(ctx, "INSERT INTO onlineaccounts(address, data, normalizedonlinebalance, updround, votelastvalid) VALUES(?, ?, ?, ?, ?)") if err != nil { return err } defer insertOnlineAcct.Close() var updateAcct *sql.Stmt - updateAcct, err = tx.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE addrid = ?") + updateAcct, err = e.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE addrid = ?") if err != nil { return err } defer updateAcct.Close() var rows *sql.Rows - rows, err = tx.QueryContext(ctx, "SELECT addrid, address, data, normalizedonlinebalance FROM accountbase") + rows, err = e.QueryContext(ctx, "SELECT addrid, address, data, normalizedonlinebalance FROM accountbase") if err != nil { return err } @@ -661,10 +661,10 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre var processedAccounts uint64 var totalOnlineBaseAccounts uint64 - arw := NewAccountsSQLReaderWriter(tx) + arw := NewAccountsSQLReaderWriter(e) totalOnlineBaseAccounts, err = arw.TotalAccounts(ctx) var total uint64 - err = tx.QueryRowContext(ctx, "SELECT count(1) FROM accountbase").Scan(&total) + err = e.QueryRowContext(ctx, "SELECT count(1) FROM accountbase").Scan(&total) if err != nil { if err != sql.ErrNoRows { return err @@ -763,7 +763,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre // update accounthashes for the modified accounts if len(acctRehash) > 0 { var count uint64 - err := tx.QueryRow("SELECT count(1) FROM accounthashes").Scan(&count) + err := e.QueryRow("SELECT count(1) FROM accounthashes").Scan(&count) if err != nil { return err } @@ -772,7 +772,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre return nil } - mc, err := MakeMerkleCommitter(tx, false) + mc, err := MakeMerkleCommitter(e, false) if err != nil { return nil } @@ -811,7 +811,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre // removeEmptyAccountData removes empty AccountData msgp-encoded entries from accountbase table // and optionally returns list of addresses that were eliminated -func removeEmptyAccountData(tx *sql.Tx, queryAddresses bool) (num int64, addresses []basics.Address, err error) { +func removeEmptyAccountData(tx db.Executable, queryAddresses bool) (num int64, addresses []basics.Address, err error) { if queryAddresses { rows, qErr := tx.Query("SELECT address FROM accountbase where length(data) = 1 and data = x'80'") // empty AccountData is 0x80 if qErr != nil { @@ -856,16 +856,16 @@ func removeEmptyAccountData(tx *sql.Tx, queryAddresses bool) (num int64, address // reencodeAccounts reads all the accounts in the accountbase table, decode and reencode the account data. // if the account data is found to have a different encoding, it would update the encoded account on disk. // on return, it returns the number of modified accounts as well as an error ( if we had any ) -func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, err error) { +func reencodeAccounts(ctx context.Context, e db.Executable) (modifiedAccounts uint, err error) { modifiedAccounts = 0 scannedAccounts := 0 - updateStmt, err := tx.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE address = ?") + updateStmt, err := e.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE address = ?") if err != nil { return 0, err } - rows, err := tx.QueryContext(ctx, "SELECT address, data FROM accountbase") + rows, err := e.QueryContext(ctx, "SELECT address, data FROM accountbase") if err != nil { return } @@ -880,7 +880,7 @@ func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, e if scannedAccounts%1000 == 0 { // The return value from ResetTransactionWarnDeadline can be safely ignored here since it would only default to writing the warning // message, which would let us know that it failed anyway. - _, err = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(time.Second)) + _, err = db.ResetTransactionWarnDeadline(ctx, e, time.Now().Add(time.Second)) if err != nil { return } @@ -932,8 +932,8 @@ func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, e return } -func convertOnlineRoundParamsTail(ctx context.Context, tx *sql.Tx) error { +func convertOnlineRoundParamsTail(ctx context.Context, e db.Executable) error { // create vote last index - _, err := tx.ExecContext(ctx, createVoteLastValidIndex) + _, err := e.ExecContext(ctx, createVoteLastValidIndex) return err } diff --git a/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go b/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go index 3a024e0a98..0354a3c832 100644 --- a/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go +++ b/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go @@ -18,6 +18,7 @@ package sqlitedriver import ( "context" + "database/sql" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger/ledgercore" @@ -66,7 +67,9 @@ func (spa *stateProofVerificationReader) LookupSPContext(stateProofLastAttestedR row := spa.q.QueryRow("SELECT verificationcontext FROM stateproofverification WHERE lastattestedround=?", stateProofLastAttestedRound) var buf []byte err := row.Scan(&buf) - if err != nil { + if err == sql.ErrNoRows { + return trackerdb.ErrNotFound + } else if err != nil { return err } err = protocol.Decode(buf, &verificationContext) diff --git a/ledger/store/trackerdb/sqlitedriver/sql.go b/ledger/store/trackerdb/sqlitedriver/sql.go index c4f1639bd4..be4dd247df 100644 --- a/ledger/store/trackerdb/sqlitedriver/sql.go +++ b/ledger/store/trackerdb/sqlitedriver/sql.go @@ -53,7 +53,7 @@ type accountsSQLWriter struct { } type onlineAccountsSQLWriter struct { - insertStmt, updateStmt *sql.Stmt + insertStmt *sql.Stmt } type sqlRowRef struct { @@ -131,16 +131,11 @@ func OnlineAccountsInitDbQueries(r db.Queryable) (*onlineAccountsDbQueries, erro } // MakeOnlineAccountsSQLWriter constructs an OnlineAccountsWriter backed by sql queries. -func MakeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) { +func MakeOnlineAccountsSQLWriter(e db.Executable, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) { w = new(onlineAccountsSQLWriter) if hasAccounts { - w.insertStmt, err = tx.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)") - if err != nil { - return - } - - w.updateStmt, err = tx.Prepare("UPDATE onlineaccounts SET normalizedonlinebalance = ?, data = ?, updround = ?, votelastvalid =? WHERE rowid = ?") + w.insertStmt, err = e.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)") if err != nil { return } @@ -150,62 +145,62 @@ func MakeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccount } // MakeAccountsSQLWriter constructs an AccountsWriter backed by sql queries. -func MakeAccountsSQLWriter(tx *sql.Tx, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) { +func MakeAccountsSQLWriter(e db.Executable, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) { w = new(accountsSQLWriter) if hasAccounts { - w.deleteByRowIDStmt, err = tx.Prepare("DELETE FROM accountbase WHERE rowid=?") + w.deleteByRowIDStmt, err = e.Prepare("DELETE FROM accountbase WHERE rowid=?") if err != nil { return } - w.insertStmt, err = tx.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)") + w.insertStmt, err = e.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)") if err != nil { return } - w.updateStmt, err = tx.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?") + w.updateStmt, err = e.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?") if err != nil { return } } if hasResources { - w.deleteResourceStmt, err = tx.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?") + w.deleteResourceStmt, err = e.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?") if err != nil { return } - w.insertResourceStmt, err = tx.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") + w.insertResourceStmt, err = e.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)") if err != nil { return } - w.updateResourceStmt, err = tx.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?") + w.updateResourceStmt, err = e.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?") if err != nil { return } } if hasKvPairs { - w.upsertKvPairStmt, err = tx.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value") + w.upsertKvPairStmt, err = e.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value") if err != nil { return } - w.deleteKvPairStmt, err = tx.Prepare("DELETE FROM kvstore WHERE key=?") + w.deleteKvPairStmt, err = e.Prepare("DELETE FROM kvstore WHERE key=?") if err != nil { return } } if hasCreatables { - w.insertCreatableIdxStmt, err = tx.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)") + w.insertCreatableIdxStmt, err = e.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)") if err != nil { return } - w.deleteCreatableIdxStmt, err = tx.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?") + w.deleteCreatableIdxStmt, err = e.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?") if err != nil { return } @@ -528,7 +523,9 @@ func (qs *onlineAccountsDbQueries) LookupOnlineTotalsHistory(round basics.Round) row := qs.lookupOnlineTotalsStmt.QueryRow(round) var buf []byte err := row.Scan(&buf) - if err != nil { + if err == sql.ErrNoRows { + return trackerdb.ErrNotFound + } else if err != nil { return err } err = protocol.Decode(buf, &data) diff --git a/ledger/store/trackerdb/sqlitedriver/sql_test.go b/ledger/store/trackerdb/sqlitedriver/sql_test.go index 3fe71be4f9..096d131bb7 100644 --- a/ledger/store/trackerdb/sqlitedriver/sql_test.go +++ b/ledger/store/trackerdb/sqlitedriver/sql_test.go @@ -17,8 +17,13 @@ package sqlitedriver import ( + "context" + "database/sql" "testing" + "github.com/algorand/go-algorand/data/basics" + storetesting "github.com/algorand/go-algorand/ledger/store/testing" + "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) @@ -53,3 +58,26 @@ func TestKeyPrefixIntervalPreprocessing(t *testing.T) { require.Equal(t, tc.outputPrefixIncr, actualOutputPrefixIncr) } } + +// TestAccountsDbQueriesCreateClose tests to see that we can create the accountsDbQueries and close it. +// it also verify that double-closing it doesn't create an issue. +func TestAccountsDbQueriesCreateClose(t *testing.T) { + partitiontest.PartitionTest(t) + + dbs, _ := storetesting.DbOpenTest(t, true) + storetesting.SetDbLogging(t, dbs) + defer dbs.Close() + + err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { + AccountsInitTest(t, tx, make(map[basics.Address]basics.AccountData), protocol.ConsensusCurrentVersion) + return nil + }) + require.NoError(t, err) + qs, err := AccountsInitDbQueries(dbs.Rdb.Handle) + require.NoError(t, err) + require.NotNil(t, qs.listCreatablesStmt) + qs.Close() + require.Nil(t, qs.listCreatablesStmt) + qs.Close() + require.Nil(t, qs.listCreatablesStmt) +} diff --git a/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go b/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go new file mode 100644 index 0000000000..925c72ace4 --- /dev/null +++ b/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go @@ -0,0 +1,330 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package sqlitedriver + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +type trackerSQLStore struct { + pair db.Pair + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +// Open opens the sqlite database store +func Open(dbFilename string, dbMem bool, log logging.Logger) (store trackerdb.Store, err error) { + pair, err := db.OpenPair(dbFilename, dbMem) + if err != nil { + return + } + pair.Rdb.SetLogger(log) + pair.Wdb.SetLogger(log) + return MakeStore(pair), nil +} + +// MakeStore crates a tracker SQL db from sql db handle. +func MakeStore(pair db.Pair) trackerdb.Store { + return &trackerSQLStore{pair, &sqlReader{pair.Rdb.Handle}, &sqlWriter{pair.Wdb.Handle}, &sqlCatchpoint{pair.Wdb.Handle, pair.Wdb.IsSharedCacheConnection()}} +} + +func (s *trackerSQLStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) { + return s.pair.Wdb.SetSynchronousMode(ctx, mode, fullfsync) +} + +func (s *trackerSQLStore) IsSharedCacheConnection() bool { + return s.pair.Wdb.IsSharedCacheConnection() +} + +func (s *trackerSQLStore) Batch(fn trackerdb.BatchFn) (err error) { + return s.BatchContext(context.Background(), fn) +} + +func (s *trackerSQLStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) { + return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + return fn(ctx, &sqlBatchScope{tx, false, &sqlWriter{tx}}) + }) +} + +func (s *trackerSQLStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) { + handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &sqlBatchScope{handle, false, &sqlWriter{handle}}, nil +} + +func (s *trackerSQLStore) Snapshot(fn trackerdb.SnapshotFn) (err error) { + return s.SnapshotContext(context.Background(), fn) +} + +func (s *trackerSQLStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) { + return s.pair.Rdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + return fn(ctx, sqlSnapshotScope{tx, &sqlReader{tx}}) + }) +} + +func (s *trackerSQLStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) { + handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &sqlSnapshotScope{handle, &sqlReader{handle}}, nil +} + +func (s *trackerSQLStore) Transaction(fn trackerdb.TransactionFn) (err error) { + return s.TransactionContext(context.Background(), fn) +} + +func (s *trackerSQLStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) { + return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + return fn(ctx, &sqlTransactionScope{tx, false, &sqlReader{tx}, &sqlWriter{tx}, &sqlCatchpoint{tx, s.IsSharedCacheConnection()}}) + }) +} + +func (s *trackerSQLStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) { + handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &sqlTransactionScope{handle, false, &sqlReader{handle}, &sqlWriter{handle}, &sqlCatchpoint{handle, s.IsSharedCacheConnection()}}, nil +} + +func (s trackerSQLStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + err = s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + mgr, err = RunMigrations(ctx, tx, params, log, targetVersion) + return err + }) + return +} + +// TODO: rename: this is a sqlite specific name, this could also be used to trigger compact on KV stores. +// it seems to only be used during a v2 migration +func (s *trackerSQLStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) { + _, err = s.pair.Wdb.Vacuum(ctx) + return +} + +func (s *trackerSQLStore) ResetToV6Test(ctx context.Context) error { + var resetExprs = []string{ + `DROP TABLE IF EXISTS onlineaccounts`, + `DROP TABLE IF EXISTS txtail`, + `DROP TABLE IF EXISTS onlineroundparamstail`, + `DROP TABLE IF EXISTS catchpointfirststageinfo`, + } + + return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { + for _, stmt := range resetExprs { + _, err := tx.ExecContext(ctx, stmt) + if err != nil { + return err + } + } + return nil + }) +} + +func (s *trackerSQLStore) Close() { + s.pair.Close() +} + +type sqlReader struct { + q db.Queryable +} + +// MakeAccountsOptimizedReader implements trackerdb.Reader +func (r *sqlReader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) { + return AccountsInitDbQueries(r.q) +} + +// MakeAccountsReader implements trackerdb.Reader +func (r *sqlReader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) { + // TODO: create and use a make accounts reader that takes just a queryable + return NewAccountsSQLReader(r.q), nil +} + +// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader +func (r *sqlReader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) { + return OnlineAccountsInitDbQueries(r.q) +} + +// MakeSpVerificationCtxReader implements trackerdb.Reader +func (r *sqlReader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader { + return makeStateProofVerificationReader(r.q) +} + +type sqlWriter struct { + e db.Executable +} + +// MakeAccountsOptimizedWriter implements trackerdb.Writer +func (w *sqlWriter) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) { + return MakeAccountsSQLWriter(w.e, hasAccounts, hasResources, hasKvPairs, hasCreatables) +} + +// MakeAccountsWriter implements trackerdb.Writer +func (w *sqlWriter) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) { + return NewAccountsSQLReaderWriter(w.e), nil +} + +// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer +func (w *sqlWriter) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) { + return MakeOnlineAccountsSQLWriter(w.e, hasAccounts) +} + +// MakeSpVerificationCtxWriter implements trackerdb.Writer +func (w *sqlWriter) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter { + return makeStateProofVerificationWriter(w.e) +} + +// Testing implements trackerdb.Writer +func (w *sqlWriter) Testing() trackerdb.WriterTestExt { + return w +} + +// AccountsInitLightTest implements trackerdb.WriterTestExt +func (w *sqlWriter) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { + return AccountsInitLightTest(tb, w.e, initAccounts, proto) +} + +// AccountsInitTest implements trackerdb.WriterTestExt +func (w *sqlWriter) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { + return AccountsInitTest(tb, w.e, initAccounts, proto) +} + +// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt +func (w *sqlWriter) AccountsUpdateSchemaTest(ctx context.Context) (err error) { + return AccountsUpdateSchemaTest(ctx, w.e) +} + +// ModifyAcctBaseTest implements trackerdb.WriterTestExt +func (w *sqlWriter) ModifyAcctBaseTest() error { + return modifyAcctBaseTest(w.e) +} + +type sqlCatchpoint struct { + e db.Executable + isShared bool +} + +// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter { + return MakeCatchpointPendingHashesIterator(hashCount, c.e) +} + +// MakeCatchpointWriter implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) { + return MakeCatchpointApplier(c.e, c.isShared), nil +} + +// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter { + return MakeEncodedAccoutsBatchIter(c.e) +} + +// MakeKVsIter implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) { + return MakeKVsIter(ctx, c.e) +} + +// MakeMerkleCommitter implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) { + return MakeMerkleCommitter(c.e, staging) +} + +// MakeOrderedAccountsIter implements trackerdb.Catchpoint +func (c *sqlCatchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter { + return MakeOrderedAccountsIter(c.e, accountCount) +} + +type sqlBatchScope struct { + tx *sql.Tx + committed bool + trackerdb.Writer +} + +func (bs *sqlBatchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + return db.ResetTransactionWarnDeadline(ctx, bs.tx, deadline) +} + +func (bs *sqlBatchScope) Close() error { + if !bs.committed { + return bs.tx.Rollback() + } + return nil +} + +func (bs *sqlBatchScope) Commit() error { + err := bs.tx.Commit() + if err != nil { + return err + } + bs.committed = true + return nil +} + +type sqlSnapshotScope struct { + tx *sql.Tx + trackerdb.Reader +} + +func (ss sqlSnapshotScope) Close() error { + return ss.tx.Rollback() +} + +type sqlTransactionScope struct { + tx *sql.Tx + committed bool + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +func (txs *sqlTransactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + return RunMigrations(ctx, txs.tx, params, log, targetVersion) +} + +func (txs *sqlTransactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + return db.ResetTransactionWarnDeadline(ctx, txs.tx, deadline) +} + +func (txs *sqlTransactionScope) Close() error { + if !txs.committed { + return txs.tx.Rollback() + } + return nil +} + +func (txs *sqlTransactionScope) Commit() error { + err := txs.tx.Commit() + if err != nil { + return err + } + txs.committed = true + return nil +} diff --git a/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go b/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go deleted file mode 100644 index 74327be2bc..0000000000 --- a/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright (C) 2019-2023 Algorand, Inc. -// This file is part of go-algorand -// -// go-algorand is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// go-algorand is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with go-algorand. If not, see . - -package sqlitedriver - -import ( - "context" - "database/sql" - "os" - "testing" - "time" - - "github.com/algorand/go-algorand/config" - "github.com/algorand/go-algorand/data/basics" - "github.com/algorand/go-algorand/ledger/store/trackerdb" - "github.com/algorand/go-algorand/logging" - "github.com/algorand/go-algorand/protocol" - "github.com/algorand/go-algorand/util/db" -) - -type trackerSQLStore struct { - // expose the internals for now so we can slowly change the code depending on them - pair db.Pair -} - -type sqlBatchScope struct { - tx *sql.Tx -} - -type sqlSnapshotScope struct { - tx *sql.Tx -} - -type sqlTransactionScope struct { - tx *sql.Tx -} - -// OpenTrackerSQLStore opens the sqlite database store -func OpenTrackerSQLStore(dbFilename string, dbMem bool) (store *trackerSQLStore, err error) { - db, err := db.OpenPair(dbFilename, dbMem) - if err != nil { - return - } - - return &trackerSQLStore{db}, nil -} - -// CreateTrackerSQLStore crates a tracker SQL db from sql db handle. -func CreateTrackerSQLStore(pair db.Pair) *trackerSQLStore { - return &trackerSQLStore{pair} -} - -// SetLogger sets the Logger, mainly for unit test quietness -func (s *trackerSQLStore) SetLogger(log logging.Logger) { - s.pair.Rdb.SetLogger(log) - s.pair.Wdb.SetLogger(log) -} - -func (s *trackerSQLStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) { - return s.pair.Wdb.SetSynchronousMode(ctx, mode, fullfsync) -} - -func (s *trackerSQLStore) IsSharedCacheConnection() bool { - return s.pair.Wdb.IsSharedCacheConnection() -} - -func (s *trackerSQLStore) Batch(fn trackerdb.BatchFn) (err error) { - return s.BatchContext(context.Background(), fn) -} - -func (s *trackerSQLStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) { - return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { - return fn(ctx, sqlBatchScope{tx}) - }) -} - -func (s *trackerSQLStore) Snapshot(fn trackerdb.SnapshotFn) (err error) { - return s.SnapshotContext(context.Background(), fn) -} - -func (s *trackerSQLStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) { - return s.pair.Rdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { - return fn(ctx, sqlSnapshotScope{tx}) - }) -} - -func (s *trackerSQLStore) Transaction(fn trackerdb.TransactionFn) (err error) { - return s.TransactionContext(context.Background(), fn) -} - -func (s *trackerSQLStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) { - return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { - return fn(ctx, sqlTransactionScope{tx}) - }) -} - -func (s *trackerSQLStore) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) { - return AccountsInitDbQueries(s.pair.Rdb.Handle) -} - -func (s *trackerSQLStore) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) { - return OnlineAccountsInitDbQueries(s.pair.Rdb.Handle) -} - -func (s *trackerSQLStore) MakeCatchpointReaderWriter() (trackerdb.CatchpointReaderWriter, error) { - w := NewCatchpointSQLReaderWriter(s.pair.Wdb.Handle) - return w, nil -} - -// TODO: rename: this is a sqlite specific name, this could also be used to trigger compact on KV stores. -// it seems to only be used during a v2 migration -func (s *trackerSQLStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) { - _, err = s.pair.Wdb.Vacuum(ctx) - return -} - -func (s *trackerSQLStore) CleanupTest(dbName string, inMemory bool) { - s.pair.Close() - if !inMemory { - os.Remove(dbName) - } -} - -func (s *trackerSQLStore) ResetToV6Test(ctx context.Context) error { - var resetExprs = []string{ - `DROP TABLE IF EXISTS onlineaccounts`, - `DROP TABLE IF EXISTS txtail`, - `DROP TABLE IF EXISTS onlineroundparamstail`, - `DROP TABLE IF EXISTS catchpointfirststageinfo`, - } - - return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error { - for _, stmt := range resetExprs { - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - } - return nil - }) -} - -func (s *trackerSQLStore) Close() { - s.pair.Close() -} - -// Testing returns this scope, exposed as an interface with test functions -func (txs sqlTransactionScope) Testing() trackerdb.TestTransactionScope { - return txs -} - -func (txs sqlTransactionScope) MakeCatchpointReaderWriter() (trackerdb.CatchpointReaderWriter, error) { - return NewCatchpointSQLReaderWriter(txs.tx), nil -} - -func (txs sqlTransactionScope) MakeAccountsReaderWriter() (trackerdb.AccountsReaderWriter, error) { - return NewAccountsSQLReaderWriter(txs.tx), nil -} - -// implements Testing interface -func (txs sqlTransactionScope) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) { - return AccountsInitDbQueries(txs.tx) -} - -func (txs sqlTransactionScope) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) { - return MakeAccountsSQLWriter(txs.tx, hasAccounts, hasResources, hasKvPairs, hasCreatables) -} - -func (txs sqlTransactionScope) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (w trackerdb.OnlineAccountsWriter, err error) { - return MakeOnlineAccountsSQLWriter(txs.tx, hasAccounts) -} - -// implements Testing interface -func (txs sqlTransactionScope) MakeOnlineAccountsOptimizedReader() (r trackerdb.OnlineAccountsReader, err error) { - return OnlineAccountsInitDbQueries(txs.tx) -} - -func (txs sqlTransactionScope) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) { - return MakeMerkleCommitter(txs.tx, staging) -} - -func (txs sqlTransactionScope) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter { - return MakeOrderedAccountsIter(txs.tx, accountCount) -} - -func (txs sqlTransactionScope) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) { - return MakeKVsIter(ctx, txs.tx) -} - -func (txs sqlTransactionScope) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter { - return MakeEncodedAccoutsBatchIter(txs.tx) -} - -func (txs sqlTransactionScope) MakeSpVerificationCtxReaderWriter() trackerdb.SpVerificationCtxReaderWriter { - return makeStateProofVerificationReaderWriter(txs.tx, txs.tx) -} - -func (txs sqlTransactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { - return RunMigrations(ctx, txs.tx, params, log, targetVersion) -} - -func (txs sqlTransactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { - return db.ResetTransactionWarnDeadline(ctx, txs.tx, deadline) -} - -// implements Testing interface -func (txs sqlTransactionScope) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { - return AccountsInitTest(tb, txs.tx, initAccounts, proto) -} - -// implements Testing interface -func (txs sqlTransactionScope) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { - return AccountsInitLightTest(tb, txs.tx, initAccounts, proto) -} - -// Testing returns this scope, exposed as an interface with test functions -func (bs sqlBatchScope) Testing() trackerdb.TestBatchScope { - return bs -} - -func (bs sqlBatchScope) MakeCatchpointWriter() (trackerdb.CatchpointWriter, error) { - return NewCatchpointSQLReaderWriter(bs.tx), nil -} - -func (bs sqlBatchScope) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) { - return NewAccountsSQLReaderWriter(bs.tx), nil -} - -func (bs sqlBatchScope) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) { - return MakeAccountsSQLWriter(bs.tx, hasAccounts, hasResources, hasKvPairs, hasCreatables) -} - -// implements Testing interface -func (bs sqlBatchScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { - return RunMigrations(ctx, bs.tx, params, log, targetVersion) -} - -func (bs sqlBatchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { - return db.ResetTransactionWarnDeadline(ctx, bs.tx, deadline) -} - -// implements Testing interface -func (bs sqlBatchScope) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { - return AccountsInitTest(tb, bs.tx, initAccounts, proto) -} - -// implements Testing interface -func (bs sqlBatchScope) ModifyAcctBaseTest() error { - return modifyAcctBaseTest(bs.tx) -} - -// implements Testing interface -func (bs sqlBatchScope) AccountsUpdateSchemaTest(ctx context.Context) (err error) { - return AccountsUpdateSchemaTest(ctx, bs.tx) -} - -func (bs sqlBatchScope) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter { - return makeStateProofVerificationWriter(bs.tx) -} - -func (ss sqlSnapshotScope) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) { - return NewAccountsSQLReaderWriter(ss.tx), nil -} - -func (ss sqlSnapshotScope) MakeCatchpointReader() (trackerdb.CatchpointReader, error) { - return NewCatchpointSQLReaderWriter(ss.tx), nil -} - -func (ss sqlSnapshotScope) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter { - return MakeCatchpointPendingHashesIterator(hashCount, ss.tx) -} - -func (ss sqlSnapshotScope) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader { - return makeStateProofVerificationReader(ss.tx) -} diff --git a/ledger/store/trackerdb/sqlitedriver/testing.go b/ledger/store/trackerdb/sqlitedriver/testing.go index 8d0a61afd7..a7d32bf0ca 100644 --- a/ledger/store/trackerdb/sqlitedriver/testing.go +++ b/ledger/store/trackerdb/sqlitedriver/testing.go @@ -18,7 +18,6 @@ package sqlitedriver import ( "context" - "database/sql" "fmt" "strings" "testing" @@ -33,59 +32,55 @@ import ( "github.com/stretchr/testify/require" ) -// DbOpenTrackerTest opens a sqlite db file for testing purposes. -func DbOpenTrackerTest(t testing.TB, inMemory bool) (trackerdb.TrackerStore, string) { - fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) +// OpenForTesting opens a sqlite db file for testing purposes. +// It set the logging to the test logger and uses a tmp directory associated to the test for the db. +// The test tmp direction is automatically cleaned up by the golang test framework. +func OpenForTesting(t testing.TB, inMemory bool) (trackerdb.Store, string) { + fn := fmt.Sprintf("%s/%s.%d", t.TempDir(), strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64()) - dbs, err := db.OpenPair(fn, inMemory) + store, err := Open(fn, inMemory, logging.TestingLog(t)) require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory) - return &trackerSQLStore{dbs}, fn -} - -// SetDbTrackerTestLogging sets a testing logger on a database. -func SetDbTrackerTestLogging(t testing.TB, dbs trackerdb.TrackerStore) { - dblogger := logging.TestingLog(t) - dbs.SetLogger(dblogger) + return store, fn } // AccountsInitLightTest initializes an empty database for testing without the extra methods being called. // implements Testing interface, test function only -func AccountsInitLightTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { - newDB, err := accountsInit(tx, initAccounts, proto) +func AccountsInitLightTest(tb testing.TB, e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) { + newDB, err := accountsInit(e, initAccounts, proto) require.NoError(tb, err) return newDB, err } // modifyAcctBaseTest tweaks the database to move backards. // implements Testing interface, test function only -func modifyAcctBaseTest(tx *sql.Tx) error { - _, err := tx.Exec("update acctrounds set rnd = 1 WHERE id='acctbase' ") +func modifyAcctBaseTest(e db.Executable) error { + _, err := e.Exec("update acctrounds set rnd = 1 WHERE id='acctbase' ") return err } // AccountsInitTest initializes an empty database for testing. // implements Testing interface, test function only -func AccountsInitTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { - newDB, err := accountsInit(tx, initAccounts, config.Consensus[proto]) +func AccountsInitTest(tb testing.TB, e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) { + newDB, err := accountsInit(e, initAccounts, config.Consensus[proto]) require.NoError(tb, err) - err = accountsAddNormalizedBalance(tx, config.Consensus[proto]) + err = accountsAddNormalizedBalance(e, config.Consensus[proto]) require.NoError(tb, err) - err = accountsCreateResourceTable(context.Background(), tx) + err = accountsCreateResourceTable(context.Background(), e) require.NoError(tb, err) - err = performResourceTableMigration(context.Background(), tx, nil) + err = performResourceTableMigration(context.Background(), e, nil) require.NoError(tb, err) - err = accountsCreateOnlineAccountsTable(context.Background(), tx) + err = accountsCreateOnlineAccountsTable(context.Background(), e) require.NoError(tb, err) - err = accountsCreateTxTailTable(context.Background(), tx) + err = accountsCreateTxTailTable(context.Background(), e) require.NoError(tb, err) - err = performOnlineAccountsTableMigration(context.Background(), tx, nil, nil) + err = performOnlineAccountsTableMigration(context.Background(), e, nil, nil) require.NoError(tb, err) // since this is a test that starts from genesis, there is no tail that needs to be migrated. @@ -94,41 +89,41 @@ func AccountsInitTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address err = performTxTailTableMigration(context.Background(), nil, db.Accessor{}) require.NoError(tb, err) - err = accountsCreateOnlineRoundParamsTable(context.Background(), tx) + err = accountsCreateOnlineRoundParamsTable(context.Background(), e) require.NoError(tb, err) - err = performOnlineRoundParamsTailMigration(context.Background(), tx, db.Accessor{}, true, proto) + err = performOnlineRoundParamsTailMigration(context.Background(), e, db.Accessor{}, true, proto) require.NoError(tb, err) - err = accountsCreateBoxTable(context.Background(), tx) + err = accountsCreateBoxTable(context.Background(), e) require.NoError(tb, err) - err = performKVStoreNullBlobConversion(context.Background(), tx) + err = performKVStoreNullBlobConversion(context.Background(), e) require.NoError(tb, err) return newDB } // AccountsUpdateSchemaTest adds some empty tables for tests to work with a "v6" store. -func AccountsUpdateSchemaTest(ctx context.Context, tx *sql.Tx) (err error) { - if err := accountsCreateOnlineAccountsTable(ctx, tx); err != nil { +func AccountsUpdateSchemaTest(ctx context.Context, e db.Executable) (err error) { + if err := accountsCreateOnlineAccountsTable(ctx, e); err != nil { return err } - if err := accountsCreateTxTailTable(ctx, tx); err != nil { + if err := accountsCreateTxTailTable(ctx, e); err != nil { return err } - if err := accountsCreateOnlineRoundParamsTable(ctx, tx); err != nil { + if err := accountsCreateOnlineRoundParamsTable(ctx, e); err != nil { return err } - if err := accountsCreateCatchpointFirstStageInfoTable(ctx, tx); err != nil { + if err := accountsCreateCatchpointFirstStageInfoTable(ctx, e); err != nil { return err } // this line creates kvstore table, even if it is not required in accountDBVersion 6 -> 7 // or in later version where we need kvstore table, some tests will fail - if err := accountsCreateBoxTable(ctx, tx); err != nil { + if err := accountsCreateBoxTable(ctx, e); err != nil { return err } - if err := createStateProofVerificationTable(ctx, tx); err != nil { + if err := createStateProofVerificationTable(ctx, e); err != nil { return err } return nil diff --git a/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go b/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go index a5c74274c6..3082887816 100644 --- a/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go +++ b/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go @@ -29,6 +29,7 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/crypto/merkletrie" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/util/db" @@ -50,9 +51,9 @@ type trackerDBSchemaInitializer struct { // RunMigrations initializes the accounts DB if needed and return current account round. // as part of the initialization, it tests the current database schema version, and perform upgrade // procedures to bring it up to the database schema supported by the binary. -func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { +func RunMigrations(ctx context.Context, e db.Executable, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { // check current database version. - dbVersion, err := db.GetUserVersion(ctx, tx) + dbVersion, err := db.GetUserVersion(ctx, e) if err != nil { return trackerdb.InitParams{}, fmt.Errorf("trackerDBInitialize unable to read database schema version : %v", err) } @@ -78,61 +79,61 @@ func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log // perform the initialization/upgrade switch tu.version() { case 0: - err = tu.upgradeDatabaseSchema0(ctx, tx) + err = tu.upgradeDatabaseSchema0(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 0 : %v", err) return } case 1: - err = tu.upgradeDatabaseSchema1(ctx, tx) + err = tu.upgradeDatabaseSchema1(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 1 : %v", err) return } case 2: - err = tu.upgradeDatabaseSchema2(ctx, tx) + err = tu.upgradeDatabaseSchema2(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 2 : %v", err) return } case 3: - err = tu.upgradeDatabaseSchema3(ctx, tx) + err = tu.upgradeDatabaseSchema3(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 3 : %v", err) return } case 4: - err = tu.upgradeDatabaseSchema4(ctx, tx) + err = tu.upgradeDatabaseSchema4(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 4 : %v", err) return } case 5: - err = tu.upgradeDatabaseSchema5(ctx, tx) + err = tu.upgradeDatabaseSchema5(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 5 : %v", err) return } case 6: - err = tu.upgradeDatabaseSchema6(ctx, tx) + err = tu.upgradeDatabaseSchema6(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 6 : %v", err) return } case 7: - err = tu.upgradeDatabaseSchema7(ctx, tx) + err = tu.upgradeDatabaseSchema7(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 7 : %v", err) return } case 8: - err = tu.upgradeDatabaseSchema8(ctx, tx) + err = tu.upgradeDatabaseSchema8(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 8 : %v", err) return } case 9: - err = tu.upgradeDatabaseSchema9(ctx, tx) + err = tu.upgradeDatabaseSchema9(ctx, e) if err != nil { tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 9 : %v", err) return @@ -147,10 +148,10 @@ func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log return trackerdb.InitParams{SchemaVersion: tu.schemaVersion, VacuumOnStartup: tu.vacuumOnStartup}, nil } -func (tu *trackerDBSchemaInitializer) setVersion(ctx context.Context, tx *sql.Tx, version int32) (err error) { +func (tu *trackerDBSchemaInitializer) setVersion(ctx context.Context, e db.Executable, version int32) (err error) { oldVersion := tu.schemaVersion tu.schemaVersion = version - _, err = db.SetUserVersion(ctx, tx, tu.schemaVersion) + _, err = db.SetUserVersion(ctx, e, tu.schemaVersion) if err != nil { return fmt.Errorf("trackerDBInitialize unable to update database schema version from %d to %d: %v", oldVersion, version, err) } @@ -180,13 +181,13 @@ func (tu trackerDBSchemaInitializer) version() int32 { // The accountbase would get initialized with the au.initAccounts // The accounttotals would get initialized to align with the initialization account added to accountbase // The acctrounds would get updated to indicate that the balance matches round 0 -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context, tx *sql.Tx) (err error) { +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context, e db.Executable) (err error) { tu.log.Infof("upgradeDatabaseSchema0 initializing schema") - tu.newDatabase, err = accountsInit(tx, tu.InitAccounts, config.Consensus[tu.InitProto]) + tu.newDatabase, err = accountsInit(e, tu.InitAccounts, config.Consensus[tu.InitProto]) if err != nil { return fmt.Errorf("upgradeDatabaseSchema0 unable to initialize schema : %v", err) } - return tu.setVersion(ctx, tx, 1) + return tu.setVersion(ctx, e, 1) } // upgradeDatabaseSchema1 upgrades the database schema from version 1 to version 2 @@ -204,7 +205,7 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context // // This upgrade doesn't change any of the actual database schema ( i.e. tables, indexes ) but rather just performing // a functional update to it's content. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context, tx *sql.Tx) (err error) { +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context, e db.Executable) (err error) { var modifiedAccounts uint if tu.newDatabase { goto schemaUpdateComplete @@ -212,14 +213,13 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context // update accounts encoding. tu.log.Infof("upgradeDatabaseSchema1 verifying accounts data encoding") - modifiedAccounts, err = reencodeAccounts(ctx, tx) + modifiedAccounts, err = reencodeAccounts(ctx, e) if err != nil { return err } if modifiedAccounts > 0 { - crw := NewCatchpointSQLReaderWriter(tx) - arw := NewAccountsSQLReaderWriter(tx) + arw := NewAccountsSQLReaderWriter(e) tu.log.Infof("upgradeDatabaseSchema1 reencoded %d accounts", modifiedAccounts) @@ -230,26 +230,14 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context return fmt.Errorf("upgradeDatabaseSchema1 unable to reset account hashes : %v", err) } - tu.log.Infof("upgradeDatabaseSchema1 preparing queries") - tu.log.Infof("upgradeDatabaseSchema1 resetting prior catchpoints") - // delete the last catchpoint label if we have any. - err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateLastCatchpoint, "") - if err != nil { - return fmt.Errorf("upgradeDatabaseSchema1 unable to clear prior catchpoint : %v", err) - } - - tu.log.Infof("upgradeDatabaseSchema1 deleting stored catchpoints") - // delete catchpoints. - err = crw.DeleteStoredCatchpoints(ctx, tu.Params.DbPathPrefix) - if err != nil { - return fmt.Errorf("upgradeDatabaseSchema1 unable to delete stored catchpoints : %v", err) - } + // we no longer cleanup catchpoints here + // Note: we no longer support cleanly upgrading from version 1. } else { tu.log.Infof("upgradeDatabaseSchema1 found that no accounts needed to be reencoded") } schemaUpdateComplete: - return tu.setVersion(ctx, tx, 2) + return tu.setVersion(ctx, e, 2) } // upgradeDatabaseSchema2 upgrades the database schema from version 2 to version 3 @@ -257,30 +245,30 @@ schemaUpdateComplete: // This upgrade only enables the database vacuuming which will take place once the upgrade process is complete. // If the user has already specified the OptimizeAccountsDatabaseOnStartup flag in the configuration file, this // step becomes a no-op. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema2(ctx context.Context, tx *sql.Tx) (err error) { +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema2(ctx context.Context, e db.Executable) (err error) { if !tu.newDatabase { tu.vacuumOnStartup = true } // update version - return tu.setVersion(ctx, tx, 3) + return tu.setVersion(ctx, e, 3) } // upgradeDatabaseSchema3 upgrades the database schema from version 3 to version 4, // adding the normalizedonlinebalance column to the accountbase table. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema3(ctx context.Context, tx *sql.Tx) (err error) { - err = accountsAddNormalizedBalance(tx, config.Consensus[tu.InitProto]) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema3(ctx context.Context, e db.Executable) (err error) { + err = accountsAddNormalizedBalance(e, config.Consensus[tu.InitProto]) if err != nil { return err } // update version - return tu.setVersion(ctx, tx, 4) + return tu.setVersion(ctx, e, 4) } // upgradeDatabaseSchema4 does not change the schema but migrates data: // remove empty AccountData entries from accountbase table -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context, tx *sql.Tx) (err error) { +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context, e db.Executable) (err error) { var numDeleted int64 var addresses []basics.Address @@ -288,13 +276,13 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context goto done } - numDeleted, addresses, err = removeEmptyAccountData(tx, tu.CatchpointEnabled) + numDeleted, addresses, err = removeEmptyAccountData(e, tu.CatchpointEnabled) if err != nil { return err } if tu.CatchpointEnabled && len(addresses) > 0 { - mc, err := MakeMerkleCommitter(tx, false) + mc, err := MakeMerkleCommitter(e, false) if err != nil { // at this point record deleted and DB is pruned for account data // if hash deletion fails just log it and do not abort startup @@ -332,15 +320,15 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context done: tu.log.Infof("upgradeDatabaseSchema4: deleted %d rows", numDeleted) - return tu.setVersion(ctx, tx, 5) + return tu.setVersion(ctx, e, 5) } // upgradeDatabaseSchema5 upgrades the database schema from version 5 to version 6, // adding the resources table and clearing empty catchpoint directories. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context, tx *sql.Tx) (err error) { - arw := NewAccountsSQLReaderWriter(tx) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context, e db.Executable) (err error) { + arw := NewAccountsSQLReaderWriter(e) - err = accountsCreateResourceTable(ctx, tx) + err = accountsCreateResourceTable(ctx, e) if err != nil { return fmt.Errorf("upgradeDatabaseSchema5 unable to create resources table : %v", err) } @@ -360,7 +348,7 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context tu.log.Infof("upgradeDatabaseSchema5 upgraded %d out of %d accounts [ %3.1f%% ]", processed, total, float64(processed)*100.0/float64(total)) } - err = performResourceTableMigration(ctx, tx, migrationProcessLog) + err = performResourceTableMigration(ctx, e, migrationProcessLog) if err != nil { return fmt.Errorf("upgradeDatabaseSchema5 unable to complete data migration : %v", err) } @@ -372,46 +360,66 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context } // update version - return tu.setVersion(ctx, tx, 6) + return tu.setVersion(ctx, e, 6) } -func (tu *trackerDBSchemaInitializer) deleteUnfinishedCatchpoint(ctx context.Context, tx *sql.Tx) error { - cts := NewCatchpointSQLReaderWriter(tx) - // Delete an unfinished catchpoint if there is one. - round, err := cts.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingCatchpoint) +func (tu *trackerDBSchemaInitializer) deleteUnfinishedCatchpoint(ctx context.Context, e db.Executable) error { + // this method comes from a time when catchpoint internal state was stored alongside the ledger data. + + // Note: this is duplicate legacy** code from catchpointdb.ReadCatchpointStateUint64 + var round uint64 + err := db.Retry(func() (err error) { + var v sql.NullInt64 + err = e.QueryRowContext(ctx, "SELECT intval FROM catchpointstate WHERE id=?", catchpointdb.CatchpointStateWritingCatchpoint).Scan(&v) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return err + } + if v.Valid { + round = uint64(v.Int64) + } + return nil + }) if err != nil { return err } - if round == 0 { - return nil + + relCatchpointFilePath := filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(basics.Round(round))) + err = catchpointdb.RemoveSingleCatchpointFileFromDisk(tu.DbPathPrefix, relCatchpointFilePath) + if err != nil { + return err } - relCatchpointFilePath := filepath.Join( - trackerdb.CatchpointDirName, - trackerdb.MakeCatchpointFilePath(basics.Round(round))) - err = trackerdb.RemoveSingleCatchpointFileFromDisk(tu.DbPathPrefix, relCatchpointFilePath) + // Note: this is duplicate legacy** code from catchpointdb.WriteCatchpointStateUint64 + _, err = e.ExecContext(ctx, "DELETE FROM catchpointstate WHERE id=?", catchpointdb.CatchpointStateWritingCatchpoint) + if err != nil { + return err + } + _, err = e.ExecContext(ctx, "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)", catchpointdb.CatchpointStateWritingCatchpoint, 0) if err != nil { return err } - return cts.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingCatchpoint, 0) + return nil } // upgradeDatabaseSchema6 upgrades the database schema from version 6 to version 7, // adding a new onlineaccounts table // TODO: onlineaccounts: upgrade as needed after switching to the final table version -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context, tx *sql.Tx) (err error) { - err = accountsCreateOnlineAccountsTable(ctx, tx) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context, e db.Executable) (err error) { + err = accountsCreateOnlineAccountsTable(ctx, e) if err != nil { return err } - err = accountsCreateTxTailTable(ctx, tx) + err = accountsCreateTxTailTable(ctx, e) if err != nil { return err } - err = accountsCreateOnlineRoundParamsTable(ctx, tx) + err = accountsCreateOnlineRoundParamsTable(ctx, e) if err != nil { return err } @@ -426,54 +434,54 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context lastProgressInfoMsg = time.Now() tu.log.Infof("upgradeDatabaseSchema6 upgraded %d out of %d accounts [ %3.1f%% ]", processed, total, float64(processed)*100.0/float64(total)) } - err = performOnlineAccountsTableMigration(ctx, tx, migrationProcessLog, tu.log) + err = performOnlineAccountsTableMigration(ctx, e, migrationProcessLog, tu.log) if err != nil { return fmt.Errorf("upgradeDatabaseSchema6 unable to complete online account data migration : %w", err) } if !tu.newDatabase { - err = performTxTailTableMigration(ctx, tx, tu.BlockDb.Rdb) + err = performTxTailTableMigration(ctx, e, tu.BlockDb.Rdb) if err != nil { return fmt.Errorf("upgradeDatabaseSchema6 unable to complete transaction tail data migration : %w", err) } } - err = performOnlineRoundParamsTailMigration(ctx, tx, tu.BlockDb.Rdb, tu.newDatabase, tu.InitProto) + err = performOnlineRoundParamsTailMigration(ctx, e, tu.BlockDb.Rdb, tu.newDatabase, tu.InitProto) if err != nil { return fmt.Errorf("upgradeDatabaseSchema6 unable to complete online round params data migration : %w", err) } - err = tu.deleteUnfinishedCatchpoint(ctx, tx) + err = tu.deleteUnfinishedCatchpoint(ctx, e) if err != nil { return err } - err = accountsCreateCatchpointFirstStageInfoTable(ctx, tx) + err = accountsCreateCatchpointFirstStageInfoTable(ctx, e) if err != nil { return err } - err = accountsCreateUnfinishedCatchpointsTable(ctx, tx) + err = accountsCreateUnfinishedCatchpointsTable(ctx, e) if err != nil { return err } // update version - return tu.setVersion(ctx, tx, 7) + return tu.setVersion(ctx, e, 7) } // upgradeDatabaseSchema7 upgrades the database schema from version 7 to version 8. // adding the kvstore table for box feature support. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema7(ctx context.Context, tx *sql.Tx) (err error) { - err = accountsCreateBoxTable(ctx, tx) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema7(ctx context.Context, e db.Executable) (err error) { + err = accountsCreateBoxTable(ctx, e) if err != nil { return fmt.Errorf("upgradeDatabaseSchema7 unable to create kvstore through createTables : %v", err) } - return tu.setVersion(ctx, tx, 8) + return tu.setVersion(ctx, e, 8) } // upgradeDatabaseSchema8 upgrades the database schema from version 8 to version 9, // forcing a rebuild of the accounthashes table on betanet nodes. Otherwise it has no effect. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context, tx *sql.Tx) (err error) { - arw := NewAccountsSQLReaderWriter(tx) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context, e db.Executable) (err error) { + arw := NewAccountsSQLReaderWriter(e) betanetGenesisHash, _ := crypto.DigestFromString("TBMBVTC7W24RJNNUZCF7LWZD2NMESGZEQSMPG5XQD7JY4O7JKVWQ") if tu.GenesisHash == betanetGenesisHash && !tu.FromCatchpoint { // reset hash round to 0, forcing catchpointTracker.initializeHashes to rebuild accounthashes @@ -482,39 +490,39 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context return fmt.Errorf("upgradeDatabaseSchema8 unable to reset acctrounds table 'hashbase' round : %v", err) } } - return tu.setVersion(ctx, tx, 9) + return tu.setVersion(ctx, e, 9) } // upgradeDatabaseSchema9 upgrades the database schema from version 9 to version 10, // adding a new stateproofverification table, // scrubbing out all nil values from kvstore table and replace with empty byte slice. -func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema9(ctx context.Context, tx *sql.Tx) (err error) { - err = createStateProofVerificationTable(ctx, tx) +func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema9(ctx context.Context, e db.Executable) (err error) { + err = createStateProofVerificationTable(ctx, e) if err != nil { return err } - err = performKVStoreNullBlobConversion(ctx, tx) + err = performKVStoreNullBlobConversion(ctx, e) if err != nil { return fmt.Errorf("upgradeDatabaseSchema9 unable to replace kvstore nil entries with empty byte slices : %v", err) } - err = convertOnlineRoundParamsTail(ctx, tx) + err = convertOnlineRoundParamsTail(ctx, e) if err != nil { return fmt.Errorf("upgradeDatabaseSchema10 unable to convert onlineroundparamstail: %v", err) } // update version - return tu.setVersion(ctx, tx, 10) + return tu.setVersion(ctx, e, 10) } func removeEmptyDirsOnSchemaUpgrade(dbDirectory string) (err error) { - catchpointRootDir := filepath.Join(dbDirectory, trackerdb.CatchpointDirName) + catchpointRootDir := filepath.Join(dbDirectory, catchpointdb.CatchpointDirName) if _, err := os.Stat(catchpointRootDir); os.IsNotExist(err) { return nil } for { - emptyDirs, err := trackerdb.GetEmptyDirs(catchpointRootDir) + emptyDirs, err := catchpointdb.GetEmptyDirs(catchpointRootDir) if err != nil { return err } diff --git a/ledger/store/trackerdb/store.go b/ledger/store/trackerdb/store.go index 17e0e720aa..7001dab5bf 100644 --- a/ledger/store/trackerdb/store.go +++ b/ledger/store/trackerdb/store.go @@ -20,77 +20,152 @@ import ( "context" "time" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/encoded" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/util/db" ) -// BatchScope is the write scope to the store. -type BatchScope interface { - MakeCatchpointWriter() (CatchpointWriter, error) - MakeAccountsWriter() (AccountsWriterExt, error) - MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (AccountsWriter, error) - ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) - Testing() TestBatchScope - MakeSpVerificationCtxWriter() SpVerificationCtxWriter +// Store is the interface for the tracker db. +type Store interface { + ReaderWriter + // settings + SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) + IsSharedCacheConnection() bool + // batch support + Batch(fn BatchFn) (err error) + BatchContext(ctx context.Context, fn BatchFn) (err error) + BeginBatch(ctx context.Context) (Batch, error) + // snapshot support + Snapshot(fn SnapshotFn) (err error) + SnapshotContext(ctx context.Context, fn SnapshotFn) (err error) + BeginSnapshot(ctx context.Context) (Snapshot, error) + // transaction support + Transaction(fn TransactionFn) (err error) + TransactionContext(ctx context.Context, fn TransactionFn) (err error) + BeginTransaction(ctx context.Context) (Transaction, error) + // maintenance + Vacuum(ctx context.Context) (stats db.VacuumStats, err error) + // testing + ResetToV6Test(ctx context.Context) error + // cleanup + Close() } -// SnapshotScope is the read scope to the store. -type SnapshotScope interface { +// Reader is the interface for the trackerdb read operations. +type Reader interface { MakeAccountsReader() (AccountsReaderExt, error) - MakeCatchpointReader() (CatchpointReader, error) - MakeCatchpointPendingHashesIterator(hashCount int) CatchpointPendingHashesIter - + MakeAccountsOptimizedReader() (AccountsReader, error) + MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error) MakeSpVerificationCtxReader() SpVerificationCtxReader } -// TransactionScope is the read/write scope to the store. -type TransactionScope interface { - MakeCatchpointReaderWriter() (CatchpointReaderWriter, error) - MakeAccountsReaderWriter() (AccountsReaderWriter, error) +// Writer is the interface for the trackerdb write operations. +type Writer interface { + // trackerdb + MakeAccountsWriter() (AccountsWriterExt, error) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (AccountsWriter, error) - MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (w OnlineAccountsWriter, err error) - MakeMerkleCommitter(staging bool) (MerkleCommitter, error) + MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (OnlineAccountsWriter, error) + MakeSpVerificationCtxWriter() SpVerificationCtxWriter + // testing + Testing() WriterTestExt +} + +// Catchpoint is currently holding most of the methods related to catchpoint. +// +// TODO: we still need to do a refactoring pass on catchpoint +// +// there are two distinct set of methods present: +// - read/write ops for managing catchpoint data +// - read/write internal catchup durable state (not sure if this should not be hidden) +// - read ops on trackerdb to support building catchpoints +// - write op for applying catchpoint to trackerdb +// we should split these two sets of methods into two separate interfaces +type Catchpoint interface { + // reader (used to generate a catchpoint) + MakeCatchpointPendingHashesIterator(hashCount int) CatchpointPendingHashesIter MakeOrderedAccountsIter(accountCount int) OrderedAccountsIter MakeKVsIter(ctx context.Context) (KVsIter, error) MakeEncodedAccoutsBatchIter() EncodedAccountsBatchIter - RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error) - ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) - Testing() TestTransactionScope - MakeSpVerificationCtxReaderWriter() SpVerificationCtxReaderWriter + // writer (used to apply a catchpoint) + MakeCatchpointWriter() (CatchpointApply, error) + // reader/writer + MakeMerkleCommitter(staging bool) (MerkleCommitter, error) } -// BatchFn is the callback lambda used in `Batch`. -type BatchFn func(ctx context.Context, tx BatchScope) error - -// SnapshotFn is the callback lambda used in `Snapshot`. -type SnapshotFn func(ctx context.Context, tx SnapshotScope) error +type CatchpointApply interface { + Reset(ctx context.Context, newCatchup bool) error + Write(ctx context.Context, payload CatchpointPayload) (CatchpointReport, error) + Apply(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) error +} -// TransactionFn is the callback lambda used in `Transaction`. -type TransactionFn func(ctx context.Context, tx TransactionScope) error +type CatchpointPayload struct { + Accounts []NormalizedAccountBalance + KVRecords []encoded.KVRecordV6 +} -// TrackerStore is the interface for the tracker db. -type TrackerStore interface { - SetLogger(log logging.Logger) - SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) - IsSharedCacheConnection() bool +type CatchpointReport struct { + BalancesWriteDuration time.Duration + CreatablesWriteDuration time.Duration + HashesWriteDuration time.Duration + KVWriteDuration time.Duration +} - Batch(fn BatchFn) (err error) - BatchContext(ctx context.Context, fn BatchFn) (err error) +// ReaderWriter is the interface for the trackerdb read/write operations. +// +// Some of the operatiosn available here might not be present in neither the Reader nor the Writer interfaces. +// This is because some operations might require to be able to read and write at the same time. +type ReaderWriter interface { + Reader + Writer + // init + RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error) + // Note: at the moment, catchpoint methods are only accesible via reader/writer + Catchpoint +} - Snapshot(fn SnapshotFn) (err error) - SnapshotContext(ctx context.Context, fn SnapshotFn) (err error) +// BatchScope is an atomic write-only scope to the store. +type BatchScope interface { + Writer + ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) +} - Transaction(fn TransactionFn) (err error) - TransactionContext(ctx context.Context, fn TransactionFn) (err error) +// Batch is an atomic write-only accecssor to the store. +type Batch interface { + BatchScope + Commit() error + Close() error +} - MakeAccountsOptimizedReader() (AccountsReader, error) - MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error) +// SnapshotScope is an atomic read-only scope to the store. +type SnapshotScope interface { + Reader +} - MakeCatchpointReaderWriter() (CatchpointReaderWriter, error) +// Snapshot is an atomic read-only accecssor to the store. +type Snapshot interface { + SnapshotScope + Close() error +} - Vacuum(ctx context.Context) (stats db.VacuumStats, err error) - Close() - CleanupTest(dbName string, inMemory bool) +// TransactionScope is an atomic read/write scope to the store. +type TransactionScope interface { + ReaderWriter + ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) +} - ResetToV6Test(ctx context.Context) error +// Transaction is an atomic read/write accecssor to the store. +type Transaction interface { + TransactionScope + Commit() error + Close() error } + +// BatchFn is the callback lambda used in `Batch`. +type BatchFn func(ctx context.Context, tx BatchScope) error + +// SnapshotFn is the callback lambda used in `Snapshot`. +type SnapshotFn func(ctx context.Context, tx SnapshotScope) error + +// TransactionFn is the callback lambda used in `Transaction`. +type TransactionFn func(ctx context.Context, tx TransactionScope) error diff --git a/ledger/store/trackerdb/testdb/testdb.go b/ledger/store/trackerdb/testdb/testdb.go new file mode 100644 index 0000000000..ae85edca78 --- /dev/null +++ b/ledger/store/trackerdb/testdb/testdb.go @@ -0,0 +1,34 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testdb + +import ( + "testing" + + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/ledger/store/trackerdb/dualdriver" + "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver" + "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver" +) + +// OpenForTesting will create a testing store to be used on tests outside of the trackerdb package. +func OpenForTesting(t testing.TB) trackerdb.Store { + primaryDB, _ := sqlitedriver.OpenForTesting(t, true) + secondaryDB := pebbledbdriver.OpenForTesting(t, true) + + return dualdriver.MakeStore(primaryDB, secondaryDB) +} diff --git a/ledger/store/trackerdb/testinterface.go b/ledger/store/trackerdb/testinterface.go index 961857e874..a406375a2f 100644 --- a/ledger/store/trackerdb/testinterface.go +++ b/ledger/store/trackerdb/testinterface.go @@ -23,41 +23,26 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" ) // testinterface.go contains interface extensions specific to testing // testing interfaces should be made accessible by calling the Testing() method // on the related interface. Example: -// testTx := tx.Testing() +// testTx := tx.Testing() // these can also be inlined: -// tx.Testing.AccountsInitTest(...) - -// TestBatchScope is an interface to extend BatchScope with test-only methods -type TestBatchScope interface { - BatchScope +// tx.Testing.AccountsInitTest(...) +// WriterTestExt is an interface to extend Writer with test-only methods +type WriterTestExt interface { AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) + AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) AccountsUpdateSchemaTest(ctx context.Context) (err error) - RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error) ModifyAcctBaseTest() error } -// TestTransactionScope is an interface to extend TransactionScope with test-only methods -type TestTransactionScope interface { - TransactionScope - - MakeAccountsOptimizedReader() (AccountsReader, error) - MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error) - AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) - AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) -} - -// TestAccountsReaderExt is an interface to extend AccountsReaderExt with test-only methods -type TestAccountsReaderExt interface { - AccountsReaderExt - +// AccountsReaderTestExt is an interface to extend AccountsReaderExt with test-only methods +type AccountsReaderTestExt interface { AccountsAllTest() (bals map[basics.Address]basics.AccountData, err error) CheckCreatablesTest(t *testing.T, iteration int, expectedDbImage map[basics.CreatableIndex]ledgercore.ModifiedCreatable) } diff --git a/ledger/store/trackerdb/testsuite/README.md b/ledger/store/trackerdb/testsuite/README.md new file mode 100644 index 0000000000..15270cfde9 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/README.md @@ -0,0 +1,29 @@ +!! **DO NOT IMPORT THIS MODULE** !! + +--- + +This is the `generickv` tests implemented _outside_ the package. +This is done to avoid cirucular dependencies on some of the testing wizardry taking place. +Namely, making the tests polymorphic on each database implementation, +so we can reuse the test suite against multiple backends. + +# Adding tests to the suite + +1. Use the following signature on your tests: + +```go +func CustomTestDoingSomething(t *customT) { + // your test.. +} +``` + +The `customT` type behaves just like `testing.T` but has some extras. + +2. Register your test with the suite in the `init()` function in your test file. + +```go +func init() { + // register tests that will run on each KV implementation + registerTest("something", CustomTestDoingSomething) +} +``` \ No newline at end of file diff --git a/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go b/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go new file mode 100644 index 0000000000..3c17c96b8e --- /dev/null +++ b/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go @@ -0,0 +1,300 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "context" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("global-round-update", CustomTestRoundUpdate) + registerTest("global-totals", CustomTestTotals) + registerTest("txtail-update", CustomTestTxTail) + registerTest("online_accounts-round_params-update", CustomTestOnlineAccountParams) + registerTest("accounts-lookup_by_rowid", CustomTestAccountLookupByRowID) + registerTest("resources-lookup_by_rowid", CustomTestResourceLookupByRowID) +} + +func CustomTestRoundUpdate(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // update the round + err = aw.UpdateAccountsRound(basics.Round(1)) + require.NoError(t, err) + + // read the round + rnd, err := ar.AccountsRound() + require.NoError(t, err) + require.Equal(t, basics.Round(1), rnd) +} + +func CustomTestTotals(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + totals := ledgercore.AccountTotals{ + Online: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 42}}, + Offline: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 1000}}, + NotParticipating: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 8}}, + RewardsLevel: 9000, + } + + // update the totals + err = aw.AccountsPutTotals(totals, false) + require.NoError(t, err) + + // read the totals + readTotals, err := ar.AccountsTotals(context.Background(), false) + require.NoError(t, err) + require.Equal(t, totals, readTotals) + + // generate some staging values + stagingTotals := ledgercore.AccountTotals{ + Online: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 1}}, + Offline: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 2}}, + NotParticipating: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 3}}, + RewardsLevel: 4, + } + + // update the (staging) totals + err = aw.AccountsPutTotals(stagingTotals, true) + require.NoError(t, err) + + // read the totals + readTotals, err = ar.AccountsTotals(context.Background(), true) + require.NoError(t, err) + require.Equal(t, stagingTotals, readTotals) + + // double check the live data is still there + readTotals, err = ar.AccountsTotals(context.Background(), false) + require.NoError(t, err) + require.Equal(t, totals, readTotals) +} + +func CustomTestTxTail(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + baseRound := basics.Round(0) + roundData := []*trackerdb.TxTailRound{ + { + TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-0")))}, + }, + { + TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-1")))}, + }, + { + TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-2")))}, + }, + } + // TODO: remove this conversion once we change the API to take the actual types + var rawRoundData [][]byte + for _, tail := range roundData { + raw := protocol.Encode(tail) + rawRoundData = append(rawRoundData, raw) + } + + // write TxTail's + err = aw.TxtailNewRound(context.Background(), baseRound, rawRoundData, baseRound) + require.NoError(t, err) + + // load TxTail's (error, must be the latest round) + _, _, _, err = ar.LoadTxTail(context.Background(), basics.Round(1)) + require.Error(t, err) + + // load TxTail's + txtails, hashes, readBaseRound, err := ar.LoadTxTail(context.Background(), basics.Round(2)) + require.NoError(t, err) + require.Len(t, txtails, 3) // assert boundries + require.Equal(t, roundData[0], txtails[0]) // assert ordering + require.Len(t, hashes, 3) + require.Equal(t, basics.Round(0), readBaseRound) + + // generate some more test data + roundData = []*trackerdb.TxTailRound{ + { + TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-3")))}, + }, + } + // reset data + rawRoundData = make([][]byte, 0) + // TODO: remove this conversion once we change the API to take the actual types + for _, tail := range roundData { + raw := protocol.Encode(tail) + rawRoundData = append(rawRoundData, raw) + } + // write TxTail's (delete everything before round 2) + err = aw.TxtailNewRound(context.Background(), basics.Round(3), rawRoundData, basics.Round(2)) + require.NoError(t, err) + + // load TxTail's + txtails, hashes, readBaseRound, err = ar.LoadTxTail(context.Background(), basics.Round(3)) + require.NoError(t, err) + require.Len(t, txtails, 2) + require.Len(t, hashes, 2) + require.Equal(t, basics.Round(2), readBaseRound) +} + +func CustomTestOnlineAccountParams(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + startRound := basics.Round(0) + roundParams := []ledgercore.OnlineRoundParamsData{ + {OnlineSupply: 100}, + {OnlineSupply: 42}, + {OnlineSupply: 9000}, + } + + // clean up the db before starting with the test + // Note: some engines might start with some data built-in data for round 0 + err = aw.AccountsPruneOnlineRoundParams(basics.Round(42)) + require.NoError(t, err) + + // write round params + err = aw.AccountsPutOnlineRoundParams(roundParams, startRound) + require.NoError(t, err) + + // read round params + readParams, endRound, err := ar.AccountsOnlineRoundParams() + require.NoError(t, err) + require.Len(t, readParams, 3) // assert boundries + require.Equal(t, roundParams[0], readParams[0]) // assert ordering + require.Equal(t, basics.Round(2), endRound) // check round + + // prune params + err = aw.AccountsPruneOnlineRoundParams(basics.Round(1)) + require.NoError(t, err) + + // read round params (again, after prunning) + readParams, endRound, err = ar.AccountsOnlineRoundParams() + require.NoError(t, err) + require.Len(t, readParams, 2) // assert boundries + require.Equal(t, roundParams[1], readParams[0]) // assert ordering, and first item + require.Equal(t, basics.Round(2), endRound) // check round +} + +func CustomTestAccountLookupByRowID(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA := trackerdb.BaseAccountData{ + RewardsBase: 1000, + } + normBalanceA := dataA.NormalizedOnlineBalance(t.proto) + refA, err := aow.InsertAccount(addrA, normBalanceA, dataA) + require.NoError(t, err) + + // + // test + // + + // non-existing account + _, err = ar.LookupAccountRowID(RandomAddress()) + require.Error(t, err) + require.Equal(t, err, trackerdb.ErrNotFound) + + // read account + ref, err := ar.LookupAccountRowID(addrA) + require.NoError(t, err) + require.Equal(t, refA, ref) +} + +func CustomTestResourceLookupByRowID(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + + // generate some test data + resDataA0 := trackerdb.MakeResourcesData(0) + resDataA0.SetAssetParams(basics.AssetParams{ + Total: 100, + UnitName: "t", + AssetName: "test-asset", + Manager: addrA, + Reserve: addrA, + Freeze: addrA, + Clawback: addrA, + URL: "http://127.0.0.1/8000", + }, true) + resDataA0.SetAssetHolding(basics.AssetHolding{Amount: 10}) + aidxResA0 := basics.CreatableIndex(0) + _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + + // + // test + // + + // non-existing resource + _, err = ar.LookupResourceDataByAddrID(refAccA, basics.CreatableIndex(100)) + require.Error(t, err) + require.Equal(t, err, trackerdb.ErrNotFound) + + // read resource + data, err := ar.LookupResourceDataByAddrID(refAccA, aidxResA0) + require.NoError(t, err) + // parse the raw data + var res trackerdb.ResourcesData + err = protocol.Decode(data, &res) + require.NoError(t, err) + // assert that we got the resource + require.Equal(t, resDataA0, res) + + // read resource on nil account + _, err = ar.LookupResourceDataByAddrID(nil, basics.CreatableIndex(100)) + require.Error(t, err) + require.Equal(t, err, trackerdb.ErrNotFound) +} diff --git a/ledger/store/trackerdb/testsuite/accounts_kv_test.go b/ledger/store/trackerdb/testsuite/accounts_kv_test.go new file mode 100644 index 0000000000..0b8e74a715 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/accounts_kv_test.go @@ -0,0 +1,480 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("accounts-crud", CustomTestAccountsCrud) + registerTest("resources-crud", CustomTestResourcesCrud) + registerTest("resources-query-all", CustomTestResourcesQueryAll) + registerTest("kv-crud", CustomTestAppKVCrud) + registerTest("creatables-crud", CustomTestCreatablesCrud) + registerTest("creatables-query-all", CustomTestCreatablesQueryAll) +} + +func CustomTestAccountsCrud(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA := trackerdb.BaseAccountData{ + RewardsBase: 1000, + } + + // insert the account + normBalanceA := dataA.NormalizedOnlineBalance(t.proto) + refA, err := aow.InsertAccount(addrA, normBalanceA, dataA) + require.NoError(t, err) + + // read the account + padA, err := aor.LookupAccount(addrA) + require.NoError(t, err) + require.Equal(t, addrA, padA.Addr) // addr is present and correct + require.Equal(t, refA, padA.Ref) // same ref as when we inserted it + require.Equal(t, dataA, padA.AccountData) // same data + require.Equal(t, expectedRound, padA.Round) // db round + + // read the accounts "ref" + readRefA, err := ar.LookupAccountRowID(addrA) + require.NoError(t, err) + require.Equal(t, refA, readRefA) // same ref as when we inserted it + + // update the account + dataA.RewardsBase = 98287 + normBalanceA = dataA.NormalizedOnlineBalance(t.proto) + _, err = aow.UpdateAccount(refA, normBalanceA, dataA) + require.NoError(t, err) + + // read updated account + padA, err = aor.LookupAccount(addrA) + require.NoError(t, err) + require.Equal(t, dataA, padA.AccountData) // same updated data + + // delete account + _, err = aow.DeleteAccount(refA) + require.NoError(t, err) + + // read deleted account + // Note: this is a bit counter-intuitive but lookup returns a value + // even when the account doesnt exist. + padA, err = aor.LookupAccount(addrA) + require.NoError(t, err) + require.Equal(t, addrA, padA.Addr) // the addr is there + require.Empty(t, padA.AccountData) // no data + require.Equal(t, expectedRound, padA.Round) // db round (this is present even if record does not exist) +} + +func CustomTestResourcesCrud(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // + // pre-fill the db with an account for testing + // + + // account + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + + // + // test + // + + // generate some test data + resDataA0 := trackerdb.MakeResourcesData(0) + resDataA0.SetAssetParams(basics.AssetParams{ + Total: 100, + UnitName: "t", + AssetName: "test-asset", + Manager: addrA, + Reserve: addrA, + Freeze: addrA, + Clawback: addrA, + URL: "http://127.0.0.1/8000", + }, true) + resDataA0.SetAssetHolding(basics.AssetHolding{Amount: 10}) + aidxResA0 := basics.CreatableIndex(0) + + // insert the resource + refResA0, err := aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + require.NotNil(t, refResA0) + + // read the resource + prdA0, err := aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable) + require.NoError(t, err) + require.Equal(t, aidxResA0, prdA0.Aidx) // aidx is present and correct + require.Equal(t, refAccA, prdA0.AcctRef) // acctRef is present and correct + require.Equal(t, resDataA0, prdA0.Data) // same data + require.Equal(t, expectedRound, prdA0.Round) // db round + + // update the resource + resDataA0.Amount = 900 + _, err = aow.UpdateResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + + // read updated resource + prdA0, err = aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable) + require.NoError(t, err) + require.Equal(t, resDataA0, prdA0.Data) // same updated data + + // delete resource + _, err = aow.DeleteResource(refAccA, aidxResA0) + require.NoError(t, err) + + // read deleted resource + // Note: this is a bit counter-intuitive but lookup returns a value + // even when the account doesnt exist. + prdA0, err = aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable) + require.NoError(t, err) + require.Equal(t, aidxResA0, prdA0.Aidx) // the aidx is there + require.Nil(t, prdA0.AcctRef) // the account ref is not present + require.Equal(t, trackerdb.MakeResourcesData(0), prdA0.Data) // rnd 0, clean data + require.Equal(t, expectedRound, prdA0.Round) // db round (this is present even if record does not exist) +} + +func CustomTestResourcesQueryAll(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // + // pre-fill the db with an account for testing + // + + // account A + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + + // resource A-0 + resDataA0 := trackerdb.ResourcesData{} + aidxResA0 := basics.CreatableIndex(0) + _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + + // resource A-1 + resDataA1 := trackerdb.ResourcesData{} + aidxResA1 := basics.CreatableIndex(1) + _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1) + require.NoError(t, err) + + // + // test + // + + prs, rnd, err := aor.LookupAllResources(addrA) + require.NoError(t, err) + require.Equal(t, aidxResA0, prs[0].Aidx) + require.Equal(t, aidxResA1, prs[1].Aidx) + require.Equal(t, expectedRound, prs[0].Round) // db round (inside resources) + require.Equal(t, expectedRound, rnd) // db round (from the return) +} + +func CustomTestAppKVCrud(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, true, false) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // + // pre-fill the db with an account for testing + // + + // account + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + // resource + resDataA0 := trackerdb.ResourcesData{} + aidxResA0 := basics.CreatableIndex(0) + refResA0, err := aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + require.NotNil(t, refResA0) + + // + // test + // + + // insert the kv + kvKey := "foobar-mykey" + kvValue := []byte("1234") + err = aow.UpsertKvPair(kvKey, kvValue) + require.NoError(t, err) + + // read the kv + pv1, err := aor.LookupKeyValue(kvKey) + require.NoError(t, err) + require.Equal(t, kvValue, pv1.Value) // same data + require.Equal(t, expectedRound, pv1.Round) // db round + + // update the kv + kvValue = []byte("777") + err = aow.UpsertKvPair(kvKey, kvValue) + require.NoError(t, err) + + // read updated kv + pv1, err = aor.LookupKeyValue(kvKey) + require.NoError(t, err) + require.Equal(t, kvValue, pv1.Value) // same data + + // delete the kv + err = aow.DeleteKvPair(kvKey) + require.NoError(t, err) + + // read deleted kv + require.NoError(t, err) + + // read deleted kv + // Note: this is a bit counter-intuitive but lookup returns a value + // even when the record doesn't exist. + pv1, err = aor.LookupKeyValue(kvKey) + require.NoError(t, err) + require.Equal(t, expectedRound, pv1.Round) // db round (this is present even if record does not exist) +} + +func CustomTestCreatablesCrud(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, true) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // + // pre-fill the db with an account for testing + // + + // account A + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + + // resource A-0 + resDataA0 := trackerdb.ResourcesData{} + aidxResA0 := basics.CreatableIndex(0) + _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + + // resource A-1 + resDataA1 := trackerdb.ResourcesData{} + aidxResA1 := basics.CreatableIndex(1) + _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1) + require.NoError(t, err) + + // + // test + // + + // insert creator for A0 + resA0ctype := basics.AssetCreatable + cRefA0, err := aow.InsertCreatable(aidxResA0, resA0ctype, addrA[:]) + require.NoError(t, err) + require.NotNil(t, cRefA0) + + // insert creator for A1 + resA1ctype := basics.AppCreatable + cRefA1, err := aow.InsertCreatable(aidxResA1, resA1ctype, addrA[:]) + require.NoError(t, err) + require.NotNil(t, cRefA1) + + // lookup creator (correct ctype) + addr, ok, rnd, err := aor.LookupCreator(aidxResA0, basics.AssetCreatable) + require.NoError(t, err) + require.True(t, ok) // ok=true when it works + require.Equal(t, addrA, addr) // correct owner + require.Equal(t, expectedRound, rnd) // db round + + // lookup creator (invalid ctype) + _, ok, rnd, err = aor.LookupCreator(aidxResA0, basics.AppCreatable) + require.NoError(t, err) + require.False(t, ok) // ok=false when its doesnt match + require.Equal(t, expectedRound, rnd) // db round (this is present even if record does not exist) + + // lookup creator (unknown index) + _, ok, rnd, err = aor.LookupCreator(basics.CreatableIndex(999), basics.AppCreatable) + require.NoError(t, err) + require.False(t, ok) // ok=false when it doesn't exist + require.Equal(t, expectedRound, rnd) // db round (this is present even if record does not exist) + +} + +func CustomTestCreatablesQueryAll(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, true) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // + // pre-fill the db with an account for testing + // + + // account A + addrA := RandomAddress() + accDataA := trackerdb.BaseAccountData{RewardsBase: 1000} + refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA) + require.NoError(t, err) + + // resource A-0 + resDataA0 := trackerdb.ResourcesData{} + aidxResA0 := basics.CreatableIndex(0) + _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0) + require.NoError(t, err) + + // resource A-1 + resDataA1 := trackerdb.ResourcesData{} + aidxResA1 := basics.CreatableIndex(1) + _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1) + require.NoError(t, err) + + // resource A-2 + resDataA2 := trackerdb.ResourcesData{} + aidxResA2 := basics.CreatableIndex(2) + _, err = aow.InsertResource(refAccA, aidxResA2, resDataA2) + require.NoError(t, err) + + // creator for A0 + resA0ctype := basics.AssetCreatable + cRefA0, err := aow.InsertCreatable(aidxResA0, resA0ctype, addrA[:]) + require.NoError(t, err) + require.NotNil(t, cRefA0) + + // creator for A1 + resA1ctype := basics.AppCreatable + cRefA1, err := aow.InsertCreatable(aidxResA1, resA1ctype, addrA[:]) + require.NoError(t, err) + require.NotNil(t, cRefA1) + + // creator for A2 + resA2ctype := basics.AppCreatable + cRefA2, err := aow.InsertCreatable(aidxResA2, resA2ctype, addrA[:]) + require.NoError(t, err) + require.NotNil(t, cRefA2) + + // + // test + // + + // filter by type + cls, rnd, err := aor.ListCreatables(basics.CreatableIndex(99), 5, basics.AssetCreatable) + require.NoError(t, err) + require.Len(t, cls, 1) // only one asset + require.Equal(t, aidxResA0, cls[0].Index) // resource A-0 + require.Equal(t, expectedRound, rnd) // db round + + // with multiple results + cls, rnd, err = aor.ListCreatables(basics.CreatableIndex(99), 5, basics.AppCreatable) + require.NoError(t, err) + require.Len(t, cls, 2) // two apps + require.Equal(t, aidxResA2, cls[0].Index) // resource A-2 + require.Equal(t, aidxResA1, cls[1].Index) // resource A-1 + require.Equal(t, expectedRound, rnd) // db round + + // limit results + cls, rnd, err = aor.ListCreatables(basics.CreatableIndex(99), 1, basics.AppCreatable) + require.NoError(t, err) + require.Len(t, cls, 1) // two apps + require.Equal(t, aidxResA2, cls[0].Index) // resource A-2 + require.Equal(t, expectedRound, rnd) // db round + + // filter maxId + cls, rnd, err = aor.ListCreatables(aidxResA2, 10, basics.AppCreatable) + require.NoError(t, err) + require.Len(t, cls, 2) // only one app since to that cidx + require.Equal(t, aidxResA2, cls[0].Index) // resource A-2 (checks for order too) + require.Equal(t, aidxResA1, cls[1].Index) // resource A-1 + require.Equal(t, expectedRound, rnd) // db round + +} diff --git a/ledger/store/trackerdb/testsuite/dbsemantics_test.go b/ledger/store/trackerdb/testsuite/dbsemantics_test.go new file mode 100644 index 0000000000..b155b918a9 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/dbsemantics_test.go @@ -0,0 +1,82 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "context" + + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("db-semantics-transaction", CustomTestTransaction) +} + +// This test will ensure that transaction semantics carry the same meaning across all engine implementations. +func CustomTestTransaction(t *customT) { + aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false) + require.NoError(t, err) + + aor, err := t.db.MakeAccountsOptimizedReader() + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA := trackerdb.BaseAccountData{ + RewardsBase: 1000, + } + + // insert the account + normBalanceA := dataA.NormalizedOnlineBalance(t.proto) + refA, err := aow.InsertAccount(addrA, normBalanceA, dataA) + require.NoError(t, err) + + // + // test + // + + err = t.db.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { + // create a scoped writer + aow, err := tx.MakeAccountsOptimizedWriter(true, false, false, false) + require.NoError(t, err) + + // create a scoped reader + aor, err := tx.MakeAccountsOptimizedReader() + require.NoError(t, err) + + // read an account + padA, err := aor.LookupAccount(addrA) + require.NoError(t, err) + require.Equal(t, refA, padA.Ref) // same ref as when we inserted it + + // update the account + dataA.RewardsBase = 98287 + normBalanceA = dataA.NormalizedOnlineBalance(t.proto) + _, err = aow.UpdateAccount(refA, normBalanceA, dataA) + require.NoError(t, err) + + return nil + }) + require.NoError(t, err) + + // read the updated record outside the transaction to make sure it was commited + padA, err := aor.LookupAccount(addrA) + require.NoError(t, err) + require.Equal(t, uint64(98287), padA.AccountData.RewardsBase) // same updated data +} diff --git a/ledger/store/trackerdb/testsuite/dual_test.go b/ledger/store/trackerdb/testsuite/dual_test.go new file mode 100644 index 0000000000..68648b5f2b --- /dev/null +++ b/ledger/store/trackerdb/testsuite/dual_test.go @@ -0,0 +1,36 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb/testdb" +) + +func TestDualEngines(t *testing.T) { + dbFactory := func(proto config.ConsensusParams) dbForTests { + db := testdb.OpenForTesting(t) + seedDb(t, db) + + return db + } + + // run the suite + runGenericTestsWithDB(t, dbFactory) +} diff --git a/ledger/store/trackerdb/testsuite/migration_test.go b/ledger/store/trackerdb/testsuite/migration_test.go new file mode 100644 index 0000000000..10d9c74737 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/migration_test.go @@ -0,0 +1,140 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "context" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("db-migration-check-basic", CustomTestChecBasicMigration) + // Disabled since it's technically broken the way its written. + // registerTest("db-migration-check-with-accounts", CustomTestCheckMigrationWithAccounts) +} + +func CustomTestChecBasicMigration(t *customT) { + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // check round + round, err := ar.AccountsRound() + require.NoError(t, err) + require.Equal(t, basics.Round(0), round) // initialized to round 0 + + // check account totals + totals, err := ar.AccountsTotals(context.Background(), false) + require.NoError(t, err) + require.Equal(t, uint64(0), totals.RewardsLevel) + require.Equal(t, ledgercore.AlgoCount{}, totals.Online) + require.Equal(t, ledgercore.AlgoCount{}, totals.Offline) + require.Equal(t, ledgercore.AlgoCount{}, totals.NotParticipating) + + // check tx-tails + txTailData, hashes, baseRound, err := ar.LoadTxTail(context.Background(), basics.Round(0)) + require.NoError(t, err) + require.Len(t, txTailData, 0) // no data + require.Len(t, hashes, 0) // no data + require.Equal(t, basics.Round(1), baseRound) // (the impls return +1 at the end) + + // check online accounts + oas, err := ar.OnlineAccountsAll(99) + require.NoError(t, err) + require.Len(t, oas, 0) + + // check online round params + oparams, endRound, err := ar.AccountsOnlineRoundParams() + require.NoError(t, err) + require.Len(t, oparams, 1) + require.Equal(t, basics.Round(0), endRound) + require.Equal(t, uint64(0), oparams[0].OnlineSupply) + require.Equal(t, uint64(0), oparams[0].RewardsLevel) + require.Equal(t, protocol.ConsensusCurrentVersion, oparams[0].CurrentProtocol) +} + +func makeAccountData(status basics.Status, algos basics.MicroAlgos) basics.AccountData { + ad := basics.AccountData{Status: status, MicroAlgos: algos} + if status == basics.Online { + ad.VoteFirstValid = 1 + ad.VoteLastValid = 100_000 + } + return ad +} + +func CustomTestCheckMigrationWithAccounts(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // reset + aw.AccountsReset(context.Background()) + + initAccounts := make(map[basics.Address]basics.AccountData) + + addrA := basics.Address(crypto.Hash([]byte("a"))) + initAccounts[addrA] = makeAccountData(basics.Online, basics.MicroAlgos{Raw: 100}) + + addrB := basics.Address(crypto.Hash([]byte("b"))) + initAccounts[addrB] = makeAccountData(basics.Online, basics.MicroAlgos{Raw: 42}) + + addrC := basics.Address(crypto.Hash([]byte("c"))) + initAccounts[addrC] = makeAccountData(basics.Offline, basics.MicroAlgos{Raw: 30}) + + addrD := basics.Address(crypto.Hash([]byte("d"))) + initAccounts[addrD] = makeAccountData(basics.NotParticipating, basics.MicroAlgos{Raw: 7}) + + params := trackerdb.Params{ + InitProto: protocol.ConsensusCurrentVersion, + InitAccounts: initAccounts, + } + + // re-run migrations + _, err = t.db.RunMigrations(context.Background(), params, logging.TestingLog(t), trackerdb.AccountDBVersion) + require.NoError(t, err) + + // check account totals + totals, err := ar.AccountsTotals(context.Background(), false) + require.NoError(t, err) + require.Equal(t, uint64(0), totals.RewardsLevel) + require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 142}}, totals.Online) + require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 30}}, totals.Offline) + require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 7}}, totals.NotParticipating) + + // check online accounts + oas, err := ar.OnlineAccountsAll(99) + require.NoError(t, err) + require.Len(t, oas, 2) + + // check online round params + oparams, endRound, err := ar.AccountsOnlineRoundParams() + require.NoError(t, err) + require.Len(t, oparams, 1) + require.Equal(t, basics.Round(0), endRound) + require.Equal(t, uint64(142), oparams[0].OnlineSupply) + require.Equal(t, uint64(0), oparams[0].RewardsLevel) + require.Equal(t, protocol.ConsensusCurrentVersion, oparams[0].CurrentProtocol) +} diff --git a/ledger/store/trackerdb/testsuite/mockdb_test.go b/ledger/store/trackerdb/testsuite/mockdb_test.go new file mode 100644 index 0000000000..cc9397f2b4 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/mockdb_test.go @@ -0,0 +1,34 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "testing" + + "github.com/algorand/go-algorand/config" +) + +func TestMockDB(t *testing.T) { + dbFactory := func(proto config.ConsensusParams) dbForTests { + db := makeMockDB(proto) + + seedDb(t, db) + + return db + } + runGenericTestsWithDB(t, dbFactory) +} diff --git a/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go b/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go new file mode 100644 index 0000000000..ed444794da --- /dev/null +++ b/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go @@ -0,0 +1,554 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/protocol" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("online-accounts-write-read", CustomTestOnlineAccountsWriteRead) + registerTest("online-accounts-all", CustomTestOnlineAccountsAll) + registerTest("online-accounts-top", CustomTestAccountsOnlineTop) + registerTest("online-accounts-get-by-addr", CustomTestLookupOnlineAccountDataByAddress) + registerTest("online-accounts-history", CustomTestOnlineAccountHistory) + registerTest("online-accounts-totals", CustomTestOnlineAccountTotals) + registerTest("online-accounts-delete", CustomTestOnlineAccountsDelete) + registerTest("online-accounts-expired", CustomTestAccountsOnlineExpired) +} + +func CustomTestOnlineAccountsWriteRead(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + oar, err := t.db.MakeOnlineAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + updRoundA := uint64(400) + lastValidA := uint64(500) + dataA := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + RewardsBase: uint64(200), + } + normalizedBalA := dataA.NormalizedOnlineBalance(t.proto) + + // write + refA, err := oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA) + require.NoError(t, err) + + // read + poA, err := oar.LookupOnline(addrA, basics.Round(updRoundA)) + require.NoError(t, err) + require.Equal(t, addrA, poA.Addr) + require.Equal(t, refA, poA.Ref) + require.Equal(t, dataA, poA.AccountData) + require.Equal(t, basics.Round(updRoundA), poA.UpdRound) // check the "update round" was read + require.Equal(t, expectedRound, poA.Round) + + // write a new version + dataA.MicroAlgos = basics.MicroAlgos{Raw: uint64(321)} + normalizedBalA = dataA.NormalizedOnlineBalance(t.proto) + updRoundA = uint64(450) + _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA) + require.NoError(t, err) + + // read (latest) + poA, err = oar.LookupOnline(addrA, basics.Round(500)) + require.NoError(t, err) + require.Equal(t, dataA, poA.AccountData) // check the data is from the new version + require.Equal(t, basics.Round(updRoundA), poA.UpdRound) // check the "update round" + + // read (original) + poA, err = oar.LookupOnline(addrA, basics.Round(405)) + require.NoError(t, err) + require.Equal(t, basics.MicroAlgos{Raw: uint64(100)}, poA.AccountData.MicroAlgos) // check the data is from the new version + require.Equal(t, basics.Round(400), poA.UpdRound) // check the "update round" + + // read (at upd round) + poA, err = oar.LookupOnline(addrA, basics.Round(450)) + require.NoError(t, err) + require.Equal(t, basics.MicroAlgos{Raw: uint64(321)}, poA.AccountData.MicroAlgos) // check the data is from the new version + require.Equal(t, basics.Round(450), poA.UpdRound) // check the "update round" +} + +func CustomTestOnlineAccountHistory(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + oar, err := t.db.MakeOnlineAccountsOptimizedReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA1 := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(20)}, + RewardsBase: uint64(200), + } + normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto) + + refA1, err := oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(2), uint64(2)) + require.NoError(t, err) + + // generate some test data + dataA2 := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + RewardsBase: uint64(200), + } + normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto) + + refA2, err := oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(3), uint64(3)) + require.NoError(t, err) + + // generate some test data + addrB := RandomAddress() + dataB1 := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(75)}, + RewardsBase: uint64(200), + } + normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto) + + refB1, err := oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(3), uint64(3)) + require.NoError(t, err) + + // + // the test + // + + resultsA, rnd, err := oar.LookupOnlineHistory(addrA) + require.NoError(t, err) + require.Equal(t, expectedRound, rnd) // check the db round + require.Len(t, resultsA, 2) + require.Equal(t, basics.Round(2), resultsA[0].UpdRound) // check ordering + require.Equal(t, basics.Round(3), resultsA[1].UpdRound) // check ordering + // check item fields + require.Empty(t, resultsA[0].Round) // check the db round is not set + require.Equal(t, addrA, resultsA[0].Addr) // check addr + require.Equal(t, refA1, resultsA[0].Ref) // check ref + require.Equal(t, dataA1, resultsA[0].AccountData) // check data + // check ref is valid on all + require.Equal(t, refA2, resultsA[1].Ref) // check ref + + // check for B + resultsB, _, err := oar.LookupOnlineHistory(addrB) + require.NoError(t, err) + require.Len(t, resultsB, 1) + require.Equal(t, addrB, resultsB[0].Addr) // check addr + require.Equal(t, refB1, resultsB[0].Ref) // check ref +} + +func CustomTestOnlineAccountsAll(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + voteLastValid := uint64(0) + + // generate some test data + addrA := basics.Address(crypto.Hash([]byte("a"))) + dataA0 := trackerdb.BaseOnlineAccountData{ + MicroAlgos: basics.MicroAlgos{Raw: uint64(200)}, + } + _, err = oaw.InsertOnlineAccount(addrA, dataA0.NormalizedOnlineBalance(t.proto), dataA0, 0, voteLastValid) + require.NoError(t, err) + + dataA1 := trackerdb.BaseOnlineAccountData{ + MicroAlgos: basics.MicroAlgos{Raw: uint64(250)}, + } + _, err = oaw.InsertOnlineAccount(addrA, dataA1.NormalizedOnlineBalance(t.proto), dataA1, 1, voteLastValid) + require.NoError(t, err) + + addrB := basics.Address(crypto.Hash([]byte("b"))) + dataB := trackerdb.BaseOnlineAccountData{ + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + } + _, err = oaw.InsertOnlineAccount(addrB, dataB.NormalizedOnlineBalance(t.proto), dataB, 0, voteLastValid) + require.NoError(t, err) + + addrC := basics.Address(crypto.Hash([]byte("c"))) + dataC := trackerdb.BaseOnlineAccountData{ + MicroAlgos: basics.MicroAlgos{Raw: uint64(30)}, + } + _, err = oaw.InsertOnlineAccount(addrC, dataC.NormalizedOnlineBalance(t.proto), dataC, 0, voteLastValid) + require.NoError(t, err) + + // + // test + // + + // read all accounts (with max accounts) + poA, err := ar.OnlineAccountsAll(2) + require.NoError(t, err) + require.Len(t, poA, 3) // account A has 2 records + 1 record from account B + + require.Equal(t, addrA, poA[0].Addr) + require.Equal(t, basics.MicroAlgos{Raw: uint64(200)}, poA[0].AccountData.MicroAlgos) + require.Equal(t, basics.Round(0), poA[0].UpdRound) + + require.Equal(t, addrA, poA[1].Addr) + require.Equal(t, basics.MicroAlgos{Raw: uint64(250)}, poA[1].AccountData.MicroAlgos) + require.Equal(t, basics.Round(1), poA[1].UpdRound) + + require.Equal(t, addrB, poA[2].Addr) + require.Equal(t, basics.MicroAlgos{Raw: uint64(100)}, poA[2].AccountData.MicroAlgos) + require.Equal(t, basics.Round(0), poA[2].UpdRound) +} + +func CustomTestAccountsOnlineTop(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + var testData []basics.Address + updRound := uint64(0) + for i := 0; i < 10; i++ { + addr := RandomAddress() + microAlgos := basics.MicroAlgos{Raw: uint64(10 + i*100)} + rewardBase := uint64(200 + i) + lastValid := uint64(500 + i) + data := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: microAlgos, + RewardsBase: rewardBase, + } + normalizedBal := data.NormalizedOnlineBalance(t.proto) + + // write + _, err := oaw.InsertOnlineAccount(addr, normalizedBal, data, updRound, lastValid) + require.NoError(t, err) + + testData = append(testData, addr) + } + + // read (all) + poA, err := ar.AccountsOnlineTop(basics.Round(0), 0, 10, t.proto) + require.NoError(t, err) + require.Contains(t, poA, testData[9]) // most money + require.Contains(t, poA, testData[0]) // least money + + // read (just a few) + poA, err = ar.AccountsOnlineTop(basics.Round(0), 1, 2, t.proto) + require.NoError(t, err) + require.Len(t, poA, 2) + require.Contains(t, poA, testData[8]) // (second most money, we skipped 1) + require.Contains(t, poA, testData[7]) // (third, we only have 2 items) +} + +func CustomTestLookupOnlineAccountDataByAddress(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + updRoundA := uint64(400) + lastValidA := uint64(500) + dataA := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + RewardsBase: uint64(200), + } + normalizedBalA := dataA.NormalizedOnlineBalance(t.proto) + + refA, err := oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA) + require.NoError(t, err) + + // + // test + // + + // check non-existing account + nonExistingAddr := RandomAddress() + _, _, err = ar.LookupOnlineAccountDataByAddress(nonExistingAddr) + require.Error(t, err) + require.Equal(t, trackerdb.ErrNotFound, err) // check the error type + + // read existing addr + readRef, readData, err := ar.LookupOnlineAccountDataByAddress(addrA) + require.NoError(t, err) + require.Equal(t, refA, readRef) // check ref is the same + // the method returns raw bytes, parse them + var badA trackerdb.BaseOnlineAccountData + err = protocol.Decode(readData, &badA) + require.NoError(t, err) + require.Equal(t, dataA, badA) +} + +func CustomTestOnlineAccountTotals(t *customT) { + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + oaor, err := t.db.MakeOnlineAccountsOptimizedReader() + require.NoError(t, err) + + // generate some test data + roundParams := []ledgercore.OnlineRoundParamsData{ + {OnlineSupply: 100}, + {OnlineSupply: 42}, + {OnlineSupply: 9000}, + } + err = aw.AccountsPutOnlineRoundParams(roundParams, basics.Round(3)) + require.NoError(t, err) + + // + // test + // + + // lookup totals + totals, err := oaor.LookupOnlineTotalsHistory(basics.Round(4)) + require.NoError(t, err) + require.Equal(t, basics.MicroAlgos{Raw: uint64(42)}, totals) + + // lookup not found + _, err = oaor.LookupOnlineTotalsHistory(basics.Round(121)) + require.Error(t, err) + require.Equal(t, trackerdb.ErrNotFound, err) +} + +func CustomTestOnlineAccountsDelete(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + oar, err := t.db.MakeOnlineAccountsOptimizedReader() + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + aw, err := t.db.MakeAccountsWriter() + require.NoError(t, err) + + // set round to 3 + // Note: this will be used to check that we read the round + expectedRound := basics.Round(3) + err = aw.UpdateAccountsRound(expectedRound) + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA1 := trackerdb.BaseOnlineAccountData{ + // some value so its NOT empty + BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(20)}, + RewardsBase: uint64(200), + } + normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(0), uint64(2)) + require.NoError(t, err) + + // generate some test data + dataA2 := trackerdb.BaseOnlineAccountData{ + // some value so its NOT empty + BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + RewardsBase: uint64(200), + } + normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(1), uint64(3)) + require.NoError(t, err) + + // generate some test data + addrB := RandomAddress() + dataB1 := trackerdb.BaseOnlineAccountData{ + // some value so its NOT empty + BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(75)}, + RewardsBase: uint64(200), + } + normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(2), uint64(3)) + require.NoError(t, err) + + // timeline + // round 0: A touched [0] + // round 1: A touched [1,0] + // round 2: B touched [2] + A remains [1,0] + + // + // the test + // + + // delete before 0 (no changes) + err = aw.OnlineAccountsDelete(basics.Round(0)) + require.NoError(t, err) + + // check they are all there + oas, err := ar.AccountsOnlineTop(basics.Round(0), 0, 10, t.proto) + require.NoError(t, err) + require.Len(t, oas, 1) + require.Equal(t, oas[addrA].MicroAlgos, basics.MicroAlgos{Raw: uint64(20)}) // check item + // read the accounts directly + poaA, err := oar.LookupOnline(addrA, basics.Round(0)) + require.NoError(t, err) + require.NotNil(t, poaA.Ref) // A was found + + // delete before round 1 + err = aw.OnlineAccountsDelete(basics.Round(1)) + require.NoError(t, err) + + // check they are all there + oas, err = ar.AccountsOnlineTop(basics.Round(1), 0, 10, t.proto) + require.NoError(t, err) + require.Len(t, oas, 1) + require.Equal(t, oas[addrA].MicroAlgos, basics.MicroAlgos{Raw: uint64(100)}) // check item + // read the accounts directly + poaA, err = oar.LookupOnline(addrA, basics.Round(1)) + require.NoError(t, err) + require.NotNil(t, poaA.Ref) // A was found + + // delete before round 2 + err = aw.OnlineAccountsDelete(basics.Round(2)) + require.NoError(t, err) + + // check they are all there + oas, err = ar.AccountsOnlineTop(basics.Round(2), 0, 10, t.proto) + require.NoError(t, err) + require.Len(t, oas, 2) + require.Equal(t, oas[addrB].MicroAlgos, basics.MicroAlgos{Raw: uint64(75)}) // check item + // read the accounts directly + poaA, err = oar.LookupOnline(addrA, basics.Round(2)) + require.NoError(t, err) + require.NotNil(t, poaA.Ref) // A is still found, the latest record is kept + require.Equal(t, basics.Round(1), poaA.UpdRound) // the latest we find is at 1 + poaB, err := oar.LookupOnline(addrB, basics.Round(2)) + require.NoError(t, err) + require.NotNil(t, poaB.Ref) // B was found +} + +func CustomTestAccountsOnlineExpired(t *customT) { + oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true) + require.NoError(t, err) + + ar, err := t.db.MakeAccountsReader() + require.NoError(t, err) + + // generate some test data + addrA := RandomAddress() + dataA1 := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(2)}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(20)}, + RewardsBase: uint64(0), + } + normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(0), uint64(2)) + require.NoError(t, err) + + // generate some test data + dataA2 := trackerdb.BaseOnlineAccountData{ + BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(5)}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(100)}, + RewardsBase: uint64(0), + } + normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(1), uint64(5)) + require.NoError(t, err) + + // generate some test data + addrB := RandomAddress() + dataB1 := trackerdb.BaseOnlineAccountData{ + // some value so its NOT empty + BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(7)}, + MicroAlgos: basics.MicroAlgos{Raw: uint64(75)}, + RewardsBase: uint64(0), + } + normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto) + + _, err = oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(2), uint64(7)) + require.NoError(t, err) + + // timeline + // round 0: A touched [0] // A expires at 2 + // round 1: A touched [1,0] // A expires at 5 + // round 2: B touched [2] + A remains [1,0] // A expires at 5, B expires at 7 + + // + // the test + // + + // read (none) + expAccts, err := ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(0), t.proto, 0) + require.NoError(t, err) + require.Empty(t, expAccts) + + // read (at acct round, voteRnd > lastValid) + expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(4), t.proto, 0) + require.NoError(t, err) + require.Len(t, expAccts, 1) + require.Equal(t, expAccts[addrA].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(20)}) // check item + + // read (at acct round, voteRnd = lastValid) + expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(2), t.proto, 0) + require.NoError(t, err) + require.Empty(t, expAccts) + + // read (at acct round, voteRnd < lastValid) + expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(1), t.proto, 0) + require.NoError(t, err) + require.Empty(t, expAccts) + + // read (take latest exp value) + expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(1), basics.Round(4), t.proto, 0) + require.NoError(t, err) + require.Len(t, expAccts, 0) + + // read (all) + expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(3), basics.Round(20), t.proto, 0) + require.Len(t, expAccts, 2) + require.Equal(t, expAccts[addrA].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(100)}) // check item + require.Equal(t, expAccts[addrB].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(75)}) // check item +} diff --git a/ledger/store/trackerdb/testsuite/pebbledb_test.go b/ledger/store/trackerdb/testsuite/pebbledb_test.go new file mode 100644 index 0000000000..56f614b7e9 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/pebbledb_test.go @@ -0,0 +1,44 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "fmt" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver" + "github.com/algorand/go-algorand/logging" + "github.com/stretchr/testify/require" +) + +func TestPebbleDB(t *testing.T) { + dbFactory := func(proto config.ConsensusParams) dbForTests { + // create a tmp dir for the db, the testing runtime will clean it up automatically + dir := fmt.Sprintf("%s/db", t.TempDir()) + + db, err := pebbledbdriver.Open(dir, false, proto, logging.TestingLog(t)) + require.NoError(t, err) + + seedDb(t, db) + + return db + } + + // run the suite + runGenericTestsWithDB(t, dbFactory) +} diff --git a/ledger/store/trackerdb/testsuite/sqlitedb_test.go b/ledger/store/trackerdb/testsuite/sqlitedb_test.go new file mode 100644 index 0000000000..c4a3d1a7b8 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/sqlitedb_test.go @@ -0,0 +1,43 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "fmt" + "testing" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver" + "github.com/algorand/go-algorand/logging" + "github.com/stretchr/testify/require" +) + +func TestSqliteDB(t *testing.T) { + dbFactory := func(config.ConsensusParams) dbForTests { + // create a tmp dir for the db, the testing runtime will clean it up automatically + fn := fmt.Sprintf("%s/tracker-db.sqlite", t.TempDir()) + db, err := sqlitedriver.Open(fn, false, logging.TestingLog(t)) + require.NoError(t, err) + + seedDb(t, db) + + return db + } + + // run the suite + runGenericTestsWithDB(t, dbFactory) +} diff --git a/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go b/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go new file mode 100644 index 0000000000..bc42141735 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go @@ -0,0 +1,130 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +import ( + "context" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/stretchr/testify/require" +) + +func init() { + // register tests that will run on each KV implementation + registerTest("stateproofs-crud", CustomTestStateproofsReadWrite) + registerTest("stateproofs-query-all", CustomTestStateproofsQueryAll) +} + +func CustomTestStateproofsReadWrite(t *customT) { + spw := t.db.MakeSpVerificationCtxWriter() + spr := t.db.MakeSpVerificationCtxReader() + + // + // test + // + + // store no items + err := spw.StoreSPContexts(context.Background(), []*ledgercore.StateProofVerificationContext{}) + require.NoError(t, err) + + // store some items + vcs := []*ledgercore.StateProofVerificationContext{ + { + LastAttestedRound: basics.Round(0), + OnlineTotalWeight: basics.MicroAlgos{Raw: 42}, + }, + { + LastAttestedRound: basics.Round(1), + OnlineTotalWeight: basics.MicroAlgos{Raw: 100}, + }, + { + LastAttestedRound: basics.Round(2), + OnlineTotalWeight: basics.MicroAlgos{Raw: 200}, + }, + } + err = spw.StoreSPContexts(context.Background(), vcs) + require.NoError(t, err) + + // read non-existing item + vc, err := spr.LookupSPContext(basics.Round(9000)) + require.Error(t, err) + require.Equal(t, trackerdb.ErrNotFound, err) + + // read back a single item + vc, err = spr.LookupSPContext(basics.Round(0)) + require.NoError(t, err) + require.Equal(t, basics.Round(0), vc.LastAttestedRound) // check round is set + require.Equal(t, basics.MicroAlgos{Raw: 42}, vc.OnlineTotalWeight) // check payload is read + + // delete some items + err = spw.DeleteOldSPContexts(context.Background(), basics.Round(1)) + require.NoError(t, err) + + // read delete items + vc, err = spr.LookupSPContext(basics.Round(0)) + require.Error(t, err) + require.Equal(t, trackerdb.ErrNotFound, err) + + // read back remaining items + vc, err = spr.LookupSPContext(basics.Round(1)) + require.NoError(t, err) + require.Equal(t, basics.Round(1), vc.LastAttestedRound) // check round is set + require.Equal(t, basics.MicroAlgos{Raw: 100}, vc.OnlineTotalWeight) // check payload is read + + // read back remaining items + vc, err = spr.LookupSPContext(basics.Round(2)) + require.NoError(t, err) + require.Equal(t, basics.Round(2), vc.LastAttestedRound) // check round is set + require.Equal(t, basics.MicroAlgos{Raw: 200}, vc.OnlineTotalWeight) // check payload is read +} + +func CustomTestStateproofsQueryAll(t *customT) { + spw := t.db.MakeSpVerificationCtxWriter() + spr := t.db.MakeSpVerificationCtxReader() + + // prepare the test with some data + // store some items + vcs := []*ledgercore.StateProofVerificationContext{ + { + LastAttestedRound: basics.Round(0), + OnlineTotalWeight: basics.MicroAlgos{Raw: 42}, + }, + { + LastAttestedRound: basics.Round(1), + OnlineTotalWeight: basics.MicroAlgos{Raw: 100}, + }, + { + LastAttestedRound: basics.Round(2), + OnlineTotalWeight: basics.MicroAlgos{Raw: 200}, + }, + } + err := spw.StoreSPContexts(context.Background(), vcs) + require.NoError(t, err) + + // + // test + // + + // read all data + result, err := spr.GetAllSPContexts(context.Background()) + require.NoError(t, err) + require.Len(t, result, 3) // check all items are present + require.Equal(t, basics.Round(0), result[0].LastAttestedRound) // check first item + require.Equal(t, basics.Round(2), result[2].LastAttestedRound) // check last item +} diff --git a/ledger/store/trackerdb/testsuite/utils_test.go b/ledger/store/trackerdb/testsuite/utils_test.go new file mode 100644 index 0000000000..342b8a5b89 --- /dev/null +++ b/ledger/store/trackerdb/testsuite/utils_test.go @@ -0,0 +1,440 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package testsuite + +// A collection of utility functions and types to write the tests in this module. + +import ( + "bytes" + "context" + "io" + "sort" + "testing" + "time" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/ledger/store/trackerdb" + "github.com/algorand/go-algorand/ledger/store/trackerdb/generickv" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/algorand/go-algorand/util/db" + "github.com/stretchr/testify/require" +) + +type customT struct { + db dbForTests + proto config.ConsensusParams + *testing.T +} + +type dbForTests interface { + // generickv.KvWrite + // generickv.KvRead + trackerdb.Store +} + +type genericTestEntry struct { + name string + f func(*customT) +} + +// list of tests to be run on each KV DB implementation +var genericTests []genericTestEntry + +// registerTest registers the given test with the suite +func registerTest(name string, f func(*customT)) { + genericTests = append(genericTests, genericTestEntry{name, f}) +} + +// runGenericTestsWithDB runs a generic set of tests on the given database +func runGenericTestsWithDB(t *testing.T, dbFactory func(config.ConsensusParams) (db dbForTests)) { + proto := config.Consensus[protocol.ConsensusCurrentVersion] + for _, entry := range genericTests { + // run each test defined in the suite using the Golang subtest + t.Run(entry.name, func(t *testing.T) { + partitiontest.PartitionTest(t) + // instantiate a new db for each test + entry.f(&customT{dbFactory(proto), proto, t}) + }) + } +} + +func seedDb(t *testing.T, db dbForTests) { + params := trackerdb.Params{InitProto: protocol.ConsensusCurrentVersion} + _, err := db.RunMigrations(context.Background(), params, logging.TestingLog(t), trackerdb.AccountDBVersion) + require.NoError(t, err) +} + +// RandomAddress generates a random address +// +// TODO: this method is defined in ledgertesting, should be moved up to basics so it can be used in more places +func RandomAddress() basics.Address { + var addr basics.Address + crypto.RandBytes(addr[:]) + return addr +} + +type mockDB struct { + kvs kvstore + proto config.ConsensusParams + // use the generickv implementations + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint +} + +func makeMockDB(proto config.ConsensusParams) trackerdb.Store { + kvs := kvstore{data: make(map[string][]byte)} + var db trackerdb.Store + db = &mockDB{ + kvs, + proto, + generickv.MakeReader(&kvs, proto), + generickv.MakeWriter(db, &kvs, &kvs), + generickv.MakeCatchpoint(), + } + return db +} + +func (db *mockDB) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) { + // TODO + return nil +} + +func (db *mockDB) IsSharedCacheConnection() bool { + return false +} + +// RunMigrations implements trackerdb.Store +func (db *mockDB) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + // create a anonym struct that impls the interface for the migration runner + aux := struct { + *mockDB + *kvstore + }{db, &db.kvs} + return generickv.RunMigrations(ctx, aux, params, targetVersion) +} + +// Batch implements trackerdb.Store +func (db *mockDB) Batch(fn trackerdb.BatchFn) (err error) { + return db.BatchContext(context.Background(), fn) +} + +// BatchContext implements trackerdb.Store +func (db *mockDB) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) { + handle, err := db.BeginBatch(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the batch + err = fn(ctx, handle) + if err != nil { + return + } + + // commit the batch + err = handle.Commit() + if err != nil { + return + } + + return err +} + +// BeginBatch implements trackerdb.Store +func (db *mockDB) BeginBatch(ctx context.Context) (trackerdb.Batch, error) { + scope := mockBatch{db} + return &struct { + mockBatch + trackerdb.Writer + }{scope, generickv.MakeWriter(db, &scope, &db.kvs)}, nil +} + +// Snapshot implements trackerdb.Store +func (db *mockDB) Snapshot(fn trackerdb.SnapshotFn) (err error) { + return db.SnapshotContext(context.Background(), fn) +} + +// SnapshotContext implements trackerdb.Store +func (db *mockDB) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) { + handle, err := db.BeginSnapshot(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the snapshot + err = fn(ctx, handle) + if err != nil { + return + } + + return err +} + +// BeginSnapshot implements trackerdb.Store +func (db *mockDB) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) { + scope := mockSnapshot{db} + return &struct { + mockSnapshot + trackerdb.Reader + }{scope, generickv.MakeReader(&scope, db.proto)}, nil +} + +// Transaction implements trackerdb.Store +func (db *mockDB) Transaction(fn trackerdb.TransactionFn) (err error) { + return db.TransactionContext(context.Background(), fn) +} + +// TransactionContext implements trackerdb.Store +func (db *mockDB) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) { + handle, err := db.BeginTransaction(ctx) + if err != nil { + return + } + defer handle.Close() + + // run the transaction + err = fn(ctx, handle) + if err != nil { + return + } + + // commit the transaction + err = handle.Commit() + if err != nil { + return + } + + return err +} + +// BeginTransaction implements trackerdb.Store +func (db *mockDB) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) { + scope := mockTransaction{db, db.proto} + + return &struct { + mockTransaction + trackerdb.Reader + trackerdb.Writer + trackerdb.Catchpoint + }{scope, generickv.MakeReader(&scope, db.proto), generickv.MakeWriter(db, &scope, &scope), generickv.MakeCatchpoint()}, nil +} + +func (db *mockDB) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) { + // TODO + return stats, nil +} + +func (db *mockDB) ResetToV6Test(ctx context.Context) error { + // TODO + return nil +} + +func (db *mockDB) Close() { + // TODO +} + +type kvstore struct { + data map[string][]byte +} + +func (kvs *kvstore) Set(key, value []byte) error { + kvs.data[string(key)] = value + return nil +} + +func (kvs *kvstore) Get(key []byte) (data []byte, closer io.Closer, err error) { + data, ok := kvs.data[string(key)] + if !ok { + err = trackerdb.ErrNotFound + return + } + return data, io.NopCloser(bytes.NewReader(data)), nil +} + +func (kvs *kvstore) NewIter(low, high []byte, reverse bool) generickv.KvIter { + // + var keys []string + + slow := string(low) + shigh := string(high) + + for k := range kvs.data { + if k > slow && k < shigh { + keys = append(keys, k) + } + } + + sort.Strings(keys) + if reverse { + for i, j := 0, len(keys)-1; i < j; i, j = i+1, j-1 { + keys[i], keys[j] = keys[j], keys[i] + } + } + + return &mockIter{kvs, keys, -1} +} + +func (kvs *kvstore) Delete(key []byte) error { + delete(kvs.data, string(key)) + return nil +} + +func (kvs *kvstore) DeleteRange(start, end []byte) error { + var toDelete []string + for k := range kvs.data { + if k > string(start) && k < string(end) { + toDelete = append(toDelete, k) + } + } + for i := range toDelete { + delete(kvs.data, toDelete[i]) + } + return nil +} + +type mockSnapshot struct { + db *mockDB +} + +func (ss mockSnapshot) Get(key []byte) (value []byte, closer io.Closer, err error) { + return ss.db.kvs.Get(key) +} + +func (ss mockSnapshot) NewIter(low, high []byte, reverse bool) generickv.KvIter { + return ss.db.kvs.NewIter(low, high, reverse) +} +func (ss mockSnapshot) Close() error { + return nil +} + +type mockTransaction struct { + db *mockDB + proto config.ConsensusParams +} + +func (txs mockTransaction) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) { + // create a anonym struct that impls the interface for the migration runner + aux := struct { + *mockDB + *kvstore + }{txs.db, &txs.db.kvs} + return generickv.RunMigrations(ctx, aux, params, targetVersion) +} + +func (txs mockTransaction) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + return time.Now(), nil +} + +func (txs mockTransaction) Close() error { + return nil +} + +func (txs mockTransaction) Commit() error { + return nil +} + +func (txs mockTransaction) Set(key, value []byte) error { + return txs.db.kvs.Set(key, value) +} + +func (txs mockTransaction) Get(key []byte) (value []byte, closer io.Closer, err error) { + return txs.db.kvs.Get(key) +} + +func (txs mockTransaction) NewIter(low, high []byte, reverse bool) generickv.KvIter { + return txs.db.kvs.NewIter(low, high, reverse) +} + +func (txs mockTransaction) Delete(key []byte) error { + return txs.db.kvs.Delete(key) +} + +func (txs mockTransaction) DeleteRange(start, end []byte) error { + return txs.db.kvs.DeleteRange(start, end) +} + +type mockBatch struct { + db *mockDB +} + +func (bs mockBatch) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) { + return time.Now(), nil +} + +func (bs mockBatch) Close() error { + return nil +} + +func (bs mockBatch) Commit() error { + return nil +} + +func (bs mockBatch) Set(key, value []byte) error { + return bs.db.kvs.Set(key, value) +} + +func (bs mockBatch) Delete(key []byte) error { + return bs.db.kvs.Delete(key) +} + +func (bs mockBatch) DeleteRange(start, end []byte) error { + return bs.db.kvs.DeleteRange(start, end) +} + +type mockIter struct { + kvs *kvstore + keys []string + curr int +} + +func (iter *mockIter) Next() bool { + if iter.curr < len(iter.keys)-1 { + iter.curr++ + return true + } + iter.curr = -1 + return false +} + +func (iter *mockIter) Key() []byte { + return []byte(iter.keys[iter.curr]) +} + +func (iter *mockIter) KeySlice() generickv.Slice { + return nil +} + +func (iter *mockIter) Value() ([]byte, error) { + return iter.kvs.data[iter.keys[iter.curr]], nil +} + +func (iter *mockIter) ValueSlice() (generickv.Slice, error) { + return nil, nil +} + +func (iter *mockIter) Valid() bool { + return iter.curr != -1 +} + +func (iter *mockIter) Close() {} diff --git a/ledger/tracker.go b/ledger/tracker.go index 39ea9d4b18..be672cb9f8 100644 --- a/ledger/tracker.go +++ b/ledger/tracker.go @@ -30,6 +30,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/eval" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/ledger/store/catchpointdb" "github.com/algorand/go-algorand/ledger/store/trackerdb" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" @@ -134,7 +135,8 @@ type ledgerTracker interface { // ledgerForTracker defines the part of the ledger that a tracker can // access. This is particularly useful for testing trackers in isolation. type ledgerForTracker interface { - trackerDB() trackerdb.TrackerStore + trackerDB() trackerdb.Store + catchpointDB() catchpointdb.Store blockDB() db.Pair trackerLog() logging.Logger trackerEvalVerified(bookkeeping.Block, eval.LedgerForEvaluator) (ledgercore.StateDelta, error) @@ -174,7 +176,7 @@ type trackerRegistry struct { // cached to avoid SQL queries. dbRound basics.Round - dbs trackerdb.TrackerStore + dbs trackerdb.Store log logging.Logger // the synchronous mode that would be used for the account database. @@ -554,7 +556,7 @@ func (tr *trackerRegistry) commitRound(dcc *deferredCommitContext) error { start := time.Now() ledgerCommitroundCount.Inc(nil) err = tr.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -566,7 +568,7 @@ func (tr *trackerRegistry) commitRound(dcc *deferredCommitContext) error { } } - return arw.UpdateAccountsRound(dbRound + basics.Round(offset)) + return aw.UpdateAccountsRound(dbRound + basics.Round(offset)) }) ledgerCommitroundMicros.AddMicrosecondsSince(start, nil) diff --git a/ledger/trackerdb.go b/ledger/trackerdb.go index 8955a27c0f..bf23075813 100644 --- a/ledger/trackerdb.go +++ b/ledger/trackerdb.go @@ -38,44 +38,57 @@ func trackerDBInitialize(l ledgerForTracker, catchpointEnabled bool, dbPathPrefi return } - err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { - arw, err := tx.MakeAccountsReaderWriter() - if err != nil { - return err - } + tp := trackerdb.Params{ + InitAccounts: l.GenesisAccounts(), + InitProto: l.GenesisProtoVersion(), + GenesisHash: l.GenesisHash(), + FromCatchpoint: false, + CatchpointEnabled: catchpointEnabled, + DbPathPrefix: dbPathPrefix, + BlockDb: bdbs, + } - tp := trackerdb.Params{ - InitAccounts: l.GenesisAccounts(), - InitProto: l.GenesisProtoVersion(), - GenesisHash: l.GenesisHash(), - FromCatchpoint: false, - CatchpointEnabled: catchpointEnabled, - DbPathPrefix: dbPathPrefix, - BlockDb: bdbs, - } - var err0 error - mgr, err0 = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion) - if err0 != nil { - return err0 - } - lastBalancesRound, err := arw.AccountsRound() - if err != nil { - return err - } - // Check for blocks DB and tracker DB un-sync - if lastBalancesRound > lastestBlockRound { - log.Warnf("trackerDBInitialize: resetting accounts DB (on round %v, but blocks DB's latest is %v)", lastBalancesRound, lastestBlockRound) - err0 = arw.AccountsReset(ctx) - if err0 != nil { - return err0 + // run migrations + mgr, err = dbs.RunMigrations(context.Background(), tp, log, trackerdb.AccountDBVersion) + if err != nil { + return + } + + // create reader for db + ar, err := dbs.MakeAccountsReader() + if err != nil { + return + } + + // check current round + lastBalancesRound, err := ar.AccountsRound() + if err != nil { + return + } + + // Check for blocks DB and tracker DB un-sync + if lastBalancesRound > lastestBlockRound { + log.Warnf("trackerDBInitialize: resetting accounts DB (on round %v, but blocks DB's latest is %v)", lastBalancesRound, lastestBlockRound) + err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error { + var aw trackerdb.AccountsWriterExt + aw, err = tx.MakeAccountsWriter() + if err != nil { + return err + } + err = aw.AccountsReset(ctx) + if err != nil { + return err } - mgr, err0 = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion) - if err0 != nil { - return err0 + mgr, err = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion) + if err != nil { + return err } + return nil + }) + if err != nil { + return } - return nil - }) + } return } diff --git a/ledger/txtail.go b/ledger/txtail.go index a86a8af5bc..7d71ea27ef 100644 --- a/ledger/txtail.go +++ b/ledger/txtail.go @@ -273,7 +273,7 @@ func (t *txTail) prepareCommit(dcc *deferredCommitContext) (err error) { } func (t *txTail) commitRound(ctx context.Context, tx trackerdb.TransactionScope, dcc *deferredCommitContext) error { - arw, err := tx.MakeAccountsReaderWriter() + aw, err := tx.MakeAccountsWriter() if err != nil { return err } @@ -282,7 +282,7 @@ func (t *txTail) commitRound(ctx context.Context, tx trackerdb.TransactionScope, // the formula is similar to the committedUpTo: rnd + 1 - retain size forgetBeforeRound := (dcc.newBase() + 1).SubSaturate(basics.Round(dcc.txTailRetainSize)) baseRound := dcc.oldBase + 1 - if err := arw.TxtailNewRound(ctx, baseRound, dcc.txTailDeltas, forgetBeforeRound); err != nil { + if err := aw.TxtailNewRound(ctx, baseRound, dcc.txTailDeltas, forgetBeforeRound); err != nil { return fmt.Errorf("txTail: unable to persist new round %d : %w", baseRound, err) } return nil diff --git a/ledger/txtail_test.go b/ledger/txtail_test.go index a21af51208..ed75b591c9 100644 --- a/ledger/txtail_test.go +++ b/ledger/txtail_test.go @@ -150,7 +150,7 @@ func (t *txTailTestLedger) initialize(ts *testing.T, protoVersion protocol.Conse // create a corresponding blockdb. inMemory := true t.blockDBs, _ = storetesting.DbOpenTest(ts, inMemory) - t.trackerDBs, _ = sqlitedriver.DbOpenTrackerTest(ts, inMemory) + t.trackerDBs, _ = sqlitedriver.OpenForTesting(ts, inMemory) t.protoVersion = protoVersion err := t.trackerDBs.Batch(func(transactionCtx context.Context, tx trackerdb.BatchScope) (err error) { diff --git a/test/testdata/configs/config-v28.json b/test/testdata/configs/config-v28.json new file mode 100644 index 0000000000..5ab6675299 --- /dev/null +++ b/test/testdata/configs/config-v28.json @@ -0,0 +1,118 @@ +{ + "Version": 28, + "AccountUpdatesStatsInterval": 5000000000, + "AccountsRebuildSynchronousMode": 1, + "AgreementIncomingBundlesQueueLength": 15, + "AgreementIncomingProposalsQueueLength": 50, + "AgreementIncomingVotesQueueLength": 20000, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BlockServiceCustomFallbackEndpoints": "", + "BroadcastConnectionsLimit": -1, + "CadaverDirectory": "", + "CadaverSizeTarget": 0, + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupBlockValidateMode": 0, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "DNSBootstrapID": ".algorand.network", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DeadlockDetectionThreshold": 30, + "DisableLedgerLRUCache": false, + "DisableLocalhostConnectionRateLimit": true, + "DisableNetworking": false, + "DisableOutgoingConnectionThrottling": false, + "EnableAccountUpdatesStats": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableBlockServiceFallbackToArchiver": true, + "EnableCatchupFromArchiveServers": false, + "EnableDeveloperAPI": false, + "EnableExperimentalAPI": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableRuntimeMetrics": false, + "EnableTopAccountsReporting": false, + "EnableTxBacklogRateLimiting": false, + "EnableUsageLog": false, + "EnableVerbosedTransactionSyncLogging": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceFetchTransactions": false, + "ForceRelayMessages": false, + "GossipFanout": 4, + "HeartbeatUpdateInterval": 600, + "IncomingConnectionsLimit": 2400, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "IsIndexerActive": false, + "LedgerSynchronousMode": 2, + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogSizeLimit": 1073741824, + "MaxAPIBoxPerApplication": 100000, + "MaxAPIResourcesPerAccount": 100000, + "MaxAcctLookback": 4, + "MaxCatchpointDownloadDuration": 7200000000000, + "MaxConnectionsPerIP": 15, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkMessageTraceServer": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "ParticipationKeysRefreshInterval": 60000000000, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "ProposalAssemblyTime": 500000000, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "ReservedFDs": 256, + "RestConnectionsHardLimit": 2048, + "RestConnectionsSoftLimit": 1024, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "StorageEngine": "sqlite", + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TransactionSyncDataExchangeRate": 0, + "TransactionSyncSignificantMessageThreshold": 0, + "TxBacklogReservedCapacityPerPeer": 20, + "TxBacklogServiceRateWindowSeconds": 10, + "TxBacklogSize": 26000, + "TxIncomingFilteringFlags": 1, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 75000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "", + "VerifiedTranscationsCacheSize": 150000 +} diff --git a/tools/block-generator/go.mod b/tools/block-generator/go.mod index 306627f71b..ea785629bf 100644 --- a/tools/block-generator/go.mod +++ b/tools/block-generator/go.mod @@ -23,28 +23,47 @@ require ( github.com/algorand/oapi-codegen v1.12.0-algorand.0 // indirect github.com/algorand/websocket v1.4.6 // indirect github.com/aws/aws-sdk-go v1.33.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cockroachdb/errors v1.8.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 // indirect + github.com/cockroachdb/redact v1.0.8 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/consensys/gnark-crypto v0.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 // indirect github.com/dchest/siphash v1.2.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.11.13 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-sqlite3 v1.10.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/miekg/dns v1.1.41 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/olivere/elastic v6.2.14+incompatible // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.12.0 // indirect + github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.1.0 // indirect + golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 // indirect ) diff --git a/tools/block-generator/go.sum b/tools/block-generator/go.sum index bd2732c8fc..cfaa581bca 100644 --- a/tools/block-generator/go.sum +++ b/tools/block-generator/go.sum @@ -13,20 +13,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -35,7 +21,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -46,17 +31,22 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k= github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g= github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 h1:nwYN+GQ7Z5OOfZwqBO1ma7DSlP7S1YrKWICOyjkwqrc= @@ -67,59 +57,52 @@ github.com/algorand/go-deadlock v0.2.2 h1:L7AKATSUCzoeVuOgpTipfCEjdUu5ECmlje8R7l github.com/algorand/go-deadlock v0.2.2/go.mod h1:Hat1OXKqKNUcN/iv74FjGhF4hsOE2l7gOgQ9ZVIq6Fk= github.com/algorand/go-sumhash v0.1.0 h1:b/QRhyLuF//vOcicBIxBXYW8bERNoeLxieht/dUYpVg= github.com/algorand/go-sumhash v0.1.0/go.mod h1:OOe7jdDWUhLkuP1XytkK5gnLu9entAviN5DfDZh6XAc= -github.com/algorand/graphtrace v0.1.0/go.mod h1:HscLQrzBdH1BH+5oehs3ICd8SYcXvnSL9BjfTu8WHCc= github.com/algorand/msgp v1.1.53 h1:D6HKLyvLE6ltfsf8Apsrc+kqYb/CcOZEAfh1DpkPrNg= github.com/algorand/msgp v1.1.53/go.mod h1:5K3d58/poT5fPmtiwuQft6GjgSrVEM46KoXdLrID8ZU= github.com/algorand/oapi-codegen v1.12.0-algorand.0 h1:W9PvED+wAJc+9EeXPONnA+0zE9UhynEqoDs4OgAxKhk= github.com/algorand/oapi-codegen v1.12.0-algorand.0/go.mod h1:tIWJ9K/qrLDVDt5A1p82UmxZIEGxv2X+uoujdhEAL48= github.com/algorand/websocket v1.4.6 h1:I0kV4EYwatuUrKtNiwzYYgojgwh6pksDmlqntKG2Woc= github.com/algorand/websocket v1.4.6/go.mod h1:HJmdGzFtnlUQ4nTzZP6WrT29oGYf1t6Ybi64vROcT+M= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY= github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e h1:CHPYEbz71w8DqJ7DRIq+MXyCQsdibK08vdcQTY4ufas= -github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e/go.mod h1:6Xhs0ZlsRjXLIiSMLKafbZxML/j30pg9Z1priLuha5s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/consensys/bavard v0.1.10/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 h1:Mp4A1rMoAE45WdoVI7a7jwjiIj8eZEXkl3aYbdMJACs= +github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4/go.mod h1:rWEpkT1ud5qGG2m1HqTBRWbLgi+oodC4+BWC46uVPjw= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/gnark-crypto v0.7.0 h1:rwdy8+ssmLYRqKp+ryRRgQJl/rCq2uv+n83cOydm5UE= github.com/consensys/gnark-crypto v0.7.0/go.mod h1:KPSuJzyxkJA8xZ/+CV47tyqkr9MmpZA3PXivK4VPrVg= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/daixiang0/gci v0.3.2/go.mod h1:jaASoJmv/ykO9dAAPy31iJnreV19248qKDdVWf3QgC4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -127,57 +110,54 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4= github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -185,8 +165,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -202,12 +180,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -217,17 +194,13 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -235,138 +208,96 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo= -github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= -github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= -github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= -github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/moq v0.2.7/go.mod h1:kITsx543GOENm48TUAQyJ9+SAvFSr7iGQXPoth/VUBk= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= @@ -375,120 +306,121 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olivere/elastic v6.2.14+incompatible h1:k+KadwNP/dkXE0/eu+T6otk1+5fe0tEpPyQJ4XVm5i8= github.com/olivere/elastic v6.2.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -501,6 +433,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -513,8 +447,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -523,21 +455,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -546,7 +473,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -557,26 +484,15 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -584,18 +500,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -607,14 +512,12 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -624,18 +527,19 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -645,74 +549,44 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -720,7 +594,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -749,25 +622,12 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -785,29 +645,13 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -831,46 +675,13 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -883,22 +694,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -911,28 +706,27 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 h1:q/fZgS8MMadqFFGa8WL4Oyz+TmjiZfi8UrzWhTl8d5w= gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009/go.mod h1:O0bY1e/dSoxMYZYTHP0SWKxG5EWLEvKR9/cOjWPPMKU= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -942,7 +736,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +pgregory.net/rapid v0.4.8 h1:d+5SGZWUbJPbl3ss6tmPFqnNeQR6VDOFly+eTjwPiEw= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/tools/debug/transplanter/main.go b/tools/debug/transplanter/main.go index 86528254e8..8120f8107a 100644 --- a/tools/debug/transplanter/main.go +++ b/tools/debug/transplanter/main.go @@ -212,18 +212,18 @@ func transcribeSnappyLog(filePath string, output chan txGroupItem, wg *sync.Wait } defer file.Close() - var headerDecoder headerDecoder + var hd headerDecoder if strings.Contains(path.Base(filePath), "_v2") { - headerDecoder = decoderV2{} + hd = decoderV2{} } else { - headerDecoder = decoderV1{} + hd = decoderV1{} } snappyReader := snappy.NewReader(file) var n int for { - headers, lenMsg, err := headerDecoder.decodeHeader(snappyReader) + headers, lenMsg, err := hd.decodeHeader(snappyReader) if err == io.EOF { break } else if err != nil { @@ -395,7 +395,7 @@ func main() { os.Exit(1) } syncRound := uint64(*roundStart) - cfg.MaxAcctLookback + 1 - err := followerNode.SetSyncRound(syncRound) + err = followerNode.SetSyncRound(syncRound) if err != nil { fmt.Fprintf(os.Stderr, "Cannot configure catchup: %v", err) os.Exit(1)