diff --git a/Makefile b/Makefile
index 450ab6999c..b0e387e0e8 100644
--- a/Makefile
+++ b/Makefile
@@ -92,7 +92,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
-MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
+MSGP_GENERATE := ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklesignature ./crypto/stateproof ./data/basics ./data/transactions ./data/stateproofmsg ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./network ./node ./ledger ./ledger/ledgercore ./ledger/store/trackerdb ./ledger/store/trackerdb/generickv ./ledger/store/catchpointdb ./ledger/encoded ./stateproof ./data/account ./daemon/algod/api/spec/v2
default: build
diff --git a/cmd/catchpointdump/database.go b/cmd/catchpointdump/database.go
index 7706bcd3dd..2205b4c524 100644
--- a/cmd/catchpointdump/database.go
+++ b/cmd/catchpointdump/database.go
@@ -27,6 +27,7 @@ import (
"github.com/algorand/go-algorand/crypto/merkletrie"
"github.com/algorand/go-algorand/ledger"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
"github.com/algorand/go-algorand/util/db"
@@ -103,30 +104,14 @@ func printDbVersion(staging bool, version uint64, outFile *os.File) {
}
func getVersion(filename string, staging bool) (uint64, error) {
- dbAccessor, err := db.MakeAccessor(filename, true, false)
- if err != nil || dbAccessor.Handle == nil {
- return 0, err
- }
- defer dbAccessor.Close()
- var version uint64
- err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
- if staging {
- // writing the version of the catchpoint file start only on ver >= CatchpointFileVersionV7.
- // in case the catchpoint version does not exists ReadCatchpointStateUint64 returns 0
- cw := sqlitedriver.NewCatchpointSQLReaderWriter(tx)
- version, err = cw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion)
- return err
- }
-
- versionAsInt32, err := db.GetUserVersion(ctx, tx)
- version = uint64(versionAsInt32)
- return err
- })
- if err != nil {
+ rdb, err := db.MakeAccessor(filename, true, false)
+ if err != nil || rdb.Handle == nil {
return 0, err
}
+ catchpointDbs := catchpointdb.MakeStore(db.Pair{Rdb: rdb, Wdb: rdb})
+ defer catchpointDbs.Close()
- return version, nil
+ return catchpointDbs.GetVersion(context.Background(), staging)
}
var checkCmd = &cobra.Command{
diff --git a/config/localTemplate.go b/config/localTemplate.go
index a9dd313bb8..210e82f196 100644
--- a/config/localTemplate.go
+++ b/config/localTemplate.go
@@ -41,7 +41,7 @@ type Local struct {
// Version tracks the current version of the defaults so we can migrate old -> new
// This is specifically important whenever we decide to change the default value
// for an existing parameter. This field tag must be updated any time we add a new version.
- Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27"`
+ Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28"`
// environmental (may be overridden)
// When enabled, stores blocks indefinitely, otherwise, only the most recent blocks
@@ -510,6 +510,12 @@ type Local struct {
// EnableTxnEvalTracer turns on features in the BlockEvaluator which collect data on transactions, exposing them via algod APIs.
// It will store txn deltas created during block evaluation, potentially consuming much larger amounts of memory,
EnableTxnEvalTracer bool `version[27]:"false"`
+
+ // StorageEngine allows to control which type of storage to use for the ledger.
+ // Available options are:
+ // - sqlite (default)
+ // - pebbledb (experimental, in development)
+ StorageEngine string `version[28]:"sqlite"`
}
// DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers
diff --git a/config/local_defaults.go b/config/local_defaults.go
index 81e4a25870..fdfb77a29b 100644
--- a/config/local_defaults.go
+++ b/config/local_defaults.go
@@ -20,7 +20,7 @@
package config
var defaultLocal = Local{
- Version: 27,
+ Version: 28,
AccountUpdatesStatsInterval: 5000000000,
AccountsRebuildSynchronousMode: 1,
AgreementIncomingBundlesQueueLength: 15,
@@ -119,6 +119,7 @@ var defaultLocal = Local{
RestReadTimeoutSeconds: 15,
RestWriteTimeoutSeconds: 120,
RunHosted: false,
+ StorageEngine: "sqlite",
SuggestedFeeBlockHistory: 3,
SuggestedFeeSlidingWindowSize: 50,
TLSCertFile: "",
diff --git a/go.mod b/go.mod
index efaf373227..b4ce0b3091 100644
--- a/go.mod
+++ b/go.mod
@@ -14,6 +14,7 @@ require (
github.com/algorand/oapi-codegen v1.12.0-algorand.0
github.com/algorand/websocket v1.4.6
github.com/aws/aws-sdk-go v1.33.0
+ github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4
github.com/consensys/gnark-crypto v0.7.0
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018
github.com/dchest/siphash v1.2.1
@@ -21,6 +22,7 @@ require (
github.com/getkin/kin-openapi v0.107.0
github.com/gofrs/flock v0.7.0
github.com/golang/snappy v0.0.4
+ github.com/google/go-cmp v0.5.6
github.com/google/go-querystring v1.0.0
github.com/gorilla/mux v1.8.0
github.com/jmoiron/sqlx v1.2.0
@@ -30,7 +32,7 @@ require (
github.com/miekg/dns v1.1.27
github.com/olivere/elastic v6.2.14+incompatible
github.com/sirupsen/logrus v1.8.1
- github.com/spf13/cobra v0.0.3
+ github.com/spf13/cobra v1.0.0
github.com/stretchr/testify v1.8.1
golang.org/x/crypto v0.1.0
golang.org/x/sys v0.7.0
@@ -41,35 +43,54 @@ require (
require (
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
- github.com/cpuguy83/go-md2man v1.0.8 // indirect
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.1.2 // indirect
+ github.com/cockroachdb/errors v1.8.1 // indirect
+ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+ github.com/cockroachdb/redact v1.0.8 // indirect
+ github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
+ github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fortytw2/leaktest v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/swag v0.19.5 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/invopop/yaml v0.1.0 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
+ github.com/klauspost/compress v1.11.13 // indirect
+ github.com/kr/pretty v0.2.1 // indirect
+ github.com/kr/text v0.2.0 // indirect
github.com/labstack/gommon v0.4.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/russross/blackfriday v1.5.2 // indirect
+ github.com/prometheus/client_golang v1.12.0 // indirect
+ github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
+ github.com/prometheus/common v0.32.1 // indirect
+ github.com/prometheus/procfs v0.7.3 // indirect
+ github.com/russross/blackfriday/v2 v2.0.1 // indirect
+ github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
+ golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+ google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index f173199fbe..304e431521 100644
--- a/go.sum
+++ b/go.sum
@@ -1,6 +1,54 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
+github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
+github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k=
github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g=
github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 h1:nwYN+GQ7Z5OOfZwqBO1ma7DSlP7S1YrKWICOyjkwqrc=
@@ -21,14 +69,52 @@ github.com/algorand/websocket v1.4.6 h1:I0kV4EYwatuUrKtNiwzYYgojgwh6pksDmlqntKG2
github.com/algorand/websocket v1.4.6/go.mod h1:HJmdGzFtnlUQ4nTzZP6WrT29oGYf1t6Ybi64vROcT+M=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY=
github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e h1:CHPYEbz71w8DqJ7DRIq+MXyCQsdibK08vdcQTY4ufas=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
+github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
+github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
+github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
+github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 h1:Mp4A1rMoAE45WdoVI7a7jwjiIj8eZEXkl3aYbdMJACs=
+github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4/go.mod h1:rWEpkT1ud5qGG2m1HqTBRWbLgi+oodC4+BWC46uVPjw=
+github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
+github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/consensys/gnark-crypto v0.7.0 h1:rwdy8+ssmLYRqKp+ryRRgQJl/rCq2uv+n83cOydm5UE=
github.com/consensys/gnark-crypto v0.7.0/go.mod h1:KPSuJzyxkJA8xZ/+CV47tyqkr9MmpZA3PXivK4VPrVg=
-github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M=
-github.com/cpuguy83/go-md2man v1.0.8/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -36,12 +122,45 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=
github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
+github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
github.com/getkin/kin-openapi v0.107.0 h1:bxhL6QArW7BXQj8NjXfIJQy680NsMKd25nwhvpCXchg=
github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
+github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
@@ -49,153 +168,667 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g=
github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
+github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
+github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
+github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
+github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
+github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
+github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
+github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
+github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.9.1 h1:GliPYSpzGKlyOhqIbG8nmHBo3i1saKWFOgh41AN3b+Y=
github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8=
github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
+github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM=
+github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olivere/elastic v6.2.14+incompatible h1:k+KadwNP/dkXE0/eu+T6otk1+5fe0tEpPyQJ4XVm5i8=
github.com/olivere/elastic v6.2.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg=
+github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y=
+github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 h1:q/fZgS8MMadqFFGa8WL4Oyz+TmjiZfi8UrzWhTl8d5w=
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009/go.mod h1:O0bY1e/dSoxMYZYTHP0SWKxG5EWLEvKR9/cOjWPPMKU=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -203,5 +836,15 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
pgregory.net/rapid v0.4.8 h1:d+5SGZWUbJPbl3ss6tmPFqnNeQR6VDOFly+eTjwPiEw=
pgregory.net/rapid v0.4.8/go.mod h1:Z5PbWqjvWR1I3UGjvboUuan4fe4ZYEYNLNQLExzCoUs=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/installer/config.json.example b/installer/config.json.example
index 4ae34a5716..31da4c1ac5 100644
--- a/installer/config.json.example
+++ b/installer/config.json.example
@@ -1,5 +1,5 @@
{
- "Version": 27,
+ "Version": 28,
"AccountUpdatesStatsInterval": 5000000000,
"AccountsRebuildSynchronousMode": 1,
"AgreementIncomingBundlesQueueLength": 15,
@@ -98,6 +98,7 @@
"RestReadTimeoutSeconds": 15,
"RestWriteTimeoutSeconds": 120,
"RunHosted": false,
+ "StorageEngine": "sqlite",
"SuggestedFeeBlockHistory": 3,
"SuggestedFeeSlidingWindowSize": 50,
"TLSCertFile": "",
diff --git a/ledger/acctdeltas.go b/ledger/acctdeltas.go
index e587a8fcfa..42d5d93b2f 100644
--- a/ledger/acctdeltas.go
+++ b/ledger/acctdeltas.go
@@ -298,7 +298,7 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope,
if len(a.misses) == 0 {
return nil
}
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
@@ -317,9 +317,9 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope,
if delta.oldResource.AcctRef != nil {
acctRef = delta.oldResource.AcctRef
} else if acctRef, ok = knownAddresses[addr]; !ok {
- acctRef, err = arw.LookupAccountRowID(addr)
+ acctRef, err = ar.LookupAccountRowID(addr)
if err != nil {
- if err != sql.ErrNoRows {
+ if err != sql.ErrNoRows && err != trackerdb.ErrNotFound {
err = fmt.Errorf("base account cannot be read while processing resource for addr=%s, aidx=%d: %w", addr.String(), aidx, err)
return err
@@ -330,7 +330,7 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope,
continue
}
}
- resDataBuf, err = arw.LookupResourceDataByAddrID(acctRef, aidx)
+ resDataBuf, err = ar.LookupResourceDataByAddrID(acctRef, aidx)
switch err {
case nil:
if len(resDataBuf) > 0 {
@@ -344,6 +344,10 @@ func (a *compactResourcesDeltas) resourcesLoadOld(tx trackerdb.TransactionScope,
err = fmt.Errorf("empty resource record: addrid=%d, aidx=%d", acctRef, aidx)
return err
}
+ case trackerdb.ErrNotFound:
+ // we don't have that account, just return an empty record.
+ a.updateOld(missIdx, trackerdb.PersistedResourcesData{AcctRef: acctRef, Aidx: aidx})
+ err = nil
case sql.ErrNoRows:
// we don't have that account, just return an empty record.
a.updateOld(missIdx, trackerdb.PersistedResourcesData{AcctRef: acctRef, Aidx: aidx})
@@ -466,7 +470,7 @@ func (a *compactAccountDeltas) accountsLoadOld(tx trackerdb.TransactionScope) (e
if len(a.misses) == 0 {
return nil
}
- arw, err := tx.MakeAccountsReaderWriter()
+ arw, err := tx.MakeAccountsOptimizedReader()
if err != nil {
return err
}
@@ -476,29 +480,13 @@ func (a *compactAccountDeltas) accountsLoadOld(tx trackerdb.TransactionScope) (e
}()
for _, idx := range a.misses {
addr := a.deltas[idx].address
- ref, acctDataBuf, err := arw.LookupAccountDataByAddress(addr)
- switch err {
- case nil:
- if len(acctDataBuf) > 0 {
- persistedAcctData := &trackerdb.PersistedAccountData{Addr: addr, Ref: ref}
- err = protocol.Decode(acctDataBuf, &persistedAcctData.AccountData)
- if err != nil {
- return err
- }
- a.updateOld(idx, *persistedAcctData)
- } else {
- // to retain backward compatibility, we will treat this condition as if we don't have the account.
- a.updateOld(idx, trackerdb.PersistedAccountData{Addr: addr, Ref: ref})
- }
- case sql.ErrNoRows:
- // we don't have that account, just return an empty record.
- a.updateOld(idx, trackerdb.PersistedAccountData{Addr: addr})
- // Note: the err will be ignored in this case since `err` is being shadowed.
- // this behaviour is equivalent to `err = nil`
- default:
+ data, err := arw.LookupAccount(addr)
+ if err != nil {
// unexpected error - let the caller know that we couldn't complete the operation.
return err
}
+ // update the account
+ a.updateOld(idx, data)
}
return
}
@@ -607,7 +595,7 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco
if len(a.misses) == 0 {
return nil
}
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
@@ -616,7 +604,7 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco
}()
for _, idx := range a.misses {
addr := a.deltas[idx].address
- ref, acctDataBuf, err := arw.LookupOnlineAccountDataByAddress(addr)
+ ref, acctDataBuf, err := ar.LookupOnlineAccountDataByAddress(addr)
switch err {
case nil:
if len(acctDataBuf) > 0 {
@@ -630,6 +618,10 @@ func (a *compactOnlineAccountDeltas) accountsLoadOld(tx trackerdb.TransactionSco
// empty data means offline account
a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr, Ref: ref})
}
+ case trackerdb.ErrNotFound:
+ // we don't have that account, just return an empty record.
+ a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr})
+ // TODO: phase out sql.ErrNoRows
case sql.ErrNoRows:
// we don't have that account, just return an empty record.
a.updateOld(idx, trackerdb.PersistedOnlineAccountData{Addr: addr})
diff --git a/ledger/acctdeltas_test.go b/ledger/acctdeltas_test.go
index 2dc748225b..0342f3fc33 100644
--- a/ledger/acctdeltas_test.go
+++ b/ledger/acctdeltas_test.go
@@ -52,14 +52,14 @@ import (
)
func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round, accts map[basics.Address]basics.AccountData) {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
require.NoError(t, err)
- r, err := arw.AccountsRound()
+ r, err := ar.AccountsRound()
require.NoError(t, err)
require.Equal(t, r, rnd)
- aor, err := tx.Testing().MakeAccountsOptimizedReader()
+ aor, err := tx.MakeAccountsOptimizedReader()
require.NoError(t, err)
var totalOnline, totalOffline, totalNotPart uint64
@@ -83,11 +83,11 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round
}
}
- all, err := arw.Testing().AccountsAllTest()
+ all, err := ar.Testing().AccountsAllTest()
require.NoError(t, err)
require.Equal(t, all, accts)
- totals, err := arw.AccountsTotals(context.Background(), false)
+ totals, err := ar.AccountsTotals(context.Background(), false)
require.NoError(t, err)
require.Equal(t, totalOnline, totals.Online.Money.Raw, "mismatching total online money")
require.Equal(t, totalOffline, totals.Offline.Money.Raw)
@@ -129,7 +129,7 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round
})
for i := 0; i < len(onlineAccounts); i++ {
- dbtop, err := arw.AccountsOnlineTop(rnd, 0, uint64(i), proto)
+ dbtop, err := ar.AccountsOnlineTop(rnd, 0, uint64(i), proto)
require.NoError(t, err)
require.Equal(t, i, len(dbtop))
@@ -139,7 +139,7 @@ func checkAccounts(t *testing.T, tx trackerdb.TransactionScope, rnd basics.Round
}
}
- top, err := arw.AccountsOnlineTop(rnd, 0, uint64(len(onlineAccounts)+1), proto)
+ top, err := ar.AccountsOnlineTop(rnd, 0, uint64(len(onlineAccounts)+1), proto)
require.NoError(t, err)
require.Equal(t, len(top), len(onlineAccounts))
}
@@ -149,8 +149,7 @@ func TestAccountDBInit(t *testing.T) {
proto := config.Consensus[protocol.ConsensusCurrentVersion]
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
@@ -210,20 +209,21 @@ func TestAccountDBRound(t *testing.T) {
proto := config.Consensus[protocol.ConsensusCurrentVersion]
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
+ require.NoError(t, err)
+ aw, err := tx.MakeAccountsWriter()
require.NoError(t, err)
accts := ledgertesting.RandomAccounts(20, true)
tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion)
checkAccounts(t, tx, 0, accts)
- totals, err := arw.AccountsTotals(context.Background(), false)
+ totals, err := ar.AccountsTotals(context.Background(), false)
require.NoError(t, err)
- expectedOnlineRoundParams, endRound, err := arw.AccountsOnlineRoundParams()
+ expectedOnlineRoundParams, endRound, err := ar.AccountsOnlineRoundParams()
require.NoError(t, err)
require.Equal(t, 1, len(expectedOnlineRoundParams))
require.Equal(t, 0, int(endRound))
@@ -269,10 +269,10 @@ func TestAccountDBRound(t *testing.T) {
err = resourceUpdatesCnt.resourcesLoadOld(tx, knownAddresses)
require.NoError(t, err)
- err = arw.AccountsPutTotals(totals, false)
+ err = aw.AccountsPutTotals(totals, false)
require.NoError(t, err)
onlineRoundParams := ledgercore.OnlineRoundParamsData{RewardsLevel: totals.RewardsLevel, OnlineSupply: totals.Online.Money.Raw, CurrentProtocol: protocol.ConsensusCurrentVersion}
- err = arw.AccountsPutOnlineRoundParams([]ledgercore.OnlineRoundParamsData{onlineRoundParams}, basics.Round(i))
+ err = aw.AccountsPutOnlineRoundParams([]ledgercore.OnlineRoundParamsData{onlineRoundParams}, basics.Round(i))
require.NoError(t, err)
expectedOnlineRoundParams = append(expectedOnlineRoundParams, onlineRoundParams)
@@ -289,7 +289,7 @@ func TestAccountDBRound(t *testing.T) {
updatedOnlineAccts, err := onlineAccountsNewRound(tx, updatesOnlineCnt, proto, basics.Round(i))
require.NoError(t, err)
- err = arw.UpdateAccountsRound(basics.Round(i))
+ err = aw.UpdateAccountsRound(basics.Round(i))
require.NoError(t, err)
// TODO: calculate exact number of updates?
@@ -297,7 +297,7 @@ func TestAccountDBRound(t *testing.T) {
require.NotEmpty(t, updatedOnlineAccts)
checkAccounts(t, tx, basics.Round(i), accts)
- arw.Testing().CheckCreatablesTest(t, i, expectedDbImage)
+ ar.Testing().CheckCreatablesTest(t, i, expectedDbImage)
}
// test the accounts totals
@@ -307,11 +307,11 @@ func TestAccountDBRound(t *testing.T) {
}
expectedTotals := ledgertesting.CalculateNewRoundAccountTotals(t, updates, 0, proto, nil, ledgercore.AccountTotals{})
- actualTotals, err := arw.AccountsTotals(context.Background(), false)
+ actualTotals, err := ar.AccountsTotals(context.Background(), false)
require.NoError(t, err)
require.Equal(t, expectedTotals, actualTotals)
- actualOnlineRoundParams, endRound, err := arw.AccountsOnlineRoundParams()
+ actualOnlineRoundParams, endRound, err := ar.AccountsOnlineRoundParams()
require.NoError(t, err)
require.Equal(t, expectedOnlineRoundParams, actualOnlineRoundParams)
require.Equal(t, 9, int(endRound))
@@ -321,7 +321,7 @@ func TestAccountDBRound(t *testing.T) {
acctCb := func(addr basics.Address, data basics.AccountData) {
loaded[addr] = data
}
- count, err := arw.LoadAllFullAccounts(context.Background(), "accountbase", "resources", acctCb)
+ count, err := ar.LoadAllFullAccounts(context.Background(), "accountbase", "resources", acctCb)
require.NoError(t, err)
require.Equal(t, count, len(accts))
require.Equal(t, count, len(loaded))
@@ -366,8 +366,7 @@ func TestAccountDBInMemoryAcct(t *testing.T) {
for i, test := range tests {
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
@@ -437,8 +436,7 @@ func TestAccountDBInMemoryAcct(t *testing.T) {
func TestAccountStorageWithStateProofID(t *testing.T) {
partitiontest.PartitionTest(t)
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
@@ -604,29 +602,21 @@ func benchmarkInitBalances(b testing.TB, numAccounts int, tx trackerdb.Transacti
return
}
-func cleanupTestDb(dbs db.Pair, dbName string, inMemory bool) {
- dbs.Close()
- if !inMemory {
- os.Remove(dbName)
- }
-}
-
func benchmarkReadingAllBalances(b *testing.B, inMemory bool) {
- dbs, _ := sqlitedriver.DbOpenTrackerTest(b, true)
- dbs.SetLogger(logging.TestingLog(b))
+ dbs, _ := sqlitedriver.OpenForTesting(b, true)
defer dbs.Close()
bal := make(map[basics.Address]basics.AccountData)
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
benchmarkInitBalances(b, b.N, tx, protocol.ConsensusCurrentVersion)
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
b.ResetTimer()
// read all the balances in the database.
var err2 error
- bal, err2 = arw.Testing().AccountsAllTest()
+ bal, err2 = ar.Testing().AccountsAllTest()
require.NoError(b, err2)
return nil
})
@@ -648,9 +638,8 @@ func BenchmarkReadingAllBalancesDisk(b *testing.B) {
}
func benchmarkReadingRandomBalances(b *testing.B, inMemory bool) {
- dbs, fn := sqlitedriver.DbOpenTrackerTest(b, true)
- dbs.SetLogger(logging.TestingLog(b))
- defer dbs.CleanupTest(fn, inMemory)
+ dbs, _ := sqlitedriver.OpenForTesting(b, true)
+ defer dbs.Close()
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
accounts := benchmarkInitBalances(b, b.N, tx, protocol.ConsensusCurrentVersion)
@@ -687,30 +676,6 @@ func BenchmarkReadingRandomBalancesDisk(b *testing.B) {
benchmarkReadingRandomBalances(b, false)
}
-// TestAccountsDbQueriesCreateClose tests to see that we can create the accountsDbQueries and close it.
-// it also verify that double-closing it doesn't create an issue.
-func TestAccountsDbQueriesCreateClose(t *testing.T) {
- partitiontest.PartitionTest(t)
-
- dbs, _ := storetesting.DbOpenTest(t, true)
- storetesting.SetDbLogging(t, dbs)
- defer dbs.Close()
-
- err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
- sqlitedriver.AccountsInitTest(t, tx, make(map[basics.Address]basics.AccountData), protocol.ConsensusCurrentVersion)
- return nil
- })
- require.NoError(t, err)
- qs, err := sqlitedriver.AccountsInitDbQueries(dbs.Rdb.Handle)
- require.NoError(t, err)
- // TODO[store-refactor]: internals are opaque, once we move the the remainder of accountdb we can mvoe this too
- // require.NotNil(t, qs.listCreatablesStmt)
- qs.Close()
- // require.Nil(t, qs.listCreatablesStmt)
- qs.Close()
- // require.Nil(t, qs.listCreatablesStmt)
-}
-
func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder bool) {
proto := config.Consensus[protocol.ConsensusCurrentVersion]
genesisInitState, _ := ledgertesting.GenerateInitState(b, protocol.ConsensusCurrentVersion, 100)
@@ -771,12 +736,12 @@ func benchmarkWriteCatchpointStagingBalancesSub(b *testing.B, ascendingOrder boo
normalizedAccountBalances, err := prepareNormalizedBalancesV6(chunk.Balances, proto)
require.NoError(b, err)
b.StartTimer()
- err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
+ err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
cw, err := tx.MakeCatchpointWriter()
if err != nil {
return err
}
- err = cw.WriteCatchpointStagingBalances(ctx, normalizedAccountBalances)
+ _, err = cw.Write(ctx, trackerdb.CatchpointPayload{Accounts: normalizedAccountBalances})
return
})
@@ -813,9 +778,8 @@ func TestLookupKeysByPrefix(t *testing.T) {
partitiontest.PartitionTest(t)
t.Parallel()
- dbs, fn := sqlitedriver.DbOpenTrackerTest(t, false)
- dbs.SetLogger(logging.TestingLog(t))
- defer dbs.CleanupTest(fn, false)
+ dbs, _ := sqlitedriver.OpenForTesting(t, false)
+ defer dbs.Close()
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
// return account data, initialize DB tables from AccountsInitTest
@@ -1000,9 +964,8 @@ func TestLookupKeysByPrefix(t *testing.T) {
func BenchmarkLookupKeyByPrefix(b *testing.B) {
// learn something from BenchmarkWritingRandomBalancesDisk
- dbs, fn := sqlitedriver.DbOpenTrackerTest(b, false)
- dbs.SetLogger(logging.TestingLog(b))
- defer dbs.CleanupTest(fn, false)
+ dbs, _ := sqlitedriver.OpenForTesting(b, false)
+ defer dbs.Close()
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
// return account data, initialize DB tables from AccountsInitTest
@@ -1187,11 +1150,8 @@ func TestKVStoreNilBlobConversion(t *testing.T) {
// | Section 4: Run migration to see replace nils with empty byte slices |
// +---------------------------------------------------------------------+
- trackerDBWrapper := sqlitedriver.CreateTrackerSQLStore(dbs)
- err = trackerDBWrapper.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err0 error) {
- _, err0 = tx.RunMigrations(ctx, trackerdb.Params{}, log, targetVersion)
- return
- })
+ trackerDBWrapper := sqlitedriver.MakeStore(dbs)
+ _, err = trackerDBWrapper.RunMigrations(context.Background(), trackerdb.Params{}, log, targetVersion)
require.NoError(t, err)
// +------------------------------------------------------------------------------------------------+
@@ -1420,8 +1380,7 @@ func TestCompactResourceDeltas(t *testing.T) {
func TestLookupAccountAddressFromAddressID(t *testing.T) {
partitiontest.PartitionTest(t)
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
addrs := make([]basics.Address, 100)
@@ -1449,13 +1408,13 @@ func TestLookupAccountAddressFromAddressID(t *testing.T) {
require.NoError(t, err)
err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
for addr, addrid := range addrsids {
- retAddr, err := arw.LookupAccountAddressFromAddressID(ctx, addrid)
+ retAddr, err := ar.LookupAccountAddressFromAddressID(ctx, addrid)
if err != nil {
return err
}
@@ -1464,7 +1423,7 @@ func TestLookupAccountAddressFromAddressID(t *testing.T) {
}
}
// test fail case:
- retAddr, err := arw.LookupAccountAddressFromAddressID(ctx, nil)
+ retAddr, err := ar.LookupAccountAddressFromAddressID(ctx, nil)
if !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("unexpected error : %w", err)
@@ -2093,10 +2052,9 @@ func initBoxDatabase(b *testing.B, totalBoxes, boxSize int) (db.Pair, func(), er
}
proto := config.Consensus[protocol.ConsensusCurrentVersion]
- dbs, fn := storetesting.DbOpenTest(b, false)
- storetesting.SetDbLogging(b, dbs)
+ dbs, _ := storetesting.DbOpenTest(b, false)
cleanup := func() {
- cleanupTestDb(dbs, fn, false)
+ dbs.Close()
}
tx, err := dbs.Wdb.Handle.Begin()
@@ -2232,20 +2190,20 @@ func TestAccountOnlineQueries(t *testing.T) {
proto := config.Consensus[protocol.ConsensusCurrentVersion]
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dbs.SetLogger(logging.TestingLog(t))
+ dbs, _ := sqlitedriver.OpenForTesting(t, true)
defer dbs.Close()
err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- arw, err := tx.MakeAccountsReaderWriter()
- if err != nil {
- return err
- }
+ ar, err := tx.MakeAccountsReader()
+ require.NoError(t, err)
+
+ aw, err := tx.MakeAccountsWriter()
+ require.NoError(t, err)
var accts map[basics.Address]basics.AccountData
tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion)
- totals, err := arw.AccountsTotals(context.Background(), false)
+ totals, err := ar.AccountsTotals(context.Background(), false)
require.NoError(t, err)
var baseAccounts lruAccounts
@@ -2329,7 +2287,7 @@ func TestAccountOnlineQueries(t *testing.T) {
err = updatesOnlineCnt.accountsLoadOld(tx)
require.NoError(t, err)
- err = arw.AccountsPutTotals(totals, false)
+ err = aw.AccountsPutTotals(totals, false)
require.NoError(t, err)
updatedAccts, _, _, err := accountsNewRound(tx, updatesCnt, compactResourcesDeltas{}, nil, nil, proto, rnd)
require.NoError(t, err)
@@ -2339,7 +2297,7 @@ func TestAccountOnlineQueries(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, updatedOnlineAccts)
- err = arw.UpdateAccountsRound(rnd)
+ err = aw.UpdateAccountsRound(rnd)
require.NoError(t, err)
return
@@ -2367,12 +2325,12 @@ func TestAccountOnlineQueries(t *testing.T) {
refoaB3 := round3poads[0].Ref
refoaC3 := round3poads[1].Ref
- queries, err := tx.Testing().MakeOnlineAccountsOptimizedReader()
+ queries, err := tx.MakeOnlineAccountsOptimizedReader()
require.NoError(t, err)
// check round 1
rnd := basics.Round(1)
- online, err := arw.AccountsOnlineTop(rnd, 0, 10, proto)
+ online, err := ar.AccountsOnlineTop(rnd, 0, 10, proto)
require.NoError(t, err)
require.Equal(t, 2, len(online))
require.NotContains(t, online, addrC)
@@ -2411,7 +2369,7 @@ func TestAccountOnlineQueries(t *testing.T) {
// check round 2
rnd = basics.Round(2)
- online, err = arw.AccountsOnlineTop(rnd, 0, 10, proto)
+ online, err = ar.AccountsOnlineTop(rnd, 0, 10, proto)
require.NoError(t, err)
require.Equal(t, 1, len(online))
require.NotContains(t, online, addrA)
@@ -2444,7 +2402,7 @@ func TestAccountOnlineQueries(t *testing.T) {
// check round 3
rnd = basics.Round(3)
- online, err = arw.AccountsOnlineTop(rnd, 0, 10, proto)
+ online, err = ar.AccountsOnlineTop(rnd, 0, 10, proto)
require.NoError(t, err)
require.Equal(t, 1, len(online))
require.NotContains(t, online, addrA)
@@ -2475,7 +2433,7 @@ func TestAccountOnlineQueries(t *testing.T) {
require.Equal(t, dataC3.AccountBaseData.MicroAlgos, paod.AccountData.MicroAlgos)
require.Equal(t, voteIDC, paod.AccountData.VoteID)
- paods, err := arw.OnlineAccountsAll(0)
+ paods, err := ar.OnlineAccountsAll(0)
require.NoError(t, err)
require.Equal(t, 5, len(paods))
@@ -2512,20 +2470,20 @@ func TestAccountOnlineQueries(t *testing.T) {
checkAddrC()
checkAddrA()
- paods, err = arw.OnlineAccountsAll(3)
+ paods, err = ar.OnlineAccountsAll(3)
require.NoError(t, err)
require.Equal(t, 5, len(paods))
checkAddrB()
checkAddrC()
checkAddrA()
- paods, err = arw.OnlineAccountsAll(2)
+ paods, err = ar.OnlineAccountsAll(2)
require.NoError(t, err)
require.Equal(t, 3, len(paods))
checkAddrB()
checkAddrC()
- paods, err = arw.OnlineAccountsAll(1)
+ paods, err = ar.OnlineAccountsAll(1)
require.NoError(t, err)
require.Equal(t, 2, len(paods))
checkAddrB()
diff --git a/ledger/acctonline.go b/ledger/acctonline.go
index ec65592d3e..1e9f55f629 100644
--- a/ledger/acctonline.go
+++ b/ledger/acctonline.go
@@ -60,7 +60,7 @@ type cachedOnlineAccount struct {
// onlineAccounts tracks history of online accounts
type onlineAccounts struct {
// Connection to the database.
- dbs trackerdb.TrackerStore
+ dbs trackerdb.Store
// Prepared SQL statements for fast accounts DB lookups.
accountsq trackerdb.OnlineAccountsReader
@@ -433,23 +433,23 @@ func (ao *onlineAccounts) commitRound(ctx context.Context, tx trackerdb.Transact
return err
}
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
- err = arw.OnlineAccountsDelete(dcc.onlineAccountsForgetBefore)
+ err = aw.OnlineAccountsDelete(dcc.onlineAccountsForgetBefore)
if err != nil {
return err
}
- err = arw.AccountsPutOnlineRoundParams(dcc.onlineRoundParams, dcc.oldBase+1)
+ err = aw.AccountsPutOnlineRoundParams(dcc.onlineRoundParams, dcc.oldBase+1)
if err != nil {
return err
}
// delete all entries all older than maxBalLookback (or votersLookback) rounds ago
- err = arw.AccountsPruneOnlineRoundParams(dcc.onlineAccountsForgetBefore)
+ err = aw.AccountsPruneOnlineRoundParams(dcc.onlineAccountsForgetBefore)
return
}
diff --git a/ledger/acctonline_test.go b/ledger/acctonline_test.go
index 0bee1f35fc..13d222a62b 100644
--- a/ledger/acctonline_test.go
+++ b/ledger/acctonline_test.go
@@ -18,7 +18,6 @@ package ledger
import (
"context"
- "database/sql"
"fmt"
"sort"
"strconv"
@@ -80,7 +79,7 @@ func commitSyncPartial(t *testing.T, oa *onlineAccounts, ml *mockLedgerForTracke
require.NoError(t, err)
}
err := ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -92,7 +91,7 @@ func commitSyncPartial(t *testing.T, oa *onlineAccounts, ml *mockLedgerForTracke
}
}
- return arw.UpdateAccountsRound(newBase)
+ return aw.UpdateAccountsRound(newBase)
})
require.NoError(t, err)
}()
@@ -1316,7 +1315,7 @@ func TestAcctOnlineVotersLongerHistory(t *testing.T) {
require.NoError(t, err)
_, err = oa.onlineTotalsEx(lowest - 1)
- require.ErrorIs(t, err, sql.ErrNoRows)
+ require.ErrorIs(t, err, trackerdb.ErrNotFound)
// ensure the cache size for addrA does not have more entries than maxBalLookback + 1
// +1 comes from the deletion before X without checking account state at X
diff --git a/ledger/acctupdates.go b/ledger/acctupdates.go
index 9c16e09279..d2850a51a8 100644
--- a/ledger/acctupdates.go
+++ b/ledger/acctupdates.go
@@ -151,7 +151,7 @@ type modifiedKvValue struct {
type accountUpdates struct {
// Connection to the database.
- dbs trackerdb.TrackerStore
+ dbs trackerdb.Store
// Optimized reader for fast accounts DB lookups.
accountsq trackerdb.AccountsReader
@@ -1685,12 +1685,12 @@ func (au *accountUpdates) commitRound(ctx context.Context, tx trackerdb.Transact
dcc.stats.OldAccountPreloadDuration = time.Duration(time.Now().UnixNano()) - dcc.stats.OldAccountPreloadDuration
}
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
- err = arw.AccountsPutTotals(dcc.roundTotals, false)
+ err = aw.AccountsPutTotals(dcc.roundTotals, false)
if err != nil {
return err
}
diff --git a/ledger/acctupdates_test.go b/ledger/acctupdates_test.go
index 0faaa4de69..c70b969c54 100644
--- a/ledger/acctupdates_test.go
+++ b/ledger/acctupdates_test.go
@@ -37,8 +37,10 @@ import (
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger/eval"
"github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/testdb"
ledgertesting "github.com/algorand/go-algorand/ledger/testing"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
@@ -51,7 +53,8 @@ var testPoolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
var testSinkAddr = basics.Address{0x2c, 0x2a, 0x6c, 0xe9, 0xa9, 0xa7, 0xc2, 0x8c, 0x22, 0x95, 0xfd, 0x32, 0x4f, 0x77, 0xa5, 0x4, 0x8b, 0x42, 0xc2, 0xb7, 0xa8, 0x54, 0x84, 0xb6, 0x80, 0xb1, 0xe1, 0x3d, 0x59, 0x9b, 0xeb, 0x36}
type mockLedgerForTracker struct {
- dbs trackerdb.TrackerStore
+ dbs trackerdb.Store
+ catchpointDbs catchpointdb.Store
blocks []blockEntry
deltas []ledgercore.StateDelta
log logging.Logger
@@ -110,8 +113,7 @@ func setupAccts(niter int) []map[basics.Address]basics.AccountData {
}
func makeMockLedgerForTrackerWithLogger(t testing.TB, inMemory bool, initialBlocksCount int, consensusVersion protocol.ConsensusVersion, accts []map[basics.Address]basics.AccountData, l logging.Logger) *mockLedgerForTracker {
- dbs, fileName := sqlitedriver.DbOpenTrackerTest(t, inMemory)
- dbs.SetLogger(l)
+ dbs, fileName := sqlitedriver.OpenForTesting(t, inMemory)
blocks := randomInitChain(consensusVersion, initialBlocksCount)
deltas := make([]ledgercore.StateDelta, initialBlocksCount)
@@ -182,7 +184,7 @@ func (ml *mockLedgerForTracker) fork(t testing.TB) *mockLedgerForTracker {
dbs.Rdb.SetLogger(dblogger)
dbs.Wdb.SetLogger(dblogger)
- newLedgerTracker.dbs = sqlitedriver.CreateTrackerSQLStore(dbs)
+ newLedgerTracker.dbs = sqlitedriver.MakeStore(dbs)
return newLedgerTracker
}
@@ -253,10 +255,14 @@ func (ml *mockLedgerForTracker) BlockHdr(rnd basics.Round) (bookkeeping.BlockHea
return ml.blocks[int(rnd)].block.BlockHeader, nil
}
-func (ml *mockLedgerForTracker) trackerDB() trackerdb.TrackerStore {
+func (ml *mockLedgerForTracker) trackerDB() trackerdb.Store {
return ml.dbs
}
+func (ml *mockLedgerForTracker) catchpointDB() catchpointdb.Store {
+ return ml.catchpointDbs
+}
+
func (ml *mockLedgerForTracker) blockDB() db.Pair {
return db.Pair{}
}
@@ -297,13 +303,13 @@ func (au *accountUpdates) allBalances(rnd basics.Round) (bals map[basics.Address
return
}
- err = au.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
+ err = au.dbs.Snapshot(func(ctx context.Context, tx trackerdb.SnapshotScope) error {
var err0 error
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
- bals, err0 = arw.Testing().AccountsAllTest()
+ bals, err0 = ar.Testing().AccountsAllTest()
return err0
})
if err != nil {
@@ -1033,64 +1039,69 @@ func TestListCreatables(t *testing.T) {
numElementsPerSegement := 25
// set up the database
- dbs, _ := sqlitedriver.DbOpenTrackerTest(t, true)
- dblogger := logging.TestingLog(t)
- dbs.SetLogger(dblogger)
+ dbs := testdb.OpenForTesting(t)
defer dbs.Close()
- err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- proto := config.Consensus[protocol.ConsensusCurrentVersion]
+ proto := config.Consensus[protocol.ConsensusCurrentVersion]
+ err := dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
accts := make(map[basics.Address]basics.AccountData)
_ = tx.Testing().AccountsInitTest(t, accts, protocol.ConsensusCurrentVersion)
- require.NoError(t, err)
+ return
+ })
+ require.NoError(t, err)
- au := &accountUpdates{}
- au.accountsq, err = tx.Testing().MakeAccountsOptimizedReader()
- require.NoError(t, err)
+ au := &accountUpdates{}
+ au.accountsq, err = dbs.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
- // ******* All results are obtained from the cache. Empty database *******
- // ******* No deletes *******
- // get random data. Initial batch, no deletes
- ctbsList, randomCtbs := randomCreatables(numElementsPerSegement)
- expectedDbImage := make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable)
- ctbsWithDeletes := randomCreatableSampling(1, ctbsList, randomCtbs,
- expectedDbImage, numElementsPerSegement)
- // set the cache
- au.creatables = ctbsWithDeletes
- listAndCompareComb(t, au, expectedDbImage)
-
- // ******* All results are obtained from the database. Empty cache *******
- // ******* No deletes *******
- // sync with the database
- var updates compactAccountDeltas
- var resUpdates compactResourcesDeltas
+ // ******* All results are obtained from the cache. Empty database *******
+ // ******* No deletes *******
+ // get random data. Initial batch, no deletes
+ ctbsList, randomCtbs := randomCreatables(numElementsPerSegement)
+ expectedDbImage := make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable)
+ ctbsWithDeletes := randomCreatableSampling(1, ctbsList, randomCtbs,
+ expectedDbImage, numElementsPerSegement)
+ // set the cache
+ au.creatables = ctbsWithDeletes
+ listAndCompareComb(t, au, expectedDbImage)
+
+ // ******* All results are obtained from the database. Empty cache *******
+ // ******* No deletes *******
+ // sync with the database
+ var updates compactAccountDeltas
+ var resUpdates compactResourcesDeltas
+ err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
_, _, _, err = accountsNewRound(tx, updates, resUpdates, nil, ctbsWithDeletes, proto, basics.Round(1))
require.NoError(t, err)
- // nothing left in cache
- au.creatables = make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable)
- listAndCompareComb(t, au, expectedDbImage)
-
- // ******* Results are obtained from the database and from the cache *******
- // ******* No deletes in the database. *******
- // ******* Data in the database deleted in the cache *******
- au.creatables = randomCreatableSampling(2, ctbsList, randomCtbs,
- expectedDbImage, numElementsPerSegement)
- listAndCompareComb(t, au, expectedDbImage)
-
- // ******* Results are obtained from the database and from the cache *******
- // ******* Deletes are in the database and in the cache *******
- // sync with the database. This has deletes synced to the database.
+ return
+ })
+ require.NoError(t, err)
+
+ // nothing left in cache
+ au.creatables = make(map[basics.CreatableIndex]ledgercore.ModifiedCreatable)
+ listAndCompareComb(t, au, expectedDbImage)
+
+ // ******* Results are obtained from the database and from the cache *******
+ // ******* No deletes in the database. *******
+ // ******* Data in the database deleted in the cache *******
+ au.creatables = randomCreatableSampling(2, ctbsList, randomCtbs,
+ expectedDbImage, numElementsPerSegement)
+ listAndCompareComb(t, au, expectedDbImage)
+
+ // ******* Results are obtained from the database and from the cache *******
+ // ******* Deletes are in the database and in the cache *******
+ // sync with the database. This has deletes synced to the database.
+ err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
_, _, _, err = accountsNewRound(tx, updates, resUpdates, nil, au.creatables, proto, basics.Round(1))
require.NoError(t, err)
- // get new creatables in the cache. There will be deleted in the cache from the previous batch.
- au.creatables = randomCreatableSampling(3, ctbsList, randomCtbs,
- expectedDbImage, numElementsPerSegement)
- listAndCompareComb(t, au, expectedDbImage)
-
return
})
require.NoError(t, err)
+ // get new creatables in the cache. There will be deleted in the cache from the previous batch.
+ au.creatables = randomCreatableSampling(3, ctbsList, randomCtbs,
+ expectedDbImage, numElementsPerSegement)
+ listAndCompareComb(t, au, expectedDbImage)
}
func TestBoxNamesByAppIDs(t *testing.T) {
@@ -2190,7 +2201,7 @@ func TestAcctUpdatesResources(t *testing.T) {
err := au.prepareCommit(dcc)
require.NoError(t, err)
err = ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -2199,7 +2210,7 @@ func TestAcctUpdatesResources(t *testing.T) {
if err != nil {
return err
}
- err = arw.UpdateAccountsRound(newBase)
+ err = aw.UpdateAccountsRound(newBase)
return err
})
require.NoError(t, err)
@@ -2477,7 +2488,7 @@ func auCommitSync(t *testing.T, rnd basics.Round, au *accountUpdates, ml *mockLe
err := au.prepareCommit(dcc)
require.NoError(t, err)
err = ml.trackers.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -2486,7 +2497,7 @@ func auCommitSync(t *testing.T, rnd basics.Round, au *accountUpdates, ml *mockLe
if err != nil {
return err
}
- err = arw.UpdateAccountsRound(newBase)
+ err = aw.UpdateAccountsRound(newBase)
return err
})
require.NoError(t, err)
diff --git a/ledger/archival_test.go b/ledger/archival_test.go
index de483e227c..8dda76b2c2 100644
--- a/ledger/archival_test.go
+++ b/ledger/archival_test.go
@@ -40,6 +40,7 @@ import (
"github.com/algorand/go-algorand/ledger/eval"
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/ledger/store/blockdb"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
@@ -76,10 +77,14 @@ func (wl *wrappedLedger) Latest() basics.Round {
return wl.l.Latest()
}
-func (wl *wrappedLedger) trackerDB() trackerdb.TrackerStore {
+func (wl *wrappedLedger) trackerDB() trackerdb.Store {
return wl.l.trackerDB()
}
+func (wl *wrappedLedger) catchpointDB() catchpointdb.Store {
+ return wl.l.catchpointDB()
+}
+
func (wl *wrappedLedger) blockDB() db.Pair {
return wl.l.blockDB()
}
diff --git a/ledger/catchpointtracker.go b/ledger/catchpointtracker.go
index a24a0439c4..349eba0466 100644
--- a/ledger/catchpointtracker.go
+++ b/ledger/catchpointtracker.go
@@ -42,6 +42,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/logging/telemetryspec"
@@ -110,8 +111,8 @@ type catchpointTracker struct {
log logging.Logger
// Connection to the database.
- dbs trackerdb.TrackerStore
- catchpointStore trackerdb.CatchpointReaderWriter
+ dbs trackerdb.Store
+ catchpointStore catchpointdb.Store
// The last catchpoint label that was written to the database. Should always align with what's in the database.
// note that this is the last catchpoint *label* and not the catchpoint file.
@@ -227,27 +228,34 @@ func (ct *catchpointTracker) finishFirstStage(ctx context.Context, dbRound basic
}
}
- return ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
+ var info catchpointdb.CatchpointFirstStageInfo
+ err := ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
+ info, err = ct.recordFirstStageInfo(ctx, tx, &catchpointGenerationStats, dbRound, totalKVs, totalAccounts, totalChunks, biggestChunkLen, spVerificationHash)
+ return err
+ })
+ if err != nil {
+ // failed to record first stage info
+ return err
+ }
- err = ct.recordFirstStageInfo(ctx, tx, &catchpointGenerationStats, dbRound, totalKVs, totalAccounts, totalChunks, biggestChunkLen, spVerificationHash)
+ err = ct.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) error {
+ // save the catchpoint info
+ err = tx.InsertOrReplaceCatchpointFirstStageInfo(ctx, dbRound, &info)
if err != nil {
return err
}
- // Clear the db record.
- return crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingFirstStageInfo, 0)
+ // reset catchpoint staging
+ return tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateWritingFirstStageInfo, 0)
})
+ return err
}
// Possibly finish generating first stage catchpoint db record and data file after
// a crash.
func (ct *catchpointTracker) finishFirstStageAfterCrash(dbRound basics.Round) error {
v, err := ct.catchpointStore.ReadCatchpointStateUint64(
- context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo)
+ context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo)
if err != nil {
return err
}
@@ -257,9 +265,9 @@ func (ct *catchpointTracker) finishFirstStageAfterCrash(dbRound basics.Round) er
// First, delete the unfinished data file.
relCatchpointDataFilePath := filepath.Join(
- trackerdb.CatchpointDirName,
+ catchpointdb.CatchpointDirName,
makeCatchpointDataFilePath(dbRound))
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath)
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath)
if err != nil {
return err
}
@@ -276,9 +284,9 @@ func (ct *catchpointTracker) finishCatchpointsAfterCrash(catchpointLookback uint
for _, record := range records {
// First, delete the unfinished catchpoint file.
relCatchpointFilePath := filepath.Join(
- trackerdb.CatchpointDirName,
- trackerdb.MakeCatchpointFilePath(basics.Round(record.Round)))
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath)
+ catchpointdb.CatchpointDirName,
+ catchpointdb.MakeCatchpointFilePath(basics.Round(record.Round)))
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath)
if err != nil {
return err
}
@@ -302,7 +310,7 @@ func (ct *catchpointTracker) recoverFromCrash(dbRound basics.Round) error {
ctx := context.Background()
catchpointLookback, err := ct.catchpointStore.ReadCatchpointStateUint64(
- ctx, trackerdb.CatchpointStateCatchpointLookback)
+ ctx, catchpointdb.CatchpointStateCatchpointLookback)
if err != nil {
return err
}
@@ -333,7 +341,7 @@ func (ct *catchpointTracker) recoverFromCrash(dbRound basics.Round) error {
func (ct *catchpointTracker) loadFromDisk(l ledgerForTracker, dbRound basics.Round) (err error) {
ct.log = l.trackerLog()
ct.dbs = l.trackerDB()
- ct.catchpointStore, err = l.trackerDB().MakeCatchpointReaderWriter()
+ ct.catchpointStore = l.catchpointDB()
if err != nil {
return err
}
@@ -357,7 +365,7 @@ func (ct *catchpointTracker) loadFromDisk(l ledgerForTracker, dbRound basics.Rou
}
ct.lastCatchpointLabel, err = ct.catchpointStore.ReadCatchpointStateString(
- context.Background(), trackerdb.CatchpointStateLastCatchpoint)
+ context.Background(), catchpointdb.CatchpointStateLastCatchpoint)
if err != nil {
return
}
@@ -512,11 +520,7 @@ func (ct *catchpointTracker) commitRound(ctx context.Context, tx trackerdb.Trans
}
}()
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -556,34 +560,42 @@ func (ct *catchpointTracker) commitRound(ctx context.Context, tx trackerdb.Trans
dcc.stats.MerkleTrieUpdateDuration = now - dcc.stats.MerkleTrieUpdateDuration
}
- err = arw.UpdateAccountsHashRound(ctx, treeTargetRound)
+ err = aw.UpdateAccountsHashRound(ctx, treeTargetRound)
if err != nil {
return err
}
- if dcc.catchpointFirstStage {
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingFirstStageInfo, 1)
- if err != nil {
- return err
- }
- }
+ return nil
+}
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchpointLookback, dcc.catchpointLookback)
- if err != nil {
- return err
- }
+func (ct *catchpointTracker) postCommit(ctx context.Context, dcc *deferredCommitContext) {
+ err := ct.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) {
+ if dcc.catchpointFirstStage {
+ // set state as dirty
+ err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateWritingFirstStageInfo, 1)
+ if err != nil {
+ return err
+ }
+ }
- for _, round := range ct.calculateCatchpointRounds(&dcc.deferredCommitRange) {
- err = crw.InsertUnfinishedCatchpoint(ctx, round, dcc.committedRoundDigests[round-dcc.oldBase-1])
+ err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchpointLookback, dcc.catchpointLookback)
if err != nil {
return err
}
- }
- return nil
-}
+ for _, round := range ct.calculateCatchpointRounds(&dcc.deferredCommitRange) {
+ err = tx.InsertUnfinishedCatchpoint(ctx, round, dcc.committedRoundDigests[round-dcc.oldBase-1])
+ if err != nil {
+ return err
+ }
+ }
+
+ return
+ })
+ if err != nil {
+ ct.log.Errorf("failed to update catchpoint db post commit: %v", err)
+ }
-func (ct *catchpointTracker) postCommit(ctx context.Context, dcc *deferredCommitContext) {
if ct.balancesTrie != nil {
_, err := ct.balancesTrie.Evict(false)
if err != nil {
@@ -728,7 +740,7 @@ func repackCatchpoint(ctx context.Context, header CatchpointFileHeader, biggestC
// Create a catchpoint (a label and possibly a file with db record) and remove
// the unfinished catchpoint record.
-func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound basics.Round, round basics.Round, dataInfo trackerdb.CatchpointFirstStageInfo, blockHash crypto.Digest) error {
+func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound basics.Round, round basics.Round, dataInfo catchpointdb.CatchpointFirstStageInfo, blockHash crypto.Digest) error {
startTime := time.Now()
labelMaker := ledgercore.MakeCatchpointLabelMakerCurrent(round, &blockHash, &dataInfo.TrieBalancesHash, dataInfo.Totals, &dataInfo.StateProofVerificationHash)
label := ledgercore.MakeLabel(labelMaker)
@@ -738,7 +750,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound
round, accountsRound, label)
err := ct.catchpointStore.WriteCatchpointStateString(
- ctx, trackerdb.CatchpointStateLastCatchpoint, label)
+ ctx, catchpointdb.CatchpointStateLastCatchpoint, label)
if err != nil {
return err
}
@@ -751,7 +763,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound
return nil
}
- catchpointDataFilePath := filepath.Join(ct.dbDirectory, trackerdb.CatchpointDirName)
+ catchpointDataFilePath := filepath.Join(ct.dbDirectory, catchpointdb.CatchpointDirName)
catchpointDataFilePath =
filepath.Join(catchpointDataFilePath, makeCatchpointDataFilePath(accountsRound))
@@ -778,7 +790,7 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound
}
relCatchpointFilePath :=
- filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(round))
+ filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(round))
absCatchpointFilePath := filepath.Join(ct.dbDirectory, relCatchpointFilePath)
err = os.MkdirAll(filepath.Dir(absCatchpointFilePath), 0700)
@@ -796,18 +808,11 @@ func (ct *catchpointTracker) createCatchpoint(ctx context.Context, accountsRound
return err
}
- err = ct.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
-
- err = ct.recordCatchpointFile(ctx, crw, round, relCatchpointFilePath, fileInfo.Size())
- if err != nil {
- return err
- }
- return crw.DeleteUnfinishedCatchpoint(ctx, round)
- })
+ err = ct.recordCatchpointFile(ctx, ct.catchpointStore, round, relCatchpointFilePath, fileInfo.Size())
+ if err != nil {
+ return err
+ }
+ err = ct.catchpointStore.DeleteUnfinishedCatchpoint(ctx, round)
if err != nil {
return err
}
@@ -887,8 +892,8 @@ func (ct *catchpointTracker) pruneFirstStageRecordsData(ctx context.Context, max
for _, round := range rounds {
relCatchpointDataFilePath :=
- filepath.Join(trackerdb.CatchpointDirName, makeCatchpointDataFilePath(round))
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath)
+ filepath.Join(catchpointdb.CatchpointDirName, makeCatchpointDataFilePath(round))
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointDataFilePath)
if err != nil {
return err
}
@@ -1096,18 +1101,18 @@ func (ct *catchpointTracker) isWritingCatchpointDataFile() bool {
// Generates a (first stage) catchpoint data file.
// The file is built in the following order:
-// - Catchpoint file header (named content.msgpack). The header is generated and appended to the file at the end of the
-// second stage of catchpoint generation.
-// - State proof verification data chunk (named stateProofVerificationContext.msgpack).
-// - Balance and KV chunk (named balances.x.msgpack).
-// ...
-// - Balance and KV chunk (named balances.x.msgpack).
+// - Catchpoint file header (named content.msgpack). The header is generated and appended to the file at the end of the
+// second stage of catchpoint generation.
+// - State proof verification data chunk (named stateProofVerificationContext.msgpack).
+// - Balance and KV chunk (named balances.x.msgpack).
+// ...
+// - Balance and KV chunk (named balances.x.msgpack).
func (ct *catchpointTracker) generateCatchpointData(ctx context.Context, accountsRound basics.Round, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails) (totalKVs, totalAccounts, totalChunks, biggestChunkLen uint64, spVerificationHash crypto.Digest, err error) {
ct.log.Debugf("catchpointTracker.generateCatchpointData() writing catchpoint accounts for round %d", accountsRound)
startTime := time.Now()
- catchpointDataFilePath := filepath.Join(ct.dbDirectory, trackerdb.CatchpointDirName)
+ catchpointDataFilePath := filepath.Join(ct.dbDirectory, catchpointdb.CatchpointDirName)
catchpointDataFilePath =
filepath.Join(catchpointDataFilePath, makeCatchpointDataFilePath(accountsRound))
@@ -1199,25 +1204,25 @@ func (ct *catchpointTracker) generateCatchpointData(ctx context.Context, account
return catchpointWriter.totalKVs, catchpointWriter.totalAccounts, catchpointWriter.chunkNum, catchpointWriter.biggestChunkLen, spVerificationHash, nil
}
-func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx trackerdb.TransactionScope, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails, accountsRound basics.Round, totalKVs uint64, totalAccounts uint64, totalChunks uint64, biggestChunkLen uint64, stateProofVerificationHash crypto.Digest) error {
- arw, err := tx.MakeAccountsReaderWriter()
+func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx trackerdb.TransactionScope, catchpointGenerationStats *telemetryspec.CatchpointGenerationEventDetails, accountsRound basics.Round, totalKVs uint64, totalAccounts uint64, totalChunks uint64, biggestChunkLen uint64, stateProofVerificationHash crypto.Digest) (info catchpointdb.CatchpointFirstStageInfo, err error) {
+ ar, err := tx.MakeAccountsReader()
if err != nil {
- return err
+ return
}
- accountTotals, err := arw.AccountsTotals(ctx, false)
+ accountTotals, err := ar.AccountsTotals(ctx, false)
if err != nil {
- return err
+ return
}
mc, err := tx.MakeMerkleCommitter(false)
if err != nil {
- return err
+ return info, err
}
if ct.balancesTrie == nil {
trie, trieErr := merkletrie.MakeTrie(mc, trackerdb.TrieMemoryConfig)
if trieErr != nil {
- return trieErr
+ return info, trieErr
}
ct.balancesTrie = trie
} else {
@@ -1226,15 +1231,10 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke
trieBalancesHash, err := ct.balancesTrie.RootHash()
if err != nil {
- return err
- }
-
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
+ return
}
- info := trackerdb.CatchpointFirstStageInfo{
+ info = catchpointdb.CatchpointFirstStageInfo{
Totals: accountTotals,
TotalAccounts: totalAccounts,
TotalKVs: totalKVs,
@@ -1244,11 +1244,6 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke
StateProofVerificationHash: stateProofVerificationHash,
}
- err = crw.InsertOrReplaceCatchpointFirstStageInfo(ctx, accountsRound, &info)
- if err != nil {
- return err
- }
-
catchpointGenerationStats.MerkleTrieRootHash = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(trieBalancesHash[:])
catchpointGenerationStats.SPVerificationCtxsHash = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(stateProofVerificationHash[:])
ct.log.EventWithDetails(telemetryspec.Accounts, telemetryspec.CatchpointGenerationEvent, catchpointGenerationStats)
@@ -1262,7 +1257,7 @@ func (ct *catchpointTracker) recordFirstStageInfo(ctx context.Context, tx tracke
With("MerkleTrieRootHash", catchpointGenerationStats.MerkleTrieRootHash).
With("SPVerificationCtxsHash", catchpointGenerationStats.SPVerificationCtxsHash).
Infof("Catchpoint data file was generated")
- return nil
+ return
}
func makeCatchpointDataFilePath(accountsRound basics.Round) string {
@@ -1273,7 +1268,7 @@ func makeCatchpointDataFilePath(accountsRound basics.Round) string {
// after a successful insert operation to the database, it would delete up to 2 old entries, as needed.
// deleting 2 entries while inserting single entry allow us to adjust the size of the backing storage and have the
// database and storage realign.
-func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw trackerdb.CatchpointReaderWriter, round basics.Round, relCatchpointFilePath string, fileSize int64) (err error) {
+func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw catchpointdb.ReaderWriter, round basics.Round, relCatchpointFilePath string, fileSize int64) (err error) {
if ct.catchpointFileHistoryLength != 0 {
err = crw.StoreCatchpoint(ctx, round, relCatchpointFilePath, "", fileSize)
if err != nil {
@@ -1281,7 +1276,7 @@ func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw track
return
}
} else {
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath)
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, relCatchpointFilePath)
if err != nil {
ct.log.Warnf("catchpointTracker.recordCatchpointFile() unable to remove file (%s): %v", relCatchpointFilePath, err)
return
@@ -1296,7 +1291,7 @@ func (ct *catchpointTracker) recordCatchpointFile(ctx context.Context, crw track
return fmt.Errorf("unable to delete catchpoint file, getOldestCatchpointFiles failed : %v", err)
}
for round, fileToDelete := range filesToDelete {
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, fileToDelete)
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(ct.dbDirectory, fileToDelete)
if err != nil {
return err
}
@@ -1316,20 +1311,12 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS
ledgerGetcatchpointCount.Inc(nil)
// TODO: we need to generalize this, check @cce PoC PR, he has something
// somewhat broken for some KVs..
- err := ct.dbs.Snapshot(func(ctx context.Context, tx trackerdb.SnapshotScope) (err error) {
- cr, err := tx.MakeCatchpointReader()
- if err != nil {
- return err
- }
-
- dbFileName, _, fileSize, err = cr.GetCatchpoint(ctx, round)
- return
- })
- ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil)
+ dbFileName, _, fileSize, err := ct.catchpointStore.GetCatchpoint(context.Background(), round)
if err != nil && err != sql.ErrNoRows {
// we had some sql error.
return nil, fmt.Errorf("catchpointTracker.GetCatchpointStream() unable to lookup catchpoint %d: %v", round, err)
}
+ ledgerGetcatchpointMicros.AddMicrosecondsSince(start, nil)
if dbFileName != "" {
catchpointPath := filepath.Join(ct.dbDirectory, dbFileName)
file, openErr := os.OpenFile(catchpointPath, os.O_RDONLY, 0666)
@@ -1340,14 +1327,10 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS
if os.IsNotExist(openErr) {
// the database told us that we have this file.. but we couldn't find it.
// delete it from the database.
- crw, err2 := ct.dbs.MakeCatchpointReaderWriter()
- if err2 != nil {
- return nil, err2
- }
- err2 = ct.recordCatchpointFile(context.Background(), crw, round, "", 0)
- if err2 != nil {
- ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to delete missing catchpoint entry: %v", err2)
- return nil, err2
+ err = ct.recordCatchpointFile(context.Background(), ct.catchpointStore, round, "", 0)
+ if err != nil {
+ ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to delete missing catchpoint entry: %v", err)
+ return nil, err
}
return nil, ledgercore.ErrNoEntry{}
@@ -1358,7 +1341,7 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS
// if the database doesn't know about that round, see if we have that file anyway:
relCatchpointFilePath :=
- filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(round))
+ filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(round))
absCatchpointFilePath := filepath.Join(ct.dbDirectory, relCatchpointFilePath)
file, err := os.OpenFile(absCatchpointFilePath, os.O_RDONLY, 0666)
if err == nil && file != nil {
@@ -1368,11 +1351,7 @@ func (ct *catchpointTracker) GetCatchpointStream(round basics.Round) (ReadCloseS
// we couldn't get the stat, so just return with the file.
return &readCloseSizer{ReadCloser: file, size: -1}, nil //nolint:nilerr // intentionally ignoring Stat error
}
- crw, err := ct.dbs.MakeCatchpointReaderWriter()
- if err != nil {
- return nil, err
- }
- err = ct.recordCatchpointFile(context.Background(), crw, round, relCatchpointFilePath, fileInfo.Size())
+ err = ct.recordCatchpointFile(context.Background(), ct.catchpointStore, round, relCatchpointFilePath, fileInfo.Size())
if err != nil {
ct.log.Warnf("catchpointTracker.GetCatchpointStream() unable to save missing catchpoint entry: %v", err)
}
@@ -1388,11 +1367,17 @@ func (ct *catchpointTracker) catchpointEnabled() bool {
// initializeHashes initializes account/resource/kv hashes.
// as part of the initialization, it tests if a hash table matches to account base and updates the former.
func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb.TransactionScope, rnd basics.Round) error {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
- hashRound, err := arw.AccountsHashRound(ctx)
+
+ aw, err := tx.MakeAccountsWriter()
+ if err != nil {
+ return err
+ }
+
+ hashRound, err := ar.AccountsHashRound(ctx)
if err != nil {
return err
}
@@ -1400,7 +1385,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb.
if hashRound != rnd {
// if the hashed round is different then the base round, something was modified, and the accounts aren't in sync
// with the hashes.
- err = arw.ResetAccountHashes(ctx)
+ err = aw.ResetAccountHashes(ctx)
if err != nil {
return err
}
@@ -1457,7 +1442,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb.
if !added {
// we need to translate the "addrid" into actual account address so that
// we can report the failure.
- addr, lErr := arw.LookupAccountAddressFromAddressID(ctx, acct.AccountRef)
+ addr, lErr := ar.LookupAccountAddressFromAddressID(ctx, acct.AccountRef)
if lErr != nil {
ct.log.Warnf("initializeHashes attempted to add duplicate acct hash '%s' to merkle trie for account id %d : %v", hex.EncodeToString(acct.Digest), acct.AccountRef, lErr)
} else {
@@ -1541,7 +1526,7 @@ func (ct *catchpointTracker) initializeHashes(ctx context.Context, tx trackerdb.
}
// we've just updated the merkle trie, update the hashRound to reflect that.
- err = arw.UpdateAccountsHashRound(ctx, rnd)
+ err = aw.UpdateAccountsHashRound(ctx, rnd)
if err != nil {
return fmt.Errorf("initializeHashes was unable to update the account hash round to %d: %v", rnd, err)
}
diff --git a/ledger/catchpointtracker_test.go b/ledger/catchpointtracker_test.go
index 84a103300f..a1e286b3cf 100644
--- a/ledger/catchpointtracker_test.go
+++ b/ledger/catchpointtracker_test.go
@@ -40,6 +40,7 @@ import (
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/data/transactions"
"github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
ledgertesting "github.com/algorand/go-algorand/ledger/testing"
"github.com/algorand/go-algorand/logging"
@@ -95,7 +96,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) {
filesToCreate := 4
temporaryDirectory := t.TempDir()
- catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName)
err := os.Mkdir(catchpointsDirectory, 0777)
require.NoError(t, err)
@@ -103,7 +104,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) {
// Create the catchpoint files with dummy data
for i := 0; i < filesToCreate; i++ {
- fileName := filepath.Join(trackerdb.CatchpointDirName, fmt.Sprintf("%d.catchpoint", i))
+ fileName := filepath.Join(catchpointdb.CatchpointDirName, fmt.Sprintf("%d.catchpoint", i))
data := []byte{byte(i), byte(i + 1), byte(i + 2)}
err = os.WriteFile(filepath.Join(temporaryDirectory, fileName), data, 0666)
require.NoError(t, err)
@@ -129,7 +130,7 @@ func TestCatchpointGetCatchpointStream(t *testing.T) {
require.Equal(t, int64(3), len)
// File deleted, but record in the database
- err = os.Remove(filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName, "2.catchpoint"))
+ err = os.Remove(filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName, "2.catchpoint"))
require.NoError(t, err)
reader, err = ct.GetCatchpointStream(basics.Round(2))
require.Equal(t, ledgercore.ErrNoEntry{}, err)
@@ -175,7 +176,7 @@ func TestCatchpointsDeleteStored(t *testing.T) {
dummyCatchpointFiles := make([]string, dummyCatchpointFilesToCreate)
for i := 0; i < dummyCatchpointFilesToCreate; i++ {
file := fmt.Sprintf("%s%c%d%c%d%cdummy_catchpoint_file-%d",
- trackerdb.CatchpointDirName, os.PathSeparator,
+ catchpointdb.CatchpointDirName, os.PathSeparator,
i/10, os.PathSeparator,
i/2, os.PathSeparator,
i)
@@ -215,7 +216,7 @@ func TestCatchpointsDeleteStoredOnSchemaUpdate(t *testing.T) {
return
}
temporaryDirectroy := t.TempDir()
- tempCatchpointDir := filepath.Join(temporaryDirectroy, trackerdb.CatchpointDirName)
+ tempCatchpointDir := filepath.Join(temporaryDirectroy, catchpointdb.CatchpointDirName)
// creating empty catchpoint directories
emptyDirPath := path.Join(tempCatchpointDir, "2f", "e1")
@@ -252,7 +253,7 @@ func TestCatchpointsDeleteStoredOnSchemaUpdate(t *testing.T) {
_, err = trackerDBInitialize(ml, true, ct.dbDirectory)
require.NoError(t, err)
- emptyDirs, err := trackerdb.GetEmptyDirs(tempCatchpointDir)
+ emptyDirs, err := catchpointdb.GetEmptyDirs(tempCatchpointDir)
require.NoError(t, err)
onlyTempDirEmpty := len(emptyDirs) == 0
require.Equal(t, onlyTempDirEmpty, true)
@@ -320,7 +321,7 @@ func TestRecordCatchpointFile(t *testing.T) {
require.NoError(t, err)
require.Equal(t, conf.CatchpointFileHistoryLength, numberOfCatchpointFiles)
- emptyDirs, err := trackerdb.GetEmptyDirs(temporaryDirectory)
+ emptyDirs, err := catchpointdb.GetEmptyDirs(temporaryDirectory)
require.NoError(t, err)
onlyCatchpointDirEmpty := len(emptyDirs) == 0 ||
(len(emptyDirs) == 1 && emptyDirs[0] == temporaryDirectory)
@@ -335,7 +336,7 @@ func createCatchpoint(t *testing.T, ct *catchpointTracker, accountsRound basics.
require.Equal(t, calculateStateProofVerificationHash(t, ml), stateProofVerificationHash)
- err = ct.createCatchpoint(context.Background(), accountsRound, round, trackerdb.CatchpointFirstStageInfo{BiggestChunkLen: biggestChunkLen}, crypto.Digest{})
+ err = ct.createCatchpoint(context.Background(), accountsRound, round, catchpointdb.CatchpointFirstStageInfo{BiggestChunkLen: biggestChunkLen}, crypto.Digest{})
require.NoError(t, err)
}
@@ -401,9 +402,7 @@ func writeDummySpVerification(t *testing.T, nextIndexForContext uint64, numberOf
e.LastAttestedRound = basics.Round(nextIndexForContext + i)
contexts[i] = &e
}
- writer := tx.MakeSpVerificationCtxReaderWriter()
-
- return writer.StoreSPContexts(ctx, contexts[:])
+ return tx.MakeSpVerificationCtxWriter().StoreSPContexts(ctx, contexts[:])
})
require.NoError(t, err)
}
@@ -423,7 +422,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) {
ct.initialize(cfg, ".")
temporaryDirectroy := b.TempDir()
- catchpointsDirectory := filepath.Join(temporaryDirectroy, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(temporaryDirectroy, catchpointdb.CatchpointDirName)
err := os.Mkdir(catchpointsDirectory, 0777)
require.NoError(b, err)
@@ -436,7 +435,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) {
// at this point, the database was created. We want to fill the accounts data
accountsNumber := 6000000 * b.N
err = ml.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -457,7 +456,7 @@ func BenchmarkLargeCatchpointDataWriting(b *testing.B) {
}
}
- return arw.UpdateAccountsHashRound(ctx, 1)
+ return aw.UpdateAccountsHashRound(ctx, 1)
})
require.NoError(b, err)
@@ -1049,7 +1048,7 @@ func TestCatchpointFirstStageInfoPruning(t *testing.T) {
defer ct.close()
temporaryDirectory := t.TempDir()
- catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName)
err := os.Mkdir(catchpointsDirectory, 0777)
require.NoError(t, err)
@@ -1147,7 +1146,7 @@ func TestCatchpointFirstStagePersistence(t *testing.T) {
defer ml.Close()
tempDirectory := t.TempDir()
- catchpointsDirectory := filepath.Join(tempDirectory, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(tempDirectory, catchpointdb.CatchpointDirName)
cfg := config.GetDefaultLocal()
cfg.CatchpointInterval = 4
@@ -1192,12 +1191,11 @@ func TestCatchpointFirstStagePersistence(t *testing.T) {
defer ml2.Close()
ml.Close()
- cps2, err := ml2.dbs.MakeCatchpointReaderWriter()
- require.NoError(t, err)
+ cps2 := ml2.catchpointDB()
// Insert unfinished first stage record.
err = cps2.WriteCatchpointStateUint64(
- context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo, 1)
+ context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo, 1)
require.NoError(t, err)
// Delete the database record.
@@ -1221,7 +1219,7 @@ func TestCatchpointFirstStagePersistence(t *testing.T) {
// Check that the unfinished first stage record is deleted.
v, err := ct2.catchpointStore.ReadCatchpointStateUint64(
- context.Background(), trackerdb.CatchpointStateWritingFirstStageInfo)
+ context.Background(), catchpointdb.CatchpointStateWritingFirstStageInfo)
require.NoError(t, err)
require.Zero(t, v)
}
@@ -1248,7 +1246,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) {
defer ml.Close()
tempDirectory := t.TempDir()
- catchpointsDirectory := filepath.Join(tempDirectory, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(tempDirectory, catchpointdb.CatchpointDirName)
cfg := config.GetDefaultLocal()
cfg.CatchpointInterval = 4
@@ -1271,7 +1269,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) {
firstStageRound := secondStageRound - basics.Round(protoParams.CatchpointLookback)
catchpointDataFilePath :=
filepath.Join(catchpointsDirectory, makeCatchpointDataFilePath(firstStageRound))
- var firstStageInfo trackerdb.CatchpointFirstStageInfo
+ var firstStageInfo catchpointdb.CatchpointFirstStageInfo
var catchpointData []byte
// Add blocks until the first catchpoint round.
@@ -1312,7 +1310,7 @@ func TestCatchpointSecondStagePersistence(t *testing.T) {
// Check that the data file exists.
catchpointFilePath :=
- filepath.Join(catchpointsDirectory, trackerdb.MakeCatchpointFilePath(secondStageRound))
+ filepath.Join(catchpointsDirectory, catchpointdb.MakeCatchpointFilePath(secondStageRound))
info, err := os.Stat(catchpointFilePath)
require.NoError(t, err)
@@ -1331,20 +1329,19 @@ func TestCatchpointSecondStagePersistence(t *testing.T) {
err = os.WriteFile(catchpointDataFilePath, catchpointData, 0644)
require.NoError(t, err)
- cps2, err := ml2.dbs.MakeCatchpointReaderWriter()
- require.NoError(t, err)
+ cw2 := ml2.catchpointDB()
// Restore the first stage database record.
- err = cps2.InsertOrReplaceCatchpointFirstStageInfo(context.Background(), firstStageRound, &firstStageInfo)
+ err = cw2.InsertOrReplaceCatchpointFirstStageInfo(context.Background(), firstStageRound, &firstStageInfo)
require.NoError(t, err)
// Insert unfinished catchpoint record.
- err = cps2.InsertUnfinishedCatchpoint(
+ err = cw2.InsertUnfinishedCatchpoint(
context.Background(), secondStageRound, crypto.Digest{})
require.NoError(t, err)
// Delete the catchpoint file database record.
- err = cps2.StoreCatchpoint(
+ err = cw2.StoreCatchpoint(
context.Background(), secondStageRound, "", "", 0)
require.NoError(t, err)
@@ -1523,8 +1520,7 @@ func TestCatchpointSecondStageDeletesUnfinishedCatchpointRecordAfterRestart(t *t
defer ml2.Close()
ml.Close()
- cps2, err := ml2.dbs.MakeCatchpointReaderWriter()
- require.NoError(t, err)
+ cps2 := ml2.catchpointDB()
// Sanity check: first stage record should be deleted.
_, exists, err := cps2.SelectCatchpointFirstStageInfo(context.Background(), firstStageRound)
@@ -1811,7 +1807,7 @@ func TestCatchpointLargeAccountCountCatchpointGeneration(t *testing.T) {
ct := newCatchpointTracker(t, ml, conf, ".")
temporaryDirectory := t.TempDir()
- catchpointsDirectory := filepath.Join(temporaryDirectory, trackerdb.CatchpointDirName)
+ catchpointsDirectory := filepath.Join(temporaryDirectory, catchpointdb.CatchpointDirName)
err := os.Mkdir(catchpointsDirectory, 0777)
require.NoError(t, err)
defer os.RemoveAll(catchpointsDirectory)
diff --git a/ledger/catchpointwriter.go b/ledger/catchpointwriter.go
index df8b72ba6f..4c22e035b0 100644
--- a/ledger/catchpointwriter.go
+++ b/ledger/catchpointwriter.go
@@ -108,17 +108,17 @@ func (data catchpointStateProofVerificationContext) ToBeHashed() (protocol.HashI
}
func makeCatchpointWriter(ctx context.Context, filePath string, tx trackerdb.TransactionScope, maxResourcesPerChunk int) (*catchpointWriter, error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsReader()
if err != nil {
return nil, err
}
- totalAccounts, err := arw.TotalAccounts(ctx)
+ totalAccounts, err := aw.TotalAccounts(ctx)
if err != nil {
return nil, err
}
- totalKVs, err := arw.TotalKVs(ctx)
+ totalKVs, err := aw.TotalKVs(ctx)
if err != nil {
return nil, err
}
@@ -161,7 +161,7 @@ func (cw *catchpointWriter) Abort() error {
}
func (cw *catchpointWriter) WriteStateProofVerificationContext() (crypto.Digest, error) {
- rawData, err := cw.tx.MakeSpVerificationCtxReaderWriter().GetAllSPContexts(cw.ctx)
+ rawData, err := cw.tx.MakeSpVerificationCtxReader().GetAllSPContexts(cw.ctx)
if err != nil {
return crypto.Digest{}, err
}
diff --git a/ledger/catchpointwriter_test.go b/ledger/catchpointwriter_test.go
index db54b4527c..efb3ca86e6 100644
--- a/ledger/catchpointwriter_test.go
+++ b/ledger/catchpointwriter_test.go
@@ -284,7 +284,7 @@ func TestBasicCatchpointWriter(t *testing.T) {
require.Equal(t, uint64(len(accts)), uint64(len(chunk.Balances)))
}
-func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath string, filepath string, maxResourcesPerChunk int) CatchpointFileHeader {
+func testWriteCatchpoint(t *testing.T, rdb trackerdb.Store, datapath string, filepath string, maxResourcesPerChunk int) CatchpointFileHeader {
var totalAccounts uint64
var totalChunks uint64
var biggestChunkLen uint64
@@ -300,7 +300,7 @@ func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath stri
return err
}
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
@@ -318,11 +318,11 @@ func testWriteCatchpoint(t *testing.T, rdb trackerdb.TrackerStore, datapath stri
totalAccounts = writer.totalAccounts
totalChunks = writer.chunkNum
biggestChunkLen = writer.biggestChunkLen
- accountsRnd, err = arw.AccountsRound()
+ accountsRnd, err = ar.AccountsRound()
if err != nil {
return
}
- totals, err = arw.AccountsTotals(ctx, false)
+ totals, err = ar.AccountsTotals(ctx, false)
return
})
require.NoError(t, err)
@@ -420,12 +420,12 @@ func TestCatchpointReadDatabaseOverflowSingleAccount(t *testing.T) {
cw, err := makeCatchpointWriter(context.Background(), catchpointDataFilePath, tx, maxResourcesPerChunk)
require.NoError(t, err)
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
- expectedTotalResources, err := arw.TotalResources(ctx)
+ expectedTotalResources, err := ar.TotalResources(ctx)
if err != nil {
return err
}
@@ -506,17 +506,17 @@ func TestCatchpointReadDatabaseOverflowAccounts(t *testing.T) {
catchpointDataFilePath := filepath.Join(temporaryDirectory, "15.data")
err = ml.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
- expectedTotalAccounts, err := arw.TotalAccounts(ctx)
+ expectedTotalAccounts, err := ar.TotalAccounts(ctx)
if err != nil {
return err
}
- expectedTotalResources, err := arw.TotalResources(ctx)
+ expectedTotalResources, err := ar.TotalResources(ctx)
if err != nil {
return err
}
@@ -602,7 +602,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) {
ctx := context.Background()
err = l.trackerDBs.TransactionContext(ctx, func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return nil
}
@@ -618,7 +618,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) {
require.NotEmpty(t, h1)
// reset hashes
- err = arw.ResetAccountHashes(ctx)
+ err = aw.ResetAccountHashes(ctx)
require.NoError(t, err)
// rebuild the MT
@@ -664,7 +664,7 @@ func TestFullCatchpointWriterOverflowAccounts(t *testing.T) {
require.NoError(t, err)
}
-func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess trackerdb.TrackerStore, filepath string) *Ledger {
+func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess trackerdb.Store, filepath string) *Ledger {
// create a ledger.
var initState ledgercore.InitState
initState.Block.CurrentProtocol = protocol.ConsensusCurrentVersion
@@ -686,17 +686,17 @@ func testNewLedgerFromCatchpoint(t *testing.T, catchpointWriterReadAccess tracke
err = accessor.BuildMerkleTrie(context.Background(), nil)
require.NoError(t, err)
- err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) error {
+ err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
cw, err := tx.MakeCatchpointWriter()
if err != nil {
return err
}
- return cw.ApplyCatchpointStagingBalances(ctx, 0, 0)
+ return cw.Apply(ctx, 0, 0)
})
require.NoError(t, err)
- balanceTrieStats := func(db trackerdb.TrackerStore) merkletrie.Stats {
+ balanceTrieStats := func(db trackerdb.Store) merkletrie.Stats {
var stats merkletrie.Stats
err = db.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
committer, err := tx.MakeMerkleCommitter(false)
diff --git a/ledger/catchupaccessor.go b/ledger/catchupaccessor.go
index 990c977dac..fe8bd0e7c4 100644
--- a/ledger/catchupaccessor.go
+++ b/ledger/catchupaccessor.go
@@ -34,6 +34,7 @@ import (
"github.com/algorand/go-algorand/ledger/encoded"
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/ledger/store/blockdb"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
@@ -95,102 +96,30 @@ type CatchpointCatchupAccessor interface {
}
type stagingWriter interface {
- writeBalances(context.Context, []trackerdb.NormalizedAccountBalance) error
- writeCreatables(context.Context, []trackerdb.NormalizedAccountBalance) error
- writeHashes(context.Context, []trackerdb.NormalizedAccountBalance) error
- writeKVs(context.Context, []encoded.KVRecordV6) error
- isShared() bool
+ write(context.Context, trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error)
}
type stagingWriterImpl struct {
- wdb trackerdb.TrackerStore
+ wdb trackerdb.Store
}
-func (w *stagingWriterImpl) writeBalances(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
- return crw.WriteCatchpointStagingBalances(ctx, balances)
- })
-}
-
-func (w *stagingWriterImpl) writeKVs(ctx context.Context, kvrs []encoded.KVRecordV6) error {
- return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
-
- keys := make([][]byte, len(kvrs))
- values := make([][]byte, len(kvrs))
- hashes := make([][]byte, len(kvrs))
- for i := 0; i < len(kvrs); i++ {
- keys[i] = kvrs[i].Key
-
- // Since `encoded.KVRecordV6` is `omitempty` and `omitemptyarray`,
- // when we have an instance of `encoded.KVRecordV6` with nil value,
- // an empty box is unmarshalled to have `nil` value,
- // while this might be mistaken to be a box deletion.
- //
- // We don't want to mistake this to be a deleted box:
- // We are (and should be) during Fast Catchup (FC)
- // writing to DB with empty byte string, rather than writing nil.
- //
- // This matters in sqlite3,
- // for sqlite3 differs on writing nil byte slice to table from writing []byte{}:
- // - writing nil byte slice is true that `value is NULL`
- // - writing []byte{} is false on `value is NULL`.
- //
- // For the sake of consistency, we convert nil to []byte{}.
- //
- // Also, from a round by round catchup perspective,
- // when we delete a box, in accountsNewRoundImpl method,
- // the kv pair with value = nil will be deleted from kvstore table.
- // Thus, it seems more consistent and appropriate to write as []byte{}.
-
- if kvrs[i].Value == nil {
- kvrs[i].Value = []byte{}
- }
- values[i] = kvrs[i].Value
- hashes[i] = trackerdb.KvHashBuilderV6(string(keys[i]), values[i])
- }
-
- return crw.WriteCatchpointStagingKVs(ctx, keys, values, hashes)
- })
-}
-
-func (w *stagingWriterImpl) writeCreatables(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
-
- return crw.WriteCatchpointStagingCreatable(ctx, balances)
- })
-}
-
-func (w *stagingWriterImpl) writeHashes(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- return w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- crw, err := tx.MakeCatchpointReaderWriter()
+func (w *stagingWriterImpl) write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) {
+ var report trackerdb.CatchpointReport
+ err := w.wdb.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
+ crw, err := tx.MakeCatchpointWriter()
if err != nil {
return err
}
-
- return crw.WriteCatchpointStagingHashes(ctx, balances)
+ report, err = crw.Write(ctx, payload)
+ return err
})
-}
-
-func (w *stagingWriterImpl) isShared() bool {
- return w.wdb.IsSharedCacheConnection()
+ return report, err
}
// catchpointCatchupAccessorImpl is the concrete implementation of the CatchpointCatchupAccessor interface
type catchpointCatchupAccessorImpl struct {
ledger *Ledger
- catchpointStore trackerdb.CatchpointReaderWriter
+ catchpointStore catchpointdb.Store
stagingWriter stagingWriter
@@ -242,10 +171,9 @@ type CatchupAccessorClientLedger interface {
// MakeCatchpointCatchupAccessor creates a CatchpointCatchupAccessor given a ledger
func MakeCatchpointCatchupAccessor(ledger *Ledger, log logging.Logger) CatchpointCatchupAccessor {
- crw, _ := ledger.trackerDB().MakeCatchpointReaderWriter()
return &catchpointCatchupAccessorImpl{
ledger: ledger,
- catchpointStore: crw,
+ catchpointStore: ledger.catchpointDB(),
stagingWriter: &stagingWriterImpl{wdb: ledger.trackerDB()},
log: log,
}
@@ -254,9 +182,9 @@ func MakeCatchpointCatchupAccessor(ledger *Ledger, log logging.Logger) Catchpoin
// GetState returns the current state of the catchpoint catchup
func (c *catchpointCatchupAccessorImpl) GetState(ctx context.Context) (state CatchpointCatchupState, err error) {
var istate uint64
- istate, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState)
+ istate, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState)
if err != nil {
- return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err)
+ return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err)
}
state = CatchpointCatchupState(istate)
return
@@ -267,18 +195,18 @@ func (c *catchpointCatchupAccessorImpl) SetState(ctx context.Context, state Catc
if state < CatchpointCatchupStateInactive || state > catchpointCatchupStateLast {
return fmt.Errorf("invalid catchpoint catchup state provided : %d", state)
}
- err = c.catchpointStore.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, uint64(state))
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, uint64(state))
if err != nil {
- return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err)
+ return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err)
}
return
}
// GetLabel returns the current catchpoint catchup label
func (c *catchpointCatchupAccessorImpl) GetLabel(ctx context.Context) (label string, err error) {
- label, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel)
+ label, err = c.catchpointStore.ReadCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel)
if err != nil {
- return "", fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err)
+ return "", fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err)
}
return
}
@@ -290,9 +218,9 @@ func (c *catchpointCatchupAccessorImpl) SetLabel(ctx context.Context, label stri
if err != nil {
return
}
- err = c.catchpointStore.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, label)
+ err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, label)
if err != nil {
- return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err)
+ return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err)
}
return
}
@@ -304,38 +232,51 @@ func (c *catchpointCatchupAccessorImpl) ResetStagingBalances(ctx context.Context
}
start := time.Now()
ledgerResetstagingbalancesCount.Inc(nil)
- err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
+ err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
crw, err := tx.MakeCatchpointWriter()
if err != nil {
return err
}
- err = crw.ResetCatchpointStagingBalances(ctx, newCatchup)
+ err = crw.Reset(ctx, newCatchup)
if err != nil {
return fmt.Errorf("unable to reset catchpoint catchup balances : %v", err)
}
- if !newCatchup {
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, 0)
+ return
+ })
+ if err != nil {
+ // failed to reset staging balances
+ ledgerResetstagingbalancesMicros.AddMicrosecondsSince(start, nil)
+ return
+ }
+
+ if !newCatchup {
+ // reset catchup internal durable state
+ err = c.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) {
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, 0)
if err != nil {
return err
}
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, 0)
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, 0)
if err != nil {
return err
}
- err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, "")
+ err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, "")
if err != nil {
return err
}
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, 0)
+
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, 0)
if err != nil {
- return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err)
+ return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err)
}
- }
- return
- })
+
+ return
+ })
+ }
+
ledgerResetstagingbalancesMicros.AddMicrosecondsSince(start, nil)
return
}
@@ -423,33 +364,41 @@ func (c *catchpointCatchupAccessorImpl) processStagingContent(ctx context.Contex
// TotalAccounts, TotalAccounts, Catchpoint, BlockHeaderDigest, BalancesRound
start := time.Now()
ledgerProcessstagingcontentCount.Inc(nil)
- err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
- cw, err := tx.MakeCatchpointWriter()
- if err != nil {
- return err
- }
- err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion, fileHeader.Version)
- if err != nil {
- return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup version '%s': %v", trackerdb.CatchpointStateCatchupVersion, err)
- }
- aw, err := tx.MakeAccountsWriter()
+
+ // update catchup internal durable state
+ err = c.catchpointStore.Batch(func(ctx context.Context, tx catchpointdb.BatchScope) (err error) {
+ err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupVersion, fileHeader.Version)
if err != nil {
- return err
+ return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup version '%s': %v", catchpointdb.CatchpointStateCatchupVersion, err)
}
-
- err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, uint64(fileHeader.BlocksRound))
+ err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, uint64(fileHeader.BlocksRound))
if err != nil {
- return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err)
+ return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBlockRound, err)
}
if fileHeader.Version >= CatchpointFileVersionV6 {
- err = cw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound, uint64(fileHeader.BlocksRound))
+ err = tx.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound, uint64(fileHeader.BlocksRound))
if err != nil {
- return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupHashRound, err)
+ return fmt.Errorf("CatchpointCatchupAccessorImpl::processStagingContent: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupHashRound, err)
}
}
- err = aw.AccountsPutTotals(fileHeader.Totals, true)
return
})
+ if err != nil {
+ // failed to update catchup internal state
+ ledgerProcessstagingcontentMicros.AddMicrosecondsSince(start, nil)
+ return err
+ }
+
+ // put totals on the ledger
+ err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
+ aw, err := tx.MakeAccountsWriter()
+ if err != nil {
+ return err
+ }
+
+ return aw.AccountsPutTotals(fileHeader.Totals, true)
+ })
+
ledgerProcessstagingcontentMicros.AddMicrosecondsSince(start, nil)
if err == nil {
progress.SeenHeader = true
@@ -588,101 +537,49 @@ func (c *catchpointCatchupAccessorImpl) processStagingBalances(ctx context.Conte
}
}
- wg := sync.WaitGroup{}
-
- var errBalances error
- var errCreatables error
- var errHashes error
- var errKVs error
- var durBalances time.Duration
- var durCreatables time.Duration
- var durHashes time.Duration
- var durKVs time.Duration
-
- // start the balances writer
- wg.Add(1)
- go func() {
- defer wg.Done()
- writeBalancesStart := time.Now()
- errBalances = c.stagingWriter.writeBalances(ctx, normalizedAccountBalances)
- durBalances = time.Since(writeBalancesStart)
- }()
-
- // on a in-memory database, wait for the writer to finish before starting the new writer
- if c.stagingWriter.isShared() {
- wg.Wait()
- }
-
- // starts the creatables writer
- wg.Add(1)
- go func() {
- defer wg.Done()
- hasCreatables := false
- for _, accBal := range normalizedAccountBalances {
- for _, res := range accBal.Resources {
- if res.IsOwning() {
- hasCreatables = true
- break
- }
- }
- }
- if hasCreatables {
- writeCreatablesStart := time.Now()
- errCreatables = c.stagingWriter.writeCreatables(ctx, normalizedAccountBalances)
- durCreatables = time.Since(writeCreatablesStart)
+ // fix empty KV's
+ for i := range chunkKVs {
+ // Since `encoded.KVRecordV6` is `omitempty` and `omitemptyarray`,
+ // when we have an instance of `encoded.KVRecordV6` with nil value,
+ // an empty box is unmarshalled to have `nil` value,
+ // while this might be mistaken to be a box deletion.
+ //
+ // We don't want to mistake this to be a deleted box:
+ // We are (and should be) during Fast Catchup (FC)
+ // writing to DB with empty byte string, rather than writing nil.
+ //
+ // This matters in sqlite3,
+ // for sqlite3 differs on writing nil byte slice to table from writing []byte{}:
+ // - writing nil byte slice is true that `value is NULL`
+ // - writing []byte{} is false on `value is NULL`.
+ //
+ // For the sake of consistency, we convert nil to []byte{}.
+ //
+ // Also, from a round by round catchup perspective,
+ // when we delete a box, in accountsNewRoundImpl method,
+ // the kv pair with value = nil will be deleted from kvstore table.
+ // Thus, it seems more consistent and appropriate to write as []byte{}.
+ if chunkKVs[i].Value == nil {
+ chunkKVs[i].Value = []byte{}
}
- }()
-
- // on a in-memory database, wait for the writer to finish before starting the new writer
- if c.stagingWriter.isShared() {
- wg.Wait()
}
- // start the accounts pending hashes writer
- wg.Add(1)
- go func() {
- defer wg.Done()
- writeHashesStart := time.Now()
- errHashes = c.stagingWriter.writeHashes(ctx, normalizedAccountBalances)
- durHashes = time.Since(writeHashesStart)
- }()
-
- // on a in-memory database, wait for the writer to finish before starting the new writer
- if c.stagingWriter.isShared() {
- wg.Wait()
- }
-
- // start the kv store writer
- wg.Add(1)
- go func() {
- defer wg.Done()
-
- writeKVsStart := time.Now()
- errKVs = c.stagingWriter.writeKVs(ctx, chunkKVs)
- durKVs = time.Since(writeKVsStart)
- }()
-
- wg.Wait()
-
- if errBalances != nil {
- return errBalances
- }
- if errCreatables != nil {
- return errCreatables
- }
- if errHashes != nil {
- return errHashes
- }
- if errKVs != nil {
- return errKVs
+ // apply catchpoint
+ var report trackerdb.CatchpointReport
+ report, err = c.stagingWriter.write(ctx, trackerdb.CatchpointPayload{
+ Accounts: normalizedAccountBalances,
+ KVRecords: chunkKVs,
+ })
+ if err != nil {
+ return err
}
- progress.BalancesWriteDuration += durBalances
- progress.CreatablesWriteDuration += durCreatables
- progress.HashesWriteDuration += durHashes
- progress.KVWriteDuration += durKVs
-
ledgerProcessstagingbalancesMicros.AddMicrosecondsSince(start, nil)
+
+ progress.BalancesWriteDuration += report.BalancesWriteDuration
+ progress.CreatablesWriteDuration += report.CreatablesWriteDuration
+ progress.HashesWriteDuration += report.HashesWriteDuration
+ progress.KVWriteDuration += report.KVWriteDuration
progress.ProcessedBytes += uint64(len(bytes))
progress.ProcessedKVs += uint64(len(chunkKVs))
for _, acctBal := range normalizedAccountBalances {
@@ -723,23 +620,6 @@ func countHashes(hashes [][]byte) (accountCount, kvCount uint64) {
// BuildMerkleTrie would process the catchpointpendinghashes and insert all the items in it into the merkle trie
func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, progressUpdates func(uint64, uint64)) (err error) {
dbs := c.ledger.trackerDB()
- err = dbs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
- crw, err := tx.MakeCatchpointWriter()
- if err != nil {
- return err
- }
-
- // creating the index can take a while, so ensure we don't generate false alerts for no good reason.
- _, err = tx.ResetTransactionWarnDeadline(ctx, time.Now().Add(120*time.Second))
- if err != nil {
- return err
- }
-
- return crw.CreateCatchpointStagingHashesIndex(ctx)
- })
- if err != nil {
- return
- }
wg := sync.WaitGroup{}
wg.Add(2)
@@ -754,7 +634,7 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro
defer wg.Done()
defer close(writerQueue)
- dbErr := dbs.Snapshot(func(transactionCtx context.Context, tx trackerdb.SnapshotScope) (err error) {
+ dbErr := dbs.Transaction(func(transactionCtx context.Context, tx trackerdb.TransactionScope) (err error) {
it := tx.MakeCatchpointPendingHashesIterator(trieRebuildAccountChunkSize)
var hashes [][]byte
for {
@@ -919,9 +799,9 @@ func (c *catchpointCatchupAccessorImpl) BuildMerkleTrie(ctx context.Context, pro
// GetCatchupBlockRound returns the latest block round matching the current catchpoint
func (c *catchpointCatchupAccessorImpl) GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error) {
var iRound uint64
- iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound)
+ iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound)
if err != nil {
- return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchpointLookback, err)
+ return 0, fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchpointLookback, err)
}
return basics.Round(iRound), nil
}
@@ -935,27 +815,27 @@ func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl
var catchpointLabel string
var version uint64
- catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel)
+ catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel)
if err != nil {
- return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err)
+ return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupLabel, err)
}
- version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion)
+ version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupVersion)
if err != nil {
return fmt.Errorf("unable to retrieve catchpoint version: %v", err)
}
var iRound uint64
- iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound)
+ iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound)
if err != nil {
- return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err)
+ return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBlockRound, err)
}
blockRound = basics.Round(iRound)
start := time.Now()
ledgerVerifycatchpointCount.Inc(nil)
err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
@@ -976,12 +856,12 @@ func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, bl
return fmt.Errorf("unable to get trie root hash: %v", err)
}
- totals, err = arw.AccountsTotals(ctx, true)
+ totals, err = ar.AccountsTotals(ctx, true)
if err != nil {
return fmt.Errorf("unable to get accounts totals: %v", err)
}
- rawStateProofVerificationContext, err = tx.MakeSpVerificationCtxReaderWriter().GetAllSPContextsFromCatchpointTbl(ctx)
+ rawStateProofVerificationContext, err = tx.MakeSpVerificationCtxReader().GetAllSPContextsFromCatchpointTbl(ctx)
if err != nil {
return fmt.Errorf("unable to get state proof verification data: %v", err)
}
@@ -1026,18 +906,10 @@ func (c *catchpointCatchupAccessorImpl) StoreBalancesRound(ctx context.Context,
balancesRound := blk.Round() - basics.Round(catchpointLookback)
start := time.Now()
ledgerStorebalancesroundCount.Inc(nil)
- err = c.ledger.trackerDB().Batch(func(ctx context.Context, tx trackerdb.BatchScope) (err error) {
- crw, err := tx.MakeCatchpointWriter()
- if err != nil {
- return err
- }
-
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, uint64(balancesRound))
- if err != nil {
- return fmt.Errorf("CatchpointCatchupAccessorImpl::StoreBalancesRound: unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBalancesRound, err)
- }
- return
- })
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, uint64(balancesRound))
+ if err != nil {
+ return fmt.Errorf("CatchpointCatchupAccessorImpl::StoreBalancesRound: unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupBalancesRound, err)
+ }
ledgerStorebalancesroundMicros.AddMicrosecondsSince(start, nil)
return
}
@@ -1126,37 +998,41 @@ func (c *catchpointCatchupAccessorImpl) CompleteCatchup(ctx context.Context) (er
func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err error) {
start := time.Now()
ledgerCatchpointFinishBalsCount.Inc(nil)
- err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- crw, err := tx.MakeCatchpointReaderWriter()
- if err != nil {
- return err
- }
- arw, err := tx.MakeAccountsReaderWriter()
- if err != nil {
- return err
- }
+ balancesRound, err := c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound)
+ if err != nil {
+ return err
+ }
+
+ hashRound, err := c.catchpointStore.ReadCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound)
+ if err != nil {
+ return err
+ }
- var balancesRound, hashRound uint64
- var totals ledgercore.AccountTotals
+ err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
+ // get current totals
- balancesRound, err = crw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound)
+ ar, err := tx.MakeAccountsReader()
if err != nil {
return err
}
- hashRound, err = crw.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound)
+ totals, err := ar.AccountsTotals(ctx, true)
if err != nil {
return err
}
- totals, err = arw.AccountsTotals(ctx, true)
+ //
+ // reset trackerdb
+ //
+
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
if hashRound == 0 {
- err = arw.ResetAccountHashes(ctx)
+ err = aw.ResetAccountHashes(ctx)
if err != nil {
return err
}
@@ -1168,7 +1044,7 @@ func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err
// it might be necessary to restore it into the latest database version. To do that, one
// will need to run the 6->7 migration code manually here or in a similar function to create
// onlineaccounts and other V7 tables.
- err = arw.AccountsReset(ctx)
+ err = aw.AccountsReset(ctx)
if err != nil {
return err
}
@@ -1188,50 +1064,69 @@ func (c *catchpointCatchupAccessorImpl) finishBalances(ctx context.Context) (err
}
}
- err = crw.ApplyCatchpointStagingBalances(ctx, basics.Round(balancesRound), basics.Round(hashRound))
- if err != nil {
- return err
- }
+ //
+ // apply catchpoint
+ //
- err = arw.AccountsPutTotals(totals, false)
+ crw, err := tx.MakeCatchpointWriter()
if err != nil {
return err
}
- err = crw.ResetCatchpointStagingBalances(ctx, false)
+ err = crw.Apply(ctx, basics.Round(balancesRound), basics.Round(hashRound))
if err != nil {
return err
}
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBalancesRound, 0)
+ err = aw.AccountsPutTotals(totals, false)
if err != nil {
return err
}
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound, 0)
+ err = crw.Reset(ctx, false)
if err != nil {
return err
}
- err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel, "")
- if err != nil {
- return err
- }
+ return nil
+ })
- if hashRound != 0 {
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupHashRound, 0)
- if err != nil {
- return err
- }
- }
+ if err != nil {
+ // apply failed
+ ledgerCatchpointFinishBalsMicros.AddMicrosecondsSince(start, nil)
+ return err
+ }
+
+ // apply succeded
+ // reset catchup durable state
+
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBalancesRound, 0)
+ if err != nil {
+ return err
+ }
+
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupBlockRound, 0)
+ if err != nil {
+ return err
+ }
- err = crw.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupState, 0)
+ err = c.catchpointStore.WriteCatchpointStateString(ctx, catchpointdb.CatchpointStateCatchupLabel, "")
+ if err != nil {
+ return err
+ }
+
+ if hashRound != 0 {
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupHashRound, 0)
if err != nil {
- return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupState, err)
+ return err
}
+ }
+
+ err = c.catchpointStore.WriteCatchpointStateUint64(ctx, catchpointdb.CatchpointStateCatchupState, 0)
+ if err != nil {
+ return fmt.Errorf("unable to write catchpoint catchup state '%s': %v", catchpointdb.CatchpointStateCatchupState, err)
+ }
- return
- })
ledgerCatchpointFinishBalsMicros.AddMicrosecondsSince(start, nil)
return err
}
diff --git a/ledger/catchupaccessor_test.go b/ledger/catchupaccessor_test.go
index a97377bd3e..b4c249d772 100644
--- a/ledger/catchupaccessor_test.go
+++ b/ledger/catchupaccessor_test.go
@@ -527,20 +527,8 @@ type testStagingWriter struct {
hashes map[[4 + crypto.DigestSize]byte]int
}
-func (w *testStagingWriter) writeBalances(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- return nil
-}
-
-func (w *testStagingWriter) writeCreatables(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- return nil
-}
-
-func (w *testStagingWriter) writeKVs(ctx context.Context, kvrs []encoded.KVRecordV6) error {
- return nil
-}
-
-func (w *testStagingWriter) writeHashes(ctx context.Context, balances []trackerdb.NormalizedAccountBalance) error {
- for _, bal := range balances {
+func (w *testStagingWriter) write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) {
+ for _, bal := range payload.Accounts {
for _, hash := range bal.AccountHashes {
var key [4 + crypto.DigestSize]byte
require.Len(w.t, hash, 4+crypto.DigestSize)
@@ -548,11 +536,7 @@ func (w *testStagingWriter) writeHashes(ctx context.Context, balances []trackerd
w.hashes[key] = w.hashes[key] + 1
}
}
- return nil
-}
-
-func (w *testStagingWriter) isShared() bool {
- return false
+ return trackerdb.CatchpointReport{}, nil
}
// makeTestCatchpointCatchupAccessor creates a CatchpointCatchupAccessor given a ledger
diff --git a/ledger/ledger.go b/ledger/ledger.go
index 0c77470873..31a8b04d85 100644
--- a/ledger/ledger.go
+++ b/ledger/ledger.go
@@ -37,7 +37,9 @@ import (
"github.com/algorand/go-algorand/ledger/eval"
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/ledger/store/blockdb"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver"
"github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
@@ -51,8 +53,9 @@ type Ledger struct {
// Database connections to the DBs storing blocks and tracker state.
// We use potentially different databases to avoid SQLite contention
// during catchup.
- trackerDBs trackerdb.TrackerStore
- blockDBs db.Pair
+ trackerDBs trackerdb.Store
+ catchpointDbs catchpointdb.Store
+ blockDBs db.Pair
// blockQ is the buffer of added blocks that will be flushed to
// persistent storage
@@ -144,14 +147,11 @@ func OpenLedger(
}
}()
- l.trackerDBs, l.blockDBs, err = openLedgerDB(dbPathPrefix, dbMem)
+ l.trackerDBs, l.blockDBs, err = openLedgerDB(dbPathPrefix, dbMem, cfg, log)
if err != nil {
err = fmt.Errorf("OpenLedger.openLedgerDB %v", err)
return nil, err
}
- l.trackerDBs.SetLogger(log)
- l.blockDBs.Rdb.SetLogger(log)
- l.blockDBs.Wdb.SetLogger(log)
l.setSynchronousMode(context.Background(), l.synchronousMode)
@@ -284,12 +284,9 @@ func (l *Ledger) verifyMatchingGenesisHash() (err error) {
return
}
-func openLedgerDB(dbPathPrefix string, dbMem bool) (trackerDBs trackerdb.TrackerStore, blockDBs db.Pair, err error) {
+func openLedgerDB(dbPathPrefix string, dbMem bool, cfg config.Local, log logging.Logger) (trackerDBs trackerdb.Store, blockDBs db.Pair, err error) {
// Backwards compatibility: we used to store both blocks and tracker
// state in a single SQLite db file.
- var trackerDBFilename string
- var blockDBFilename string
-
if !dbMem {
commonDBFilename := dbPathPrefix + ".sqlite"
_, err = os.Stat(commonDBFilename)
@@ -302,20 +299,36 @@ func openLedgerDB(dbPathPrefix string, dbMem bool) (trackerDBs trackerdb.Tracker
}
}
- trackerDBFilename = dbPathPrefix + ".tracker.sqlite"
- blockDBFilename = dbPathPrefix + ".block.sqlite"
-
outErr := make(chan error, 2)
go func() {
var lerr error
- trackerDBs, lerr = sqlitedriver.OpenTrackerSQLStore(trackerDBFilename, dbMem)
+ switch cfg.StorageEngine {
+ case "pebbledb":
+ dir := dbPathPrefix + "/tracker"
+ trackerDBs, lerr = pebbledbdriver.Open(dir, dbMem, config.Consensus[protocol.ConsensusCurrentVersion], log)
+ // pebbledb needs to be explicitly configured.
+ // anything else will initialize a sqlite engine.
+ case "sqlite":
+ fallthrough
+ default:
+ file := dbPathPrefix + ".tracker.sqlite"
+ trackerDBs, lerr = sqlitedriver.Open(file, dbMem, log)
+ }
+
outErr <- lerr
}()
go func() {
var lerr error
+ blockDBFilename := dbPathPrefix + ".block.sqlite"
blockDBs, lerr = db.OpenPair(blockDBFilename, dbMem)
- outErr <- lerr
+ if lerr != nil {
+ outErr <- lerr
+ return
+ }
+ blockDBs.Rdb.SetLogger(log)
+ blockDBs.Wdb.SetLogger(log)
+ outErr <- nil
}()
err = <-outErr
@@ -805,10 +818,15 @@ func (l *Ledger) GetCatchpointStream(round basics.Round) (ReadCloseSizer, error)
}
// ledgerForTracker methods
-func (l *Ledger) trackerDB() trackerdb.TrackerStore {
+func (l *Ledger) trackerDB() trackerdb.Store {
return l.trackerDBs
}
+// ledgerForTracker methods
+func (l *Ledger) catchpointDB() catchpointdb.Store {
+ return l.catchpointDbs
+}
+
// ledgerForTracker methods
func (l *Ledger) blockDB() db.Pair {
return l.blockDBs
diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go
index e9a413678c..5165fcfa06 100644
--- a/ledger/ledger_test.go
+++ b/ledger/ledger_test.go
@@ -43,6 +43,7 @@ import (
"github.com/algorand/go-algorand/data/transactions/verify"
"github.com/algorand/go-algorand/ledger/eval"
"github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
ledgertesting "github.com/algorand/go-algorand/ledger/testing"
"github.com/algorand/go-algorand/logging"
@@ -2277,7 +2278,7 @@ func TestLedgerReloadTxTailHistoryAccess(t *testing.T) {
// reset tables and re-init again, similary to the catchpount apply code
// since the ledger has only genesis accounts, this recreates them
- err = l.trackerDBs.Batch(func(ctx context.Context, tx trackerdb.BatchScope) error {
+ err = l.trackerDBs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
arw, err := tx.MakeAccountsWriter()
if err != nil {
return err
@@ -2296,7 +2297,7 @@ func TestLedgerReloadTxTailHistoryAccess(t *testing.T) {
DbPathPrefix: l.catchpoint.dbDirectory,
BlockDb: l.blockDBs,
}
- _, err0 = tx.Testing().RunMigrations(ctx, tp, l.log, preReleaseDBVersion /*target database version*/)
+ _, err0 = tx.RunMigrations(ctx, tp, l.log, preReleaseDBVersion /*target database version*/)
if err0 != nil {
return err0
}
@@ -2439,7 +2440,7 @@ func TestLedgerMigrateV6ShrinkDeltas(t *testing.T) {
cfg.MaxAcctLookback = proto.MaxBalLookback
log := logging.TestingLog(t)
log.SetLevel(logging.Info) // prevent spamming with ledger.AddValidatedBlock debug message
- trackerDB, blockDB, err := openLedgerDB(dbName, inMem)
+ trackerDB, blockDB, err := openLedgerDB(dbName, inMem, cfg, log)
require.NoError(t, err)
defer func() {
trackerDB.Close()
@@ -3301,7 +3302,7 @@ func TestLedgerCatchpointSPVerificationTracker(t *testing.T) {
catchpointAccessor, accessorProgress := initializeTestCatchupAccessor(t, l, uint64(len(initkeys)))
- relCatchpointFilePath := filepath.Join(trackerdb.CatchpointDirName, trackerdb.MakeCatchpointFilePath(basics.Round(cfg.CatchpointInterval)))
+ relCatchpointFilePath := filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(basics.Round(cfg.CatchpointInterval)))
catchpointData := readCatchpointFile(t, relCatchpointFilePath)
err = catchpointAccessor.ProcessStagingBalances(context.Background(), catchpointData[1].headerName, catchpointData[1].data, &accessorProgress)
diff --git a/ledger/spverificationtracker.go b/ledger/spverificationtracker.go
index d98974897d..cca86fcca9 100644
--- a/ledger/spverificationtracker.go
+++ b/ledger/spverificationtracker.go
@@ -134,7 +134,7 @@ func (spt *spVerificationTracker) commitRound(ctx context.Context, tx trackerdb.
}
if dcc.spVerification.lastDeleteIndex >= 0 {
- err = tx.MakeSpVerificationCtxReaderWriter().DeleteOldSPContexts(ctx, dcc.spVerification.earliestLastAttestedRound)
+ err = tx.MakeSpVerificationCtxWriter().DeleteOldSPContexts(ctx, dcc.spVerification.earliestLastAttestedRound)
}
return err
@@ -146,7 +146,7 @@ func commitSPContexts(ctx context.Context, tx trackerdb.TransactionScope, commit
ptrToCtxs[i] = &commitData[i].verificationContext
}
- return tx.MakeSpVerificationCtxReaderWriter().StoreSPContexts(ctx, ptrToCtxs)
+ return tx.MakeSpVerificationCtxWriter().StoreSPContexts(ctx, ptrToCtxs)
}
func (spt *spVerificationTracker) postCommit(_ context.Context, dcc *deferredCommitContext) {
diff --git a/ledger/spverificationtracker_test.go b/ledger/spverificationtracker_test.go
index e0f073fe4d..a88c05f0ed 100644
--- a/ledger/spverificationtracker_test.go
+++ b/ledger/spverificationtracker_test.go
@@ -424,7 +424,7 @@ func TestStateProofVerificationTracker_LookupVerificationContext(t *testing.T) {
_, err := spt.LookupVerificationContext(basics.Round(0))
a.ErrorIs(err, errSPVerificationContextNotFound)
- a.ErrorContains(err, "no rows")
+ a.ErrorContains(err, "not found")
finalLastAttestedRound := basics.Round(defaultStateProofInterval + contextToAdd*defaultStateProofInterval)
_, err = spt.LookupVerificationContext(finalLastAttestedRound + basics.Round(defaultStateProofInterval))
diff --git a/ledger/store/catchpointdb/catchpoint.go b/ledger/store/catchpointdb/catchpoint.go
new file mode 100644
index 0000000000..a361e81f7d
--- /dev/null
+++ b/ledger/store/catchpointdb/catchpoint.go
@@ -0,0 +1,157 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package catchpointdb
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+)
+
+// CatchpointDirName represents the directory name in which all the catchpoints files are stored
+var CatchpointDirName = "catchpoints"
+
+// CatchpointState is used to store catchpoint related variables into the catchpointstate table.
+//
+//msgp:ignore CatchpointState
+type CatchpointState string
+
+const (
+ // CatchpointStateLastCatchpoint is written by a node once a catchpoint label is created for a round
+ CatchpointStateLastCatchpoint = CatchpointState("lastCatchpoint")
+ // CatchpointStateWritingFirstStageInfo state variable is set to 1 if catchpoint's first stage is unfinished,
+ // and is 0 otherwise. Used to clear / restart the first stage after a crash.
+ // This key is set in the same db transaction as the account updates, so the
+ // unfinished first stage corresponds to the current db round.
+ CatchpointStateWritingFirstStageInfo = CatchpointState("writingFirstStageInfo")
+ // CatchpointStateWritingCatchpoint if there is an unfinished catchpoint, this state variable is set to
+ // the catchpoint's round. Otherwise, it is set to 0.
+ // DEPRECATED.
+ CatchpointStateWritingCatchpoint = CatchpointState("writingCatchpoint")
+ // CatchpointStateCatchupState is the state of the catchup process. The variable is stored only during the catchpoint catchup process, and removed afterward.
+ CatchpointStateCatchupState = CatchpointState("catchpointCatchupState")
+ // CatchpointStateCatchupLabel is the label to which the currently catchpoint catchup process is trying to catchup to.
+ CatchpointStateCatchupLabel = CatchpointState("catchpointCatchupLabel")
+ // CatchpointStateCatchupBlockRound is the block round that is associated with the current running catchpoint catchup.
+ CatchpointStateCatchupBlockRound = CatchpointState("catchpointCatchupBlockRound")
+ // CatchpointStateCatchupBalancesRound is the balance round that is associated with the current running catchpoint catchup. Typically it would be
+ // equal to CatchpointStateCatchupBlockRound - 320.
+ CatchpointStateCatchupBalancesRound = CatchpointState("catchpointCatchupBalancesRound")
+ // CatchpointStateCatchupHashRound is the round that is associated with the hash of the merkle trie. Normally, it's identical to CatchpointStateCatchupBalancesRound,
+ // however, it could differ when we catchup from a catchpoint that was created using a different version : in this case,
+ // we set it to zero in order to reset the merkle trie. This would force the merkle trie to be re-build on startup ( if needed ).
+ CatchpointStateCatchupHashRound = CatchpointState("catchpointCatchupHashRound")
+ // CatchpointStateCatchpointLookback is the number of rounds we keep catchpoints for
+ CatchpointStateCatchpointLookback = CatchpointState("catchpointLookback")
+ // CatchpointStateCatchupVersion is the catchpoint version which the currently catchpoint catchup process is trying to catchup to.
+ CatchpointStateCatchupVersion = CatchpointState("catchpointCatchupVersion")
+)
+
+// UnfinishedCatchpointRecord represents a stored record of an unfinished catchpoint.
+type UnfinishedCatchpointRecord struct {
+ Round basics.Round
+ BlockHash crypto.Digest
+}
+
+// CatchpointFirstStageInfo For the `catchpointfirststageinfo` table.
+type CatchpointFirstStageInfo struct {
+ _struct struct{} `codec:",omitempty,omitemptyarray"`
+
+ Totals ledgercore.AccountTotals `codec:"accountTotals"`
+ TrieBalancesHash crypto.Digest `codec:"trieBalancesHash"`
+ // Total number of accounts in the catchpoint data file. Only set when catchpoint
+ // data files are generated.
+ TotalAccounts uint64 `codec:"accountsCount"`
+
+ // Total number of accounts in the catchpoint data file. Only set when catchpoint
+ // data files are generated.
+ TotalKVs uint64 `codec:"kvsCount"`
+
+ // Total number of chunks in the catchpoint data file. Only set when catchpoint
+ // data files are generated.
+ TotalChunks uint64 `codec:"chunksCount"`
+ // BiggestChunkLen is the size in the bytes of the largest chunk, used when re-packing.
+ BiggestChunkLen uint64 `codec:"biggestChunk"`
+
+ // StateProofVerificationHash is the hash of the state proof verification data contained in the catchpoint data file.
+ StateProofVerificationHash crypto.Digest `codec:"spVerificationHash"`
+}
+
+// MakeCatchpointFilePath builds the path of a catchpoint file.
+func MakeCatchpointFilePath(round basics.Round) string {
+ irnd := int64(round) / 256
+ outStr := ""
+ for irnd > 0 {
+ outStr = filepath.Join(outStr, fmt.Sprintf("%02x", irnd%256))
+ irnd = irnd / 256
+ }
+ outStr = filepath.Join(outStr, strconv.FormatInt(int64(round), 10)+".catchpoint")
+ return outStr
+}
+
+// RemoveSingleCatchpointFileFromDisk removes a single catchpoint file from the disk. this function does not leave empty directories
+func RemoveSingleCatchpointFileFromDisk(dbDirectory, fileToDelete string) (err error) {
+ absCatchpointFileName := filepath.Join(dbDirectory, fileToDelete)
+ err = os.Remove(absCatchpointFileName)
+ if err == nil || os.IsNotExist(err) {
+ // it's ok if the file doesn't exist.
+ err = nil
+ } else {
+ // we can't delete the file, abort -
+ return fmt.Errorf("unable to delete old catchpoint file '%s' : %v", absCatchpointFileName, err)
+ }
+ splitedDirName := strings.Split(fileToDelete, string(os.PathSeparator))
+
+ var subDirectoriesToScan []string
+ //build a list of all the subdirs
+ currentSubDir := ""
+ for _, element := range splitedDirName {
+ currentSubDir = filepath.Join(currentSubDir, element)
+ subDirectoriesToScan = append(subDirectoriesToScan, currentSubDir)
+ }
+
+ // iterating over the list of directories. starting from the sub dirs and moving up.
+ // skipping the file itself.
+ for i := len(subDirectoriesToScan) - 2; i >= 0; i-- {
+ absSubdir := filepath.Join(dbDirectory, subDirectoriesToScan[i])
+ if _, err := os.Stat(absSubdir); os.IsNotExist(err) {
+ continue
+ }
+
+ isEmpty, err := isDirEmpty(absSubdir)
+ if err != nil {
+ return fmt.Errorf("unable to read old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err)
+ }
+ if isEmpty {
+ err = os.Remove(absSubdir)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ return fmt.Errorf("unable to delete old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err)
+ }
+ }
+ }
+
+ return nil
+}
diff --git a/ledger/store/catchpointdb/catchpointdb.go b/ledger/store/catchpointdb/catchpointdb.go
new file mode 100644
index 0000000000..4e639001a3
--- /dev/null
+++ b/ledger/store/catchpointdb/catchpointdb.go
@@ -0,0 +1,182 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package catchpointdb
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+)
+
+type Store interface {
+ ReaderWriter
+ // init
+ RunMigrations(ctx context.Context, targetVersion int32) (err error)
+ // batch support
+ Batch(fn BatchFn) (err error)
+ BatchContext(ctx context.Context, fn BatchFn) (err error)
+ // root methods
+ DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error)
+ // cleanup
+ Close()
+}
+
+// BatchFn is the callback lambda used in `Batch`.
+type BatchFn func(ctx context.Context, tx BatchScope) error
+
+// BatchScope is an atomic write-only scope to the store.
+type BatchScope interface {
+ Writer
+}
+
+// Writer is the write interface for the catchpoint store
+type Writer interface {
+ MetadataWriter
+ GeneratorWriter
+ CatchupWriter
+}
+
+// GeneratorWriter all writes needs into the catchpoint store during catchpoint generation
+type GeneratorWriter interface {
+ CatchpointDB_catchpointstate_Writer
+ CatchpointDB_unfinishedcatchpoints_Writer
+ CatchpointDB_catchpointfirststageinfo_Writer
+}
+
+// CatchupWriter all catchpoint processing writes needed during catchpoint catchup
+type CatchupWriter interface {
+ CatchpointDB_catchpointstate_Writer
+}
+
+// Reader is the read interface for the catchpoint store
+type Reader interface {
+ GetVersion(ctx context.Context, staging bool) (uint64, error)
+
+ MetadataReader
+ GeneratorReader
+ CatchupReader
+}
+
+// GeneratorReader all read needs from the catchpoint store during catchpoint generation
+type GeneratorReader interface {
+ CatchpointDB_catchpointstate_Reader
+ CatchpointDB_unfinishedcatchpoints_Reader
+ CatchpointDB_catchpointfirststageinfo_Reader
+}
+
+// CatchupReader all catchpoint processing reads needed during catchpoint catchup
+type CatchupReader interface {
+ CatchpointDB_catchpointstate_Reader
+}
+
+// ReaderWriter combines the Reader and Writer
+type ReaderWriter interface {
+ Reader
+ Writer
+}
+
+/// -------------
+/// catchpoints metadata register
+
+type MetadataWriter interface {
+ StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error)
+}
+
+type MetadataReader interface {
+ GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error)
+ GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error)
+}
+
+///------------------------
+/// catchup runtime (durable)
+/// could this be part of apply?
+
+// used during:
+// - mainly in catchpoint application, with a couple uses in catchpoint generation
+// - used in catchpoint tracker during commit round
+// - used to signal that the db state is ready for being read by the catchpoint generator
+// - CatchpointStateWritingFirstStageInfo, catchpoint is unfinished (dirty flag)
+// - CatchpointStateCatchpointLookback, handle protocol changes during crash recovery
+//
+// decision here:
+// we could simplify the code, and restart cathcpoint from a clean state after a crash
+// and not need to resume where we left of
+
+// During Generation:
+// - CatchpointStateWritingFirstStageInfo (write)
+// - CatchpointStateCatchpointLookback (write)
+// - CatchpointStateLastCatchpoint (write)
+// During Apply:
+// - CatchpointStateCatchupVersion .. version (write)
+// - CatchpointStateCatchupState .. state (write)
+// - CatchpointStateCatchupBlockRound .. block round (write)
+// - CatchpointStateCatchupHashRound .. hash round (write)
+// - CatchpointStateCatchupBalancesRound .. balances round (write)
+// - CatchpointStateCatchupLabel .. label (write)
+type CatchpointDB_catchpointstate_Writer interface {
+ WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error)
+ WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error)
+}
+
+// During Generation:
+// - CatchpointStateWritingFirstStageInfo (read)
+// - CatchpointStateCatchpointLookback (read)
+// - CatchpointStateLastCatchpoint (read)
+// During Apply:
+// - CatchpointStateCatchupVersion .. version (read)
+// - CatchpointStateCatchupState .. state (read)
+// - CatchpointStateCatchupBlockRound .. block round (read)
+// - CatchpointStateCatchupHashRound .. hash round (read)
+// - CatchpointStateCatchupBalancesRound .. balances round (read)
+// - CatchpointStateCatchupLabel .. label (read)
+// CatchpointDump?:
+// - CatchpointStateCatchupVersion .. version (read)
+type CatchpointDB_catchpointstate_Reader interface {
+ ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error)
+ ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error)
+}
+
+// During Generation:
+// - InsertUnfinishedCatchpoint
+// - DeleteUnfinishedCatchpoint
+type CatchpointDB_unfinishedcatchpoints_Writer interface {
+ InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error
+ DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error
+}
+
+// During Generation:
+// - SelectUnfinishedCatchpoints
+type CatchpointDB_unfinishedcatchpoints_Reader interface {
+ SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error)
+}
+
+// During Generation:
+// - InsertOrReplaceCatchpointFirstStageInfo
+// - DeleteOldCatchpointFirstStageInfo
+type CatchpointDB_catchpointfirststageinfo_Writer interface {
+ InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error
+ DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error
+}
+
+// During Generation:
+// - SelectCatchpointFirstStageInfo
+// - SelectOldCatchpointFirstStageInfoRounds
+type CatchpointDB_catchpointfirststageinfo_Reader interface {
+ SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool /*exists*/, error)
+ SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error)
+}
diff --git a/ledger/store/catchpointdb/catchpointdb_impl.go b/ledger/store/catchpointdb/catchpointdb_impl.go
new file mode 100644
index 0000000000..a47cc79b22
--- /dev/null
+++ b/ledger/store/catchpointdb/catchpointdb_impl.go
@@ -0,0 +1,101 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package catchpointdb
+
+import (
+ "context"
+ "database/sql"
+
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/util/db"
+)
+
+var LatestSchemaVersion int32 = 1
+
+type catchpointStore struct {
+ pair db.Pair
+ Reader
+ Writer
+}
+
+func Open(dbFilename string, dbMem bool, log logging.Logger) (Store, error) {
+ pair, err := db.OpenPair(dbFilename, dbMem)
+ if err != nil {
+ return nil, err
+ }
+ pair.Rdb.SetLogger(log)
+ pair.Wdb.SetLogger(log)
+ return MakeStore(pair), nil
+}
+
+func MakeStore(pair db.Pair) Store {
+ return &catchpointStore{pair, makeReader(pair.Rdb.Handle), makeWriter(pair.Wdb.Handle)}
+}
+
+// RunMigrations implements Store
+func (*catchpointStore) RunMigrations(ctx context.Context, targetVersion int32) (err error) {
+ panic("unimplemented")
+}
+
+// Batch implements Store
+func (store *catchpointStore) Batch(fn BatchFn) (err error) {
+ return store.BatchContext(context.Background(), fn)
+}
+
+// BatchContext implements Store
+func (store *catchpointStore) BatchContext(ctx context.Context, fn BatchFn) (err error) {
+ return store.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ return fn(ctx, &batchScope{makeWriter(tx)})
+ })
+}
+
+// DeleteStoredCatchpoints iterates over the storedcatchpoints table and deletes all the files stored on disk.
+// once all the files have been deleted, it would go ahead and remove the entries from the table.
+func (store *catchpointStore) DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) {
+ catchpointsFilesChunkSize := 50
+ for {
+ fileNames, err := store.GetOldestCatchpointFiles(ctx, catchpointsFilesChunkSize, 0)
+ if err != nil {
+ return err
+ }
+ if len(fileNames) == 0 {
+ break
+ }
+
+ for round, fileName := range fileNames {
+ err = RemoveSingleCatchpointFileFromDisk(dbDirectory, fileName)
+ if err != nil {
+ return err
+ }
+ // clear the entry from the database
+ err = store.StoreCatchpoint(ctx, round, "", "", 0)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// Close implements Store
+func (store *catchpointStore) Close() {
+ store.pair.Close()
+}
+
+type batchScope struct {
+ Writer
+}
diff --git a/ledger/store/trackerdb/sqlitedriver/catchpoint_test.go b/ledger/store/catchpointdb/catchpointdb_test.go
similarity index 64%
rename from ledger/store/trackerdb/sqlitedriver/catchpoint_test.go
rename to ledger/store/catchpointdb/catchpointdb_test.go
index e2c57c0f01..de5e222040 100644
--- a/ledger/store/trackerdb/sqlitedriver/catchpoint_test.go
+++ b/ledger/store/catchpointdb/catchpointdb_test.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with go-algorand. If not, see .
-package sqlitedriver
+package catchpointdb
import (
"context"
@@ -24,7 +24,6 @@ import (
"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/data/basics"
storetesting "github.com/algorand/go-algorand/ledger/store/testing"
- "github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/test/partitiontest"
"github.com/stretchr/testify/require"
)
@@ -33,47 +32,46 @@ import (
func TestCatchpointFirstStageInfoTable(t *testing.T) {
partitiontest.PartitionTest(t)
- dbs, _ := storetesting.DbOpenTest(t, true)
+ pair, _ := storetesting.DbOpenTest(t, true)
+ dbs := MakeStore(pair)
defer dbs.Close()
ctx := context.Background()
- err := accountsCreateCatchpointFirstStageInfoTable(ctx, dbs.Wdb.Handle)
+ err := dbs.RunMigrations(ctx, LatestSchemaVersion)
require.NoError(t, err)
- crw := NewCatchpointSQLReaderWriter(dbs.Wdb.Handle)
-
for _, round := range []basics.Round{4, 6, 8} {
- info := trackerdb.CatchpointFirstStageInfo{
+ info := CatchpointFirstStageInfo{
TotalAccounts: uint64(round) * 10,
}
- err = crw.InsertOrReplaceCatchpointFirstStageInfo(ctx, round, &info)
+ err = dbs.InsertOrReplaceCatchpointFirstStageInfo(ctx, round, &info)
require.NoError(t, err)
}
for _, round := range []basics.Round{4, 6, 8} {
- info, exists, err := crw.SelectCatchpointFirstStageInfo(ctx, round)
+ info, exists, err := dbs.SelectCatchpointFirstStageInfo(ctx, round)
require.NoError(t, err)
require.True(t, exists)
- infoExpected := trackerdb.CatchpointFirstStageInfo{
+ infoExpected := CatchpointFirstStageInfo{
TotalAccounts: uint64(round) * 10,
}
require.Equal(t, infoExpected, info)
}
- _, exists, err := crw.SelectCatchpointFirstStageInfo(ctx, 7)
+ _, exists, err := dbs.SelectCatchpointFirstStageInfo(ctx, 7)
require.NoError(t, err)
require.False(t, exists)
- rounds, err := crw.SelectOldCatchpointFirstStageInfoRounds(ctx, 6)
+ rounds, err := dbs.SelectOldCatchpointFirstStageInfoRounds(ctx, 6)
require.NoError(t, err)
require.Equal(t, []basics.Round{4, 6}, rounds)
- err = crw.DeleteOldCatchpointFirstStageInfo(ctx, 6)
+ err = dbs.DeleteOldCatchpointFirstStageInfo(ctx, 6)
require.NoError(t, err)
- rounds, err = crw.SelectOldCatchpointFirstStageInfoRounds(ctx, 9)
+ rounds, err = dbs.SelectOldCatchpointFirstStageInfoRounds(ctx, 9)
require.NoError(t, err)
require.Equal(t, []basics.Round{8}, rounds)
}
@@ -81,28 +79,26 @@ func TestCatchpointFirstStageInfoTable(t *testing.T) {
func TestUnfinishedCatchpointsTable(t *testing.T) {
partitiontest.PartitionTest(t)
- dbs, _ := storetesting.DbOpenTest(t, true)
+ pair, _ := storetesting.DbOpenTest(t, true)
+ dbs := MakeStore(pair)
defer dbs.Close()
- cts := NewCatchpointSQLReaderWriter(dbs.Wdb.Handle)
-
- err := accountsCreateUnfinishedCatchpointsTable(
- context.Background(), dbs.Wdb.Handle)
+ err := dbs.RunMigrations(context.Background(), LatestSchemaVersion)
require.NoError(t, err)
var d3 crypto.Digest
rand.Read(d3[:])
- err = cts.InsertUnfinishedCatchpoint(context.Background(), 3, d3)
+ err = dbs.InsertUnfinishedCatchpoint(context.Background(), 3, d3)
require.NoError(t, err)
var d5 crypto.Digest
rand.Read(d5[:])
- err = cts.InsertUnfinishedCatchpoint(context.Background(), 5, d5)
+ err = dbs.InsertUnfinishedCatchpoint(context.Background(), 5, d5)
require.NoError(t, err)
- ret, err := cts.SelectUnfinishedCatchpoints(context.Background())
+ ret, err := dbs.SelectUnfinishedCatchpoints(context.Background())
require.NoError(t, err)
- expected := []trackerdb.UnfinishedCatchpointRecord{
+ expected := []UnfinishedCatchpointRecord{
{
Round: 3,
BlockHash: d3,
@@ -114,12 +110,12 @@ func TestUnfinishedCatchpointsTable(t *testing.T) {
}
require.Equal(t, expected, ret)
- err = cts.DeleteUnfinishedCatchpoint(context.Background(), 3)
+ err = dbs.DeleteUnfinishedCatchpoint(context.Background(), 3)
require.NoError(t, err)
- ret, err = cts.SelectUnfinishedCatchpoints(context.Background())
+ ret, err = dbs.SelectUnfinishedCatchpoints(context.Background())
require.NoError(t, err)
- expected = []trackerdb.UnfinishedCatchpointRecord{
+ expected = []UnfinishedCatchpointRecord{
{
Round: 5,
BlockHash: d5,
diff --git a/ledger/store/catchpointdb/msgp_gen.go b/ledger/store/catchpointdb/msgp_gen.go
new file mode 100644
index 0000000000..9ddeabc697
--- /dev/null
+++ b/ledger/store/catchpointdb/msgp_gen.go
@@ -0,0 +1,261 @@
+package catchpointdb
+
+// Code generated by github.com/algorand/msgp DO NOT EDIT.
+
+import (
+ "github.com/algorand/msgp/msgp"
+)
+
+// The following msgp objects are implemented in this file:
+// CatchpointFirstStageInfo
+// |-----> (*) MarshalMsg
+// |-----> (*) CanMarshalMsg
+// |-----> (*) UnmarshalMsg
+// |-----> (*) CanUnmarshalMsg
+// |-----> (*) Msgsize
+// |-----> (*) MsgIsZero
+//
+
+// MarshalMsg implements msgp.Marshaler
+func (z *CatchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) {
+ o = msgp.Require(b, z.Msgsize())
+ // omitempty: check for empty values
+ zb0001Len := uint32(7)
+ var zb0001Mask uint8 /* 8 bits */
+ if (*z).Totals.MsgIsZero() {
+ zb0001Len--
+ zb0001Mask |= 0x2
+ }
+ if (*z).TotalAccounts == 0 {
+ zb0001Len--
+ zb0001Mask |= 0x4
+ }
+ if (*z).BiggestChunkLen == 0 {
+ zb0001Len--
+ zb0001Mask |= 0x8
+ }
+ if (*z).TotalChunks == 0 {
+ zb0001Len--
+ zb0001Mask |= 0x10
+ }
+ if (*z).TotalKVs == 0 {
+ zb0001Len--
+ zb0001Mask |= 0x20
+ }
+ if (*z).StateProofVerificationHash.MsgIsZero() {
+ zb0001Len--
+ zb0001Mask |= 0x40
+ }
+ if (*z).TrieBalancesHash.MsgIsZero() {
+ zb0001Len--
+ zb0001Mask |= 0x80
+ }
+ // variable map header, size zb0001Len
+ o = append(o, 0x80|uint8(zb0001Len))
+ if zb0001Len != 0 {
+ if (zb0001Mask & 0x2) == 0 { // if not empty
+ // string "accountTotals"
+ o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
+ o = (*z).Totals.MarshalMsg(o)
+ }
+ if (zb0001Mask & 0x4) == 0 { // if not empty
+ // string "accountsCount"
+ o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+ o = msgp.AppendUint64(o, (*z).TotalAccounts)
+ }
+ if (zb0001Mask & 0x8) == 0 { // if not empty
+ // string "biggestChunk"
+ o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b)
+ o = msgp.AppendUint64(o, (*z).BiggestChunkLen)
+ }
+ if (zb0001Mask & 0x10) == 0 { // if not empty
+ // string "chunksCount"
+ o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+ o = msgp.AppendUint64(o, (*z).TotalChunks)
+ }
+ if (zb0001Mask & 0x20) == 0 { // if not empty
+ // string "kvsCount"
+ o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+ o = msgp.AppendUint64(o, (*z).TotalKVs)
+ }
+ if (zb0001Mask & 0x40) == 0 { // if not empty
+ // string "spVerificationHash"
+ o = append(o, 0xb2, 0x73, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68)
+ o = (*z).StateProofVerificationHash.MarshalMsg(o)
+ }
+ if (zb0001Mask & 0x80) == 0 { // if not empty
+ // string "trieBalancesHash"
+ o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68)
+ o = (*z).TrieBalancesHash.MarshalMsg(o)
+ }
+ }
+ return
+}
+
+func (_ *CatchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool {
+ _, ok := (z).(*CatchpointFirstStageInfo)
+ return ok
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *CatchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zb0001 int
+ var zb0002 bool
+ zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if _, ok := err.(msgp.TypeError); ok {
+ zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ if zb0001 > 0 {
+ zb0001--
+ bts, err = (*z).Totals.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "Totals")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "TotalAccounts")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "TotalKVs")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "TotalChunks")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "StateProofVerificationHash")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ err = msgp.ErrTooManyArrayFields(zb0001)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array")
+ return
+ }
+ }
+ } else {
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ if zb0002 {
+ (*z) = CatchpointFirstStageInfo{}
+ }
+ for zb0001 > 0 {
+ zb0001--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ switch string(field) {
+ case "accountTotals":
+ bts, err = (*z).Totals.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "Totals")
+ return
+ }
+ case "trieBalancesHash":
+ bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "TrieBalancesHash")
+ return
+ }
+ case "accountsCount":
+ (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "TotalAccounts")
+ return
+ }
+ case "kvsCount":
+ (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "TotalKVs")
+ return
+ }
+ case "chunksCount":
+ (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "TotalChunks")
+ return
+ }
+ case "biggestChunk":
+ (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "BiggestChunkLen")
+ return
+ }
+ case "spVerificationHash":
+ bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "StateProofVerificationHash")
+ return
+ }
+ default:
+ err = msgp.ErrNoField(string(field))
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+func (_ *CatchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool {
+ _, ok := (z).(*CatchpointFirstStageInfo)
+ return ok
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *CatchpointFirstStageInfo) Msgsize() (s int) {
+ s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size + 19 + (*z).StateProofVerificationHash.Msgsize()
+ return
+}
+
+// MsgIsZero returns whether this is a zero value
+func (z *CatchpointFirstStageInfo) MsgIsZero() bool {
+ return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) && ((*z).StateProofVerificationHash.MsgIsZero())
+}
diff --git a/ledger/store/catchpointdb/msgp_gen_test.go b/ledger/store/catchpointdb/msgp_gen_test.go
new file mode 100644
index 0000000000..1b416cdd8e
--- /dev/null
+++ b/ledger/store/catchpointdb/msgp_gen_test.go
@@ -0,0 +1,75 @@
+//go:build !skip_msgp_testing
+// +build !skip_msgp_testing
+
+package catchpointdb
+
+// Code generated by github.com/algorand/msgp DO NOT EDIT.
+
+import (
+ "testing"
+
+ "github.com/algorand/msgp/msgp"
+
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/test/partitiontest"
+)
+
+func TestMarshalUnmarshalCatchpointFirstStageInfo(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ v := CatchpointFirstStageInfo{}
+ bts := v.MarshalMsg(nil)
+ left, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+ }
+
+ left, err = msgp.Skip(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+ }
+}
+
+func TestRandomizedEncodingCatchpointFirstStageInfo(t *testing.T) {
+ protocol.RunEncodingTest(t, &CatchpointFirstStageInfo{})
+}
+
+func BenchmarkMarshalMsgCatchpointFirstStageInfo(b *testing.B) {
+ v := CatchpointFirstStageInfo{}
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ v.MarshalMsg(nil)
+ }
+}
+
+func BenchmarkAppendMsgCatchpointFirstStageInfo(b *testing.B) {
+ v := CatchpointFirstStageInfo{}
+ bts := make([]byte, 0, v.Msgsize())
+ bts = v.MarshalMsg(bts[0:0])
+ b.SetBytes(int64(len(bts)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ bts = v.MarshalMsg(bts[0:0])
+ }
+}
+
+func BenchmarkUnmarshalCatchpointFirstStageInfo(b *testing.B) {
+ v := CatchpointFirstStageInfo{}
+ bts := v.MarshalMsg(nil)
+ b.ReportAllocs()
+ b.SetBytes(int64(len(bts)))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
diff --git a/ledger/store/catchpointdb/reader.go b/ledger/store/catchpointdb/reader.go
new file mode 100644
index 0000000000..82c60a14ca
--- /dev/null
+++ b/ledger/store/catchpointdb/reader.go
@@ -0,0 +1,227 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package catchpointdb
+
+import (
+ "context"
+ "database/sql"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/util/db"
+)
+
+type reader struct {
+ q db.Queryable
+}
+
+func makeReader(q db.Queryable) Reader {
+ return &reader{q}
+}
+
+// Version implements Reader
+func (r *reader) GetVersion(ctx context.Context, staging bool) (version uint64, err error) {
+ if staging {
+ // writing the version of the catchpoint file start only on ver >= CatchpointFileVersionV7.
+ // in case the catchpoint version does not exists ReadCatchpointStateUint64 returns 0
+ version, err = r.ReadCatchpointStateUint64(context.Background(), CatchpointStateCatchupVersion)
+ if err != nil {
+ return 0, err
+ }
+ return version, err
+ }
+
+ versionAsInt32, err := db.GetUserVersion(ctx, r.q)
+ if err != nil {
+ return 0, err
+ }
+ version = uint64(versionAsInt32)
+
+ return
+}
+
+// GetCatchpoint implements Reader
+func (r *reader) GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) {
+ err = r.q.QueryRowContext(ctx, "SELECT filename, catchpoint, filesize FROM storedcatchpoints WHERE round=?", int64(round)).Scan(&fileName, &catchpoint, &fileSize)
+ return
+}
+
+// GetOldestCatchpointFiles implements Reader
+func (r *reader) GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) {
+ err = db.Retry(func() (err error) {
+ query := "SELECT round, filename FROM storedcatchpoints WHERE pinned = 0 and round <= COALESCE((SELECT round FROM storedcatchpoints WHERE pinned = 0 ORDER BY round DESC LIMIT ?, 1),0) ORDER BY round ASC LIMIT ?"
+ rows, err := r.q.QueryContext(ctx, query, filesToKeep, fileCount)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+
+ fileNames = make(map[basics.Round]string)
+ for rows.Next() {
+ var fileName string
+ var round basics.Round
+ err = rows.Scan(&round, &fileName)
+ if err != nil {
+ return err
+ }
+ fileNames[round] = fileName
+ }
+
+ return rows.Err()
+ })
+ if err != nil {
+ fileNames = nil
+ }
+ return
+}
+
+// ReadCatchpointStateString implements Reader
+func (r *reader) ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error) {
+ err = db.Retry(func() (err error) {
+ query := "SELECT strval FROM catchpointstate WHERE id=?"
+ var v sql.NullString
+ err = r.q.QueryRowContext(ctx, query, stateName).Scan(&v)
+ if err == sql.ErrNoRows {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+
+ if v.Valid {
+ val = v.String
+ }
+ return nil
+ })
+ return val, err
+}
+
+// ReadCatchpointStateUint64 implements Reader
+func (r *reader) ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error) {
+ err = db.Retry(func() (err error) {
+ query := "SELECT intval FROM catchpointstate WHERE id=?"
+ var v sql.NullInt64
+ err = r.q.QueryRowContext(ctx, query, stateName).Scan(&v)
+ if err == sql.ErrNoRows {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if v.Valid {
+ val = uint64(v.Int64)
+ }
+ return nil
+ })
+ return val, err
+}
+
+// SelectUnfinishedCatchpoints implements Reader
+func (r *reader) SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error) {
+ var res []UnfinishedCatchpointRecord
+
+ f := func() error {
+ query := "SELECT round, blockhash FROM unfinishedcatchpoints ORDER BY round"
+ rows, err := r.q.QueryContext(ctx, query)
+ if err != nil {
+ return err
+ }
+
+ // Clear `res` in case this function is repeated.
+ res = res[:0]
+ for rows.Next() {
+ var record UnfinishedCatchpointRecord
+ var blockHash []byte
+ err = rows.Scan(&record.Round, &blockHash)
+ if err != nil {
+ return err
+ }
+ copy(record.BlockHash[:], blockHash)
+ res = append(res, record)
+ }
+
+ return nil
+ }
+ err := db.Retry(f)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
+
+// SelectCatchpointFirstStageInfo implements Reader
+func (r *reader) SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool, error) {
+ var data []byte
+ f := func() error {
+ query := "SELECT info FROM catchpointfirststageinfo WHERE round=?"
+ err := r.q.QueryRowContext(ctx, query, round).Scan(&data)
+ if err == sql.ErrNoRows {
+ data = nil
+ return nil
+ }
+ return err
+ }
+ err := db.Retry(f)
+ if err != nil {
+ return CatchpointFirstStageInfo{}, false, err
+ }
+
+ if data == nil {
+ return CatchpointFirstStageInfo{}, false, nil
+ }
+
+ var res CatchpointFirstStageInfo
+ err = protocol.Decode(data, &res)
+ if err != nil {
+ return CatchpointFirstStageInfo{}, false, err
+ }
+
+ return res, true, nil
+}
+
+// SelectOldCatchpointFirstStageInfoRounds implements Reader
+func (r *reader) SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) {
+ var res []basics.Round
+
+ f := func() error {
+ query := "SELECT round FROM catchpointfirststageinfo WHERE round <= ?"
+ rows, err := r.q.QueryContext(ctx, query, maxRound)
+ if err != nil {
+ return err
+ }
+
+ // Clear `res` in case this function is repeated.
+ res = res[:0]
+ for rows.Next() {
+ var r basics.Round
+ err = rows.Scan(&r)
+ if err != nil {
+ return err
+ }
+ res = append(res, r)
+ }
+
+ return nil
+ }
+ err := db.Retry(f)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
diff --git a/ledger/store/trackerdb/utils.go b/ledger/store/catchpointdb/utils.go
similarity index 98%
rename from ledger/store/trackerdb/utils.go
rename to ledger/store/catchpointdb/utils.go
index 0d9e8b4d8a..2a665c2ab0 100644
--- a/ledger/store/trackerdb/utils.go
+++ b/ledger/store/catchpointdb/utils.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with go-algorand. If not, see .
-package trackerdb
+package catchpointdb
import (
"io"
diff --git a/ledger/store/catchpointdb/writer.go b/ledger/store/catchpointdb/writer.go
new file mode 100644
index 0000000000..3cd405704e
--- /dev/null
+++ b/ledger/store/catchpointdb/writer.go
@@ -0,0 +1,127 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package catchpointdb
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/util/db"
+)
+
+type writer struct {
+ e db.Executable
+}
+
+func makeWriter(e db.Executable) Writer {
+ return &writer{e}
+}
+
+// StoreCatchpoint implements Writer
+func (w *writer) StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) {
+ err = db.Retry(func() (err error) {
+ query := "DELETE FROM storedcatchpoints WHERE round=?"
+ _, err = w.e.ExecContext(ctx, query, round)
+ if err != nil || (fileName == "" && catchpoint == "" && fileSize == 0) {
+ return err
+ }
+
+ query = "INSERT INTO storedcatchpoints(round, filename, catchpoint, filesize, pinned) VALUES(?, ?, ?, ?, 0)"
+ _, err = w.e.ExecContext(ctx, query, round, fileName, catchpoint, fileSize)
+ return err
+ })
+ return
+}
+
+// WriteCatchpointStateString implements Writer
+func (w *writer) WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error) {
+ err = db.Retry(func() (err error) {
+ if setValue == "" {
+ return deleteCatchpointStateImpl(ctx, w.e, stateName)
+ }
+
+ // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case.
+ query := "INSERT OR REPLACE INTO catchpointstate(id, strval) VALUES(?, ?)"
+ _, err = w.e.ExecContext(ctx, query, stateName, setValue)
+ return err
+ })
+ return err
+}
+
+// WriteCatchpointStateUint64 implements Writer
+func (w *writer) WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error) {
+ err = db.Retry(func() (err error) {
+ if setValue == 0 {
+ return deleteCatchpointStateImpl(ctx, w.e, stateName)
+ }
+
+ // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case.
+ query := "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)"
+ _, err = w.e.ExecContext(ctx, query, stateName, setValue)
+ return err
+ })
+ return err
+}
+
+// DeleteUnfinishedCatchpoint implements Writer
+func (w *writer) DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error {
+ f := func() error {
+ query := "DELETE FROM unfinishedcatchpoints WHERE round = ?"
+ _, err := w.e.ExecContext(ctx, query, round)
+ return err
+ }
+ return db.Retry(f)
+}
+
+// InsertUnfinishedCatchpoint implements Writer
+func (w *writer) InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error {
+ f := func() error {
+ query := "INSERT INTO unfinishedcatchpoints(round, blockhash) VALUES(?, ?)"
+ _, err := w.e.ExecContext(ctx, query, round, blockHash[:])
+ return err
+ }
+ return db.Retry(f)
+}
+
+// DeleteOldCatchpointFirstStageInfo implements Writer
+func (w *writer) DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error {
+ f := func() error {
+ query := "DELETE FROM catchpointfirststageinfo WHERE round <= ?"
+ _, err := w.e.ExecContext(ctx, query, maxRoundToDelete)
+ return err
+ }
+ return db.Retry(f)
+}
+
+// InsertOrReplaceCatchpointFirstStageInfo implements Writer
+func (w *writer) InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error {
+ infoSerialized := protocol.Encode(info)
+ f := func() error {
+ query := "INSERT OR REPLACE INTO catchpointfirststageinfo(round, info) VALUES(?, ?)"
+ _, err := w.e.ExecContext(ctx, query, round, infoSerialized)
+ return err
+ }
+ return db.Retry(f)
+}
+
+func deleteCatchpointStateImpl(ctx context.Context, e db.Executable, stateName CatchpointState) error {
+ query := "DELETE FROM catchpointstate WHERE id=?"
+ _, err := e.ExecContext(ctx, query, stateName)
+ return err
+}
diff --git a/ledger/store/merkle_committer.go b/ledger/store/merkle_committer.go
deleted file mode 100644
index bc7502dac6..0000000000
--- a/ledger/store/merkle_committer.go
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright (C) 2019-2023 Algorand, Inc.
-// This file is part of go-algorand
-//
-// go-algorand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// go-algorand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with go-algorand. If not, see .
-
-package store
-
-import "database/sql"
-
-// MerkleCommitter allows storing and loading merkletrie pages from a sqlite database.
-//
-//msgp:ignore MerkleCommitter
-type MerkleCommitter struct {
- tx *sql.Tx
- deleteStmt *sql.Stmt
- insertStmt *sql.Stmt
- selectStmt *sql.Stmt
-}
-
-// MakeMerkleCommitter creates a MerkleCommitter object that implements the merkletrie.Committer interface allowing storing and loading
-// merkletrie pages from a sqlite database.
-func MakeMerkleCommitter(tx *sql.Tx, staging bool) (mc *MerkleCommitter, err error) {
- mc = &MerkleCommitter{tx: tx}
- accountHashesTable := "accounthashes"
- if staging {
- accountHashesTable = "catchpointaccounthashes"
- }
- mc.deleteStmt, err = tx.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?")
- if err != nil {
- return nil, err
- }
- mc.insertStmt, err = tx.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)")
- if err != nil {
- return nil, err
- }
- mc.selectStmt, err = tx.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?")
- if err != nil {
- return nil, err
- }
- return mc, nil
-}
-
-// StorePage is the merkletrie.Committer interface implementation, stores a single page in a sqlite database table.
-func (mc *MerkleCommitter) StorePage(page uint64, content []byte) error {
- if len(content) == 0 {
- _, err := mc.deleteStmt.Exec(page)
- return err
- }
- _, err := mc.insertStmt.Exec(page, content)
- return err
-}
-
-// LoadPage is the merkletrie.Committer interface implementation, load a single page from a sqlite database table.
-func (mc *MerkleCommitter) LoadPage(page uint64) (content []byte, err error) {
- err = mc.selectStmt.QueryRow(page).Scan(&content)
- if err == sql.ErrNoRows {
- content = nil
- err = nil
- return
- } else if err != nil {
- return nil, err
- }
- return content, nil
-}
diff --git a/ledger/store/testing/helpers.go b/ledger/store/testing/helpers.go
index 34ba3e3ffb..b9e41ee286 100644
--- a/ledger/store/testing/helpers.go
+++ b/ledger/store/testing/helpers.go
@@ -29,7 +29,7 @@ import (
// DbOpenTest opens a db file for testing purposes.
func DbOpenTest(t testing.TB, inMemory bool) (db.Pair, string) {
- fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64())
+ fn := fmt.Sprintf("%s/%s.%d", t.TempDir(), strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64())
dbs, err := db.OpenPair(fn, inMemory)
require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory)
return dbs, fn
diff --git a/ledger/store/trackerdb/catchpoint.go b/ledger/store/trackerdb/catchpoint.go
index dfbacbd86f..1af7ad1e5d 100644
--- a/ledger/store/trackerdb/catchpoint.go
+++ b/ledger/store/trackerdb/catchpoint.go
@@ -17,16 +17,8 @@
package trackerdb
import (
- "fmt"
- "os"
- "path/filepath"
- "strconv"
- "strings"
-
- "github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/crypto/merkletrie"
"github.com/algorand/go-algorand/data/basics"
- "github.com/algorand/go-algorand/ledger/ledgercore"
)
// TrieMemoryConfig is the memory configuration setup used for the merkle trie.
@@ -45,51 +37,6 @@ var MerkleCommitterNodesPerPage = int64(116)
// value was calibrated using BenchmarkCalibrateCacheNodeSize
var TrieCachedNodesCount = 9000
-// CatchpointDirName represents the directory name in which all the catchpoints files are stored
-var CatchpointDirName = "catchpoints"
-
-// CatchpointState is used to store catchpoint related variables into the catchpointstate table.
-//
-//msgp:ignore CatchpointState
-type CatchpointState string
-
-const (
- // CatchpointStateLastCatchpoint is written by a node once a catchpoint label is created for a round
- CatchpointStateLastCatchpoint = CatchpointState("lastCatchpoint")
- // CatchpointStateWritingFirstStageInfo state variable is set to 1 if catchpoint's first stage is unfinished,
- // and is 0 otherwise. Used to clear / restart the first stage after a crash.
- // This key is set in the same db transaction as the account updates, so the
- // unfinished first stage corresponds to the current db round.
- CatchpointStateWritingFirstStageInfo = CatchpointState("writingFirstStageInfo")
- // CatchpointStateWritingCatchpoint if there is an unfinished catchpoint, this state variable is set to
- // the catchpoint's round. Otherwise, it is set to 0.
- // DEPRECATED.
- CatchpointStateWritingCatchpoint = CatchpointState("writingCatchpoint")
- // CatchpointStateCatchupState is the state of the catchup process. The variable is stored only during the catchpoint catchup process, and removed afterward.
- CatchpointStateCatchupState = CatchpointState("catchpointCatchupState")
- // CatchpointStateCatchupLabel is the label to which the currently catchpoint catchup process is trying to catchup to.
- CatchpointStateCatchupLabel = CatchpointState("catchpointCatchupLabel")
- // CatchpointStateCatchupBlockRound is the block round that is associated with the current running catchpoint catchup.
- CatchpointStateCatchupBlockRound = CatchpointState("catchpointCatchupBlockRound")
- // CatchpointStateCatchupBalancesRound is the balance round that is associated with the current running catchpoint catchup. Typically it would be
- // equal to CatchpointStateCatchupBlockRound - 320.
- CatchpointStateCatchupBalancesRound = CatchpointState("catchpointCatchupBalancesRound")
- // CatchpointStateCatchupHashRound is the round that is associated with the hash of the merkle trie. Normally, it's identical to CatchpointStateCatchupBalancesRound,
- // however, it could differ when we catchup from a catchpoint that was created using a different version : in this case,
- // we set it to zero in order to reset the merkle trie. This would force the merkle trie to be re-build on startup ( if needed ).
- CatchpointStateCatchupHashRound = CatchpointState("catchpointCatchupHashRound")
- // CatchpointStateCatchpointLookback is the number of rounds we keep catchpoints for
- CatchpointStateCatchpointLookback = CatchpointState("catchpointLookback")
- // CatchpointStateCatchupVersion is the catchpoint version which the currently catchpoint catchup process is trying to catchup to.
- CatchpointStateCatchupVersion = CatchpointState("catchpointCatchupVersion")
-)
-
-// UnfinishedCatchpointRecord represents a stored record of an unfinished catchpoint.
-type UnfinishedCatchpointRecord struct {
- Round basics.Round
- BlockHash crypto.Digest
-}
-
// NormalizedAccountBalance is a staging area for a catchpoint file account information before it's being added to the catchpoint staging tables.
type NormalizedAccountBalance struct {
// The public key address to which the account belongs.
@@ -110,86 +57,3 @@ type NormalizedAccountBalance struct {
// partial balance indicates that the original account balance was split into multiple parts in catchpoint creation time
PartialBalance bool
}
-
-// CatchpointFirstStageInfo For the `catchpointfirststageinfo` table.
-type CatchpointFirstStageInfo struct {
- _struct struct{} `codec:",omitempty,omitemptyarray"`
-
- Totals ledgercore.AccountTotals `codec:"accountTotals"`
- TrieBalancesHash crypto.Digest `codec:"trieBalancesHash"`
- // Total number of accounts in the catchpoint data file. Only set when catchpoint
- // data files are generated.
- TotalAccounts uint64 `codec:"accountsCount"`
-
- // Total number of accounts in the catchpoint data file. Only set when catchpoint
- // data files are generated.
- TotalKVs uint64 `codec:"kvsCount"`
-
- // Total number of chunks in the catchpoint data file. Only set when catchpoint
- // data files are generated.
- TotalChunks uint64 `codec:"chunksCount"`
- // BiggestChunkLen is the size in the bytes of the largest chunk, used when re-packing.
- BiggestChunkLen uint64 `codec:"biggestChunk"`
-
- // StateProofVerificationHash is the hash of the state proof verification data contained in the catchpoint data file.
- StateProofVerificationHash crypto.Digest `codec:"spVerificationHash"`
-}
-
-// MakeCatchpointFilePath builds the path of a catchpoint file.
-func MakeCatchpointFilePath(round basics.Round) string {
- irnd := int64(round) / 256
- outStr := ""
- for irnd > 0 {
- outStr = filepath.Join(outStr, fmt.Sprintf("%02x", irnd%256))
- irnd = irnd / 256
- }
- outStr = filepath.Join(outStr, strconv.FormatInt(int64(round), 10)+".catchpoint")
- return outStr
-}
-
-// RemoveSingleCatchpointFileFromDisk removes a single catchpoint file from the disk. this function does not leave empty directories
-func RemoveSingleCatchpointFileFromDisk(dbDirectory, fileToDelete string) (err error) {
- absCatchpointFileName := filepath.Join(dbDirectory, fileToDelete)
- err = os.Remove(absCatchpointFileName)
- if err == nil || os.IsNotExist(err) {
- // it's ok if the file doesn't exist.
- err = nil
- } else {
- // we can't delete the file, abort -
- return fmt.Errorf("unable to delete old catchpoint file '%s' : %v", absCatchpointFileName, err)
- }
- splitedDirName := strings.Split(fileToDelete, string(os.PathSeparator))
-
- var subDirectoriesToScan []string
- //build a list of all the subdirs
- currentSubDir := ""
- for _, element := range splitedDirName {
- currentSubDir = filepath.Join(currentSubDir, element)
- subDirectoriesToScan = append(subDirectoriesToScan, currentSubDir)
- }
-
- // iterating over the list of directories. starting from the sub dirs and moving up.
- // skipping the file itself.
- for i := len(subDirectoriesToScan) - 2; i >= 0; i-- {
- absSubdir := filepath.Join(dbDirectory, subDirectoriesToScan[i])
- if _, err := os.Stat(absSubdir); os.IsNotExist(err) {
- continue
- }
-
- isEmpty, err := isDirEmpty(absSubdir)
- if err != nil {
- return fmt.Errorf("unable to read old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err)
- }
- if isEmpty {
- err = os.Remove(absSubdir)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- return fmt.Errorf("unable to delete old catchpoint directory '%s' : %v", subDirectoriesToScan[i], err)
- }
- }
- }
-
- return nil
-}
diff --git a/ledger/store/trackerdb/dualdriver/accounts_reader.go b/ledger/store/trackerdb/dualdriver/accounts_reader.go
new file mode 100644
index 0000000000..cda1a0687c
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/accounts_reader.go
@@ -0,0 +1,209 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/google/go-cmp/cmp"
+)
+
+type accountsReader struct {
+ primary trackerdb.AccountsReader
+ secondary trackerdb.AccountsReader
+}
+
+// Close implements trackerdb.AccountsReader
+func (ar *accountsReader) Close() {
+ ar.primary.Close()
+ ar.secondary.Close()
+}
+
+// ListCreatables implements trackerdb.AccountsReader
+func (ar *accountsReader) ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) {
+ resultsP, dbRoundP, errP := ar.primary.ListCreatables(maxIdx, maxResults, ctype)
+ resultsS, dbRoundS, errS := ar.secondary.ListCreatables(maxIdx, maxResults, ctype)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(resultsP, resultsS) {
+ err = ErrInconsistentResult
+ return
+ }
+ if dbRoundP != dbRoundS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return resultsP, dbRoundP, nil
+}
+
+// LookupAccount implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupAccount(addr basics.Address) (data trackerdb.PersistedAccountData, err error) {
+ dataP, errP := ar.primary.LookupAccount(addr)
+ dataS, errS := ar.secondary.LookupAccount(addr)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ ref, err := coalesceAccountRefs(dataP.Ref, dataS.Ref)
+ if err != nil {
+ return
+ }
+ // update ref in results
+ // Note: this is safe because the refs are engine specific
+ dataP.Ref = ref
+ dataS.Ref = ref
+ // check results match
+ if dataP != dataS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return dataP, nil
+}
+
+// LookupAllResources implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupAllResources(addr basics.Address) (data []trackerdb.PersistedResourcesData, rnd basics.Round, err error) {
+ dataP, rndP, errP := ar.primary.LookupAllResources(addr)
+ dataS, rndS, errS := ar.secondary.LookupAllResources(addr)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ if len(dataP) != len(dataS) {
+ err = ErrInconsistentResult
+ return
+ }
+ var ref trackerdb.AccountRef
+ for i := range dataP {
+ ref, err = coalesceAccountRefs(dataP[i].AcctRef, dataS[i].AcctRef)
+ if err != nil {
+ return data, rnd, err
+ }
+ // update ref in results
+ dataP[i].AcctRef = ref
+ dataS[i].AcctRef = ref
+ }
+ // check results match
+ if !cmp.Equal(dataP, dataS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ if rndP != rndS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return dataP, rndP, nil
+}
+
+// LookupCreator implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) {
+ addrP, okP, dbRoundP, errP := ar.primary.LookupCreator(cidx, ctype)
+ addrS, okS, dbRoundS, errS := ar.secondary.LookupCreator(cidx, ctype)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if addrP != addrS {
+ err = ErrInconsistentResult
+ return
+ }
+ if okP != okS {
+ err = ErrInconsistentResult
+ return
+ }
+ if dbRoundP != dbRoundS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return addrP, okP, dbRoundP, nil
+}
+
+// LookupKeyValue implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupKeyValue(key string) (pv trackerdb.PersistedKVData, err error) {
+ pvP, errP := ar.primary.LookupKeyValue(key)
+ pvS, errS := ar.secondary.LookupKeyValue(key)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(pvP, pvS) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return pvP, nil
+}
+
+// LookupKeysByPrefix implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) {
+ roundP, errP := ar.primary.LookupKeysByPrefix(prefix, maxKeyNum, results, resultCount)
+ roundS, errS := ar.secondary.LookupKeysByPrefix(prefix, maxKeyNum, results, resultCount)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if roundP != roundS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return roundP, nil
+}
+
+// LookupResources implements trackerdb.AccountsReader
+func (ar *accountsReader) LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data trackerdb.PersistedResourcesData, err error) {
+ dataP, errP := ar.primary.LookupResources(addr, aidx, ctype)
+ dataS, errS := ar.secondary.LookupResources(addr, aidx, ctype)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ ref, err := coalesceAccountRefs(dataP.AcctRef, dataS.AcctRef)
+ if err != nil {
+ return
+ }
+ // update ref in results
+ // Note: this is safe because the refs are engine specific
+ dataP.AcctRef = ref
+ dataS.AcctRef = ref
+ // check results match
+ if !cmp.Equal(dataP, dataS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return dataP, nil
+}
diff --git a/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go b/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go
new file mode 100644
index 0000000000..6a67c0db53
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/accounts_reader_ext.go
@@ -0,0 +1,356 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/google/go-cmp/cmp"
+)
+
+type accountsReaderExt struct {
+ primary trackerdb.AccountsReaderExt
+ secondary trackerdb.AccountsReaderExt
+}
+
+// AccountsHashRound implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) {
+ hashrndP, errP := ar.primary.AccountsHashRound(ctx)
+ hashrndS, errS := ar.secondary.AccountsHashRound(ctx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if hashrndP != hashrndS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return hashrndP, nil
+}
+
+// AccountsOnlineRoundParams implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error) {
+ onlineRoundParamsDataP, endRoundP, errP := ar.primary.AccountsOnlineRoundParams()
+ onlineRoundParamsDataS, endRoundS, errS := ar.secondary.AccountsOnlineRoundParams()
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(onlineRoundParamsDataP, onlineRoundParamsDataS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ if endRoundP != endRoundS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return onlineRoundParamsDataP, endRoundP, nil
+}
+
+// AccountsOnlineTop implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (onlineAccounts map[basics.Address]*ledgercore.OnlineAccount, err error) {
+ onlineAccountsP, errP := ar.primary.AccountsOnlineTop(rnd, offset, n, proto)
+ onlineAccountsS, errS := ar.secondary.AccountsOnlineTop(rnd, offset, n, proto)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(onlineAccountsP, onlineAccountsS) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return onlineAccountsP, nil
+}
+
+// AccountsRound implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) AccountsRound() (rnd basics.Round, err error) {
+ rndP, errP := ar.primary.AccountsRound()
+ rndS, errS := ar.secondary.AccountsRound()
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if rndP != rndS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return rndP, nil
+}
+
+// AccountsTotals implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error) {
+ totalsP, errP := ar.primary.AccountsTotals(ctx, catchpointStaging)
+ totalsS, errS := ar.secondary.AccountsTotals(ctx, catchpointStaging)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if totalsP != totalsS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return totalsP, nil
+}
+
+// LoadAllFullAccounts implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error) {
+ countP, errP := ar.primary.LoadAllFullAccounts(ctx, balancesTable, resourcesTable, acctCb)
+ countS, errS := ar.secondary.LoadAllFullAccounts(ctx, balancesTable, resourcesTable, acctCb)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if countP != countS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return countP, nil
+}
+
+// LoadTxTail implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*trackerdb.TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error) {
+ roundDataP, roundHashP, baseRoundP, errP := ar.primary.LoadTxTail(ctx, dbRound)
+ roundDataS, roundHashS, baseRoundS, errS := ar.secondary.LoadTxTail(ctx, dbRound)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(roundDataP, roundDataS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ if !cmp.Equal(roundHashP, roundHashS) {
+ err = ErrInconsistentResult
+ return
+ }
+ if baseRoundP != baseRoundS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return roundDataP, roundHashP, baseRoundP, nil
+}
+
+// LookupAccountAddressFromAddressID implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LookupAccountAddressFromAddressID(ctx context.Context, ref trackerdb.AccountRef) (address basics.Address, err error) {
+ addressP, errP := ar.primary.LookupAccountAddressFromAddressID(ctx, ref)
+ addressS, errS := ar.secondary.LookupAccountAddressFromAddressID(ctx, ref)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if addressP != addressS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return addressP, nil
+}
+
+// LookupAccountRowID implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LookupAccountRowID(addr basics.Address) (ref trackerdb.AccountRef, err error) {
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, errP := ar.primary.LookupAccountRowID(addr)
+ refS, errS := ar.secondary.LookupAccountRowID(addr)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ return accountRef{refP, refS}, nil
+}
+
+// LookupOnlineAccountDataByAddress implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) {
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, dataP, errP := ar.primary.LookupOnlineAccountDataByAddress(addr)
+ refS, dataS, errS := ar.secondary.LookupOnlineAccountDataByAddress(addr)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(dataP, dataS) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return onlineAccountRef{refP, refS}, dataP, nil
+}
+
+// LookupResourceDataByAddrID implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) LookupResourceDataByAddrID(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) {
+ if accRef == nil {
+ return data, trackerdb.ErrNotFound
+ }
+ // parse ref
+ xRef := accRef.(accountRef)
+ // lookup
+ dataP, errP := ar.primary.LookupResourceDataByAddrID(xRef.primary, aidx)
+ dataS, errS := ar.secondary.LookupResourceDataByAddrID(xRef.secondary, aidx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(dataP, dataS) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return dataP, nil
+}
+
+// OnlineAccountsAll implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) OnlineAccountsAll(maxAccounts uint64) (accounts []trackerdb.PersistedOnlineAccountData, err error) {
+ accountsP, errP := ar.primary.OnlineAccountsAll(maxAccounts)
+ accountsS, errS := ar.secondary.OnlineAccountsAll(maxAccounts)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ if len(accountsP) != len(accountsS) {
+ err = ErrInconsistentResult
+ return
+ }
+ var ref trackerdb.OnlineAccountRef
+ for i := range accountsP {
+ ref, err = coalesceOnlineAccountRefs(accountsP[i].Ref, accountsS[i].Ref)
+ if err != nil {
+ return accounts, err
+ }
+ // update ref in results
+ accountsP[i].Ref = ref
+ accountsS[i].Ref = ref
+ }
+ // check results match
+ if !cmp.Equal(accountsP, accountsS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return accountsP, nil
+}
+
+// ExpiredOnlineAccountsForRound implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) ExpiredOnlineAccountsForRound(rnd basics.Round, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (expAccounts map[basics.Address]*ledgercore.OnlineAccountData, err error) {
+ expAccountsP, errP := ar.primary.ExpiredOnlineAccountsForRound(rnd, voteRnd, proto, rewardsLevel)
+ expAccountsS, errS := ar.secondary.ExpiredOnlineAccountsForRound(rnd, voteRnd, proto, rewardsLevel)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if !cmp.Equal(expAccountsP, expAccountsS) {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return expAccountsP, nil
+}
+
+// Testing implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) Testing() trackerdb.AccountsReaderTestExt {
+ // TODO
+ return nil
+}
+
+// TotalAccounts implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) TotalAccounts(ctx context.Context) (total uint64, err error) {
+ totalP, errP := ar.primary.TotalAccounts(ctx)
+ totalS, errS := ar.secondary.TotalAccounts(ctx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if totalP != totalS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return totalP, nil
+}
+
+// TotalKVs implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) TotalKVs(ctx context.Context) (total uint64, err error) {
+ totalP, errP := ar.primary.TotalKVs(ctx)
+ totalS, errS := ar.secondary.TotalKVs(ctx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if totalP != totalS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return totalP, nil
+}
+
+// TotalResources implements trackerdb.AccountsReaderExt
+func (ar *accountsReaderExt) TotalResources(ctx context.Context) (total uint64, err error) {
+ totalP, errP := ar.primary.TotalResources(ctx)
+ totalS, errS := ar.secondary.TotalResources(ctx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if totalP != totalS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return totalP, nil
+}
diff --git a/ledger/store/trackerdb/dualdriver/accounts_writer.go b/ledger/store/trackerdb/dualdriver/accounts_writer.go
new file mode 100644
index 0000000000..318ccd36c0
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/accounts_writer.go
@@ -0,0 +1,170 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type accountsWriter struct {
+ primary trackerdb.AccountsWriter
+ secondary trackerdb.AccountsWriter
+}
+
+// Close implements trackerdb.AccountsWriter
+func (aw *accountsWriter) Close() {
+ aw.primary.Close()
+ aw.secondary.Close()
+}
+
+// DeleteAccount implements trackerdb.AccountsWriter
+func (aw *accountsWriter) DeleteAccount(accRef trackerdb.AccountRef) (rowsAffected int64, err error) {
+ // parse ref
+ xRef := accRef.(accountRef)
+ // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines
+ rowsAffectedP, errP := aw.primary.DeleteAccount(xRef.primary)
+ _, errS := aw.secondary.DeleteAccount(xRef.secondary)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ return rowsAffectedP, nil
+}
+
+// DeleteCreatable implements trackerdb.AccountsWriter
+func (aw *accountsWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) {
+ // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines
+ rowsAffectedP, errP := aw.primary.DeleteCreatable(cidx, ctype)
+ _, errS := aw.secondary.DeleteCreatable(cidx, ctype)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ return rowsAffectedP, nil
+}
+
+// DeleteKvPair implements trackerdb.AccountsWriter
+func (aw *accountsWriter) DeleteKvPair(key string) error {
+ errP := aw.primary.DeleteKvPair(key)
+ errS := aw.secondary.DeleteKvPair(key)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// DeleteResource implements trackerdb.AccountsWriter
+func (aw *accountsWriter) DeleteResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (rowsAffected int64, err error) {
+ // parse ref
+ xRef := accRef.(accountRef)
+ // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines
+ rowsAffectedP, errP := aw.primary.DeleteResource(xRef.primary, aidx)
+ _, errS := aw.secondary.DeleteResource(xRef.secondary, aidx)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ return rowsAffectedP, nil
+}
+
+// InsertAccount implements trackerdb.AccountsWriter
+func (aw *accountsWriter) InsertAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseAccountData) (ref trackerdb.AccountRef, err error) {
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, errP := aw.primary.InsertAccount(addr, normBalance, data)
+ refS, errS := aw.secondary.InsertAccount(addr, normBalance, data)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // return ref
+ return accountRef{refP, refS}, nil
+}
+
+// InsertCreatable implements trackerdb.AccountsWriter
+func (aw *accountsWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (ref trackerdb.CreatableRef, err error) {
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, errP := aw.primary.InsertCreatable(cidx, ctype, creator)
+ refS, errS := aw.secondary.InsertCreatable(cidx, ctype, creator)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // return ref
+ return creatableRef{refP, refS}, nil
+}
+
+// InsertResource implements trackerdb.AccountsWriter
+func (aw *accountsWriter) InsertResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (ref trackerdb.ResourceRef, err error) {
+ // parse ref
+ xRef := accRef.(accountRef)
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, errP := aw.primary.InsertResource(xRef.primary, aidx, data)
+ refS, errS := aw.secondary.InsertResource(xRef.secondary, aidx, data)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // return ref
+ return resourceRef{refP, refS}, nil
+}
+
+// UpdateAccount implements trackerdb.AccountsWriter
+func (aw *accountsWriter) UpdateAccount(accRef trackerdb.AccountRef, normBalance uint64, data trackerdb.BaseAccountData) (rowsAffected int64, err error) {
+ // parse ref
+ xRef := accRef.(accountRef)
+ // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines
+ rowsAffectedP, errP := aw.primary.UpdateAccount(xRef.primary, normBalance, data)
+ _, errS := aw.secondary.UpdateAccount(xRef.secondary, normBalance, data)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ return rowsAffectedP, nil
+}
+
+// UpdateResource implements trackerdb.AccountsWriter
+func (aw *accountsWriter) UpdateResource(accRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (rowsAffected int64, err error) {
+ // parse ref
+ xRef := accRef.(accountRef)
+ // Note: rowsAffected is ignored because it is not possible to determine this correctly in all engines
+ rowsAffectedP, errP := aw.primary.UpdateResource(xRef.primary, aidx, data)
+ _, errS := aw.secondary.UpdateResource(xRef.secondary, aidx, data)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ return rowsAffectedP, nil
+}
+
+// UpsertKvPair implements trackerdb.AccountsWriter
+func (aw *accountsWriter) UpsertKvPair(key string, value []byte) error {
+ errP := aw.primary.UpsertKvPair(key, value)
+ errS := aw.secondary.UpsertKvPair(key, value)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
diff --git a/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go b/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go
new file mode 100644
index 0000000000..27b74381f0
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/accounts_writer_ext.go
@@ -0,0 +1,101 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type accountsWriterExt struct {
+ primary trackerdb.AccountsWriterExt
+ secondary trackerdb.AccountsWriterExt
+}
+
+// AccountsPruneOnlineRoundParams implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error {
+ errP := aw.primary.AccountsPruneOnlineRoundParams(deleteBeforeRound)
+ errS := aw.secondary.AccountsPruneOnlineRoundParams(deleteBeforeRound)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// AccountsPutOnlineRoundParams implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error {
+ errP := aw.primary.AccountsPutOnlineRoundParams(onlineRoundParamsData, startRound)
+ errS := aw.secondary.AccountsPutOnlineRoundParams(onlineRoundParamsData, startRound)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// AccountsPutTotals implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) AccountsPutTotals(totals ledgercore.AccountTotals, catchpointStaging bool) error {
+ errP := aw.primary.AccountsPutTotals(totals, catchpointStaging)
+ errS := aw.secondary.AccountsPutTotals(totals, catchpointStaging)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// AccountsReset implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) AccountsReset(ctx context.Context) error {
+ errP := aw.primary.AccountsReset(ctx)
+ errS := aw.secondary.AccountsReset(ctx)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// OnlineAccountsDelete implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) OnlineAccountsDelete(forgetBefore basics.Round) (err error) {
+ errP := aw.primary.OnlineAccountsDelete(forgetBefore)
+ errS := aw.secondary.OnlineAccountsDelete(forgetBefore)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// ResetAccountHashes implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) ResetAccountHashes(ctx context.Context) (err error) {
+ errP := aw.primary.ResetAccountHashes(ctx)
+ errS := aw.secondary.ResetAccountHashes(ctx)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// TxtailNewRound implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error {
+ errP := aw.primary.TxtailNewRound(ctx, baseRound, roundData, forgetBeforeRound)
+ errS := aw.secondary.TxtailNewRound(ctx, baseRound, roundData, forgetBeforeRound)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// UpdateAccountsHashRound implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error) {
+ errP := aw.primary.UpdateAccountsHashRound(ctx, hashRound)
+ errS := aw.secondary.UpdateAccountsHashRound(ctx, hashRound)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// UpdateAccountsRound implements trackerdb.AccountsWriterExt
+func (aw *accountsWriterExt) UpdateAccountsRound(rnd basics.Round) (err error) {
+ errP := aw.primary.UpdateAccountsRound(rnd)
+ errS := aw.secondary.UpdateAccountsRound(rnd)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
diff --git a/ledger/store/trackerdb/dualdriver/dualdriver.go b/ledger/store/trackerdb/dualdriver/dualdriver.go
new file mode 100644
index 0000000000..64692a9d7e
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/dualdriver.go
@@ -0,0 +1,498 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+ "errors"
+ "reflect"
+ "time"
+
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/util/db"
+ "github.com/google/go-cmp/cmp"
+)
+
+// ErrInconsistentResult is returned when the two stores return different results.
+var ErrInconsistentResult = errors.New("inconsistent results between store engines")
+
+var allowAllUnexported = cmp.Exporter(func(f reflect.Type) bool { return true })
+
+type trackerStore struct {
+ primary trackerdb.Store
+ secondary trackerdb.Store
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+// MakeStore creates a dual tracker store that verifies that both stores return the same results.
+func MakeStore(primary trackerdb.Store, secondary trackerdb.Store) trackerdb.Store {
+ return &trackerStore{primary, secondary, &reader{primary, secondary}, &writer{primary, secondary}, &catchpoint{primary, secondary}}
+}
+
+func (s *trackerStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) {
+ errP := s.primary.SetSynchronousMode(ctx, mode, fullfsync)
+ errS := s.secondary.SetSynchronousMode(ctx, mode, fullfsync)
+ return coalesceErrors(errP, errS)
+}
+
+func (s *trackerStore) IsSharedCacheConnection() bool {
+ // Note: this is not something to check for being equal but rather keep the most conservative answer.
+ return s.primary.IsSharedCacheConnection() || s.secondary.IsSharedCacheConnection()
+}
+
+func (s *trackerStore) Batch(fn trackerdb.BatchFn) (err error) {
+ return s.BatchContext(context.Background(), fn)
+}
+
+func (s *trackerStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) {
+ handle, err := s.BeginBatch(ctx)
+ if err != nil {
+ return err
+ }
+ defer handle.Close()
+
+ err = fn(ctx, handle)
+ if err != nil {
+ return err
+ }
+
+ return handle.Commit()
+}
+
+func (s *trackerStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) {
+ primary, err := s.primary.BeginBatch(ctx)
+ if err != nil {
+ return nil, err
+ }
+ secondary, err := s.secondary.BeginBatch(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &batch{primary, secondary, &writer{primary, secondary}}, nil
+}
+
+func (s *trackerStore) Snapshot(fn trackerdb.SnapshotFn) (err error) {
+ return s.SnapshotContext(context.Background(), fn)
+}
+
+func (s *trackerStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) {
+ handle, err := s.BeginSnapshot(ctx)
+ if err != nil {
+ return err
+ }
+ defer handle.Close()
+
+ err = fn(ctx, handle)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *trackerStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) {
+ primary, err := s.primary.BeginSnapshot(ctx)
+ if err != nil {
+ return nil, err
+ }
+ secondary, err := s.secondary.BeginSnapshot(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &snapshot{primary, secondary, &reader{primary, secondary}}, nil
+}
+
+func (s *trackerStore) Transaction(fn trackerdb.TransactionFn) (err error) {
+ return s.TransactionContext(context.Background(), fn)
+}
+
+func (s *trackerStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) error {
+ handle, err := s.BeginTransaction(ctx)
+ if err != nil {
+ return err
+ }
+ defer handle.Close()
+
+ err = fn(ctx, handle)
+ if err != nil {
+ return err
+ }
+
+ return handle.Commit()
+}
+
+func (s *trackerStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) {
+ primary, err := s.primary.BeginTransaction(ctx)
+ if err != nil {
+ return nil, err
+ }
+ secondary, err := s.secondary.BeginTransaction(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &transaction{primary, secondary, &reader{primary, secondary}, &writer{primary, secondary}, &catchpoint{primary, secondary}}, nil
+}
+
+// RunMigrations implements trackerdb.Transaction
+func (s *trackerStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ paramsP, errP := s.primary.RunMigrations(ctx, params, log, targetVersion)
+ paramsS, errS := s.secondary.RunMigrations(ctx, params, log, targetVersion)
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results
+ if paramsP != paramsS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary result
+ return paramsP, nil
+}
+
+func (s *trackerStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) {
+ // ignore the stats
+ // Note: this is a SQL specific operation, so the are unlikely to match
+ stats, errP := s.primary.Vacuum(ctx)
+ _, errS := s.secondary.Vacuum(ctx)
+ err = coalesceErrors(errP, errS)
+ return
+}
+
+func (s *trackerStore) ResetToV6Test(ctx context.Context) error {
+ // TODO
+ return nil
+}
+
+func (s *trackerStore) Close() {
+ s.primary.Close()
+ s.secondary.Close()
+}
+
+type reader struct {
+ primary trackerdb.Reader
+ secondary trackerdb.Reader
+}
+
+// MakeAccountsOptimizedReader implements trackerdb.Reader
+func (r *reader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) {
+ primary, errP := r.primary.MakeAccountsOptimizedReader()
+ secondary, errS := r.secondary.MakeAccountsOptimizedReader()
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &accountsReader{primary, secondary}, nil
+}
+
+// MakeAccountsReader implements trackerdb.Reader
+func (r *reader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) {
+ primary, errP := r.primary.MakeAccountsReader()
+ secondary, errS := r.secondary.MakeAccountsReader()
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &accountsReaderExt{primary, secondary}, nil
+}
+
+// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader
+func (r *reader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) {
+ primary, errP := r.primary.MakeOnlineAccountsOptimizedReader()
+ secondary, errS := r.secondary.MakeOnlineAccountsOptimizedReader()
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &onlineAccountsReader{primary, secondary}, nil
+}
+
+// MakeSpVerificationCtxReader implements trackerdb.Reader
+func (r *reader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader {
+ primary := r.primary.MakeSpVerificationCtxReader()
+ secondary := r.secondary.MakeSpVerificationCtxReader()
+ return &stateproofReader{primary, secondary}
+}
+
+type writer struct {
+ primary trackerdb.Writer
+ secondary trackerdb.Writer
+}
+
+// MakeAccountsOptimizedWriter implements trackerdb.Writer
+func (w *writer) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) {
+ primary, errP := w.primary.MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables)
+ secondary, errS := w.secondary.MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables)
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &accountsWriter{primary, secondary}, nil
+}
+
+// MakeAccountsWriter implements trackerdb.Writer
+func (w *writer) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) {
+ primary, errP := w.primary.MakeAccountsWriter()
+ secondary, errS := w.secondary.MakeAccountsWriter()
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &accountsWriterExt{primary, secondary}, nil
+}
+
+// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer
+func (w *writer) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) {
+ primary, errP := w.primary.MakeOnlineAccountsOptimizedWriter(hasAccounts)
+ secondary, errS := w.secondary.MakeOnlineAccountsOptimizedWriter(hasAccounts)
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ return &onlineAccountsWriter{primary, secondary}, nil
+}
+
+// MakeSpVerificationCtxWriter implements trackerdb.Writer
+func (w *writer) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter {
+ primary := w.primary.MakeSpVerificationCtxWriter()
+ secondary := w.secondary.MakeSpVerificationCtxWriter()
+ return &stateproofWriter{primary, secondary}
+}
+
+// Testing implements trackerdb.Writer
+func (w *writer) Testing() trackerdb.WriterTestExt {
+ primary := w.primary.Testing()
+ secondary := w.secondary.Testing()
+ return &writerForTesting{primary, secondary}
+}
+
+type catchpoint struct {
+ primary trackerdb.Catchpoint
+ secondary trackerdb.Catchpoint
+}
+
+// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint
+func (*catchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter {
+ // TODO: catchpoint
+ return nil
+}
+
+// MakeCatchpointWriter implements trackerdb.Catchpoint
+func (*catchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) {
+ // TODO: catchpoint
+ return nil, nil
+}
+
+// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter {
+ // TODO: catchpoint
+ return nil
+}
+
+// MakeKVsIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) {
+ // TODO: catchpoint
+ return nil, nil
+}
+
+// MakeMerkleCommitter implements trackerdb.Catchpoint
+func (*catchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) {
+ // TODO: catchpoint
+ return nil, nil
+}
+
+// MakeOrderedAccountsIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter {
+ // TODO: catchpoint
+ return nil
+}
+
+type batch struct {
+ primary trackerdb.Batch
+ secondary trackerdb.Batch
+ trackerdb.Writer
+}
+
+// ResetTransactionWarnDeadline implements trackerdb.Batch
+func (b *batch) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ _, _ = b.primary.ResetTransactionWarnDeadline(ctx, deadline)
+ _, _ = b.secondary.ResetTransactionWarnDeadline(ctx, deadline)
+ // ignore results, this is very engine specific
+ return
+}
+
+// Commit implements trackerdb.Batch
+func (b *batch) Commit() error {
+ errP := b.primary.Commit()
+ errS := b.secondary.Commit()
+ // errors are unlikely to match between engines
+ return coalesceErrors(errP, errS)
+}
+
+// Close implements trackerdb.Batch
+func (b *batch) Close() error {
+ errP := b.primary.Close()
+ errS := b.secondary.Close()
+ // errors are unlikely to match between engines
+ return coalesceErrors(errP, errS)
+}
+
+type transaction struct {
+ primary trackerdb.Transaction
+ secondary trackerdb.Transaction
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+// ResetTransactionWarnDeadline implements trackerdb.Transaction
+func (tx *transaction) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ _, _ = tx.primary.ResetTransactionWarnDeadline(ctx, deadline)
+ _, _ = tx.secondary.ResetTransactionWarnDeadline(ctx, deadline)
+ // ignore results, this is very engine specific
+ return
+}
+
+// RunMigrations implements trackerdb.Transaction
+func (tx *transaction) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ paramsP, errP := tx.primary.RunMigrations(ctx, params, log, targetVersion)
+ paramsS, errS := tx.secondary.RunMigrations(ctx, params, log, targetVersion)
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results
+ if paramsP != paramsS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary result
+ return paramsP, nil
+}
+
+// Commit implements trackerdb.Transaction
+func (tx *transaction) Commit() error {
+ errP := tx.primary.Commit()
+ errS := tx.secondary.Commit()
+ // errors are unlikely to match between engines
+ return coalesceErrors(errP, errS)
+}
+
+// Close implements trackerdb.Transaction
+func (tx *transaction) Close() error {
+ errP := tx.primary.Close()
+ errS := tx.secondary.Close()
+ // errors are unlikely to match between engines
+ return coalesceErrors(errP, errS)
+}
+
+type snapshot struct {
+ primary trackerdb.Snapshot
+ secondary trackerdb.Snapshot
+ trackerdb.Reader
+}
+
+// Close implements trackerdb.Snapshot
+func (s *snapshot) Close() error {
+ errP := s.primary.Close()
+ errS := s.secondary.Close()
+ // errors are unlikely to match between engines
+ return coalesceErrors(errP, errS)
+}
+
+//
+// refs
+//
+
+type accountRef struct {
+ primary trackerdb.AccountRef
+ secondary trackerdb.AccountRef
+}
+
+// AccountRefMarker implements trackerdb.AccountRef
+func (accountRef) AccountRefMarker() {}
+
+func coalesceAccountRefs(primary, secondary trackerdb.AccountRef) (trackerdb.AccountRef, error) {
+ if primary != nil && secondary != nil {
+ return accountRef{primary, secondary}, nil
+ } else if primary == nil && secondary == nil {
+ // all good, ref is nil
+ return nil, nil
+ } else {
+ // ref mismatch
+ return nil, ErrInconsistentResult
+ }
+}
+
+type onlineAccountRef struct {
+ primary trackerdb.OnlineAccountRef
+ secondary trackerdb.OnlineAccountRef
+}
+
+// OnlineAccountRefMarker implements trackerdb.OnlineAccountRef
+func (onlineAccountRef) OnlineAccountRefMarker() {}
+
+func coalesceOnlineAccountRefs(primary, secondary trackerdb.OnlineAccountRef) (trackerdb.OnlineAccountRef, error) {
+ if primary != nil && secondary != nil {
+ return onlineAccountRef{primary, secondary}, nil
+ } else if primary == nil && secondary == nil {
+ // all good, ref is nil
+ return nil, nil
+ } else {
+ // ref mismatch
+ return nil, ErrInconsistentResult
+ }
+}
+
+type resourceRef struct {
+ primary trackerdb.ResourceRef
+ secondary trackerdb.ResourceRef
+}
+
+// ResourceRefMarker implements trackerdb.ResourceRef
+func (resourceRef) ResourceRefMarker() {}
+
+type creatableRef struct {
+ primary trackerdb.CreatableRef
+ secondary trackerdb.CreatableRef
+}
+
+// CreatableRefMarker implements trackerdb.CreatableRef
+func (creatableRef) CreatableRefMarker() {}
+
+//
+// helpers
+//
+
+func coalesceErrors(errP error, errS error) error {
+ if errP == nil && errS != nil {
+ logging.Base().Error("secondary engine error, ", errS)
+ return ErrInconsistentResult
+ }
+ if errP != nil && errS == nil {
+ logging.Base().Error("primary engine error, ", errP)
+ return ErrInconsistentResult
+ }
+ // happy case (no errors)
+ if errP == nil && errS == nil {
+ return nil
+ }
+ // happy case (both errored)
+ return errP
+}
diff --git a/ledger/store/trackerdb/dualdriver/online_accounts_reader.go b/ledger/store/trackerdb/dualdriver/online_accounts_reader.go
new file mode 100644
index 0000000000..f933639911
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/online_accounts_reader.go
@@ -0,0 +1,116 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/google/go-cmp/cmp"
+)
+
+type onlineAccountsReader struct {
+ primary trackerdb.OnlineAccountsReader
+ secondary trackerdb.OnlineAccountsReader
+}
+
+// Close implements trackerdb.OnlineAccountsReader
+func (oar *onlineAccountsReader) Close() {
+ oar.primary.Close()
+ oar.secondary.Close()
+}
+
+// LookupOnline implements trackerdb.OnlineAccountsReader
+func (oar *onlineAccountsReader) LookupOnline(addr basics.Address, rnd basics.Round) (data trackerdb.PersistedOnlineAccountData, err error) {
+ dataP, errP := oar.primary.LookupOnline(addr, rnd)
+ dataS, errS := oar.secondary.LookupOnline(addr, rnd)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ ref, err := coalesceOnlineAccountRefs(dataP.Ref, dataS.Ref)
+ if err != nil {
+ return
+ }
+ // update ref in results
+ // Note: this is safe because the refs are engine specific
+ dataP.Ref = ref
+ dataS.Ref = ref
+ // check results match
+ if dataP != dataS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return dataP, nil
+}
+
+// LookupOnlineHistory implements trackerdb.OnlineAccountsReader
+func (oar *onlineAccountsReader) LookupOnlineHistory(addr basics.Address) (result []trackerdb.PersistedOnlineAccountData, rnd basics.Round, err error) {
+ resultP, rndP, errP := oar.primary.LookupOnlineHistory(addr)
+ resultS, rndS, errS := oar.secondary.LookupOnlineHistory(addr)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // coalesce refs
+ if len(resultP) != len(resultS) {
+ err = ErrInconsistentResult
+ return
+ }
+ var ref trackerdb.OnlineAccountRef
+ for i := range resultP {
+ ref, err = coalesceOnlineAccountRefs(resultP[i].Ref, resultS[i].Ref)
+ if err != nil {
+ return result, rnd, err
+ }
+ // update ref in results
+ resultP[i].Ref = ref
+ resultS[i].Ref = ref
+ }
+ // check results match
+ if !cmp.Equal(resultP, resultS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return
+ }
+ if rndP != rndS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return resultP, rndP, nil
+}
+
+// LookupOnlineTotalsHistory implements trackerdb.OnlineAccountsReader
+func (oar *onlineAccountsReader) LookupOnlineTotalsHistory(round basics.Round) (algos basics.MicroAlgos, err error) {
+ algosP, errP := oar.primary.LookupOnlineTotalsHistory(round)
+ algosS, errS := oar.secondary.LookupOnlineTotalsHistory(round)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if algosP != algosS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return algosP, nil
+}
diff --git a/ledger/store/trackerdb/dualdriver/online_accounts_writer.go b/ledger/store/trackerdb/dualdriver/online_accounts_writer.go
new file mode 100644
index 0000000000..823ed54eee
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/online_accounts_writer.go
@@ -0,0 +1,47 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type onlineAccountsWriter struct {
+ primary trackerdb.OnlineAccountsWriter
+ secondary trackerdb.OnlineAccountsWriter
+}
+
+// Close implements trackerdb.OnlineAccountsWriter
+func (oaw *onlineAccountsWriter) Close() {
+ oaw.primary.Close()
+ oaw.secondary.Close()
+}
+
+// InsertOnlineAccount implements trackerdb.OnlineAccountsWriter
+func (oaw *onlineAccountsWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (ref trackerdb.OnlineAccountRef, err error) {
+ // Note: we do not check the refs since they are internal to the engines and wont match
+ refP, errP := oaw.primary.InsertOnlineAccount(addr, normBalance, data, updRound, voteLastValid)
+ refS, errS := oaw.secondary.InsertOnlineAccount(addr, normBalance, data, updRound, voteLastValid)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // return ref
+ return onlineAccountRef{refP, refS}, nil
+}
diff --git a/ledger/store/trackerdb/dualdriver/stateproof_reader.go b/ledger/store/trackerdb/dualdriver/stateproof_reader.go
new file mode 100644
index 0000000000..1ed61b4ebc
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/stateproof_reader.go
@@ -0,0 +1,85 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/google/go-cmp/cmp"
+)
+
+type stateproofReader struct {
+ primary trackerdb.SpVerificationCtxReader
+ secondary trackerdb.SpVerificationCtxReader
+}
+
+// LookupSPContext implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) LookupSPContext(stateProofLastAttestedRound basics.Round) (*ledgercore.StateProofVerificationContext, error) {
+ vcP, errP := r.primary.LookupSPContext(stateProofLastAttestedRound)
+ vcS, errS := r.secondary.LookupSPContext(stateProofLastAttestedRound)
+ // coalesce errors
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ // check results match
+ if !cmp.Equal(vcP, vcS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return nil, err
+ }
+ // return primary results
+ return vcP, nil
+}
+
+// GetAllSPContexts implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) GetAllSPContexts(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) {
+ resultsP, errP := r.primary.GetAllSPContexts(ctx)
+ resultsS, errS := r.secondary.GetAllSPContexts(ctx)
+ // coalesce errors
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ // check results match
+ if !cmp.Equal(resultsP, resultsS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return nil, err
+ }
+ // return primary results
+ return resultsP, nil
+}
+
+// GetAllSPContextsFromCatchpointTbl implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) GetAllSPContextsFromCatchpointTbl(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) {
+ resultsP, errP := r.primary.GetAllSPContextsFromCatchpointTbl(ctx)
+ resultsS, errS := r.secondary.GetAllSPContextsFromCatchpointTbl(ctx)
+ // coalesce errors
+ err := coalesceErrors(errP, errS)
+ if err != nil {
+ return nil, err
+ }
+ // check results match
+ if !cmp.Equal(resultsP, resultsS, allowAllUnexported) {
+ err = ErrInconsistentResult
+ return nil, err
+ }
+ // return primary results
+ return resultsP, nil
+}
diff --git a/ledger/store/trackerdb/dualdriver/stateproof_writer.go b/ledger/store/trackerdb/dualdriver/stateproof_writer.go
new file mode 100644
index 0000000000..208ce3cbd4
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/stateproof_writer.go
@@ -0,0 +1,53 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type stateproofWriter struct {
+ primary trackerdb.SpVerificationCtxWriter
+ secondary trackerdb.SpVerificationCtxWriter
+}
+
+// StoreSPContexts implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) StoreSPContexts(ctx context.Context, verificationContext []*ledgercore.StateProofVerificationContext) error {
+ errP := w.primary.StoreSPContexts(ctx, verificationContext)
+ errS := w.secondary.StoreSPContexts(ctx, verificationContext)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// StoreSPContextsToCatchpointTbl implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) StoreSPContextsToCatchpointTbl(ctx context.Context, verificationContexts []ledgercore.StateProofVerificationContext) error {
+ errP := w.primary.StoreSPContextsToCatchpointTbl(ctx, verificationContexts)
+ errS := w.secondary.StoreSPContextsToCatchpointTbl(ctx, verificationContexts)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
+
+// DeleteOldSPContexts implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) DeleteOldSPContexts(ctx context.Context, earliestLastAttestedRound basics.Round) error {
+ errP := w.primary.DeleteOldSPContexts(ctx, earliestLastAttestedRound)
+ errS := w.secondary.DeleteOldSPContexts(ctx, earliestLastAttestedRound)
+ // coalesce errors
+ return coalesceErrors(errP, errS)
+}
diff --git a/ledger/store/trackerdb/dualdriver/transaction_for_testing.go b/ledger/store/trackerdb/dualdriver/transaction_for_testing.go
new file mode 100644
index 0000000000..18ba888a52
--- /dev/null
+++ b/ledger/store/trackerdb/dualdriver/transaction_for_testing.go
@@ -0,0 +1,68 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package dualdriver
+
+import (
+ "context"
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+type writerForTesting struct {
+ primary trackerdb.WriterTestExt
+ secondary trackerdb.WriterTestExt
+}
+
+// AccountsInitLightTest implements trackerdb.WriterTestExt
+func (tx *writerForTesting) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabse bool, err error) {
+ newDatabaseP, errP := tx.primary.AccountsInitLightTest(tb, initAccounts, proto)
+ newDatabaseS, errS := tx.secondary.AccountsInitLightTest(tb, initAccounts, proto)
+ // coalesce errors
+ err = coalesceErrors(errP, errS)
+ if err != nil {
+ return
+ }
+ // check results match
+ if newDatabaseP != newDatabaseS {
+ err = ErrInconsistentResult
+ return
+ }
+ // return primary results
+ return newDatabaseP, nil
+}
+
+// AccountsInitTest implements trackerdb.WriterTestExt
+func (tx *writerForTesting) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
+ newDatabaseP := tx.primary.AccountsInitTest(tb, initAccounts, proto)
+ tx.secondary.AccountsInitTest(tb, initAccounts, proto)
+ // return primary results
+ return newDatabaseP
+}
+
+// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt
+func (*writerForTesting) AccountsUpdateSchemaTest(ctx context.Context) (err error) {
+ panic("unimplemented")
+}
+
+// ModifyAcctBaseTest implements trackerdb.WriterTestExt
+func (*writerForTesting) ModifyAcctBaseTest() error {
+ panic("unimplemented")
+}
diff --git a/ledger/store/trackerdb/generickv/accounts_ext_reader.go b/ledger/store/trackerdb/generickv/accounts_ext_reader.go
new file mode 100644
index 0000000000..553d143a45
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/accounts_ext_reader.go
@@ -0,0 +1,521 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "fmt"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+func (r *accountsReader) AccountsRound() (rnd basics.Round, err error) {
+ // SQL at time of impl:
+ //
+ // "SELECT rnd FROM acctrounds WHERE id='acctbase'"
+
+ // read round entry
+ value, closer, err := r.kvr.Get(roundKey())
+ if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ // parse the bytes into a u64
+ rnd = basics.Round(binary.BigEndian.Uint64(value))
+
+ return
+}
+
+func (r *accountsReader) AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error) {
+ // read round entry
+ value, closer, err := r.kvr.Get(totalsKey(catchpointStaging))
+ if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ err = protocol.Decode(value, &totals)
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (r *accountsReader) AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error) {
+ // TODO: catchpoint
+ return
+}
+
+func (r *accountsReader) LookupAccountAddressFromAddressID(ctx context.Context, ref trackerdb.AccountRef) (address basics.Address, err error) {
+ // TODO: catchpoint
+ return
+}
+
+func (r *accountsReader) LookupAccountRowID(addr basics.Address) (ref trackerdb.AccountRef, err error) {
+ // Note: [perf] this is not a very cheap operation since we have to pull up the entire record
+ acc, err := r.LookupAccount(addr)
+ if err != nil {
+ return
+ }
+
+ if acc.Ref == nil {
+ return nil, trackerdb.ErrNotFound
+ }
+
+ return acc.Ref, nil
+}
+
+func (r *accountsReader) LookupResourceDataByAddrID(accRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) {
+ // TODO: this can probably get removed in favor of LookupResources
+ // the only issue here is that the only caller of this is not doing anything with the ctype
+ // so we might have to change the signature of LookupResources to skip the ctype, which might be reasonable
+ if accRef == nil {
+ return data, trackerdb.ErrNotFound
+ }
+ xref := accRef.(accountRef)
+
+ value, closer, err := r.kvr.Get(resourceKey(xref.addr, aidx))
+ if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ return value, nil
+}
+
+func (r *accountsReader) TotalResources(ctx context.Context) (total uint64, err error) {
+ // TODO: catchpoint
+ return
+}
+
+func (r *accountsReader) TotalAccounts(ctx context.Context) (total uint64, err error) {
+ // TODO: catchpoint
+ return
+}
+
+func (r *accountsReader) TotalKVs(ctx context.Context) (total uint64, err error) {
+ // TODO: catchpoint
+ return
+}
+
+// TODO: this replicates some functionality from LookupOnlineHistory, implemented for onlineAccountsReader
+func (r *accountsReader) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) {
+ low := onlineAccountOnlyPartialKey(addr)
+ high := onlineAccountOnlyPartialKey(addr)
+ high[len(high)-1]++
+ iter := r.kvr.NewIter(low, high, true)
+ defer iter.Close()
+
+ if iter.Next() {
+ // key is -- =
+ key := iter.Key()
+
+ rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1
+ u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+
+ addrOffset := len(kvPrefixOnlineAccount) + 1
+ var addr basics.Address
+ copy(addr[:], key[addrOffset:addrOffset+32])
+
+ data, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ var oa trackerdb.BaseOnlineAccountData
+ err = protocol.Decode(data, &oa)
+ if err != nil {
+ return
+ }
+
+ ref = onlineAccountRef{
+ addr: addr,
+ round: basics.Round(u64Rnd),
+ normBalance: oa.NormalizedOnlineBalance(r.proto),
+ }
+ } else {
+ err = trackerdb.ErrNotFound
+ return
+ }
+
+ return
+}
+
+// AccountsOnlineTop returns the top n online accounts starting at position offset
+// (that is, the top offset'th account through the top offset+n-1'th account).
+//
+// The accounts are sorted by their normalized balance and address. The normalized
+// balance has to do with the reward parts of online account balances. See the
+// normalization procedure in AccountData.NormalizedOnlineBalance().
+//
+// Note that this does not check if the accounts have a vote key valid for any
+// particular round (past, present, or future).
+func (r *accountsReader) AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (data map[basics.Address]*ledgercore.OnlineAccount, err error) {
+ // The SQL before the impl
+ // SELECT
+ // address, normalizedonlinebalance, data, max(updround) FROM onlineaccounts
+ // WHERE updround <= ?
+ // GROUP BY address HAVING normalizedonlinebalance > 0
+ // ORDER BY normalizedonlinebalance DESC, address
+ // DESC LIMIT ?
+ // OFFSET ?
+
+ // initialize return map
+ data = make(map[basics.Address]*ledgercore.OnlineAccount)
+
+ // prepare iter over online accounts (by balance)
+ low := []byte(kvPrefixOnlineAccountBalance)
+ low = append(low, "-"...)
+ high := onlineAccountBalanceOnlyPartialKey(rnd)
+ high[len(high)-1]++
+ // reverse order iterator to get high-to-low
+ iter := r.kvr.NewIter(low, high, true)
+ defer iter.Close()
+
+ var value []byte
+
+ // first, drop the results from 0 to the offset
+ for i := uint64(0); i < offset; i++ {
+ iter.Next()
+ }
+
+ // add the other results to the map
+ for i := uint64(0); i < n; i++ {
+ // if no more results, return early
+ if !iter.Next() {
+ return
+ }
+
+ // key is --- =
+ key := iter.Key()
+
+ // TODO: make this cleaner
+ // get the offset where the address starts
+ offset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1
+
+ // extract address
+ var addr basics.Address
+ copy(addr[:], key[offset:])
+
+ // skip if already in map
+ if _, ok := data[addr]; ok {
+ continue
+ }
+
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ oa := trackerdb.BaseOnlineAccountData{}
+ err = protocol.Decode(value, &oa)
+ if err != nil {
+ return
+ }
+ // load the data as a ledgercore OnlineAccount
+ data[addr] = &ledgercore.OnlineAccount{
+ Address: addr,
+ MicroAlgos: oa.MicroAlgos,
+ RewardsBase: oa.RewardsBase,
+ NormalizedOnlineBalance: oa.NormalizedOnlineBalance(proto),
+ VoteFirstValid: oa.VoteFirstValid,
+ VoteLastValid: oa.VoteLastValid,
+ StateProofID: oa.StateProofID,
+ }
+ }
+ return
+}
+
+func (r *accountsReader) AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error) {
+ // The SQL at the time of writing:
+ //
+ // SELECT rnd, data FROM onlineroundparamstail ORDER BY rnd ASC
+
+ start := []byte(kvOnlineAccountRoundParams + "-")
+ end := []byte(kvOnlineAccountRoundParams + ".")
+ iter := r.kvr.NewIter(start, end, false)
+ defer iter.Close()
+
+ var value []byte
+
+ for iter.Next() {
+ // read the key
+ // schema: -
+ key := iter.Key()
+
+ // extract the round from the key
+ rndOffset := len(kvOnlineAccountRoundParams) + 1
+ u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+ // assign current item round as endRound
+ endRound = basics.Round(u64Rnd)
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return nil, endRound, err
+ }
+
+ // decode the param
+ roundParams := ledgercore.OnlineRoundParamsData{}
+ err = protocol.Decode(value, &roundParams)
+ if err != nil {
+ return nil, endRound, err
+ }
+
+ // add the params to the return list
+ onlineRoundParamsData = append(onlineRoundParamsData, roundParams)
+ }
+
+ return
+}
+
+// OnlineAccountsAll returns all online accounts up to a provided maximum
+// the returned list of PersistedOnlineAccountData includes all of the available
+// data for each included account in ascending order of account and round
+// (example [account-1-round-1, account1-round-2, ..., account2-round-1, ...])
+func (r *accountsReader) OnlineAccountsAll(maxAccounts uint64) ([]trackerdb.PersistedOnlineAccountData, error) {
+ // The SQL at the time of impl:
+ //
+ // SELECT rowid, address, updround, data
+ // FROM onlineaccounts
+ // ORDER BY address, updround ASC
+ //
+ // Note: the SQL implementation does not seem to load the current db round to the resulting objects
+
+ // read the current db round
+ var round basics.Round
+ round, err := r.AccountsRound()
+ if err != nil {
+ return nil, err
+ }
+
+ low := []byte(kvPrefixOnlineAccount + "-")
+ high := []byte(kvPrefixOnlineAccount + ".")
+ iter := r.kvr.NewIter(low, high, false)
+ defer iter.Close()
+
+ result := make([]trackerdb.PersistedOnlineAccountData, 0, maxAccounts)
+
+ var value []byte
+ var updround uint64
+
+ // keep track of the most recently seen account so we can tally up the total number seen
+ lastAddr := basics.Address{}
+ seen := uint64(0)
+
+ for iter.Next() {
+ pitem := trackerdb.PersistedOnlineAccountData{Round: round}
+
+ // schema: --
+ key := iter.Key()
+
+ addrOffset := len(kvPrefixOnlineAccount) + 1
+ var addr basics.Address
+ copy(addr[:], key[addrOffset:addrOffset+32])
+ // extract updround, it's the last section after the "-"
+
+ rndOffset := addrOffset + 32 + 1
+ updround = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+
+ // load addr, round and data into the persisted item
+ pitem.Addr = addr
+ pitem.UpdRound = basics.Round(updround)
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return nil, err
+ }
+ // decode raw value
+ err = protocol.Decode(value, &pitem.AccountData)
+ if err != nil {
+ return nil, err
+ }
+ // set ref
+ normBalance := pitem.AccountData.NormalizedOnlineBalance(r.proto)
+ pitem.Ref = onlineAccountRef{addr, normBalance, pitem.UpdRound}
+ // if maxAccounts is supplied, potentially stop reading data if we've collected enough
+ if maxAccounts > 0 {
+ // we have encountered a new address
+ if !bytes.Equal(addr[:], lastAddr[:]) {
+ copy(lastAddr[:], addr[:])
+ seen++
+ }
+ // this newest account seen is beyond the maxAccounts requested, meaning we've seen all the data we need
+ if seen > maxAccounts {
+ break
+ }
+ }
+
+ // append entry to accum
+ result = append(result, pitem)
+ }
+
+ return result, nil
+}
+
+// ExpiredOnlineAccountsForRound implements trackerdb.AccountsReaderExt
+func (r *accountsReader) ExpiredOnlineAccountsForRound(rnd basics.Round, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (data map[basics.Address]*ledgercore.OnlineAccountData, err error) {
+ // The SQL at the time of writing:
+ //
+ // SELECT address, data, max(updround)
+ // FROM onlineaccounts
+ // WHERE updround <= ? <---- ? = rnd
+ // GROUP BY address
+ // HAVING votelastvalid < ? and votelastvalid > 0 <---- ? = voteRnd
+ // ORDER BY address
+
+ // initialize return map
+ data = make(map[basics.Address]*ledgercore.OnlineAccountData)
+ expired := make(map[basics.Address]struct{})
+
+ // prepare iter over online accounts (by balance)
+ low := []byte(kvPrefixOnlineAccountBalance)
+ low = append(low, "-"...)
+ high := onlineAccountBalanceOnlyPartialKey(rnd)
+ high[len(high)-1]++
+ // reverse order iterator to get high-to-low
+ iter := r.kvr.NewIter(low, high, true)
+ defer iter.Close()
+
+ var value []byte
+
+ // add the other results to the map
+ for iter.Next() {
+ // key is --- =
+ key := iter.Key()
+
+ // get the addrOffset where the address starts
+ addrOffset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1
+
+ // extract address
+ var addr basics.Address
+ copy(addr[:], key[addrOffset:])
+
+ // skip if already in map
+ // we keep only the one with `max(updround)`
+ // the reverse iter makes us hit the max first
+ if _, ok := data[addr]; ok {
+ continue
+ }
+ // when the a ccount is expired we do not add it to the data
+ // but we might have an older version that is not expired show up
+ // this would be wrong, so we skip those accounts if the latest version is expired
+ if _, ok := expired[addr]; ok {
+ continue
+ }
+
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ oa := trackerdb.BaseOnlineAccountData{}
+ err = protocol.Decode(value, &oa)
+ if err != nil {
+ return
+ }
+
+ // filter by vote expiration
+ // sql: HAVING votelastvalid < ? and votelastvalid > 0
+ // Note: we might have to add an extra index during insert if this doing this in memory becomes a perf issue
+ if !(oa.VoteLastValid < voteRnd && oa.VoteLastValid > 0) {
+ expired[addr] = struct{}{}
+ continue
+ }
+
+ // load the data as a ledgercore OnlineAccount
+ oadata := oa.GetOnlineAccountData(proto, rewardsLevel)
+ data[addr] = &oadata
+ }
+
+ return
+}
+
+func (r *accountsReader) LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*trackerdb.TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error) {
+ // The SQL at the time of writing:
+ //
+ // "SELECT rnd, data FROM txtail ORDER BY rnd DESC"
+
+ start := []byte(kvTxTail + "-")
+ end := []byte(kvTxTail + ".")
+ iter := r.kvr.NewIter(start, end, true)
+ defer iter.Close()
+
+ var value []byte
+
+ expectedRound := dbRound
+ for iter.Next() {
+ // read the key
+ key := iter.Key()
+
+ // extract the txTail round from the key
+ rndOffset := len(kvTxTail) + 1
+ u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+ round := basics.Round(u64Rnd)
+ // check that we are on the right round
+ if round != expectedRound {
+ return nil, nil, 0, fmt.Errorf("txtail table contain unexpected round %d; round %d was expected", round, expectedRound)
+ }
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return nil, nil, 0, err
+ }
+
+ // decode the TxTail
+ tail := &trackerdb.TxTailRound{}
+ err = protocol.Decode(value, tail)
+ if err != nil {
+ return nil, nil, 0, err
+ }
+
+ // add the tail
+ roundData = append(roundData, tail)
+ // add the hash
+ roundHash = append(roundHash, crypto.Hash(value))
+
+ // step the round down (we expect the "previous" round next..)
+ expectedRound--
+ }
+
+ // reverse the array ordering in-place so that it would be incremental order.
+ for i := 0; i < len(roundData)/2; i++ {
+ roundData[i], roundData[len(roundData)-i-1] = roundData[len(roundData)-i-1], roundData[i]
+ roundHash[i], roundHash[len(roundHash)-i-1] = roundHash[len(roundHash)-i-1], roundHash[i]
+ }
+ return roundData, roundHash, expectedRound + 1, nil
+}
+
+func (r *accountsReader) LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error) {
+ // TODO: catchpoint CLI
+ return
+}
+
+func (r *accountsReader) Testing() trackerdb.AccountsReaderTestExt {
+ // TODO: this can wait
+ return nil
+}
diff --git a/ledger/store/trackerdb/generickv/accounts_ext_writer.go b/ledger/store/trackerdb/generickv/accounts_ext_writer.go
new file mode 100644
index 0000000000..bdb05b9138
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/accounts_ext_writer.go
@@ -0,0 +1,265 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+ "encoding/binary"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+func (w *accountsWriter) AccountsReset(ctx context.Context) error {
+ // TODO: catchpoint
+ return nil
+}
+
+func (w *accountsWriter) ResetAccountHashes(ctx context.Context) (err error) {
+ // TODO: catchpoint
+ return
+}
+
+func (w *accountsWriter) TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error {
+ // The SQL at the time fo writing:
+ //
+ // for i, data := range roundData:
+ // the inserted rnd value is baseRound + i
+ //
+ // INSERT INTO txtail(rnd, data) VALUES(?, ?)
+ //
+ // then it also cleans up everything before `forgetBeforeRound`:
+ //
+ // DELETE FROM txtail WHERE rnd < ?
+
+ // insert the new txTail's
+ for i, data := range roundData {
+ rnd := basics.Round(int(baseRound) + i)
+ err := w.kvw.Set(txTailKey(rnd), data)
+ if err != nil {
+ return err
+ }
+ }
+
+ // delete old ones
+ start := []byte(kvTxTail + "-")
+ end := txTailKey(forgetBeforeRound)
+ err := w.kvw.DeleteRange(start, end)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (w *accountsWriter) UpdateAccountsRound(rnd basics.Round) (err error) {
+ // The SQL at the time of writing:
+ //
+ // UPDATE acctrounds SET rnd=? WHERE id='acctbase' AND rnd",
+
+ // TODO: read the row for sanity? wont help the kv with race conditions, but we will need it for test parity
+ // inside a batch we wont have a read ptr..
+
+ // write round entry
+ raw := bigEndianUint64(uint64(rnd))
+ err = w.kvw.Set(roundKey(), raw)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (w *accountsWriter) UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error) {
+ // TODO: catchpoint
+ return nil
+}
+
+func (w *accountsWriter) AccountsPutTotals(totals ledgercore.AccountTotals, catchpointStaging bool) (err error) {
+ // The SQL at the time of impl:
+ //
+ // id := ""
+ // if catchpointStaging {
+ // id = "catchpointStaging"
+ // }
+ // "REPLACE INTO accounttotals
+ // (id, online, onlinerewardunits, offline, offlinerewardunits, notparticipating, notparticipatingrewardunits, rewardslevel)
+ // VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
+
+ // write totals entry
+ raw := protocol.Encode(&totals)
+ err = w.kvw.Set(totalsKey(catchpointStaging), raw)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (w *accountsWriter) OnlineAccountsDelete(forgetBefore basics.Round) (err error) {
+ // The SQL at the time of impl:
+ //
+ // SELECT
+ // rowid, address, updRound, data
+ // FROM onlineaccounts
+ // WHERE updRound < ?
+ // ORDER BY address, updRound DESC
+ //
+ // The it would delete by rowid in chunks with:
+ //
+ // DELETE FROM onlineaccounts WHERE rowid IN (..)
+
+ // On the KV implmentation:
+ //
+ // We have two ranges of keys associated with online accounts:
+ // - the `onlineAccountKey(address, round)` -> "-".join(kvPrefixOnlineAccount, addr, round)
+ // - and the `onlineAccountBalanceKey(round, normBalance, addr) -> "-".join(kvPrefixOnlineAccountBalance, round, normBalance, addr)
+
+ // 1. read from the `onlineAccountBalanceKey` range since we can the addr's that will need to be deleted
+ start := []byte(kvPrefixOnlineAccountBalance + "-")
+ end := []byte(kvPrefixOnlineAccountBalance + "-")
+ end = append(end, bigEndianUint64(uint64(forgetBefore))...)
+ iter := w.kvr.NewIter(start, end, true)
+ defer iter.Close()
+
+ toDeletePrimaryIndex := make([]struct {
+ basics.Address
+ basics.Round
+ }, 0)
+
+ toDeleteSecondaryIndex := make([][]byte, 0)
+
+ var prevAddr basics.Address
+
+ for iter.Next() {
+ // read the key
+ // schema: ---
+ key := iter.Key()
+
+ // extract the round from the key (offset: 1)
+ rndOffset := len(kvPrefixOnlineAccountBalance) + 1
+ u64Rnd := binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+ round := basics.Round(u64Rnd)
+
+ // get the offset where the address starts
+ addrOffset := len(kvPrefixOnlineAccountBalance) + 1 + 8 + 1 + 8 + 1
+ var addr basics.Address
+ copy(addr[:], key[addrOffset:addrOffset+32])
+
+ if addr != prevAddr {
+ // new address
+ // if the first (latest) entry is
+ // - offline then delete all
+ // - online then safe to delete all previous except this first (latest)
+
+ // reset the state
+ prevAddr = addr
+
+ // delete on voting empty
+ var oad trackerdb.BaseOnlineAccountData
+ var data []byte
+ data, err = iter.Value()
+ if err != nil {
+ return err
+ }
+ err = protocol.Decode(data, &oad)
+ if err != nil {
+ return err
+ }
+ if oad.IsVotingEmpty() {
+ // delete this and all subsequent
+ toDeletePrimaryIndex = append(toDeletePrimaryIndex, struct {
+ basics.Address
+ basics.Round
+ }{addr, round})
+ toDeleteSecondaryIndex = append(toDeleteSecondaryIndex, key)
+ }
+
+ // restart the loop
+ // if there are some subsequent entries, they will deleted on the next iteration
+ // if no subsequent entries, the loop will reset the state and the latest entry does not get deleted
+ continue
+ }
+
+ // mark the item for deletion
+ toDeletePrimaryIndex = append(toDeletePrimaryIndex, struct {
+ basics.Address
+ basics.Round
+ }{addr, round})
+ toDeleteSecondaryIndex = append(toDeleteSecondaryIndex, key)
+ }
+
+ // 2. delete the individual addr+round entries
+ for _, item := range toDeletePrimaryIndex {
+ // TODO: [perf] we might be able to optimize this with a SingleDelete call
+ err = w.kvw.Delete(onlineAccountKey(item.Address, item.Round))
+ if err != nil {
+ return
+ }
+ }
+
+ // 3. delete the range from `onlineAccountBalanceKey`
+ for _, key := range toDeleteSecondaryIndex {
+ // TODO: [perf] we might be able to optimize this with a SingleDelete call
+ err = w.kvw.Delete(key)
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+func (w *accountsWriter) AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error {
+ // The SQL at the time of impl:
+ //
+ // for i, data := range onlineRoundParamsData {
+ // the inserted rnd value is startRound + i
+ //
+ // INSERT INTO onlineroundparamstail (rnd, data) VALUES (?, ?)
+ //
+
+ // insert the round params
+ for i := range onlineRoundParamsData {
+ rnd := basics.Round(int(startRound) + i)
+ raw := protocol.Encode(&onlineRoundParamsData[i])
+ err := w.kvw.Set(onlineAccountRoundParamsKey(rnd), raw)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (w *accountsWriter) AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error {
+ // The SQL at the time of impl:
+ //
+ // DELETE FROM onlineroundparamstail WHERE rnd
+
+ // delete old ones
+ start := []byte(kvOnlineAccountRoundParams + "-")
+ end := onlineAccountRoundParamsKey(deleteBeforeRound)
+ err := w.kvw.DeleteRange(start, end)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/ledger/store/trackerdb/generickv/accounts_reader.go b/ledger/store/trackerdb/generickv/accounts_reader.go
new file mode 100644
index 0000000000..b6806a109a
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/accounts_reader.go
@@ -0,0 +1,405 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "encoding/binary"
+ "fmt"
+ "io"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+// KvRead is a low level KV db interface for reading.
+type KvRead interface {
+ Get(key []byte) ([]byte, io.Closer, error)
+ NewIter(low, high []byte, reverse bool) KvIter
+}
+
+// KvIter is a low level KV iterator.
+type KvIter interface {
+ Next() bool
+ Key() []byte
+ KeySlice() Slice
+ Value() ([]byte, error)
+ ValueSlice() (Slice, error)
+ Valid() bool
+ Close()
+}
+
+// Slice is a low level slice used during the KV iterator.
+type Slice interface {
+ Data() []byte
+ Free()
+ Size() int
+ Exists() bool
+}
+
+type accountsReader struct {
+ kvr KvRead
+ proto config.ConsensusParams
+}
+
+// MakeAccountsReader returns a kv db agnostic AccountsReader.
+func MakeAccountsReader(kvr KvRead, proto config.ConsensusParams) *accountsReader {
+ return &accountsReader{kvr, proto}
+}
+
+func (r *accountsReader) LookupAccount(addr basics.Address) (data trackerdb.PersistedAccountData, err error) {
+ // SQL impl at time of writing:
+ //
+ // SELECT
+ // accountbase.rowid,
+ // acctrounds.rnd,
+ // accountbase.data
+ // FROM acctrounds
+ // LEFT JOIN accountbase ON address=?
+ // WHERE id='acctbase'
+
+ data.Addr = addr
+
+ // read the current db round
+ data.Round, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ value, closer, err := r.kvr.Get(accountKey(addr))
+ if err == trackerdb.ErrNotFound {
+ // Note: the SQL implementation returns a data value and no error even when the account does not exist.
+ return data, nil
+ } else if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ err = protocol.Decode(value, &data.AccountData)
+ if err != nil {
+ return
+ }
+
+ data.Ref = accountRef{addr}
+
+ return
+}
+
+func (r *accountsReader) LookupResources(addr basics.Address, aidx basics.CreatableIndex, ctype basics.CreatableType) (data trackerdb.PersistedResourcesData, err error) {
+ data.Aidx = aidx
+
+ // read the current db round
+ data.Round, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ value, closer, err := r.kvr.Get(resourceKey(addr, aidx))
+ if err == trackerdb.ErrNotFound {
+ // Note: the SQL implementation returns a data value and no error even when the account does not exist.
+ data.Data = trackerdb.MakeResourcesData(0)
+ return data, nil
+ } else if err != nil {
+ err = fmt.Errorf("unable to query resource data for address %v aidx %v ctype %v : %w", addr, aidx, ctype, err)
+ return
+ }
+ defer closer.Close()
+
+ err = protocol.Decode(value, &data.Data)
+ if err != nil {
+ return
+ }
+
+ // Note: the ctype is not filtered during the query, but rather asserted to be what the caller expected
+ if ctype == basics.AssetCreatable && !data.Data.IsAsset() {
+ err = fmt.Errorf("lookupResources asked for an asset but got %v", data.Data)
+ }
+ if ctype == basics.AppCreatable && !data.Data.IsApp() {
+ err = fmt.Errorf("lookupResources asked for an app but got %v", data.Data)
+ }
+
+ data.AcctRef = accountRef{addr}
+
+ return
+}
+
+func (r *accountsReader) LookupAllResources(addr basics.Address) (data []trackerdb.PersistedResourcesData, rnd basics.Round, err error) {
+ low := resourceAddrOnlyPartialKey(addr)
+ high := resourceAddrOnlyPartialKey(addr)
+ high[len(high)-1]++
+
+ iter := r.kvr.NewIter(low, high, false)
+ defer iter.Close()
+
+ var value []byte
+ var aidx uint64
+
+ // read the current db round
+ rnd, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ for iter.Next() {
+ pitem := trackerdb.PersistedResourcesData{AcctRef: accountRef{addr}, Round: rnd}
+
+ // read the key to parse the aidx
+ // key is -- =
+ key := iter.Key()
+
+ aidxOffset := len(kvPrefixResource) + 1 + 32 + 1
+ aidx = binary.BigEndian.Uint64(key[aidxOffset : aidxOffset+8])
+ pitem.Aidx = basics.CreatableIndex(aidx)
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+ // decode raw value
+ err = protocol.Decode(value, &pitem.Data)
+ if err != nil {
+ return
+ }
+ // append entry to accum
+ data = append(data, pitem)
+ }
+
+ return
+}
+
+func (r *accountsReader) LookupKeyValue(key string) (pv trackerdb.PersistedKVData, err error) {
+ // read the current db round
+ pv.Round, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ value, closer, err := r.kvr.Get(appKvKey(key))
+ if err == trackerdb.ErrNotFound {
+ // Note: the SQL implementation returns a data value and no error even when the account does not exist.
+ return pv, nil
+ } else if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ pv.Value = value
+
+ return
+}
+
+// TODO: lifted from sql.go, we might want to refactor it
+func keyPrefixIntervalPreprocessing(prefix []byte) ([]byte, []byte) {
+ if prefix == nil {
+ prefix = []byte{}
+ }
+ prefixIncr := make([]byte, len(prefix))
+ copy(prefixIncr, prefix)
+ for i := len(prefix) - 1; i >= 0; i-- {
+ currentByteIncr := int(prefix[i]) + 1
+ if currentByteIncr > 0xFF {
+ prefixIncr = prefixIncr[:len(prefixIncr)-1]
+ continue
+ }
+ prefixIncr[i] = byte(currentByteIncr)
+ return prefix, prefixIncr
+ }
+ return prefix, nil
+}
+
+func (r *accountsReader) LookupKeysByPrefix(prefix string, maxKeyNum uint64, results map[string]bool, resultCount uint64) (round basics.Round, err error) {
+ // SQL at time of writing:
+ //
+ // SELECT acctrounds.rnd, kvstore.key
+ // FROM acctrounds LEFT JOIN kvstore ON kvstore.key >= ? AND kvstore.key < ?
+ // WHERE id='acctbase'
+
+ // read the current db round
+ round, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ start, end := keyPrefixIntervalPreprocessing([]byte(prefix))
+
+ iter := r.kvr.NewIter(start, end, false)
+ defer iter.Close()
+
+ var value []byte
+
+ for iter.Next() {
+ // end iteration if we reached max results
+ if resultCount == maxKeyNum {
+ return
+ }
+
+ // read the key
+ key := string(iter.Key())
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ // mark if the key has data on the result map
+ results[key] = len(value) > 0
+
+ // inc results in range
+ resultCount++
+ }
+
+ return
+}
+
+func (r *accountsReader) ListCreatables(maxIdx basics.CreatableIndex, maxResults uint64, ctype basics.CreatableType) (results []basics.CreatableLocator, dbRound basics.Round, err error) {
+ // The old SQL impl:
+ //
+ // SELECT
+ // acctrounds.rnd,
+ // assetcreators.asset,
+ // assetcreators.creator
+ // FROM acctrounds
+ // LEFT JOIN assetcreators ON assetcreators.asset <= ? AND assetcreators.ctype = ?
+ // WHERE acctrounds.id='acctbase'
+ // ORDER BY assetcreators.asset desc
+ // LIMIT ?
+
+ // read the current db round
+ dbRound, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ start := []byte(fmt.Sprintf("%s-", kvPrefixCreatorIndex))
+ end := creatableKey(basics.CreatableIndex(uint64(maxIdx) + 1))
+
+ // assets are returned in descending order of cidx
+ iter := r.kvr.NewIter(start, end, true)
+ defer iter.Close()
+
+ var value []byte
+ var resultCount uint64
+ var cidx uint64
+
+ for iter.Next() {
+ // end iteration if we reached max results
+ if resultCount == maxResults {
+ return
+ }
+
+ // read the key
+ // schema: -
+ key := iter.Key()
+
+ // extract cidx
+ cidxOffset := len(kvPrefixCreatorIndex) + 1
+ cidx = binary.BigEndian.Uint64(key[cidxOffset : cidxOffset+8])
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ // decode the raw value
+ var entry creatableEntry
+ err = protocol.Decode(value, &entry)
+ if err != nil {
+ return
+ }
+
+ // TODO: the ctype as part of key makes this filterable during the iter
+ if entry.Ctype != ctype {
+ continue
+ }
+
+ // create the "creatable" struct
+ cl := basics.CreatableLocator{Type: ctype, Index: basics.CreatableIndex(cidx)}
+ copy(cl.Creator[:], entry.CreatorAddr)
+
+ // add it to the the results
+ results = append(results, cl)
+
+ // inc results in range
+ resultCount++
+ }
+
+ // read the current db round
+ dbRound, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (r *accountsReader) LookupCreator(cidx basics.CreatableIndex, ctype basics.CreatableType) (addr basics.Address, ok bool, dbRound basics.Round, err error) {
+ // The old SQL impl:
+ //
+ // SELECT
+ // acctrounds.rnd,
+ // assetcreators.creator
+ // FROM acctrounds
+ // LEFT JOIN assetcreators ON asset = ? AND ctype = ?
+ // WHERE id='acctbase'
+
+ // read the current db round
+ dbRound, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ value, closer, err := r.kvr.Get(creatableKey(cidx))
+ if err == trackerdb.ErrNotFound {
+ // the record does not exist
+ // clean up the error and just return ok=false
+ err = nil
+ ok = false
+ return
+ } else if err != nil {
+ return
+ }
+ defer closer.Close()
+
+ // decode the raw value
+ var entry creatableEntry
+ err = protocol.Decode(value, &entry)
+ if err != nil {
+ return
+ }
+
+ // assert that the ctype is the one expected
+ if entry.Ctype != ctype {
+ ok = false
+ return
+ }
+
+ // copy the addr to the return
+ copy(addr[:], entry.CreatorAddr)
+
+ // mark result as ok
+ ok = true
+
+ return
+}
+
+func (r *accountsReader) Close() {
+
+}
diff --git a/ledger/store/trackerdb/generickv/accounts_writer.go b/ledger/store/trackerdb/generickv/accounts_writer.go
new file mode 100644
index 0000000000..b2f808a436
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/accounts_writer.go
@@ -0,0 +1,188 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+// KvWrite is a low level KV db interface for writing.
+type KvWrite interface {
+ Set(key, value []byte) error
+ Delete(key []byte) error
+ DeleteRange(start, end []byte) error
+}
+
+type accountsWriter struct {
+ kvw KvWrite
+ kvr KvRead
+}
+
+type accountRef struct {
+ addr basics.Address
+}
+
+func (ref accountRef) AccountRefMarker() {}
+
+type resourceRef struct {
+ addr basics.Address
+ aidx basics.CreatableIndex
+}
+
+func (ref resourceRef) ResourceRefMarker() {}
+
+type creatableRef struct {
+ cidx basics.CreatableIndex
+}
+
+func (ref creatableRef) CreatableRefMarker() {}
+
+// MakeAccountsWriter returns a kv db agnostic AccountsWriter.
+// TODO: we should discuss what is the best approach for this `kvr KvRead`.
+// the problem is that `OnlineAccountsDelete` requires reading and writing.
+// the cleanest approach is to move that method to a separate interface, and have it only be available on 'transactions'.
+// although a snapshot+batch should be able to support it too since its all reads, then writes.
+func MakeAccountsWriter(kvw KvWrite, kvr KvRead) *accountsWriter {
+ return &accountsWriter{kvw, kvr}
+}
+
+func (w *accountsWriter) InsertAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseAccountData) (ref trackerdb.AccountRef, err error) {
+ // write account entry
+ raw := protocol.Encode(&data)
+ err = w.kvw.Set(accountKey(addr), raw)
+ if err != nil {
+ return nil, err
+ }
+
+ return accountRef{addr}, nil
+}
+
+func (w *accountsWriter) DeleteAccount(ref trackerdb.AccountRef) (rowsAffected int64, err error) {
+ xref := ref.(accountRef)
+
+ // delete account entry
+ err = w.kvw.Delete(accountKey(xref.addr))
+ if err != nil {
+ return 0, err
+ }
+
+ return 1, nil
+}
+
+func (w *accountsWriter) UpdateAccount(ref trackerdb.AccountRef, normBalance uint64, data trackerdb.BaseAccountData) (rowsAffected int64, err error) {
+ xref := ref.(accountRef)
+
+ // overwrite account entry
+ raw := protocol.Encode(&data)
+ err = w.kvw.Set(accountKey(xref.addr), raw)
+ if err != nil {
+ return 0, err
+ }
+
+ return 1, nil
+}
+
+func (w *accountsWriter) InsertResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (ref trackerdb.ResourceRef, err error) {
+ xref := acctRef.(accountRef)
+
+ // write resource entry
+ raw := protocol.Encode(&data)
+ err = w.kvw.Set(resourceKey(xref.addr, aidx), raw)
+ if err != nil {
+ return nil, err
+ }
+
+ return resourceRef{xref.addr, aidx}, nil
+}
+
+func (w *accountsWriter) DeleteResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex) (rowsAffected int64, err error) {
+ xref := acctRef.(accountRef)
+
+ // delete resource entry
+ err = w.kvw.Delete(resourceKey(xref.addr, aidx))
+ if err != nil {
+ return 0, err
+ }
+
+ return 1, nil
+}
+
+func (w *accountsWriter) UpdateResource(acctRef trackerdb.AccountRef, aidx basics.CreatableIndex, data trackerdb.ResourcesData) (rowsAffected int64, err error) {
+ xref := acctRef.(accountRef)
+
+ // update resource entry
+ raw := protocol.Encode(&data)
+ err = w.kvw.Set(resourceKey(xref.addr, aidx), raw)
+ if err != nil {
+ return 0, err
+ }
+
+ return 1, nil
+}
+
+func (w *accountsWriter) UpsertKvPair(key string, value []byte) error {
+ // upsert kv entry
+ err := w.kvw.Set(appKvKey(key), value)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (w *accountsWriter) DeleteKvPair(key string) error {
+ // delete kv entry
+ err := w.kvw.Delete(appKvKey(key))
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+type creatableEntry struct {
+ _struct struct{} `codec:",omitempty,omitemptyarray"`
+ Ctype basics.CreatableType
+ CreatorAddr []byte
+}
+
+func (w *accountsWriter) InsertCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType, creator []byte) (ref trackerdb.CreatableRef, err error) {
+ // insert creatable entry
+ raw := protocol.Encode(&creatableEntry{Ctype: ctype, CreatorAddr: creator})
+ err = w.kvw.Set(creatableKey(cidx), raw)
+ if err != nil {
+ return
+ }
+
+ return creatableRef{cidx}, nil
+}
+
+func (w *accountsWriter) DeleteCreatable(cidx basics.CreatableIndex, ctype basics.CreatableType) (rowsAffected int64, err error) {
+ // delete creatable entry
+ err = w.kvw.Delete(creatableKey(cidx))
+ if err != nil {
+ return 0, err
+ }
+
+ return 1, nil
+}
+
+func (w *accountsWriter) Close() {
+
+}
diff --git a/ledger/store/trackerdb/generickv/catchpoint.go b/ledger/store/trackerdb/generickv/catchpoint.go
new file mode 100644
index 0000000000..18b4e209fa
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/catchpoint.go
@@ -0,0 +1,60 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type catchpoint struct{}
+
+// MakeCatchpoint returns a trackerdb.Catchpoint for a KV
+func MakeCatchpoint() trackerdb.Catchpoint {
+ return &catchpoint{}
+}
+
+// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint
+func (*catchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter {
+ panic("unimplemented")
+}
+
+// MakeCatchpointWriter implements trackerdb.Catchpoint
+func (*catchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) {
+ panic("unimplemented")
+}
+
+// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter {
+ panic("unimplemented")
+}
+
+// MakeKVsIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) {
+ panic("unimplemented")
+}
+
+// MakeMerkleCommitter implements trackerdb.Catchpoint
+func (*catchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) {
+ panic("unimplemented")
+}
+
+// MakeOrderedAccountsIter implements trackerdb.Catchpoint
+func (*catchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter {
+ panic("unimplemented")
+}
diff --git a/ledger/store/trackerdb/generickv/init_accounts.go b/ledger/store/trackerdb/generickv/init_accounts.go
new file mode 100644
index 0000000000..af587af105
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/init_accounts.go
@@ -0,0 +1,60 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/stretchr/testify/require"
+)
+
+type dbForInit interface {
+ trackerdb.Store
+ KvRead
+ KvWrite
+}
+
+// AccountsInitTest initializes the database for testing with the given accounts.
+func AccountsInitTest(tb testing.TB, db dbForInit, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
+ params := trackerdb.Params{
+ InitAccounts: initAccounts,
+ InitProto: proto,
+ }
+ _, err := RunMigrations(context.Background(), db, params, trackerdb.AccountDBVersion)
+ require.NoError(tb, err)
+ return true
+}
+
+// AccountsInitLightTest initializes the database for testing with the given accounts.
+//
+// This is duplicate due to a specific legacy test in accdeltas_test.go.
+// TODO: remove the need for this.
+func AccountsInitLightTest(tb testing.TB, db dbForInit, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
+ params := trackerdb.Params{
+ InitAccounts: initAccounts,
+ // TODO: how do we get the correct version from the proto arg?
+ InitProto: protocol.ConsensusCurrentVersion,
+ }
+ _, err = RunMigrations(context.Background(), db, params, trackerdb.AccountDBVersion)
+ require.NoError(tb, err)
+ return true, nil
+}
diff --git a/ledger/store/trackerdb/generickv/migrations.go b/ledger/store/trackerdb/generickv/migrations.go
new file mode 100644
index 0000000000..9c8e942e39
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/migrations.go
@@ -0,0 +1,225 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+ "encoding/binary"
+ "fmt"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+func getSchemaVersion(ctx context.Context, kvr KvRead) (int32, error) {
+ // read version entry
+ value, closer, err := kvr.Get(schemaVersionKey())
+ if err == trackerdb.ErrNotFound {
+ // ignore the error, return version 0
+ return 0, nil
+ } else if err != nil {
+ return 0, err
+ }
+ defer closer.Close()
+
+ // parse the bytes into a i32
+ version := int32(binary.BigEndian.Uint32(value))
+
+ return version, nil
+}
+
+func setSchemaVersion(ctx context.Context, kvw KvWrite, version int32) error {
+ // write version entry
+ raw := bigEndianUint32(uint32(version))
+ err := kvw.Set(schemaVersionKey(), raw)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+type dbForMigrations interface {
+ trackerdb.Store
+ KvRead
+ KvWrite
+}
+
+// RunMigrations runs the migrations on the store up to the target version.
+func RunMigrations(ctx context.Context, db dbForMigrations, params trackerdb.Params, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+
+ dbVersion, err := getSchemaVersion(ctx, db)
+ if err != nil {
+ return
+ }
+
+ mgr.SchemaVersion = dbVersion
+ mgr.VacuumOnStartup = false
+
+ migrator := &migrator{
+ currentVersion: dbVersion,
+ targetVersion: targetVersion,
+ params: params,
+ db: db,
+ }
+
+ err = migrator.Migrate(ctx)
+ if err != nil {
+ return
+ }
+
+ mgr.SchemaVersion = migrator.currentVersion
+
+ return mgr, nil
+}
+
+type migrator struct {
+ currentVersion int32
+ targetVersion int32
+ params trackerdb.Params
+ db dbForMigrations
+}
+
+func (m *migrator) Migrate(ctx context.Context) error {
+ // we cannot rollback
+ if m.currentVersion > m.targetVersion {
+ return nil
+ }
+ // upgrade the db one version at at time
+ for m.currentVersion < m.targetVersion {
+ // run next version upgrade
+ switch m.currentVersion {
+ case 0: // initial version
+ err := m.initialVersion(ctx)
+ if err != nil {
+ return err
+ }
+ default:
+ // any other version we do nothing
+ return nil
+ }
+ }
+ return nil
+}
+
+func (m *migrator) setVersion(ctx context.Context, version int32) error {
+ // update crrent version in the db
+ err := setSchemaVersion(ctx, m.db, version)
+ if err != nil {
+ return err
+ }
+ // update current version in the migrator
+ m.currentVersion = version
+ return nil
+}
+
+func (m *migrator) initialVersion(ctx context.Context) error {
+ proto := config.Consensus[m.params.InitProto]
+
+ // TODO: make this a batch scope
+ err := m.db.TransactionContext(ctx, func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
+ aow, err := tx.MakeAccountsOptimizedWriter(true, false, false, false)
+ if err != nil {
+ return err
+ }
+
+ oaow, err := tx.MakeOnlineAccountsOptimizedWriter(true)
+ if err != nil {
+ return err
+ }
+
+ aw, err := tx.MakeAccountsWriter()
+ if err != nil {
+ return err
+ }
+
+ updRound := basics.Round(0)
+
+ // mark the db as round 0
+ err = aw.UpdateAccountsRound(updRound)
+ if err != nil {
+ return err
+ }
+
+ var ot basics.OverflowTracker
+ var totals ledgercore.AccountTotals
+
+ // insert initial accounts
+ for addr, account := range m.params.InitAccounts {
+ // build a trackerdb.BaseAccountData to pass to the DB
+ var bad trackerdb.BaseAccountData
+ bad.SetAccountData(&account)
+ // insert the account
+ _, err = aow.InsertAccount(addr, account.NormalizedOnlineBalance(proto), bad)
+ if err != nil {
+ return err
+ }
+
+ // build a ledgercore.AccountData to track the totals
+ ad := ledgercore.ToAccountData(account)
+ // track the totals
+ totals.AddAccount(proto, ad, &ot)
+
+ // insert online account (if online)
+ if bad.Status == basics.Online {
+ var baseOnlineAD trackerdb.BaseOnlineAccountData
+ baseOnlineAD.BaseVotingData = bad.BaseVotingData
+ baseOnlineAD.MicroAlgos = bad.MicroAlgos
+ baseOnlineAD.RewardsBase = bad.RewardsBase
+
+ _, err = oaow.InsertOnlineAccount(addr, account.NormalizedOnlineBalance(proto), baseOnlineAD, uint64(updRound), uint64(baseOnlineAD.VoteLastValid))
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // make sure we didn't overflow
+ if ot.Overflowed {
+ return fmt.Errorf("overflow computing totals")
+ }
+
+ // insert the totals
+ err = aw.AccountsPutTotals(totals, false)
+ if err != nil {
+ return err
+ }
+
+ // insert online params
+ params := []ledgercore.OnlineRoundParamsData{
+ {
+ OnlineSupply: totals.Online.Money.Raw,
+ RewardsLevel: totals.RewardsLevel,
+ CurrentProtocol: m.params.InitProto,
+ },
+ }
+ err = aw.AccountsPutOnlineRoundParams(params, basics.Round(0))
+ if err != nil {
+ return err
+ }
+
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+
+ // KV store starts at version 10
+ return m.setVersion(ctx, 10)
+}
diff --git a/ledger/store/trackerdb/generickv/msgp_gen.go b/ledger/store/trackerdb/generickv/msgp_gen.go
new file mode 100644
index 0000000000..fd825ca99d
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/msgp_gen.go
@@ -0,0 +1,146 @@
+package generickv
+
+// Code generated by github.com/algorand/msgp DO NOT EDIT.
+
+import (
+ "github.com/algorand/msgp/msgp"
+)
+
+// The following msgp objects are implemented in this file:
+// creatableEntry
+// |-----> (*) MarshalMsg
+// |-----> (*) CanMarshalMsg
+// |-----> (*) UnmarshalMsg
+// |-----> (*) CanUnmarshalMsg
+// |-----> (*) Msgsize
+// |-----> (*) MsgIsZero
+//
+
+// MarshalMsg implements msgp.Marshaler
+func (z *creatableEntry) MarshalMsg(b []byte) (o []byte) {
+ o = msgp.Require(b, z.Msgsize())
+ // omitempty: check for empty values
+ zb0001Len := uint32(2)
+ var zb0001Mask uint8 /* 3 bits */
+ if len((*z).CreatorAddr) == 0 {
+ zb0001Len--
+ zb0001Mask |= 0x1
+ }
+ if (*z).Ctype.MsgIsZero() {
+ zb0001Len--
+ zb0001Mask |= 0x2
+ }
+ // variable map header, size zb0001Len
+ o = append(o, 0x80|uint8(zb0001Len))
+ if zb0001Len != 0 {
+ if (zb0001Mask & 0x1) == 0 { // if not empty
+ // string "CreatorAddr"
+ o = append(o, 0xab, 0x43, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x41, 0x64, 0x64, 0x72)
+ o = msgp.AppendBytes(o, (*z).CreatorAddr)
+ }
+ if (zb0001Mask & 0x2) == 0 { // if not empty
+ // string "Ctype"
+ o = append(o, 0xa5, 0x43, 0x74, 0x79, 0x70, 0x65)
+ o = (*z).Ctype.MarshalMsg(o)
+ }
+ }
+ return
+}
+
+func (_ *creatableEntry) CanMarshalMsg(z interface{}) bool {
+ _, ok := (z).(*creatableEntry)
+ return ok
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *creatableEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+ var field []byte
+ _ = field
+ var zb0001 int
+ var zb0002 bool
+ zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+ if _, ok := err.(msgp.TypeError); ok {
+ zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ if zb0001 > 0 {
+ zb0001--
+ bts, err = (*z).Ctype.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "Ctype")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ zb0001--
+ (*z).CreatorAddr, bts, err = msgp.ReadBytesBytes(bts, (*z).CreatorAddr)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array", "CreatorAddr")
+ return
+ }
+ }
+ if zb0001 > 0 {
+ err = msgp.ErrTooManyArrayFields(zb0001)
+ if err != nil {
+ err = msgp.WrapError(err, "struct-from-array")
+ return
+ }
+ }
+ } else {
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ if zb0002 {
+ (*z) = creatableEntry{}
+ }
+ for zb0001 > 0 {
+ zb0001--
+ field, bts, err = msgp.ReadMapKeyZC(bts)
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ switch string(field) {
+ case "Ctype":
+ bts, err = (*z).Ctype.UnmarshalMsg(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "Ctype")
+ return
+ }
+ case "CreatorAddr":
+ (*z).CreatorAddr, bts, err = msgp.ReadBytesBytes(bts, (*z).CreatorAddr)
+ if err != nil {
+ err = msgp.WrapError(err, "CreatorAddr")
+ return
+ }
+ default:
+ err = msgp.ErrNoField(string(field))
+ if err != nil {
+ err = msgp.WrapError(err)
+ return
+ }
+ }
+ }
+ }
+ o = bts
+ return
+}
+
+func (_ *creatableEntry) CanUnmarshalMsg(z interface{}) bool {
+ _, ok := (z).(*creatableEntry)
+ return ok
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *creatableEntry) Msgsize() (s int) {
+ s = 1 + 6 + (*z).Ctype.Msgsize() + 12 + msgp.BytesPrefixSize + len((*z).CreatorAddr)
+ return
+}
+
+// MsgIsZero returns whether this is a zero value
+func (z *creatableEntry) MsgIsZero() bool {
+ return ((*z).Ctype.MsgIsZero()) && (len((*z).CreatorAddr) == 0)
+}
diff --git a/ledger/store/trackerdb/generickv/msgp_gen_test.go b/ledger/store/trackerdb/generickv/msgp_gen_test.go
new file mode 100644
index 0000000000..88d79eec21
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/msgp_gen_test.go
@@ -0,0 +1,75 @@
+//go:build !skip_msgp_testing
+// +build !skip_msgp_testing
+
+package generickv
+
+// Code generated by github.com/algorand/msgp DO NOT EDIT.
+
+import (
+ "testing"
+
+ "github.com/algorand/msgp/msgp"
+
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/test/partitiontest"
+)
+
+func TestMarshalUnmarshalcreatableEntry(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ v := creatableEntry{}
+ bts := v.MarshalMsg(nil)
+ left, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+ }
+
+ left, err = msgp.Skip(bts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(left) > 0 {
+ t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+ }
+}
+
+func TestRandomizedEncodingcreatableEntry(t *testing.T) {
+ protocol.RunEncodingTest(t, &creatableEntry{})
+}
+
+func BenchmarkMarshalMsgcreatableEntry(b *testing.B) {
+ v := creatableEntry{}
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ v.MarshalMsg(nil)
+ }
+}
+
+func BenchmarkAppendMsgcreatableEntry(b *testing.B) {
+ v := creatableEntry{}
+ bts := make([]byte, 0, v.Msgsize())
+ bts = v.MarshalMsg(bts[0:0])
+ b.SetBytes(int64(len(bts)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ bts = v.MarshalMsg(bts[0:0])
+ }
+}
+
+func BenchmarkUnmarshalcreatableEntry(b *testing.B) {
+ v := creatableEntry{}
+ bts := v.MarshalMsg(nil)
+ b.ReportAllocs()
+ b.SetBytes(int64(len(bts)))
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _, err := v.UnmarshalMsg(bts)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
diff --git a/ledger/store/trackerdb/generickv/onlineaccounts_reader.go b/ledger/store/trackerdb/generickv/onlineaccounts_reader.go
new file mode 100644
index 0000000000..029b0317c2
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/onlineaccounts_reader.go
@@ -0,0 +1,165 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "encoding/binary"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+// LookupOnline pulls the Online Account data for a given account+round
+func (r *accountsReader) LookupOnline(addr basics.Address, rnd basics.Round) (data trackerdb.PersistedOnlineAccountData, err error) {
+ // SQL at the time of writing this:
+ //
+ // SELECT
+ // onlineaccounts.rowid, onlineaccounts.updround,
+ // acctrounds.rnd,
+ // onlineaccounts.data
+ // FROM acctrounds
+ // LEFT JOIN onlineaccounts ON address=? AND updround <= ?
+ // WHERE id='acctbase'
+ // ORDER BY updround DESC LIMIT 1
+
+ // read the current db round
+ data.Round, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ // read latest account up to `rnd``
+ low := onlineAccountOnlyPartialKey(addr)
+ high := onlineAccountKey(addr, rnd)
+ // inc the last byte to make it inclusive
+ high[len(high)-1]++
+ iter := r.kvr.NewIter(low, high, true)
+ defer iter.Close()
+
+ var value []byte
+ var updRound uint64
+
+ if iter.Next() {
+ // schema: --
+ key := iter.Key()
+
+ // extract updround, its the last section after the "-"
+ rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1
+ updRound = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+ if err != nil {
+ return
+ }
+ data.Addr = addr
+ data.UpdRound = basics.Round(updRound)
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+
+ // parse the value
+ err = protocol.Decode(value, &data.AccountData)
+ if err != nil {
+ return
+ }
+
+ normBalance := data.AccountData.NormalizedOnlineBalance(r.proto)
+ data.Ref = onlineAccountRef{addr, normBalance, rnd}
+
+ // we have the record, we can leave
+ return
+ }
+
+ // nothing was found
+ // Note: the SQL implementation returns a data value and no error even when the account does not exist.
+ return data, nil
+}
+
+// LookupOnlineTotalsHistory pulls the total Online Algos on a given round
+func (r *accountsReader) LookupOnlineTotalsHistory(round basics.Round) (basics.MicroAlgos, error) {
+ // SQL at the time of writing this:
+ //
+ // SELECT data FROM onlineroundparamstail WHERE rnd=?
+
+ value, closer, err := r.kvr.Get(onlineAccountRoundParamsKey(round))
+ if err != nil {
+ return basics.MicroAlgos{}, err
+ }
+ defer closer.Close()
+ data := ledgercore.OnlineRoundParamsData{}
+ err = protocol.Decode(value, &data)
+ if err != nil {
+ return basics.MicroAlgos{}, err
+ }
+ return basics.MicroAlgos{Raw: data.OnlineSupply}, nil
+}
+
+func (r *accountsReader) LookupOnlineHistory(addr basics.Address) (result []trackerdb.PersistedOnlineAccountData, rnd basics.Round, err error) {
+ low := onlineAccountOnlyPartialKey(addr)
+ high := onlineAccountOnlyPartialKey(addr)
+ high[len(high)-1]++
+ iter := r.kvr.NewIter(low, high, false)
+ defer iter.Close()
+
+ var value []byte
+ var updround uint64
+
+ // read the current db round
+ rnd, err = r.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ for iter.Next() {
+ pitem := trackerdb.PersistedOnlineAccountData{}
+
+ // schema: --
+ key := iter.Key()
+ // extract updround, its the last section after the "-"
+ rndOffset := len(kvPrefixOnlineAccount) + 1 + 32 + 1
+ updround = binary.BigEndian.Uint64(key[rndOffset : rndOffset+8])
+ if err != nil {
+ return
+ }
+ pitem.Addr = addr
+ pitem.UpdRound = basics.Round(updround)
+ // Note: for compatibility with the SQL impl, this is not included on each item
+ // pitem.Round = rnd
+
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return
+ }
+ // decode raw value
+ err = protocol.Decode(value, &pitem.AccountData)
+ if err != nil {
+ return
+ }
+
+ // set the ref
+ pitem.Ref = onlineAccountRef{addr, pitem.AccountData.NormalizedOnlineBalance(r.proto), pitem.UpdRound}
+
+ // append entry to accum
+ result = append(result, pitem)
+ }
+
+ return
+}
diff --git a/ledger/store/trackerdb/generickv/onlineaccounts_writer.go b/ledger/store/trackerdb/generickv/onlineaccounts_writer.go
new file mode 100644
index 0000000000..cca847c608
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/onlineaccounts_writer.go
@@ -0,0 +1,64 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+type onlineAccountsWriter struct {
+ kvw KvWrite
+}
+
+type onlineAccountRef struct {
+ addr basics.Address
+ normBalance uint64
+ round basics.Round
+}
+
+func (ref onlineAccountRef) OnlineAccountRefMarker() {}
+
+// MakeOnlineAccountsWriter constructs an kv agnostic OnlineAccountsWriter
+func MakeOnlineAccountsWriter(kvw KvWrite) trackerdb.OnlineAccountsWriter {
+ return &onlineAccountsWriter{kvw}
+}
+
+func (w *onlineAccountsWriter) InsertOnlineAccount(addr basics.Address, normBalance uint64, data trackerdb.BaseOnlineAccountData, updRound uint64, voteLastValid uint64) (ref trackerdb.OnlineAccountRef, err error) {
+ raw := protocol.Encode(&data)
+ rnd := basics.Round(updRound)
+
+ // write to the online account key
+ err = w.kvw.Set(onlineAccountKey(addr, rnd), raw)
+ if err != nil {
+ return nil, err
+ }
+
+ // write to the secondary account balance key
+ // TODO: this is not the most efficient use of space, but its a tradeoff with a second lookup per object.
+ // this impacts `AccountsOnlineTop`, and some experiments will be needed to determine if we do extra lookups or duplicate the values.
+ err = w.kvw.Set(onlineAccountBalanceKey(updRound, normBalance, addr), raw)
+ if err != nil {
+ return nil, err
+ }
+
+ return onlineAccountRef{addr, normBalance, rnd}, nil
+}
+
+func (w *onlineAccountsWriter) Close() {
+}
diff --git a/ledger/store/trackerdb/generickv/reader.go b/ledger/store/trackerdb/generickv/reader.go
new file mode 100644
index 0000000000..88761c6013
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/reader.go
@@ -0,0 +1,52 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+)
+
+type reader struct {
+ proto config.ConsensusParams
+ KvRead
+}
+
+// MakeReader returns a trackerdb.Reader for a KV
+func MakeReader(kvr KvRead, proto config.ConsensusParams) trackerdb.Reader {
+ return &reader{proto, kvr}
+}
+
+// MakeAccountsOptimizedReader implements trackerdb.Reader
+func (r *reader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) {
+ return MakeAccountsReader(r, r.proto), nil
+}
+
+// MakeAccountsReader implements trackerdb.Reader
+func (r *reader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) {
+ return MakeAccountsReader(r, r.proto), nil
+}
+
+// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader
+func (r *reader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) {
+ return MakeAccountsReader(r, r.proto), nil
+}
+
+// MakeSpVerificationCtxReader implements trackerdb.Reader
+func (r *reader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader {
+ return MakeStateproofReader(r)
+}
diff --git a/ledger/store/trackerdb/generickv/schema.go b/ledger/store/trackerdb/generickv/schema.go
new file mode 100644
index 0000000000..8afb41e9e2
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/schema.go
@@ -0,0 +1,171 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "encoding/binary"
+
+ "github.com/algorand/go-algorand/data/basics"
+)
+
+const (
+ kvPrefixAccount = "account"
+ kvPrefixResource = "resource"
+ kvPrefixAppKv = "appkv"
+ kvPrefixCreatorIndex = "creator"
+ kvPrefixOnlineAccount = "online_account_base"
+ kvPrefixOnlineAccountBalance = "online_account_balance"
+ kvRoundKey = "global_round"
+ kvSchemaVersionKey = "global_schema_version"
+ kvTotalsKey = "global_total"
+ kvTxTail = "txtail"
+ kvOnlineAccountRoundParams = "online_account_round_params"
+ kvPrefixStateproof = "stateproofs"
+)
+
+// return the big-endian binary encoding of a uint64
+func bigEndianUint64(v uint64) []byte {
+ ret := make([]byte, 8)
+ binary.BigEndian.PutUint64(ret, v)
+ return ret
+}
+
+// return the big-endian binary encoding of a uint32
+func bigEndianUint32(v uint32) []byte {
+ ret := make([]byte, 4)
+ binary.BigEndian.PutUint32(ret, v)
+ return ret
+}
+
+// accountKey: 4-byte prefix + 32-byte address
+func accountKey(address basics.Address) []byte {
+ ret := []byte(kvPrefixAccount)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ return ret
+}
+
+// resourceKey: 4-byte prefix + 32-byte address + 8-byte big-endian uint64
+func resourceKey(address basics.Address, aidx basics.CreatableIndex) []byte {
+ ret := []byte(kvPrefixResource)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(aidx))...)
+ return ret
+}
+
+func resourceAddrOnlyPartialKey(address basics.Address) []byte {
+ ret := []byte(kvPrefixResource)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ ret = append(ret, "-"...)
+ return ret
+}
+
+func appKvKey(key string) []byte {
+ ret := []byte(kvPrefixAppKv)
+ ret = append(ret, "-"...)
+ ret = append(ret, key...)
+ return ret
+}
+
+func creatableKey(cidx basics.CreatableIndex) []byte {
+ ret := []byte(kvPrefixCreatorIndex)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(cidx))...)
+ return ret
+}
+
+func onlineAccountKey(address basics.Address, round basics.Round) []byte {
+ ret := []byte(kvPrefixOnlineAccount)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(round))...)
+ return ret
+}
+
+func onlineAccountOnlyPartialKey(address basics.Address) []byte {
+ ret := []byte(kvPrefixOnlineAccount)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ ret = append(ret, "-"...)
+ return ret
+}
+
+// TODO: use basics.Round
+func onlineAccountBalanceKey(round uint64, normBalance uint64, address basics.Address) []byte {
+ ret := []byte(kvPrefixOnlineAccountBalance)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(round)...)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(normBalance)...)
+ ret = append(ret, "-"...)
+ ret = append(ret, address[:]...)
+ return ret
+}
+
+func onlineAccountBalanceOnlyPartialKey(round basics.Round) []byte {
+ ret := []byte(kvPrefixOnlineAccountBalance)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(round))...)
+ ret = append(ret, "-"...)
+ return ret
+}
+
+func roundKey() []byte {
+ ret := []byte(kvRoundKey)
+ return ret
+}
+
+func schemaVersionKey() []byte {
+ ret := []byte(kvSchemaVersionKey)
+ return ret
+}
+
+func totalsKey(catchpointStaging bool) []byte {
+ ret := []byte(kvTotalsKey)
+ ret = append(ret, "-"...)
+ if catchpointStaging {
+ ret = append(ret, "staging"...)
+ } else {
+ ret = append(ret, "live"...)
+ }
+ return ret
+}
+
+func txTailKey(rnd basics.Round) []byte {
+ ret := []byte(kvTxTail)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(rnd))...)
+ return ret
+}
+
+func onlineAccountRoundParamsKey(rnd basics.Round) []byte {
+ ret := []byte(kvOnlineAccountRoundParams)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(rnd))...)
+ return ret
+}
+
+func stateproofKey(rnd basics.Round) []byte {
+ ret := []byte(kvPrefixStateproof)
+ ret = append(ret, "-"...)
+ ret = append(ret, bigEndianUint64(uint64(rnd))...)
+ return ret
+}
diff --git a/ledger/store/trackerdb/generickv/stateproof_reader.go b/ledger/store/trackerdb/generickv/stateproof_reader.go
new file mode 100644
index 0000000000..0c3ed62b58
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/stateproof_reader.go
@@ -0,0 +1,104 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+type stateproofReader struct {
+ kvr KvRead
+}
+
+// MakeStateproofReader returns a trackerdb.SpVerificationCtxReader for a KV
+func MakeStateproofReader(kvr KvRead) trackerdb.SpVerificationCtxReader {
+ return &stateproofReader{kvr}
+}
+
+// LookupSPContext implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) LookupSPContext(stateProofLastAttestedRound basics.Round) (*ledgercore.StateProofVerificationContext, error) {
+ // SQL at the time of writing:
+ //
+ // SELECT
+ // verificationcontext
+ // FROM stateproofverification
+ // WHERE lastattestedround=?
+
+ value, closer, err := r.kvr.Get(stateproofKey(stateProofLastAttestedRound))
+ if err != nil {
+ return nil, err
+ }
+ defer closer.Close()
+
+ var vc ledgercore.StateProofVerificationContext
+ err = protocol.Decode(value, &vc)
+ if err != nil {
+ return nil, err
+ }
+
+ return &vc, nil
+}
+
+// GetAllSPContexts implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) GetAllSPContexts(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) {
+ // SQL at the time of writing:
+ //
+ // SELECT
+ // verificationContext
+ // FROM stateProofVerification
+ // ORDER BY lastattestedround
+
+ low := []byte(kvPrefixStateproof + "-")
+ high := []byte(kvPrefixStateproof + ".")
+ iter := r.kvr.NewIter(low, high, false)
+ defer iter.Close()
+
+ results := make([]ledgercore.StateProofVerificationContext, 0)
+
+ var value []byte
+ var err error
+ for iter.Next() {
+ // get value for current item in the iterator
+ value, err = iter.Value()
+ if err != nil {
+ return nil, err
+ }
+
+ // decode the value
+ vc := ledgercore.StateProofVerificationContext{}
+ err = protocol.Decode(value, &vc)
+ if err != nil {
+ return nil, err
+ }
+
+ // add the item to the results
+ results = append(results, vc)
+ }
+
+ return results, nil
+}
+
+// GetAllSPContextsFromCatchpointTbl implements trackerdb.SpVerificationCtxReader
+func (r *stateproofReader) GetAllSPContextsFromCatchpointTbl(ctx context.Context) ([]ledgercore.StateProofVerificationContext, error) {
+ // TODO: catchpoint
+ return nil, nil
+}
diff --git a/ledger/store/trackerdb/generickv/stateproof_writer.go b/ledger/store/trackerdb/generickv/stateproof_writer.go
new file mode 100644
index 0000000000..42565d9edc
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/stateproof_writer.go
@@ -0,0 +1,76 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+)
+
+type stateproofWriter struct {
+ kvw KvWrite
+}
+
+// MakeStateproofWriter returns a trackerdb.SpVerificationCtxWriter for a KV
+func MakeStateproofWriter(kvw KvWrite) trackerdb.SpVerificationCtxWriter {
+ return &stateproofWriter{kvw}
+}
+
+// StoreSPContexts implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) StoreSPContexts(ctx context.Context, verificationContext []*ledgercore.StateProofVerificationContext) error {
+ // SQL at the time of writing:
+ //
+ // INSERT INTO stateProofVerification
+ // (lastattestedround, verificationContext)
+ // VALUES
+ // (?, ?)
+
+ for i := range verificationContext {
+ // write stateproof entry
+ vc := verificationContext[i]
+ raw := protocol.Encode(vc)
+ err := w.kvw.Set(stateproofKey(vc.LastAttestedRound), raw)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// DeleteOldSPContexts implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) DeleteOldSPContexts(ctx context.Context, earliestLastAttestedRound basics.Round) error {
+ // SQL at the time of writing:
+ //
+ // DELETE FROM stateproofverification
+ // WHERE lastattestedround < ?
+
+ start := []byte(kvPrefixStateproof + "-")
+ end := stateproofKey(earliestLastAttestedRound)
+
+ return w.kvw.DeleteRange(start, end)
+}
+
+// StoreSPContextsToCatchpointTbl implements trackerdb.SpVerificationCtxWriter
+func (w *stateproofWriter) StoreSPContextsToCatchpointTbl(ctx context.Context, verificationContexts []ledgercore.StateProofVerificationContext) error {
+ // TODO: catchpoint
+ return nil
+}
diff --git a/ledger/store/trackerdb/generickv/writer.go b/ledger/store/trackerdb/generickv/writer.go
new file mode 100644
index 0000000000..c5d8dcfc0f
--- /dev/null
+++ b/ledger/store/trackerdb/generickv/writer.go
@@ -0,0 +1,91 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package generickv
+
+import (
+ "context"
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+ "testing"
+)
+
+type writer struct {
+ // TODO: the need for the store here is quite broken
+ // this is due to exposing RunMigrations and AccountsInit on the writers
+ // the internals of this methods completly ignore the "writer" and recreate a transaction
+ store trackerdb.Store
+ KvWrite
+ KvRead
+}
+
+// MakeWriter returns a trackerdb.Writer for a KV
+func MakeWriter(store trackerdb.Store, kvw KvWrite, kvr KvRead) trackerdb.Writer {
+ return &writer{store, kvw, kvr}
+}
+
+// MakeAccountsOptimizedWriter implements trackerdb.Writer
+func (w *writer) MakeAccountsOptimizedWriter(hasAccounts bool, hasResources bool, hasKvPairs bool, hasCreatables bool) (trackerdb.AccountsWriter, error) {
+ return MakeAccountsWriter(w, w), nil
+}
+
+// MakeAccountsWriter implements trackerdb.Writer
+func (w *writer) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) {
+ return MakeAccountsWriter(w, w), nil
+}
+
+// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer
+func (w *writer) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) {
+ return MakeOnlineAccountsWriter(w), nil
+}
+
+// MakeSpVerificationCtxWriter implements trackerdb.Writer
+func (w *writer) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter {
+ return MakeStateproofWriter(w)
+}
+
+// Testing implements trackerdb.Writer
+func (w *writer) Testing() trackerdb.WriterTestExt {
+ return &writerForTesting{w.store, w, w}
+}
+
+type writerForTesting struct {
+ trackerdb.Store
+ KvWrite
+ KvRead
+}
+
+// AccountsInitLightTest implements trackerdb.WriterTestExt
+func (w *writerForTesting) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
+ panic("unimplemented")
+}
+
+// AccountsInitTest implements trackerdb.WriterTestExt
+func (w *writerForTesting) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
+ return AccountsInitTest(tb, w, initAccounts, proto)
+}
+
+// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt
+func (w *writerForTesting) AccountsUpdateSchemaTest(ctx context.Context) (err error) {
+ panic("unimplemented")
+}
+
+// ModifyAcctBaseTest implements trackerdb.WriterTestExt
+func (w *writerForTesting) ModifyAcctBaseTest() error {
+ panic("unimplemented")
+}
diff --git a/ledger/store/trackerdb/interface.go b/ledger/store/trackerdb/interface.go
index 9e9fbb1a10..b532fa238a 100644
--- a/ledger/store/trackerdb/interface.go
+++ b/ledger/store/trackerdb/interface.go
@@ -18,6 +18,7 @@ package trackerdb
import (
"context"
+ "errors"
"github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/crypto"
@@ -26,6 +27,9 @@ import (
"github.com/algorand/go-algorand/ledger/ledgercore"
)
+// ErrNotFound is returned when a record is not found.
+var ErrNotFound = errors.New("trackerdb: not found")
+
// AccountRef is an opaque ref to an account in the db.
type AccountRef interface {
AccountRefMarker()
@@ -68,15 +72,16 @@ type AccountsWriter interface {
// AccountsWriterExt is the write interface used inside transactions and batch operations.
type AccountsWriterExt interface {
- AccountsReset(ctx context.Context) error
- ResetAccountHashes(ctx context.Context) (err error)
- TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error
+ CatchpointApplicationWriter
+
UpdateAccountsRound(rnd basics.Round) (err error)
- UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error)
AccountsPutTotals(totals ledgercore.AccountTotals, catchpointStaging bool) error
+
OnlineAccountsDelete(forgetBefore basics.Round) (err error)
AccountsPutOnlineRoundParams(onlineRoundParamsData []ledgercore.OnlineRoundParamsData, startRound basics.Round) error
AccountsPruneOnlineRoundParams(deleteBeforeRound basics.Round) error
+
+ TxtailNewRound(ctx context.Context, baseRound basics.Round, roundData [][]byte, forgetBeforeRound basics.Round) error
}
// AccountsReader is the "optimized" read interface for:
@@ -100,24 +105,38 @@ type AccountsReader interface {
// AccountsReaderExt is the read interface for:
// - accounts, resources, app kvs, creatables
type AccountsReaderExt interface {
+ CatchpointCreationReader
+
+ AccountsRound() (rnd basics.Round, err error)
AccountsTotals(ctx context.Context, catchpointStaging bool) (totals ledgercore.AccountTotals, err error)
- AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error)
- LookupAccountAddressFromAddressID(ctx context.Context, ref AccountRef) (address basics.Address, err error)
- LookupAccountDataByAddress(basics.Address) (ref AccountRef, data []byte, err error)
+
LookupAccountRowID(basics.Address) (ref AccountRef, err error)
LookupResourceDataByAddrID(accountRef AccountRef, aidx basics.CreatableIndex) (data []byte, err error)
- TotalResources(ctx context.Context) (total uint64, err error)
- TotalAccounts(ctx context.Context) (total uint64, err error)
- TotalKVs(ctx context.Context) (total uint64, err error)
- AccountsRound() (rnd basics.Round, err error)
LookupOnlineAccountDataByAddress(addr basics.Address) (ref OnlineAccountRef, data []byte, err error)
+
AccountsOnlineTop(rnd basics.Round, offset uint64, n uint64, proto config.ConsensusParams) (map[basics.Address]*ledgercore.OnlineAccount, error)
AccountsOnlineRoundParams() (onlineRoundParamsData []ledgercore.OnlineRoundParamsData, endRound basics.Round, err error)
ExpiredOnlineAccountsForRound(rnd, voteRnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (map[basics.Address]*ledgercore.OnlineAccountData, error)
OnlineAccountsAll(maxAccounts uint64) ([]PersistedOnlineAccountData, error)
+
LoadTxTail(ctx context.Context, dbRound basics.Round) (roundData []*TxTailRound, roundHash []crypto.Digest, baseRound basics.Round, err error)
+ // testing
+ Testing() AccountsReaderTestExt
+}
+
+type CatchpointApplicationWriter interface {
+ AccountsReset(ctx context.Context) error
+ ResetAccountHashes(ctx context.Context) (err error)
+ UpdateAccountsHashRound(ctx context.Context, hashRound basics.Round) (err error)
+}
+
+type CatchpointCreationReader interface {
+ AccountsHashRound(ctx context.Context) (hashrnd basics.Round, err error)
+ LookupAccountAddressFromAddressID(ctx context.Context, ref AccountRef) (address basics.Address, err error)
+ TotalResources(ctx context.Context) (total uint64, err error)
+ TotalAccounts(ctx context.Context) (total uint64, err error)
+ TotalKVs(ctx context.Context) (total uint64, err error)
LoadAllFullAccounts(ctx context.Context, balancesTable string, resourcesTable string, acctCb func(basics.Address, basics.AccountData)) (count int, err error)
- Testing() TestAccountsReaderExt
}
// AccountsReaderWriter is AccountsReader+AccountsWriter
@@ -146,52 +165,6 @@ type OnlineAccountsReader interface {
Close()
}
-// CatchpointWriter is the write interface for:
-// - catchpoints
-type CatchpointWriter interface {
- CreateCatchpointStagingHashesIndex(ctx context.Context) (err error)
-
- StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error)
-
- WriteCatchpointStateUint64(ctx context.Context, stateName CatchpointState, setValue uint64) (err error)
- WriteCatchpointStateString(ctx context.Context, stateName CatchpointState, setValue string) (err error)
-
- WriteCatchpointStagingBalances(ctx context.Context, bals []NormalizedAccountBalance) error
- WriteCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error
- WriteCatchpointStagingCreatable(ctx context.Context, bals []NormalizedAccountBalance) error
- WriteCatchpointStagingHashes(ctx context.Context, bals []NormalizedAccountBalance) error
-
- ApplyCatchpointStagingBalances(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error)
- ResetCatchpointStagingBalances(ctx context.Context, newCatchup bool) (err error)
-
- InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error
- DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error
- DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error
- InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *CatchpointFirstStageInfo) error
-
- DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error)
-}
-
-// CatchpointReader is the read interface for:
-// - catchpoints
-type CatchpointReader interface {
- GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error)
- GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error)
-
- ReadCatchpointStateUint64(ctx context.Context, stateName CatchpointState) (val uint64, err error)
- ReadCatchpointStateString(ctx context.Context, stateName CatchpointState) (val string, err error)
-
- SelectUnfinishedCatchpoints(ctx context.Context) ([]UnfinishedCatchpointRecord, error)
- SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (CatchpointFirstStageInfo, bool /*exists*/, error)
- SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error)
-}
-
-// CatchpointReaderWriter is CatchpointReader+CatchpointWriter
-type CatchpointReaderWriter interface {
- CatchpointReader
- CatchpointWriter
-}
-
// MerkleCommitter allows storing and loading merkletrie pages from a sqlite database.
type MerkleCommitter interface {
StorePage(page uint64, content []byte) error
diff --git a/ledger/store/trackerdb/msgp_gen.go b/ledger/store/trackerdb/msgp_gen.go
index e6fa865d65..41f41d0a89 100644
--- a/ledger/store/trackerdb/msgp_gen.go
+++ b/ledger/store/trackerdb/msgp_gen.go
@@ -35,14 +35,6 @@ import (
// |-----> (*) Msgsize
// |-----> (*) MsgIsZero
//
-// CatchpointFirstStageInfo
-// |-----> (*) MarshalMsg
-// |-----> (*) CanMarshalMsg
-// |-----> (*) UnmarshalMsg
-// |-----> (*) CanUnmarshalMsg
-// |-----> (*) Msgsize
-// |-----> (*) MsgIsZero
-//
// ResourceFlags
// |-----> MarshalMsg
// |-----> CanMarshalMsg
@@ -1130,250 +1122,6 @@ func (z *BaseVotingData) MsgIsZero() bool {
return ((*z).VoteID.MsgIsZero()) && ((*z).SelectionID.MsgIsZero()) && ((*z).VoteFirstValid.MsgIsZero()) && ((*z).VoteLastValid.MsgIsZero()) && ((*z).VoteKeyDilution == 0) && ((*z).StateProofID.MsgIsZero())
}
-// MarshalMsg implements msgp.Marshaler
-func (z *CatchpointFirstStageInfo) MarshalMsg(b []byte) (o []byte) {
- o = msgp.Require(b, z.Msgsize())
- // omitempty: check for empty values
- zb0001Len := uint32(7)
- var zb0001Mask uint8 /* 8 bits */
- if (*z).Totals.MsgIsZero() {
- zb0001Len--
- zb0001Mask |= 0x2
- }
- if (*z).TotalAccounts == 0 {
- zb0001Len--
- zb0001Mask |= 0x4
- }
- if (*z).BiggestChunkLen == 0 {
- zb0001Len--
- zb0001Mask |= 0x8
- }
- if (*z).TotalChunks == 0 {
- zb0001Len--
- zb0001Mask |= 0x10
- }
- if (*z).TotalKVs == 0 {
- zb0001Len--
- zb0001Mask |= 0x20
- }
- if (*z).StateProofVerificationHash.MsgIsZero() {
- zb0001Len--
- zb0001Mask |= 0x40
- }
- if (*z).TrieBalancesHash.MsgIsZero() {
- zb0001Len--
- zb0001Mask |= 0x80
- }
- // variable map header, size zb0001Len
- o = append(o, 0x80|uint8(zb0001Len))
- if zb0001Len != 0 {
- if (zb0001Mask & 0x2) == 0 { // if not empty
- // string "accountTotals"
- o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
- o = (*z).Totals.MarshalMsg(o)
- }
- if (zb0001Mask & 0x4) == 0 { // if not empty
- // string "accountsCount"
- o = append(o, 0xad, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
- o = msgp.AppendUint64(o, (*z).TotalAccounts)
- }
- if (zb0001Mask & 0x8) == 0 { // if not empty
- // string "biggestChunk"
- o = append(o, 0xac, 0x62, 0x69, 0x67, 0x67, 0x65, 0x73, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b)
- o = msgp.AppendUint64(o, (*z).BiggestChunkLen)
- }
- if (zb0001Mask & 0x10) == 0 { // if not empty
- // string "chunksCount"
- o = append(o, 0xab, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
- o = msgp.AppendUint64(o, (*z).TotalChunks)
- }
- if (zb0001Mask & 0x20) == 0 { // if not empty
- // string "kvsCount"
- o = append(o, 0xa8, 0x6b, 0x76, 0x73, 0x43, 0x6f, 0x75, 0x6e, 0x74)
- o = msgp.AppendUint64(o, (*z).TotalKVs)
- }
- if (zb0001Mask & 0x40) == 0 { // if not empty
- // string "spVerificationHash"
- o = append(o, 0xb2, 0x73, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68)
- o = (*z).StateProofVerificationHash.MarshalMsg(o)
- }
- if (zb0001Mask & 0x80) == 0 { // if not empty
- // string "trieBalancesHash"
- o = append(o, 0xb0, 0x74, 0x72, 0x69, 0x65, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x48, 0x61, 0x73, 0x68)
- o = (*z).TrieBalancesHash.MarshalMsg(o)
- }
- }
- return
-}
-
-func (_ *CatchpointFirstStageInfo) CanMarshalMsg(z interface{}) bool {
- _, ok := (z).(*CatchpointFirstStageInfo)
- return ok
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *CatchpointFirstStageInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
- var field []byte
- _ = field
- var zb0001 int
- var zb0002 bool
- zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
- if _, ok := err.(msgp.TypeError); ok {
- zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
- if err != nil {
- err = msgp.WrapError(err)
- return
- }
- if zb0001 > 0 {
- zb0001--
- bts, err = (*z).Totals.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "Totals")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "TrieBalancesHash")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "TotalAccounts")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "TotalKVs")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "TotalChunks")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "BiggestChunkLen")
- return
- }
- }
- if zb0001 > 0 {
- zb0001--
- bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array", "StateProofVerificationHash")
- return
- }
- }
- if zb0001 > 0 {
- err = msgp.ErrTooManyArrayFields(zb0001)
- if err != nil {
- err = msgp.WrapError(err, "struct-from-array")
- return
- }
- }
- } else {
- if err != nil {
- err = msgp.WrapError(err)
- return
- }
- if zb0002 {
- (*z) = CatchpointFirstStageInfo{}
- }
- for zb0001 > 0 {
- zb0001--
- field, bts, err = msgp.ReadMapKeyZC(bts)
- if err != nil {
- err = msgp.WrapError(err)
- return
- }
- switch string(field) {
- case "accountTotals":
- bts, err = (*z).Totals.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "Totals")
- return
- }
- case "trieBalancesHash":
- bts, err = (*z).TrieBalancesHash.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "TrieBalancesHash")
- return
- }
- case "accountsCount":
- (*z).TotalAccounts, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "TotalAccounts")
- return
- }
- case "kvsCount":
- (*z).TotalKVs, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "TotalKVs")
- return
- }
- case "chunksCount":
- (*z).TotalChunks, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "TotalChunks")
- return
- }
- case "biggestChunk":
- (*z).BiggestChunkLen, bts, err = msgp.ReadUint64Bytes(bts)
- if err != nil {
- err = msgp.WrapError(err, "BiggestChunkLen")
- return
- }
- case "spVerificationHash":
- bts, err = (*z).StateProofVerificationHash.UnmarshalMsg(bts)
- if err != nil {
- err = msgp.WrapError(err, "StateProofVerificationHash")
- return
- }
- default:
- err = msgp.ErrNoField(string(field))
- if err != nil {
- err = msgp.WrapError(err)
- return
- }
- }
- }
- }
- o = bts
- return
-}
-
-func (_ *CatchpointFirstStageInfo) CanUnmarshalMsg(z interface{}) bool {
- _, ok := (z).(*CatchpointFirstStageInfo)
- return ok
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *CatchpointFirstStageInfo) Msgsize() (s int) {
- s = 1 + 14 + (*z).Totals.Msgsize() + 17 + (*z).TrieBalancesHash.Msgsize() + 14 + msgp.Uint64Size + 9 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size + 19 + (*z).StateProofVerificationHash.Msgsize()
- return
-}
-
-// MsgIsZero returns whether this is a zero value
-func (z *CatchpointFirstStageInfo) MsgIsZero() bool {
- return ((*z).Totals.MsgIsZero()) && ((*z).TrieBalancesHash.MsgIsZero()) && ((*z).TotalAccounts == 0) && ((*z).TotalKVs == 0) && ((*z).TotalChunks == 0) && ((*z).BiggestChunkLen == 0) && ((*z).StateProofVerificationHash.MsgIsZero())
-}
-
// MarshalMsg implements msgp.Marshaler
func (z ResourceFlags) MarshalMsg(b []byte) (o []byte) {
o = msgp.Require(b, z.Msgsize())
diff --git a/ledger/store/trackerdb/msgp_gen_test.go b/ledger/store/trackerdb/msgp_gen_test.go
index be8a232c32..1d1be174f5 100644
--- a/ledger/store/trackerdb/msgp_gen_test.go
+++ b/ledger/store/trackerdb/msgp_gen_test.go
@@ -194,66 +194,6 @@ func BenchmarkUnmarshalBaseVotingData(b *testing.B) {
}
}
-func TestMarshalUnmarshalCatchpointFirstStageInfo(t *testing.T) {
- partitiontest.PartitionTest(t)
- v := CatchpointFirstStageInfo{}
- bts := v.MarshalMsg(nil)
- left, err := v.UnmarshalMsg(bts)
- if err != nil {
- t.Fatal(err)
- }
- if len(left) > 0 {
- t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
- }
-
- left, err = msgp.Skip(bts)
- if err != nil {
- t.Fatal(err)
- }
- if len(left) > 0 {
- t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
- }
-}
-
-func TestRandomizedEncodingCatchpointFirstStageInfo(t *testing.T) {
- protocol.RunEncodingTest(t, &CatchpointFirstStageInfo{})
-}
-
-func BenchmarkMarshalMsgCatchpointFirstStageInfo(b *testing.B) {
- v := CatchpointFirstStageInfo{}
- b.ReportAllocs()
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- v.MarshalMsg(nil)
- }
-}
-
-func BenchmarkAppendMsgCatchpointFirstStageInfo(b *testing.B) {
- v := CatchpointFirstStageInfo{}
- bts := make([]byte, 0, v.Msgsize())
- bts = v.MarshalMsg(bts[0:0])
- b.SetBytes(int64(len(bts)))
- b.ReportAllocs()
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- bts = v.MarshalMsg(bts[0:0])
- }
-}
-
-func BenchmarkUnmarshalCatchpointFirstStageInfo(b *testing.B) {
- v := CatchpointFirstStageInfo{}
- bts := v.MarshalMsg(nil)
- b.ReportAllocs()
- b.SetBytes(int64(len(bts)))
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- _, err := v.UnmarshalMsg(bts)
- if err != nil {
- b.Fatal(err)
- }
- }
-}
-
func TestMarshalUnmarshalResourcesData(t *testing.T) {
partitiontest.PartitionTest(t)
v := ResourcesData{}
diff --git a/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go b/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go
new file mode 100644
index 0000000000..619cfb3e5f
--- /dev/null
+++ b/ledger/store/trackerdb/pebbledbdriver/pebbledriver.go
@@ -0,0 +1,494 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package pebbledbdriver
+
+import (
+ "context"
+ "io"
+ "runtime"
+ "time"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/generickv"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/util/db"
+ "github.com/cockroachdb/pebble"
+ "github.com/cockroachdb/pebble/bloom"
+ "github.com/cockroachdb/pebble/vfs"
+)
+
+const (
+ // minCache is the minimum amount of memory in megabytes to allocate to pebble
+ // read and write caching, split half and half.
+ minCache = 16
+
+ // minHandles is the minimum number of files handles to allocate to the open
+ // database files.
+ minHandles = 16
+)
+
+type trackerStore struct {
+ kvs kvstore
+ proto config.ConsensusParams
+ // use the generickv implementations
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+// Open opens a Pebble db database
+func Open(dbdir string, inMem bool, proto config.ConsensusParams, log logging.Logger) (trackerdb.Store, error) {
+ cache := 100
+ handles := 64
+
+ // Ensure we have some minimal caching and file guarantees
+ if cache < minCache {
+ cache = minCache
+ }
+ if handles < minHandles {
+ handles = minHandles
+ }
+
+ // The max memtable size is limited by the uint32 offsets stored in
+ // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
+ // Taken from https://github.com/cockroachdb/pebble/blob/master/open.go#L38
+ maxMemTableSize := 4<<30 - 1 // Capped by 4 GB
+
+ memTableLimit := 2
+ memTableSize := cache * 1024 * 1024 / 2 / memTableLimit
+ if memTableSize > maxMemTableSize {
+ memTableSize = maxMemTableSize
+ }
+
+ // configure pebbledb
+ opts := &pebble.Options{
+ // logging
+ Logger: log,
+
+ // Pebble has a single combined cache area and the write
+ // buffers are taken from this too. Assign all available
+ // memory allowance for cache.
+ Cache: pebble.NewCache(int64(cache * 1024 * 1024)),
+ MaxOpenFiles: handles,
+
+ // The size of memory table(as well as the write buffer).
+ // Note, there may have more than two memory tables in the system.
+ MemTableSize: memTableSize,
+
+ // MemTableStopWritesThreshold places a hard limit on the size
+ // of the existent MemTables(including the frozen one).
+ // Note, this must be the number of tables not the size of all memtables
+ // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742
+ // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903.
+ MemTableStopWritesThreshold: memTableLimit,
+
+ // The default compaction concurrency(1 thread),
+ // Here use all available CPUs for faster compaction.
+ MaxConcurrentCompactions: func() int { return runtime.NumCPU() },
+
+ // Per-level options. Options for at least one level must be specified. The
+ // options for the last level are used for all subsequent levels.
+ Levels: []pebble.LevelOptions{
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
+ },
+ }
+ // Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130
+ // for more details.
+ opts.Experimental.ReadSamplingMultiplier = -1
+
+ if inMem {
+ opts.FS = vfs.NewMem()
+ }
+ db, err := pebble.Open(dbdir+".pebbledb", opts)
+ if err != nil {
+ return nil, err
+ }
+ // no fsync
+ wo := &pebble.WriteOptions{Sync: false}
+ kvs := kvstore{Pdb: db, wo: wo}
+ var store trackerdb.Store
+ store = &trackerStore{
+ kvs,
+ proto,
+ generickv.MakeReader(&kvs, proto),
+ generickv.MakeWriter(store, &kvs, &kvs),
+ generickv.MakeCatchpoint(),
+ }
+ return store, nil
+}
+
+// IsSharedCacheConnection implements trackerdb.Store
+func (s *trackerStore) IsSharedCacheConnection() bool {
+ return false
+}
+
+// SetSynchronousMode implements trackerdb.Store
+func (s *trackerStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) {
+ // TODO
+ return nil
+}
+
+// RunMigrations implements trackerdb.Store
+func (s *trackerStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ // create a anonym struct that impls the interface for the migration runner
+ db := struct {
+ *trackerStore
+ *kvstore
+ }{s, &s.kvs}
+ return generickv.RunMigrations(ctx, db, params, targetVersion)
+}
+
+// Batch implements trackerdb.Store
+func (s *trackerStore) Batch(fn trackerdb.BatchFn) (err error) {
+ return s.BatchContext(context.Background(), fn)
+}
+
+// BatchContext implements trackerdb.Store
+func (s *trackerStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) {
+ handle, err := s.BeginBatch(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the batch
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ // commit the batch
+ err = handle.Commit()
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginBatch implements trackerdb.Store
+func (s *trackerStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) {
+ scope := batchScope{store: s, wb: s.kvs.Pdb.NewBatch(), wo: s.kvs.wo, db: s.kvs.Pdb}
+
+ return &struct {
+ batchScope
+ trackerdb.Writer
+ }{scope, generickv.MakeWriter(s, &scope, &s.kvs)}, nil
+}
+
+// Snapshot implements trackerdb.Store
+func (s *trackerStore) Snapshot(fn trackerdb.SnapshotFn) (err error) {
+ return s.SnapshotContext(context.Background(), fn)
+}
+
+// SnapshotContext implements trackerdb.Store
+func (s *trackerStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) {
+ handle, err := s.BeginSnapshot(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the snapshot
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginSnapshot implements trackerdb.Store
+func (s *trackerStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) {
+ scope := snapshotScope{db: s.kvs.Pdb, snap: s.kvs.Pdb.NewSnapshot()}
+ return &struct {
+ snapshotScope
+ trackerdb.Reader
+ }{scope, generickv.MakeReader(&scope, s.proto)}, nil
+}
+
+// Transaction implements trackerdb.Store
+func (s *trackerStore) Transaction(fn trackerdb.TransactionFn) (err error) {
+ return s.TransactionContext(context.Background(), fn)
+}
+
+// TransactionContext implements trackerdb.Store
+func (s *trackerStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) {
+ handle, err := s.BeginTransaction(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the transaction
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ // commit the transaction
+ err = handle.Commit()
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginTransaction implements trackerdb.Store
+func (s *trackerStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) {
+ scope := transactionScope{
+ store: s,
+ db: s.kvs.Pdb,
+ wo: s.kvs.wo,
+ snap: s.kvs.Pdb.NewSnapshot(),
+ wb: s.kvs.Pdb.NewBatch(),
+ }
+
+ return &struct {
+ transactionScope
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+ }{scope, generickv.MakeReader(&scope, s.proto), generickv.MakeWriter(s, &scope, &scope), generickv.MakeCatchpoint()}, nil
+}
+
+// Vacuum implements trackerdb.Store
+func (s *trackerStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) {
+ // TODO
+ return db.VacuumStats{}, nil
+}
+
+// ResetToV6Test implements trackerdb.Store
+func (s *trackerStore) ResetToV6Test(ctx context.Context) error {
+ // TODO
+ return nil
+}
+
+// Close implements trackerdb.Store
+func (s *trackerStore) Close() {
+ s.kvs.Pdb.Close()
+}
+
+//
+// generic impls
+//
+
+func mapPebbleErrors(err error) error {
+ switch err {
+ case pebble.ErrNotFound:
+ return trackerdb.ErrNotFound
+ default:
+ return err
+ }
+}
+
+type kvstore struct {
+ Pdb *pebble.DB
+ wo *pebble.WriteOptions
+}
+
+func (s *kvstore) Set(key, value []byte) error {
+ return s.Pdb.Set(key, value, s.wo)
+}
+
+func (s *kvstore) Get(key []byte) (value []byte, closer io.Closer, err error) {
+ value, closer, err = s.Pdb.Get(key)
+ err = mapPebbleErrors(err)
+ return
+}
+
+func (s *kvstore) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ opts := pebble.IterOptions{LowerBound: low, UpperBound: high}
+ return newIter(s.Pdb.NewIter(&opts), reverse)
+}
+
+func (s *kvstore) Delete(key []byte) error {
+ return s.Pdb.Delete(key, s.wo)
+}
+
+func (s *kvstore) DeleteRange(start, end []byte) error {
+ return s.Pdb.DeleteRange(start, end, s.wo)
+}
+
+type batchScope struct {
+ // Hack: we should tray to impl without this field
+ store *trackerStore
+ db *pebble.DB
+ wo *pebble.WriteOptions
+ wb *pebble.Batch
+}
+
+func (bs batchScope) Set(key, value []byte) error {
+ return bs.wb.Set(key, value, bs.wo)
+}
+
+func (bs batchScope) Delete(key []byte) error {
+ return bs.wb.Delete(key, bs.wo)
+}
+
+func (bs batchScope) DeleteRange(start, end []byte) error {
+ return bs.wb.DeleteRange(start, end, bs.wo)
+}
+
+func (bs batchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ // noop
+ return time.Now(), nil
+}
+
+func (bs batchScope) Commit() error {
+ return bs.wb.Commit(bs.wo)
+}
+
+func (bs batchScope) Close() error {
+ return bs.wb.Close()
+}
+
+type snapshotScope struct {
+ db *pebble.DB
+ snap *pebble.Snapshot
+}
+
+func (ss snapshotScope) Get(key []byte) (value []byte, closer io.Closer, err error) {
+ value, closer, err = ss.snap.Get(key)
+ err = mapPebbleErrors(err)
+ return
+}
+
+func (ss snapshotScope) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ opts := pebble.IterOptions{LowerBound: low, UpperBound: high}
+ return newIter(ss.snap.NewIter(&opts), reverse)
+}
+
+func (ss snapshotScope) Close() error {
+ return ss.snap.Close()
+}
+
+type transactionScope struct {
+ // Hack: we should tray to impl without this field
+ store *trackerStore
+ db *pebble.DB
+ wo *pebble.WriteOptions
+ snap *pebble.Snapshot
+ wb *pebble.Batch
+}
+
+func (txs transactionScope) Set(key, value []byte) error {
+ return txs.wb.Set(key, value, txs.wo)
+}
+
+func (txs transactionScope) Get(key []byte) (value []byte, closer io.Closer, err error) {
+ value, closer, err = txs.snap.Get(key)
+ err = mapPebbleErrors(err)
+ return
+}
+
+func (txs transactionScope) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ opts := pebble.IterOptions{LowerBound: low, UpperBound: high}
+ return newIter(txs.snap.NewIter(&opts), reverse)
+}
+
+func (txs transactionScope) Delete(key []byte) error {
+ return txs.wb.Delete(key, txs.wo)
+}
+
+func (txs transactionScope) DeleteRange(start, end []byte) error {
+ return txs.wb.DeleteRange(start, end, txs.wo)
+}
+
+func (txs *transactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ // create a anonym struct that impls the interface for the migration runner
+ db := struct {
+ *trackerStore
+ *kvstore
+ }{txs.store, &txs.store.kvs}
+ return generickv.RunMigrations(ctx, db, params, targetVersion)
+}
+
+func (txs transactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ // noop
+ return time.Now(), nil
+}
+
+func (txs transactionScope) Commit() error {
+ return txs.wb.Commit(txs.wo)
+}
+
+func (txs transactionScope) Close() error {
+ txs.snap.Close() // ignore error
+ return txs.wb.Close()
+}
+
+type pebbleIter struct {
+ iter *pebble.Iterator
+ reverse bool
+ firstCall bool
+}
+
+func newIter(iter *pebble.Iterator, reverse bool) *pebbleIter {
+ return &pebbleIter{iter, reverse, true}
+}
+
+func (i *pebbleIter) Next() bool {
+ if i.firstCall {
+ i.firstCall = false
+ if i.reverse {
+ return i.iter.Last()
+ }
+ return i.iter.First()
+ }
+ if i.reverse {
+ return i.iter.Prev()
+ }
+ return i.iter.Next()
+}
+func (i *pebbleIter) Valid() bool { return i.iter.Valid() }
+func (i *pebbleIter) Close() { i.iter.Close() }
+
+func (i *pebbleIter) Key() []byte {
+ k := i.iter.Key()
+ ret := make([]byte, len(k))
+ copy(ret, k)
+ return ret
+}
+
+func (i *pebbleIter) Value() ([]byte, error) {
+ v := i.iter.Value()
+ ret := make([]byte, len(v))
+ copy(ret, v)
+ return ret, nil
+}
+
+// KeySlice is a zero copy slice only valid until iter.Next() or iter.Close() is called.
+func (i *pebbleIter) KeySlice() generickv.Slice { return pebbleSlice(i.iter.Key()) }
+
+// ValueSlice is a zero copy slice only valid until iter.Next() or iter.Close() is called.
+func (i *pebbleIter) ValueSlice() (generickv.Slice, error) { return pebbleSlice(i.iter.Value()), nil }
+
+type pebbleSlice []byte
+
+func (s pebbleSlice) Data() []byte { return s }
+func (s pebbleSlice) Free() {}
+func (s pebbleSlice) Size() int { return len(s) }
+func (s pebbleSlice) Exists() bool { return s != nil }
diff --git a/ledger/store/trackerdb/pebbledbdriver/testing.go b/ledger/store/trackerdb/pebbledbdriver/testing.go
new file mode 100644
index 0000000000..2724eb01fb
--- /dev/null
+++ b/ledger/store/trackerdb/pebbledbdriver/testing.go
@@ -0,0 +1,41 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package pebbledbdriver
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/stretchr/testify/require"
+)
+
+// OpenForTesting opens a sqlite db file for testing purposes.
+func OpenForTesting(t testing.TB, inMemory bool) trackerdb.Store {
+ proto := config.Consensus[protocol.ConsensusCurrentVersion]
+
+ // create a tmp dir for the db, the testing runtime will clean it up automatically
+ dir := fmt.Sprintf("%s/db", t.TempDir())
+
+ db, err := Open(dir, inMemory, proto, logging.TestingLog(t))
+ require.NoErrorf(t, err, "Dir : %s\nInMemory: %v", dir, inMemory)
+
+ return db
+}
diff --git a/ledger/store/trackerdb/sqlitedriver/accountsV2.go b/ledger/store/trackerdb/sqlitedriver/accountsV2.go
index a551bc4975..51e4da54f8 100644
--- a/ledger/store/trackerdb/sqlitedriver/accountsV2.go
+++ b/ledger/store/trackerdb/sqlitedriver/accountsV2.go
@@ -56,8 +56,13 @@ func NewAccountsSQLReaderWriter(e db.Executable) *accountsV2ReaderWriter {
}
}
+// NewAccountsSQLReader creates an SQL reader+writer
+func NewAccountsSQLReader(q db.Queryable) *accountsV2Reader {
+ return &accountsV2Reader{q: q, preparedStatements: make(map[string]*sql.Stmt)}
+}
+
// Testing returns this reader, exposed as an interface with test functions
-func (r *accountsV2Reader) Testing() trackerdb.TestAccountsReaderExt {
+func (r *accountsV2Reader) Testing() trackerdb.AccountsReaderTestExt {
return r
}
@@ -433,21 +438,6 @@ func (r *accountsV2Reader) LookupAccountAddressFromAddressID(ctx context.Context
return
}
-func (r *accountsV2Reader) LookupAccountDataByAddress(addr basics.Address) (ref trackerdb.AccountRef, data []byte, err error) {
- // optimize this query for repeated usage
- selectStmt, err := r.getOrPrepare("SELECT rowid, data FROM accountbase WHERE address=?")
- if err != nil {
- return
- }
-
- var rowid int64
- err = selectStmt.QueryRow(addr[:]).Scan(&rowid, &data)
- if err != nil {
- return
- }
- return sqlRowRef{rowid}, data, err
-}
-
// LookupOnlineAccountDataByAddress looks up online account data by address.
func (r *accountsV2Reader) LookupOnlineAccountDataByAddress(addr basics.Address) (ref trackerdb.OnlineAccountRef, data []byte, err error) {
// optimize this query for repeated usage
@@ -458,7 +448,10 @@ func (r *accountsV2Reader) LookupOnlineAccountDataByAddress(addr basics.Address)
var rowid int64
err = selectStmt.QueryRow(addr[:]).Scan(&rowid, &data)
- if err != nil {
+ if err == sql.ErrNoRows {
+ err = trackerdb.ErrNotFound
+ return
+ } else if err != nil {
return
}
return sqlRowRef{rowid}, data, err
@@ -474,7 +467,10 @@ func (r *accountsV2Reader) LookupAccountRowID(addr basics.Address) (ref trackerd
var rowid int64
err = addrRowidStmt.QueryRow(addr[:]).Scan(&rowid)
- if err != nil {
+ if err == sql.ErrNoRows {
+ err = trackerdb.ErrNotFound
+ return
+ } else if err != nil {
return
}
return sqlRowRef{rowid}, err
@@ -483,7 +479,7 @@ func (r *accountsV2Reader) LookupAccountRowID(addr basics.Address) (ref trackerd
// LookupResourceDataByAddrID looks up the resource data by account rowid + resource aidx.
func (r *accountsV2Reader) LookupResourceDataByAddrID(accountRef trackerdb.AccountRef, aidx basics.CreatableIndex) (data []byte, err error) {
if accountRef == nil {
- return data, sql.ErrNoRows
+ return data, trackerdb.ErrNotFound
}
addrid := accountRef.(sqlRowRef).rowid
// optimize this query for repeated usage
@@ -493,7 +489,10 @@ func (r *accountsV2Reader) LookupResourceDataByAddrID(accountRef trackerdb.Accou
}
err = selectStmt.QueryRow(addrid, aidx).Scan(&data)
- if err != nil {
+ if err == sql.ErrNoRows {
+ err = trackerdb.ErrNotFound
+ return
+ } else if err != nil {
return
}
return data, err
diff --git a/ledger/store/trackerdb/sqlitedriver/catchpoint.go b/ledger/store/trackerdb/sqlitedriver/catchpoint.go
index 388749858d..1a4009e0f0 100644
--- a/ledger/store/trackerdb/sqlitedriver/catchpoint.go
+++ b/ledger/store/trackerdb/sqlitedriver/catchpoint.go
@@ -21,290 +21,127 @@ import (
"database/sql"
"errors"
"fmt"
+ "sync"
"time"
- "github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
- "github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/util/db"
"github.com/mattn/go-sqlite3"
)
-type catchpointReader struct {
- q db.Queryable
-}
-
type catchpointWriter struct {
- e db.Executable
-}
-
-type catchpointReaderWriter struct {
- catchpointReader
- catchpointWriter
-}
-
-// NewCatchpointSQLReaderWriter creates a Catchpoint SQL reader+writer
-func NewCatchpointSQLReaderWriter(e db.Executable) *catchpointReaderWriter {
- return &catchpointReaderWriter{
- catchpointReader{q: e},
- catchpointWriter{e: e},
- }
-}
-
-func (cr *catchpointReader) GetCatchpoint(ctx context.Context, round basics.Round) (fileName string, catchpoint string, fileSize int64, err error) {
- err = cr.q.QueryRowContext(ctx, "SELECT filename, catchpoint, filesize FROM storedcatchpoints WHERE round=?", int64(round)).Scan(&fileName, &catchpoint, &fileSize)
- return
-}
-
-func (cr *catchpointReader) GetOldestCatchpointFiles(ctx context.Context, fileCount int, filesToKeep int) (fileNames map[basics.Round]string, err error) {
- err = db.Retry(func() (err error) {
- query := "SELECT round, filename FROM storedcatchpoints WHERE pinned = 0 and round <= COALESCE((SELECT round FROM storedcatchpoints WHERE pinned = 0 ORDER BY round DESC LIMIT ?, 1),0) ORDER BY round ASC LIMIT ?"
- rows, err := cr.q.QueryContext(ctx, query, filesToKeep, fileCount)
- if err != nil {
- return err
- }
- defer rows.Close()
-
- fileNames = make(map[basics.Round]string)
- for rows.Next() {
- var fileName string
- var round basics.Round
- err = rows.Scan(&round, &fileName)
- if err != nil {
- return err
- }
- fileNames[round] = fileName
- }
-
- return rows.Err()
- })
- if err != nil {
- fileNames = nil
- }
- return
+ e db.Executable
+ isShared bool
}
-func (cr *catchpointReader) ReadCatchpointStateUint64(ctx context.Context, stateName trackerdb.CatchpointState) (val uint64, err error) {
- err = db.Retry(func() (err error) {
- query := "SELECT intval FROM catchpointstate WHERE id=?"
- var v sql.NullInt64
- err = cr.q.QueryRowContext(ctx, query, stateName).Scan(&v)
- if err == sql.ErrNoRows {
- return nil
- }
- if err != nil {
- return err
- }
- if v.Valid {
- val = uint64(v.Int64)
- }
- return nil
- })
- return val, err
+// MakeCatchpointApplier creates a Catchpoint SQL reader+writer
+func MakeCatchpointApplier(e db.Executable, isShared bool) trackerdb.CatchpointApply {
+ return &catchpointWriter{e, isShared}
}
-func (cr *catchpointReader) ReadCatchpointStateString(ctx context.Context, stateName trackerdb.CatchpointState) (val string, err error) {
- err = db.Retry(func() (err error) {
- query := "SELECT strval FROM catchpointstate WHERE id=?"
- var v sql.NullString
- err = cr.q.QueryRowContext(ctx, query, stateName).Scan(&v)
- if err == sql.ErrNoRows {
- return nil
- }
- if err != nil {
- return err
- }
-
- if v.Valid {
- val = v.String
- }
- return nil
- })
- return val, err
-}
-
-func (cr *catchpointReader) SelectUnfinishedCatchpoints(ctx context.Context) ([]trackerdb.UnfinishedCatchpointRecord, error) {
- var res []trackerdb.UnfinishedCatchpointRecord
-
- f := func() error {
- query := "SELECT round, blockhash FROM unfinishedcatchpoints ORDER BY round"
- rows, err := cr.q.QueryContext(ctx, query)
- if err != nil {
- return err
- }
-
- // Clear `res` in case this function is repeated.
- res = res[:0]
- for rows.Next() {
- var record trackerdb.UnfinishedCatchpointRecord
- var blockHash []byte
- err = rows.Scan(&record.Round, &blockHash)
- if err != nil {
- return err
+// Write implements trackerdb.CatchpointApply
+func (c *catchpointWriter) Write(ctx context.Context, payload trackerdb.CatchpointPayload) (trackerdb.CatchpointReport, error) {
+ wg := sync.WaitGroup{}
+
+ var report trackerdb.CatchpointReport
+
+ var errBalances error
+ var errCreatables error
+ var errHashes error
+ var errKVs error
+
+ // start the balances writer
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ start := time.Now()
+ errBalances = c.writeCatchpointStagingBalances(ctx, payload.Accounts)
+ report.BalancesWriteDuration = time.Since(start)
+ }()
+
+ // on a in-memory database, wait for the writer to finish before starting the new writer
+ if c.isShared {
+ wg.Wait()
+ }
+
+ // starts the creatables writer
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ hasCreatables := false
+ for _, accBal := range payload.Accounts {
+ for _, res := range accBal.Resources {
+ if res.IsOwning() {
+ hasCreatables = true
+ break
+ }
}
- copy(record.BlockHash[:], blockHash)
- res = append(res, record)
}
-
- return nil
- }
- err := db.Retry(f)
- if err != nil {
- return nil, err
- }
-
- return res, nil
-}
-
-func (cr *catchpointReader) SelectCatchpointFirstStageInfo(ctx context.Context, round basics.Round) (trackerdb.CatchpointFirstStageInfo, bool /*exists*/, error) {
- var data []byte
- f := func() error {
- query := "SELECT info FROM catchpointfirststageinfo WHERE round=?"
- err := cr.q.QueryRowContext(ctx, query, round).Scan(&data)
- if err == sql.ErrNoRows {
- data = nil
- return nil
+ if hasCreatables {
+ start := time.Now()
+ errCreatables = c.writeCatchpointStagingCreatable(ctx, payload.Accounts)
+ report.CreatablesWriteDuration = time.Since(start)
}
- return err
- }
- err := db.Retry(f)
- if err != nil {
- return trackerdb.CatchpointFirstStageInfo{}, false, err
- }
-
- if data == nil {
- return trackerdb.CatchpointFirstStageInfo{}, false, nil
- }
-
- var res trackerdb.CatchpointFirstStageInfo
- err = protocol.Decode(data, &res)
- if err != nil {
- return trackerdb.CatchpointFirstStageInfo{}, false, err
- }
-
- return res, true, nil
-}
-
-func (cr *catchpointReader) SelectOldCatchpointFirstStageInfoRounds(ctx context.Context, maxRound basics.Round) ([]basics.Round, error) {
- var res []basics.Round
-
- f := func() error {
- query := "SELECT round FROM catchpointfirststageinfo WHERE round <= ?"
- rows, err := cr.q.QueryContext(ctx, query, maxRound)
- if err != nil {
- return err
+ }()
+
+ // on a in-memory database, wait for the writer to finish before starting the new writer
+ if c.isShared {
+ wg.Wait()
+ }
+
+ // start the accounts pending hashes writer
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ start := time.Now()
+ errHashes = c.writeCatchpointStagingHashes(ctx, payload.Accounts)
+ report.HashesWriteDuration = time.Since(start)
+ }()
+
+ // on a in-memory database, wait for the writer to finish before starting the new writer
+ if c.isShared {
+ wg.Wait()
+ }
+
+ // start the kv store writer
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+
+ start := time.Now()
+ keys := make([][]byte, len(payload.KVRecords))
+ values := make([][]byte, len(payload.KVRecords))
+ hashes := make([][]byte, len(payload.KVRecords))
+ for i := 0; i < len(payload.KVRecords); i++ {
+ keys[i] = payload.KVRecords[i].Key
+ values[i] = payload.KVRecords[i].Value
+ hashes[i] = trackerdb.KvHashBuilderV6(string(keys[i]), values[i])
}
+ errKVs = c.writeCatchpointStagingKVs(ctx, keys, values, hashes)
+ report.KVWriteDuration = time.Since(start)
+ }()
- // Clear `res` in case this function is repeated.
- res = res[:0]
- for rows.Next() {
- var r basics.Round
- err = rows.Scan(&r)
- if err != nil {
- return err
- }
- res = append(res, r)
- }
+ wg.Wait()
- return nil
+ if errBalances != nil {
+ return report, errBalances
}
- err := db.Retry(f)
- if err != nil {
- return nil, err
+ if errCreatables != nil {
+ return report, errCreatables
}
-
- return res, nil
-}
-
-func (cw *catchpointWriter) StoreCatchpoint(ctx context.Context, round basics.Round, fileName string, catchpoint string, fileSize int64) (err error) {
- err = db.Retry(func() (err error) {
- query := "DELETE FROM storedcatchpoints WHERE round=?"
- _, err = cw.e.ExecContext(ctx, query, round)
- if err != nil || (fileName == "" && catchpoint == "" && fileSize == 0) {
- return err
- }
-
- query = "INSERT INTO storedcatchpoints(round, filename, catchpoint, filesize, pinned) VALUES(?, ?, ?, ?, 0)"
- _, err = cw.e.ExecContext(ctx, query, round, fileName, catchpoint, fileSize)
- return err
- })
- return
-}
-
-func (cw *catchpointWriter) WriteCatchpointStateUint64(ctx context.Context, stateName trackerdb.CatchpointState, setValue uint64) (err error) {
- err = db.Retry(func() (err error) {
- if setValue == 0 {
- return deleteCatchpointStateImpl(ctx, cw.e, stateName)
- }
-
- // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case.
- query := "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)"
- _, err = cw.e.ExecContext(ctx, query, stateName, setValue)
- return err
- })
- return err
-}
-
-func (cw *catchpointWriter) WriteCatchpointStateString(ctx context.Context, stateName trackerdb.CatchpointState, setValue string) (err error) {
- err = db.Retry(func() (err error) {
- if setValue == "" {
- return deleteCatchpointStateImpl(ctx, cw.e, stateName)
- }
-
- // we don't know if there is an entry in the table for this state, so we'll insert/replace it just in case.
- query := "INSERT OR REPLACE INTO catchpointstate(id, strval) VALUES(?, ?)"
- _, err = cw.e.ExecContext(ctx, query, stateName, setValue)
- return err
- })
- return err
-}
-
-func (cw *catchpointWriter) InsertUnfinishedCatchpoint(ctx context.Context, round basics.Round, blockHash crypto.Digest) error {
- f := func() error {
- query := "INSERT INTO unfinishedcatchpoints(round, blockhash) VALUES(?, ?)"
- _, err := cw.e.ExecContext(ctx, query, round, blockHash[:])
- return err
+ if errHashes != nil {
+ return report, errHashes
}
- return db.Retry(f)
-}
-
-func (cw *catchpointWriter) DeleteUnfinishedCatchpoint(ctx context.Context, round basics.Round) error {
- f := func() error {
- query := "DELETE FROM unfinishedcatchpoints WHERE round = ?"
- _, err := cw.e.ExecContext(ctx, query, round)
- return err
- }
- return db.Retry(f)
-}
-
-func deleteCatchpointStateImpl(ctx context.Context, e db.Executable, stateName trackerdb.CatchpointState) error {
- query := "DELETE FROM catchpointstate WHERE id=?"
- _, err := e.ExecContext(ctx, query, stateName)
- return err
-}
-
-func (cw *catchpointWriter) InsertOrReplaceCatchpointFirstStageInfo(ctx context.Context, round basics.Round, info *trackerdb.CatchpointFirstStageInfo) error {
- infoSerialized := protocol.Encode(info)
- f := func() error {
- query := "INSERT OR REPLACE INTO catchpointfirststageinfo(round, info) VALUES(?, ?)"
- _, err := cw.e.ExecContext(ctx, query, round, infoSerialized)
- return err
+ if errKVs != nil {
+ return report, errKVs
}
- return db.Retry(f)
-}
-func (cw *catchpointWriter) DeleteOldCatchpointFirstStageInfo(ctx context.Context, maxRoundToDelete basics.Round) error {
- f := func() error {
- query := "DELETE FROM catchpointfirststageinfo WHERE round <= ?"
- _, err := cw.e.ExecContext(ctx, query, maxRoundToDelete)
- return err
- }
- return db.Retry(f)
+ return report, nil
}
// WriteCatchpointStagingBalances inserts all the account balances in the provided array into the catchpoint balance staging table catchpointbalances.
-func (cw *catchpointWriter) WriteCatchpointStagingBalances(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
+func (cw *catchpointWriter) writeCatchpointStagingBalances(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
selectAcctStmt, err := cw.e.PrepareContext(ctx, "SELECT rowid FROM catchpointbalances WHERE address = ?")
if err != nil {
return err
@@ -371,7 +208,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingBalances(ctx context.Context,
}
// WriteCatchpointStagingHashes inserts all the account hashes in the provided array into the catchpoint pending hashes table catchpointpendinghashes.
-func (cw *catchpointWriter) WriteCatchpointStagingHashes(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
+func (cw *catchpointWriter) writeCatchpointStagingHashes(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
insertStmt, err := cw.e.PrepareContext(ctx, "INSERT INTO catchpointpendinghashes(data) VALUES(?)")
if err != nil {
return err
@@ -399,7 +236,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingHashes(ctx context.Context, ba
// WriteCatchpointStagingCreatable inserts all the creatables in the provided array into the catchpoint asset creator staging table catchpointassetcreators.
// note that we cannot insert the resources here : in order to insert the resources, we need the rowid of the accountbase entry. This is being inserted by
// writeCatchpointStagingBalances via a separate go-routine.
-func (cw *catchpointWriter) WriteCatchpointStagingCreatable(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
+func (cw *catchpointWriter) writeCatchpointStagingCreatable(ctx context.Context, bals []trackerdb.NormalizedAccountBalance) error {
var insertCreatorsStmt *sql.Stmt
var err error
insertCreatorsStmt, err = cw.e.PrepareContext(ctx, "INSERT INTO catchpointassetcreators(asset, creator, ctype) VALUES(?, ?, ?)")
@@ -433,7 +270,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingCreatable(ctx context.Context,
// WriteCatchpointStagingKVs inserts all the KVs in the provided array into the
// catchpoint kvstore staging table catchpointkvstore, and their hashes to the pending
-func (cw *catchpointWriter) WriteCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error {
+func (cw *catchpointWriter) writeCatchpointStagingKVs(ctx context.Context, keys [][]byte, values [][]byte, hashes [][]byte) error {
insertKV, err := cw.e.PrepareContext(ctx, "INSERT INTO catchpointkvstore(key, value) VALUES(?, ?)")
if err != nil {
return err
@@ -460,7 +297,7 @@ func (cw *catchpointWriter) WriteCatchpointStagingKVs(ctx context.Context, keys
return nil
}
-func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context, newCatchup bool) (err error) {
+func (cw *catchpointWriter) Reset(ctx context.Context, newCatchup bool) (err error) {
s := []string{
"DROP TABLE IF EXISTS catchpointbalances",
"DROP TABLE IF EXISTS catchpointassetcreators",
@@ -494,6 +331,7 @@ func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context,
createNormalizedOnlineBalanceIndex(idxnameBalances, "catchpointbalances"), // should this be removed ?
createUniqueAddressBalanceIndex(idxnameAddress, "catchpointbalances"),
+ "CREATE INDEX IF NOT EXISTS catchpointpendinghashesidx ON catchpointpendinghashes(data)",
)
}
@@ -509,7 +347,7 @@ func (cw *catchpointWriter) ResetCatchpointStagingBalances(ctx context.Context,
// ApplyCatchpointStagingBalances switches the staged catchpoint catchup tables onto the actual
// tables and update the correct balance round. This is the final step in switching onto the new catchpoint round.
-func (cw *catchpointWriter) ApplyCatchpointStagingBalances(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error) {
+func (cw *catchpointWriter) Apply(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) (err error) {
stmts := []string{
"DROP TABLE IF EXISTS accountbase",
"DROP TABLE IF EXISTS assetcreators",
@@ -545,40 +383,3 @@ func (cw *catchpointWriter) ApplyCatchpointStagingBalances(ctx context.Context,
return
}
-
-// CreateCatchpointStagingHashesIndex creates an index on catchpointpendinghashes to allow faster scanning according to the hash order
-func (cw *catchpointWriter) CreateCatchpointStagingHashesIndex(ctx context.Context) (err error) {
- _, err = cw.e.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS catchpointpendinghashesidx ON catchpointpendinghashes(data)")
- if err != nil {
- return
- }
- return
-}
-
-// DeleteStoredCatchpoints iterates over the storedcatchpoints table and deletes all the files stored on disk.
-// once all the files have been deleted, it would go ahead and remove the entries from the table.
-func (crw *catchpointReaderWriter) DeleteStoredCatchpoints(ctx context.Context, dbDirectory string) (err error) {
- catchpointsFilesChunkSize := 50
- for {
- fileNames, err := crw.GetOldestCatchpointFiles(ctx, catchpointsFilesChunkSize, 0)
- if err != nil {
- return err
- }
- if len(fileNames) == 0 {
- break
- }
-
- for round, fileName := range fileNames {
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(dbDirectory, fileName)
- if err != nil {
- return err
- }
- // clear the entry from the database
- err = crw.StoreCatchpoint(ctx, round, "", "", 0)
- if err != nil {
- return err
- }
- }
- }
- return nil
-}
diff --git a/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go b/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go
index 32d2614e1d..611c67c877 100644
--- a/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go
+++ b/ledger/store/trackerdb/sqlitedriver/catchpointPendingHashesIter.go
@@ -19,27 +19,29 @@ package sqlitedriver
import (
"context"
"database/sql"
+
+ "github.com/algorand/go-algorand/util/db"
)
// catchpointPendingHashesIterator allows us to iterate over the hashes in the catchpointpendinghashes table in their order.
type catchpointPendingHashesIterator struct {
hashCount int
- tx *sql.Tx
+ q db.Queryable
rows *sql.Rows
}
// MakeCatchpointPendingHashesIterator create a pending hashes iterator that retrieves the hashes in the catchpointpendinghashes table.
-func MakeCatchpointPendingHashesIterator(hashCount int, tx *sql.Tx) *catchpointPendingHashesIterator {
+func MakeCatchpointPendingHashesIterator(hashCount int, q db.Queryable) *catchpointPendingHashesIterator {
return &catchpointPendingHashesIterator{
hashCount: hashCount,
- tx: tx,
+ q: q,
}
}
// Next returns an array containing the hashes, returning HashCount hashes at a time.
func (iterator *catchpointPendingHashesIterator) Next(ctx context.Context) (hashes [][]byte, err error) {
if iterator.rows == nil {
- iterator.rows, err = iterator.tx.QueryContext(ctx, "SELECT data FROM catchpointpendinghashes ORDER BY data")
+ iterator.rows, err = iterator.q.QueryContext(ctx, "SELECT data FROM catchpointpendinghashes ORDER BY data")
if err != nil {
return
}
diff --git a/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go b/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go
index f48a4f82c7..6dc3b67226 100644
--- a/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go
+++ b/ledger/store/trackerdb/sqlitedriver/encodedAccountsIter.go
@@ -23,12 +23,13 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/encoded"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/util/db"
"github.com/algorand/msgp/msgp"
)
// encodedAccountsBatchIter allows us to iterate over the accounts data stored in the accountbase table.
type encodedAccountsBatchIter struct {
- tx *sql.Tx
+ q db.Queryable
accountsRows *sql.Rows
resourcesRows *sql.Rows
nextBaseRow pendingBaseRow
@@ -45,21 +46,21 @@ type catchpointAccountResourceCounter struct {
}
// MakeEncodedAccoutsBatchIter creates an empty accounts batch iterator.
-func MakeEncodedAccoutsBatchIter(tx *sql.Tx) *encodedAccountsBatchIter {
- return &encodedAccountsBatchIter{tx: tx}
+func MakeEncodedAccoutsBatchIter(q db.Queryable) *encodedAccountsBatchIter {
+ return &encodedAccountsBatchIter{q: q}
}
// Next returns an array containing the account data, in the same way it appear in the database
// returning accountCount accounts data at a time.
func (iterator *encodedAccountsBatchIter) Next(ctx context.Context, accountCount int, resourceCount int) (bals []encoded.BalanceRecordV6, numAccountsProcessed uint64, err error) {
if iterator.accountsRows == nil {
- iterator.accountsRows, err = iterator.tx.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid")
+ iterator.accountsRows, err = iterator.q.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid")
if err != nil {
return
}
}
if iterator.resourcesRows == nil {
- iterator.resourcesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx")
+ iterator.resourcesRows, err = iterator.q.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx")
if err != nil {
return
}
diff --git a/ledger/store/trackerdb/sqlitedriver/kvsIter.go b/ledger/store/trackerdb/sqlitedriver/kvsIter.go
index 9cd1c9a7a1..3beb6ae0d6 100644
--- a/ledger/store/trackerdb/sqlitedriver/kvsIter.go
+++ b/ledger/store/trackerdb/sqlitedriver/kvsIter.go
@@ -19,22 +19,24 @@ package sqlitedriver
import (
"context"
"database/sql"
+
+ "github.com/algorand/go-algorand/util/db"
)
type kvsIter struct {
- tx *sql.Tx
+ q db.Queryable
rows *sql.Rows
}
// MakeKVsIter creates a KV iterator.
-func MakeKVsIter(ctx context.Context, tx *sql.Tx) (*kvsIter, error) {
- rows, err := tx.QueryContext(ctx, "SELECT key, value FROM kvstore")
+func MakeKVsIter(ctx context.Context, q db.Queryable) (*kvsIter, error) {
+ rows, err := q.QueryContext(ctx, "SELECT key, value FROM kvstore")
if err != nil {
return nil, err
}
return &kvsIter{
- tx: tx,
+ q: q,
rows: rows,
}, nil
}
diff --git a/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go b/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go
index 41589345fe..052dfdac93 100644
--- a/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go
+++ b/ledger/store/trackerdb/sqlitedriver/merkle_commiter.go
@@ -16,11 +16,15 @@
package sqlitedriver
-import "database/sql"
+import (
+ "database/sql"
+
+ "github.com/algorand/go-algorand/util/db"
+)
//msgp:ignore MerkleCommitter
type merkleCommitter struct {
- tx *sql.Tx
+ e db.Executable
deleteStmt *sql.Stmt
insertStmt *sql.Stmt
selectStmt *sql.Stmt
@@ -28,21 +32,21 @@ type merkleCommitter struct {
// MakeMerkleCommitter creates a MerkleCommitter object that implements the merkletrie.Committer interface allowing storing and loading
// merkletrie pages from a sqlite database.
-func MakeMerkleCommitter(tx *sql.Tx, staging bool) (mc *merkleCommitter, err error) {
- mc = &merkleCommitter{tx: tx}
+func MakeMerkleCommitter(e db.Executable, staging bool) (mc *merkleCommitter, err error) {
+ mc = &merkleCommitter{e: e}
accountHashesTable := "accounthashes"
if staging {
accountHashesTable = "catchpointaccounthashes"
}
- mc.deleteStmt, err = tx.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?")
+ mc.deleteStmt, err = e.Prepare("DELETE FROM " + accountHashesTable + " WHERE id=?")
if err != nil {
return nil, err
}
- mc.insertStmt, err = tx.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)")
+ mc.insertStmt, err = e.Prepare("INSERT OR REPLACE INTO " + accountHashesTable + "(id, data) VALUES(?, ?)")
if err != nil {
return nil, err
}
- mc.selectStmt, err = tx.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?")
+ mc.selectStmt, err = e.Prepare("SELECT data FROM " + accountHashesTable + " WHERE id = ?")
if err != nil {
return nil, err
}
diff --git a/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go b/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go
index cf927042c5..2752001131 100644
--- a/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go
+++ b/ledger/store/trackerdb/sqlitedriver/orderedAccountsIter.go
@@ -26,6 +26,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/util/db"
)
// orderedAccountsIter allows us to iterate over the accounts addresses in the order of the account hashes.
@@ -34,7 +35,7 @@ type orderedAccountsIter struct {
accountBaseRows *sql.Rows
hashesRows *sql.Rows
resourcesRows *sql.Rows
- tx *sql.Tx
+ e db.Executable
pendingBaseRow pendingBaseRow
pendingResourceRow pendingResourceRow
accountCount int
@@ -84,9 +85,9 @@ type pendingResourceRow struct {
// MakeOrderedAccountsIter creates an ordered account iterator. Note that due to implementation reasons,
// only a single iterator can be active at a time.
-func MakeOrderedAccountsIter(tx *sql.Tx, accountCount int) *orderedAccountsIter {
+func MakeOrderedAccountsIter(e db.Executable, accountCount int) *orderedAccountsIter {
return &orderedAccountsIter{
- tx: tx,
+ e: e,
accountCount: accountCount,
step: oaiStepStartup,
}
@@ -104,7 +105,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb
if iterator.step == oaiStepDeleteOldOrderingTable {
// although we're going to delete this table anyway when completing the iterator execution, we'll try to
// clean up any intermediate table.
- _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes")
+ _, err = iterator.e.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes")
if err != nil {
return
}
@@ -113,7 +114,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb
}
if iterator.step == oaiStepCreateOrderingTable {
// create the temporary table
- _, err = iterator.tx.ExecContext(ctx, "CREATE TABLE accountsiteratorhashes(addrid INTEGER, hash blob)")
+ _, err = iterator.e.ExecContext(ctx, "CREATE TABLE accountsiteratorhashes(addrid INTEGER, hash blob)")
if err != nil {
return
}
@@ -122,17 +123,17 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb
}
if iterator.step == oaiStepQueryAccounts {
// iterate over the existing accounts
- iterator.accountBaseRows, err = iterator.tx.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid")
+ iterator.accountBaseRows, err = iterator.e.QueryContext(ctx, "SELECT rowid, address, data FROM accountbase ORDER BY rowid")
if err != nil {
return
}
// iterate over the existing resources
- iterator.resourcesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx")
+ iterator.resourcesRows, err = iterator.e.QueryContext(ctx, "SELECT addrid, aidx, data FROM resources ORDER BY addrid, aidx")
if err != nil {
return
}
// prepare the insert statement into the temporary table
- iterator.insertStmt, err = iterator.tx.PrepareContext(ctx, "INSERT INTO accountsiteratorhashes(addrid, hash) VALUES(?, ?)")
+ iterator.insertStmt, err = iterator.e.PrepareContext(ctx, "INSERT INTO accountsiteratorhashes(addrid, hash) VALUES(?, ?)")
if err != nil {
return
}
@@ -200,7 +201,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb
if iterator.step == oaiStepCreateOrderingAccountIndex {
// create an index. It shown that even when we're making a single select statement in step 5, it would be better to have this index vs. not having it at all.
// note that this index is using the rowid of the accountsiteratorhashes table.
- _, err = iterator.tx.ExecContext(ctx, "CREATE INDEX accountsiteratorhashesidx ON accountsiteratorhashes(hash)")
+ _, err = iterator.e.ExecContext(ctx, "CREATE INDEX accountsiteratorhashesidx ON accountsiteratorhashes(hash)")
if err != nil {
iterator.Close(ctx)
return
@@ -210,7 +211,7 @@ func (iterator *orderedAccountsIter) Next(ctx context.Context) (acct []trackerdb
}
if iterator.step == oaiStepSelectFromOrderedTable {
// select the data from the ordered table
- iterator.hashesRows, err = iterator.tx.QueryContext(ctx, "SELECT addrid, hash FROM accountsiteratorhashes ORDER BY hash")
+ iterator.hashesRows, err = iterator.e.QueryContext(ctx, "SELECT addrid, hash FROM accountsiteratorhashes ORDER BY hash")
if err != nil {
iterator.Close(ctx)
@@ -272,7 +273,7 @@ func (iterator *orderedAccountsIter) Close(ctx context.Context) (err error) {
iterator.insertStmt.Close()
iterator.insertStmt = nil
}
- _, err = iterator.tx.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes")
+ _, err = iterator.e.ExecContext(ctx, "DROP TABLE IF EXISTS accountsiteratorhashes")
return
}
diff --git a/ledger/store/trackerdb/sqlitedriver/schema.go b/ledger/store/trackerdb/sqlitedriver/schema.go
index abc53675eb..ea6f577868 100644
--- a/ledger/store/trackerdb/sqlitedriver/schema.go
+++ b/ledger/store/trackerdb/sqlitedriver/schema.go
@@ -181,9 +181,9 @@ var accountsResetExprs = []string{
//
// accountsInit returns nil if either it has initialized the database
// correctly, or if the database has already been initialized.
-func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
+func accountsInit(e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
for _, tableCreate := range accountsSchema {
- _, err = tx.Exec(tableCreate)
+ _, err = e.Exec(tableCreate)
if err != nil {
return
}
@@ -191,11 +191,11 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData
// Run creatables migration if it hasn't run yet
var creatableMigrated bool
- err = tx.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated)
+ err = e.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated)
if err == sql.ErrNoRows {
// Run migration
for _, migrateCmd := range creatablesMigration {
- _, err = tx.Exec(migrateCmd)
+ _, err = e.Exec(migrateCmd)
if err != nil {
return
}
@@ -204,13 +204,13 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData
return
}
- _, err = tx.Exec("INSERT INTO acctrounds (id, rnd) VALUES ('acctbase', 0)")
+ _, err = e.Exec("INSERT INTO acctrounds (id, rnd) VALUES ('acctbase', 0)")
if err == nil {
var ot basics.OverflowTracker
var totals ledgercore.AccountTotals
for addr, data := range initAccounts {
- _, err = tx.Exec("INSERT INTO accountbase (address, data) VALUES (?, ?)",
+ _, err = e.Exec("INSERT INTO accountbase (address, data) VALUES (?, ?)",
addr[:], protocol.Encode(&data)) //nolint:gosec // Encode does not hold on to reference
if err != nil {
return true, err
@@ -224,7 +224,7 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData
return true, fmt.Errorf("overflow computing totals")
}
- arw := NewAccountsSQLReaderWriter(tx)
+ arw := NewAccountsSQLReaderWriter(e)
err = arw.AccountsPutTotals(totals, false)
if err != nil {
return true, err
@@ -245,9 +245,9 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData
// accountsAddNormalizedBalance adds the normalizedonlinebalance column
// to the accountbase table.
-func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) error {
+func accountsAddNormalizedBalance(e db.Executable, proto config.ConsensusParams) error {
var exists bool
- err := tx.QueryRow("SELECT 1 FROM pragma_table_info('accountbase') WHERE name='normalizedonlinebalance'").Scan(&exists)
+ err := e.QueryRow("SELECT 1 FROM pragma_table_info('accountbase') WHERE name='normalizedonlinebalance'").Scan(&exists)
if err == nil {
// Already exists.
return nil
@@ -257,13 +257,13 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro
}
for _, stmt := range createOnlineAccountIndex {
- _, err = tx.Exec(stmt)
+ _, err = e.Exec(stmt)
if err != nil {
return err
}
}
- rows, err := tx.Query("SELECT address, data FROM accountbase")
+ rows, err := e.Query("SELECT address, data FROM accountbase")
if err != nil {
return err
}
@@ -285,7 +285,7 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro
normBalance := data.NormalizedOnlineBalance(proto)
if normBalance > 0 {
- _, err = tx.Exec("UPDATE accountbase SET normalizedonlinebalance=? WHERE address=?", normBalance, addrbuf)
+ _, err = e.Exec("UPDATE accountbase SET normalizedonlinebalance=? WHERE address=?", normBalance, addrbuf)
if err != nil {
return err
}
@@ -296,9 +296,9 @@ func accountsAddNormalizedBalance(tx *sql.Tx, proto config.ConsensusParams) erro
}
// accountsCreateResourceTable creates the resource table in the database.
-func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error {
+func accountsCreateResourceTable(ctx context.Context, e db.Executable) error {
var exists bool
- err := tx.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('resources') WHERE name='addrid'").Scan(&exists)
+ err := e.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('resources') WHERE name='addrid'").Scan(&exists)
if err == nil {
// Already exists.
return nil
@@ -307,7 +307,7 @@ func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error {
return err
}
for _, stmt := range createResourcesTable {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return err
}
@@ -315,9 +315,9 @@ func accountsCreateResourceTable(ctx context.Context, tx *sql.Tx) error {
return nil
}
-func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error {
+func accountsCreateOnlineAccountsTable(ctx context.Context, e db.Executable) error {
var exists bool
- err := tx.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('onlineaccounts') WHERE name='address'").Scan(&exists)
+ err := e.QueryRowContext(ctx, "SELECT 1 FROM pragma_table_info('onlineaccounts') WHERE name='address'").Scan(&exists)
if err == nil {
// Already exists.
return nil
@@ -326,7 +326,7 @@ func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error {
return err
}
for _, stmt := range createOnlineAccountsTable {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return err
}
@@ -335,9 +335,9 @@ func accountsCreateOnlineAccountsTable(ctx context.Context, tx *sql.Tx) error {
}
// accountsCreateBoxTable creates the KVStore table for box-storage in the database.
-func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error {
+func accountsCreateBoxTable(ctx context.Context, e db.Executable) error {
var exists bool
- err := tx.QueryRow("SELECT 1 FROM pragma_table_info('kvstore') WHERE name='key'").Scan(&exists)
+ err := e.QueryRow("SELECT 1 FROM pragma_table_info('kvstore') WHERE name='key'").Scan(&exists)
if err == nil {
// already exists
return nil
@@ -346,7 +346,7 @@ func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error {
return err
}
for _, stmt := range createBoxTable {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return err
}
@@ -355,14 +355,14 @@ func accountsCreateBoxTable(ctx context.Context, tx *sql.Tx) error {
}
// performKVStoreNullBlobConversion scans keys with null blob value, and convert the value to `[]byte{}`.
-func performKVStoreNullBlobConversion(ctx context.Context, tx *sql.Tx) error {
- _, err := tx.ExecContext(ctx, "UPDATE kvstore SET value = '' WHERE value is NULL")
+func performKVStoreNullBlobConversion(ctx context.Context, e db.Executable) error {
+ _, err := e.ExecContext(ctx, "UPDATE kvstore SET value = '' WHERE value is NULL")
return err
}
-func accountsCreateTxTailTable(ctx context.Context, tx *sql.Tx) (err error) {
+func accountsCreateTxTailTable(ctx context.Context, e db.Executable) (err error) {
for _, stmt := range createTxTailTable {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return
}
@@ -370,9 +370,9 @@ func accountsCreateTxTailTable(ctx context.Context, tx *sql.Tx) (err error) {
return nil
}
-func accountsCreateOnlineRoundParamsTable(ctx context.Context, tx *sql.Tx) (err error) {
+func accountsCreateOnlineRoundParamsTable(ctx context.Context, e db.Executable) (err error) {
for _, stmt := range createOnlineRoundParamsTable {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return
}
@@ -396,7 +396,7 @@ func createStateProofVerificationTable(ctx context.Context, e db.Executable) err
}
// performResourceTableMigration migrate the database to use the resources table.
-func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(processed, total uint64)) (err error) {
+func performResourceTableMigration(ctx context.Context, e db.Executable, log func(processed, total uint64)) (err error) {
now := time.Now().UnixNano()
idxnameBalances := fmt.Sprintf("onlineaccountbals_idx_%d", now)
idxnameAddress := fmt.Sprintf("accountbase_address_idx_%d", now)
@@ -418,7 +418,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro
}
for _, stmt := range createNewAcctBase {
- _, err = tx.ExecContext(ctx, stmt)
+ _, err = e.ExecContext(ctx, stmt)
if err != nil {
return err
}
@@ -426,26 +426,26 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro
var insertNewAcctBase *sql.Stmt
var insertResources *sql.Stmt
var insertNewAcctBaseNormBal *sql.Stmt
- insertNewAcctBase, err = tx.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data) VALUES(?, ?)")
+ insertNewAcctBase, err = e.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data) VALUES(?, ?)")
if err != nil {
return err
}
defer insertNewAcctBase.Close()
- insertNewAcctBaseNormBal, err = tx.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data, normalizedonlinebalance) VALUES(?, ?, ?)")
+ insertNewAcctBaseNormBal, err = e.PrepareContext(ctx, "INSERT INTO accountbase_resources_migration(address, data, normalizedonlinebalance) VALUES(?, ?, ?)")
if err != nil {
return err
}
defer insertNewAcctBaseNormBal.Close()
- insertResources, err = tx.PrepareContext(ctx, "INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)")
+ insertResources, err = e.PrepareContext(ctx, "INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)")
if err != nil {
return err
}
defer insertResources.Close()
var rows *sql.Rows
- rows, err = tx.QueryContext(ctx, "SELECT address, data, normalizedonlinebalance FROM accountbase ORDER BY address")
+ rows, err = e.QueryContext(ctx, "SELECT address, data, normalizedonlinebalance FROM accountbase ORDER BY address")
if err != nil {
return err
}
@@ -457,7 +457,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro
var processedAccounts uint64
var totalBaseAccounts uint64
- arw := NewAccountsSQLReaderWriter(tx)
+ arw := NewAccountsSQLReaderWriter(e)
totalBaseAccounts, err = arw.TotalAccounts(ctx)
if err != nil {
return err
@@ -524,7 +524,7 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro
}
for _, stmt := range applyNewAcctBase {
- _, err = tx.Exec(stmt)
+ _, err = e.Exec(stmt)
if err != nil {
return err
}
@@ -532,12 +532,12 @@ func performResourceTableMigration(ctx context.Context, tx *sql.Tx, log func(pro
return nil
}
-func performTxTailTableMigration(ctx context.Context, tx *sql.Tx, blockDb db.Accessor) (err error) {
- if tx == nil {
+func performTxTailTableMigration(ctx context.Context, e db.Executable, blockDb db.Accessor) (err error) {
+ if e == nil {
return nil
}
- arw := NewAccountsSQLReaderWriter(tx)
+ arw := NewAccountsSQLReaderWriter(e)
dbRound, err := arw.AccountsRound()
if err != nil {
return fmt.Errorf("latest block number cannot be retrieved : %w", err)
@@ -596,8 +596,8 @@ func performTxTailTableMigration(ctx context.Context, tx *sql.Tx, blockDb db.Acc
return err
}
-func performOnlineRoundParamsTailMigration(ctx context.Context, tx *sql.Tx, blockDb db.Accessor, newDatabase bool, initProto protocol.ConsensusVersion) (err error) {
- arw := NewAccountsSQLReaderWriter(tx)
+func performOnlineRoundParamsTailMigration(ctx context.Context, e db.Executable, blockDb db.Accessor, newDatabase bool, initProto protocol.ConsensusVersion) (err error) {
+ arw := NewAccountsSQLReaderWriter(e)
totals, err := arw.AccountsTotals(ctx, false)
if err != nil {
return err
@@ -632,24 +632,24 @@ func performOnlineRoundParamsTailMigration(ctx context.Context, tx *sql.Tx, bloc
return arw.AccountsPutOnlineRoundParams(onlineRoundParams, rnd)
}
-func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progress func(processed, total uint64), log logging.Logger) (err error) {
+func performOnlineAccountsTableMigration(ctx context.Context, e db.Executable, progress func(processed, total uint64), log logging.Logger) (err error) {
var insertOnlineAcct *sql.Stmt
- insertOnlineAcct, err = tx.PrepareContext(ctx, "INSERT INTO onlineaccounts(address, data, normalizedonlinebalance, updround, votelastvalid) VALUES(?, ?, ?, ?, ?)")
+ insertOnlineAcct, err = e.PrepareContext(ctx, "INSERT INTO onlineaccounts(address, data, normalizedonlinebalance, updround, votelastvalid) VALUES(?, ?, ?, ?, ?)")
if err != nil {
return err
}
defer insertOnlineAcct.Close()
var updateAcct *sql.Stmt
- updateAcct, err = tx.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE addrid = ?")
+ updateAcct, err = e.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE addrid = ?")
if err != nil {
return err
}
defer updateAcct.Close()
var rows *sql.Rows
- rows, err = tx.QueryContext(ctx, "SELECT addrid, address, data, normalizedonlinebalance FROM accountbase")
+ rows, err = e.QueryContext(ctx, "SELECT addrid, address, data, normalizedonlinebalance FROM accountbase")
if err != nil {
return err
}
@@ -661,10 +661,10 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre
var processedAccounts uint64
var totalOnlineBaseAccounts uint64
- arw := NewAccountsSQLReaderWriter(tx)
+ arw := NewAccountsSQLReaderWriter(e)
totalOnlineBaseAccounts, err = arw.TotalAccounts(ctx)
var total uint64
- err = tx.QueryRowContext(ctx, "SELECT count(1) FROM accountbase").Scan(&total)
+ err = e.QueryRowContext(ctx, "SELECT count(1) FROM accountbase").Scan(&total)
if err != nil {
if err != sql.ErrNoRows {
return err
@@ -763,7 +763,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre
// update accounthashes for the modified accounts
if len(acctRehash) > 0 {
var count uint64
- err := tx.QueryRow("SELECT count(1) FROM accounthashes").Scan(&count)
+ err := e.QueryRow("SELECT count(1) FROM accounthashes").Scan(&count)
if err != nil {
return err
}
@@ -772,7 +772,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre
return nil
}
- mc, err := MakeMerkleCommitter(tx, false)
+ mc, err := MakeMerkleCommitter(e, false)
if err != nil {
return nil
}
@@ -811,7 +811,7 @@ func performOnlineAccountsTableMigration(ctx context.Context, tx *sql.Tx, progre
// removeEmptyAccountData removes empty AccountData msgp-encoded entries from accountbase table
// and optionally returns list of addresses that were eliminated
-func removeEmptyAccountData(tx *sql.Tx, queryAddresses bool) (num int64, addresses []basics.Address, err error) {
+func removeEmptyAccountData(tx db.Executable, queryAddresses bool) (num int64, addresses []basics.Address, err error) {
if queryAddresses {
rows, qErr := tx.Query("SELECT address FROM accountbase where length(data) = 1 and data = x'80'") // empty AccountData is 0x80
if qErr != nil {
@@ -856,16 +856,16 @@ func removeEmptyAccountData(tx *sql.Tx, queryAddresses bool) (num int64, address
// reencodeAccounts reads all the accounts in the accountbase table, decode and reencode the account data.
// if the account data is found to have a different encoding, it would update the encoded account on disk.
// on return, it returns the number of modified accounts as well as an error ( if we had any )
-func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, err error) {
+func reencodeAccounts(ctx context.Context, e db.Executable) (modifiedAccounts uint, err error) {
modifiedAccounts = 0
scannedAccounts := 0
- updateStmt, err := tx.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE address = ?")
+ updateStmt, err := e.PrepareContext(ctx, "UPDATE accountbase SET data = ? WHERE address = ?")
if err != nil {
return 0, err
}
- rows, err := tx.QueryContext(ctx, "SELECT address, data FROM accountbase")
+ rows, err := e.QueryContext(ctx, "SELECT address, data FROM accountbase")
if err != nil {
return
}
@@ -880,7 +880,7 @@ func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, e
if scannedAccounts%1000 == 0 {
// The return value from ResetTransactionWarnDeadline can be safely ignored here since it would only default to writing the warning
// message, which would let us know that it failed anyway.
- _, err = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(time.Second))
+ _, err = db.ResetTransactionWarnDeadline(ctx, e, time.Now().Add(time.Second))
if err != nil {
return
}
@@ -932,8 +932,8 @@ func reencodeAccounts(ctx context.Context, tx *sql.Tx) (modifiedAccounts uint, e
return
}
-func convertOnlineRoundParamsTail(ctx context.Context, tx *sql.Tx) error {
+func convertOnlineRoundParamsTail(ctx context.Context, e db.Executable) error {
// create vote last index
- _, err := tx.ExecContext(ctx, createVoteLastValidIndex)
+ _, err := e.ExecContext(ctx, createVoteLastValidIndex)
return err
}
diff --git a/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go b/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go
index 3a024e0a98..0354a3c832 100644
--- a/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go
+++ b/ledger/store/trackerdb/sqlitedriver/spVerificationAccessor.go
@@ -18,6 +18,7 @@ package sqlitedriver
import (
"context"
+ "database/sql"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/ledgercore"
@@ -66,7 +67,9 @@ func (spa *stateProofVerificationReader) LookupSPContext(stateProofLastAttestedR
row := spa.q.QueryRow("SELECT verificationcontext FROM stateproofverification WHERE lastattestedround=?", stateProofLastAttestedRound)
var buf []byte
err := row.Scan(&buf)
- if err != nil {
+ if err == sql.ErrNoRows {
+ return trackerdb.ErrNotFound
+ } else if err != nil {
return err
}
err = protocol.Decode(buf, &verificationContext)
diff --git a/ledger/store/trackerdb/sqlitedriver/sql.go b/ledger/store/trackerdb/sqlitedriver/sql.go
index c4f1639bd4..be4dd247df 100644
--- a/ledger/store/trackerdb/sqlitedriver/sql.go
+++ b/ledger/store/trackerdb/sqlitedriver/sql.go
@@ -53,7 +53,7 @@ type accountsSQLWriter struct {
}
type onlineAccountsSQLWriter struct {
- insertStmt, updateStmt *sql.Stmt
+ insertStmt *sql.Stmt
}
type sqlRowRef struct {
@@ -131,16 +131,11 @@ func OnlineAccountsInitDbQueries(r db.Queryable) (*onlineAccountsDbQueries, erro
}
// MakeOnlineAccountsSQLWriter constructs an OnlineAccountsWriter backed by sql queries.
-func MakeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) {
+func MakeOnlineAccountsSQLWriter(e db.Executable, hasAccounts bool) (w *onlineAccountsSQLWriter, err error) {
w = new(onlineAccountsSQLWriter)
if hasAccounts {
- w.insertStmt, err = tx.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)")
- if err != nil {
- return
- }
-
- w.updateStmt, err = tx.Prepare("UPDATE onlineaccounts SET normalizedonlinebalance = ?, data = ?, updround = ?, votelastvalid =? WHERE rowid = ?")
+ w.insertStmt, err = e.Prepare("INSERT INTO onlineaccounts (address, normalizedonlinebalance, data, updround, votelastvalid) VALUES (?, ?, ?, ?, ?)")
if err != nil {
return
}
@@ -150,62 +145,62 @@ func MakeOnlineAccountsSQLWriter(tx *sql.Tx, hasAccounts bool) (w *onlineAccount
}
// MakeAccountsSQLWriter constructs an AccountsWriter backed by sql queries.
-func MakeAccountsSQLWriter(tx *sql.Tx, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) {
+func MakeAccountsSQLWriter(e db.Executable, hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (w *accountsSQLWriter, err error) {
w = new(accountsSQLWriter)
if hasAccounts {
- w.deleteByRowIDStmt, err = tx.Prepare("DELETE FROM accountbase WHERE rowid=?")
+ w.deleteByRowIDStmt, err = e.Prepare("DELETE FROM accountbase WHERE rowid=?")
if err != nil {
return
}
- w.insertStmt, err = tx.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)")
+ w.insertStmt, err = e.Prepare("INSERT INTO accountbase (address, normalizedonlinebalance, data) VALUES (?, ?, ?)")
if err != nil {
return
}
- w.updateStmt, err = tx.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?")
+ w.updateStmt, err = e.Prepare("UPDATE accountbase SET normalizedonlinebalance = ?, data = ? WHERE rowid = ?")
if err != nil {
return
}
}
if hasResources {
- w.deleteResourceStmt, err = tx.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?")
+ w.deleteResourceStmt, err = e.Prepare("DELETE FROM resources WHERE addrid = ? AND aidx = ?")
if err != nil {
return
}
- w.insertResourceStmt, err = tx.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)")
+ w.insertResourceStmt, err = e.Prepare("INSERT INTO resources(addrid, aidx, data) VALUES(?, ?, ?)")
if err != nil {
return
}
- w.updateResourceStmt, err = tx.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?")
+ w.updateResourceStmt, err = e.Prepare("UPDATE resources SET data = ? WHERE addrid = ? AND aidx = ?")
if err != nil {
return
}
}
if hasKvPairs {
- w.upsertKvPairStmt, err = tx.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value")
+ w.upsertKvPairStmt, err = e.Prepare("INSERT INTO kvstore (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value")
if err != nil {
return
}
- w.deleteKvPairStmt, err = tx.Prepare("DELETE FROM kvstore WHERE key=?")
+ w.deleteKvPairStmt, err = e.Prepare("DELETE FROM kvstore WHERE key=?")
if err != nil {
return
}
}
if hasCreatables {
- w.insertCreatableIdxStmt, err = tx.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)")
+ w.insertCreatableIdxStmt, err = e.Prepare("INSERT INTO assetcreators (asset, creator, ctype) VALUES (?, ?, ?)")
if err != nil {
return
}
- w.deleteCreatableIdxStmt, err = tx.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?")
+ w.deleteCreatableIdxStmt, err = e.Prepare("DELETE FROM assetcreators WHERE asset=? AND ctype=?")
if err != nil {
return
}
@@ -528,7 +523,9 @@ func (qs *onlineAccountsDbQueries) LookupOnlineTotalsHistory(round basics.Round)
row := qs.lookupOnlineTotalsStmt.QueryRow(round)
var buf []byte
err := row.Scan(&buf)
- if err != nil {
+ if err == sql.ErrNoRows {
+ return trackerdb.ErrNotFound
+ } else if err != nil {
return err
}
err = protocol.Decode(buf, &data)
diff --git a/ledger/store/trackerdb/sqlitedriver/sql_test.go b/ledger/store/trackerdb/sqlitedriver/sql_test.go
index 3fe71be4f9..096d131bb7 100644
--- a/ledger/store/trackerdb/sqlitedriver/sql_test.go
+++ b/ledger/store/trackerdb/sqlitedriver/sql_test.go
@@ -17,8 +17,13 @@
package sqlitedriver
import (
+ "context"
+ "database/sql"
"testing"
+ "github.com/algorand/go-algorand/data/basics"
+ storetesting "github.com/algorand/go-algorand/ledger/store/testing"
+ "github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/test/partitiontest"
"github.com/stretchr/testify/require"
)
@@ -53,3 +58,26 @@ func TestKeyPrefixIntervalPreprocessing(t *testing.T) {
require.Equal(t, tc.outputPrefixIncr, actualOutputPrefixIncr)
}
}
+
+// TestAccountsDbQueriesCreateClose tests to see that we can create the accountsDbQueries and close it.
+// it also verify that double-closing it doesn't create an issue.
+func TestAccountsDbQueriesCreateClose(t *testing.T) {
+ partitiontest.PartitionTest(t)
+
+ dbs, _ := storetesting.DbOpenTest(t, true)
+ storetesting.SetDbLogging(t, dbs)
+ defer dbs.Close()
+
+ err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
+ AccountsInitTest(t, tx, make(map[basics.Address]basics.AccountData), protocol.ConsensusCurrentVersion)
+ return nil
+ })
+ require.NoError(t, err)
+ qs, err := AccountsInitDbQueries(dbs.Rdb.Handle)
+ require.NoError(t, err)
+ require.NotNil(t, qs.listCreatablesStmt)
+ qs.Close()
+ require.Nil(t, qs.listCreatablesStmt)
+ qs.Close()
+ require.Nil(t, qs.listCreatablesStmt)
+}
diff --git a/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go b/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go
new file mode 100644
index 0000000000..925c72ace4
--- /dev/null
+++ b/ledger/store/trackerdb/sqlitedriver/sqlitedriver.go
@@ -0,0 +1,330 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package sqlitedriver
+
+import (
+ "context"
+ "database/sql"
+ "testing"
+ "time"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/util/db"
+)
+
+type trackerSQLStore struct {
+ pair db.Pair
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+// Open opens the sqlite database store
+func Open(dbFilename string, dbMem bool, log logging.Logger) (store trackerdb.Store, err error) {
+ pair, err := db.OpenPair(dbFilename, dbMem)
+ if err != nil {
+ return
+ }
+ pair.Rdb.SetLogger(log)
+ pair.Wdb.SetLogger(log)
+ return MakeStore(pair), nil
+}
+
+// MakeStore crates a tracker SQL db from sql db handle.
+func MakeStore(pair db.Pair) trackerdb.Store {
+ return &trackerSQLStore{pair, &sqlReader{pair.Rdb.Handle}, &sqlWriter{pair.Wdb.Handle}, &sqlCatchpoint{pair.Wdb.Handle, pair.Wdb.IsSharedCacheConnection()}}
+}
+
+func (s *trackerSQLStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) {
+ return s.pair.Wdb.SetSynchronousMode(ctx, mode, fullfsync)
+}
+
+func (s *trackerSQLStore) IsSharedCacheConnection() bool {
+ return s.pair.Wdb.IsSharedCacheConnection()
+}
+
+func (s *trackerSQLStore) Batch(fn trackerdb.BatchFn) (err error) {
+ return s.BatchContext(context.Background(), fn)
+}
+
+func (s *trackerSQLStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) {
+ return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ return fn(ctx, &sqlBatchScope{tx, false, &sqlWriter{tx}})
+ })
+}
+
+func (s *trackerSQLStore) BeginBatch(ctx context.Context) (trackerdb.Batch, error) {
+ handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ return &sqlBatchScope{handle, false, &sqlWriter{handle}}, nil
+}
+
+func (s *trackerSQLStore) Snapshot(fn trackerdb.SnapshotFn) (err error) {
+ return s.SnapshotContext(context.Background(), fn)
+}
+
+func (s *trackerSQLStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) {
+ return s.pair.Rdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ return fn(ctx, sqlSnapshotScope{tx, &sqlReader{tx}})
+ })
+}
+
+func (s *trackerSQLStore) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) {
+ handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ return &sqlSnapshotScope{handle, &sqlReader{handle}}, nil
+}
+
+func (s *trackerSQLStore) Transaction(fn trackerdb.TransactionFn) (err error) {
+ return s.TransactionContext(context.Background(), fn)
+}
+
+func (s *trackerSQLStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) {
+ return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ return fn(ctx, &sqlTransactionScope{tx, false, &sqlReader{tx}, &sqlWriter{tx}, &sqlCatchpoint{tx, s.IsSharedCacheConnection()}})
+ })
+}
+
+func (s *trackerSQLStore) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) {
+ handle, err := s.pair.Wdb.Handle.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ return &sqlTransactionScope{handle, false, &sqlReader{handle}, &sqlWriter{handle}, &sqlCatchpoint{handle, s.IsSharedCacheConnection()}}, nil
+}
+
+func (s trackerSQLStore) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ err = s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ mgr, err = RunMigrations(ctx, tx, params, log, targetVersion)
+ return err
+ })
+ return
+}
+
+// TODO: rename: this is a sqlite specific name, this could also be used to trigger compact on KV stores.
+// it seems to only be used during a v2 migration
+func (s *trackerSQLStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) {
+ _, err = s.pair.Wdb.Vacuum(ctx)
+ return
+}
+
+func (s *trackerSQLStore) ResetToV6Test(ctx context.Context) error {
+ var resetExprs = []string{
+ `DROP TABLE IF EXISTS onlineaccounts`,
+ `DROP TABLE IF EXISTS txtail`,
+ `DROP TABLE IF EXISTS onlineroundparamstail`,
+ `DROP TABLE IF EXISTS catchpointfirststageinfo`,
+ }
+
+ return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
+ for _, stmt := range resetExprs {
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+
+func (s *trackerSQLStore) Close() {
+ s.pair.Close()
+}
+
+type sqlReader struct {
+ q db.Queryable
+}
+
+// MakeAccountsOptimizedReader implements trackerdb.Reader
+func (r *sqlReader) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) {
+ return AccountsInitDbQueries(r.q)
+}
+
+// MakeAccountsReader implements trackerdb.Reader
+func (r *sqlReader) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) {
+ // TODO: create and use a make accounts reader that takes just a queryable
+ return NewAccountsSQLReader(r.q), nil
+}
+
+// MakeOnlineAccountsOptimizedReader implements trackerdb.Reader
+func (r *sqlReader) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) {
+ return OnlineAccountsInitDbQueries(r.q)
+}
+
+// MakeSpVerificationCtxReader implements trackerdb.Reader
+func (r *sqlReader) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader {
+ return makeStateProofVerificationReader(r.q)
+}
+
+type sqlWriter struct {
+ e db.Executable
+}
+
+// MakeAccountsOptimizedWriter implements trackerdb.Writer
+func (w *sqlWriter) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) {
+ return MakeAccountsSQLWriter(w.e, hasAccounts, hasResources, hasKvPairs, hasCreatables)
+}
+
+// MakeAccountsWriter implements trackerdb.Writer
+func (w *sqlWriter) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) {
+ return NewAccountsSQLReaderWriter(w.e), nil
+}
+
+// MakeOnlineAccountsOptimizedWriter implements trackerdb.Writer
+func (w *sqlWriter) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (trackerdb.OnlineAccountsWriter, error) {
+ return MakeOnlineAccountsSQLWriter(w.e, hasAccounts)
+}
+
+// MakeSpVerificationCtxWriter implements trackerdb.Writer
+func (w *sqlWriter) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter {
+ return makeStateProofVerificationWriter(w.e)
+}
+
+// Testing implements trackerdb.Writer
+func (w *sqlWriter) Testing() trackerdb.WriterTestExt {
+ return w
+}
+
+// AccountsInitLightTest implements trackerdb.WriterTestExt
+func (w *sqlWriter) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
+ return AccountsInitLightTest(tb, w.e, initAccounts, proto)
+}
+
+// AccountsInitTest implements trackerdb.WriterTestExt
+func (w *sqlWriter) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
+ return AccountsInitTest(tb, w.e, initAccounts, proto)
+}
+
+// AccountsUpdateSchemaTest implements trackerdb.WriterTestExt
+func (w *sqlWriter) AccountsUpdateSchemaTest(ctx context.Context) (err error) {
+ return AccountsUpdateSchemaTest(ctx, w.e)
+}
+
+// ModifyAcctBaseTest implements trackerdb.WriterTestExt
+func (w *sqlWriter) ModifyAcctBaseTest() error {
+ return modifyAcctBaseTest(w.e)
+}
+
+type sqlCatchpoint struct {
+ e db.Executable
+ isShared bool
+}
+
+// MakeCatchpointPendingHashesIterator implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter {
+ return MakeCatchpointPendingHashesIterator(hashCount, c.e)
+}
+
+// MakeCatchpointWriter implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeCatchpointWriter() (trackerdb.CatchpointApply, error) {
+ return MakeCatchpointApplier(c.e, c.isShared), nil
+}
+
+// MakeEncodedAccoutsBatchIter implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter {
+ return MakeEncodedAccoutsBatchIter(c.e)
+}
+
+// MakeKVsIter implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) {
+ return MakeKVsIter(ctx, c.e)
+}
+
+// MakeMerkleCommitter implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) {
+ return MakeMerkleCommitter(c.e, staging)
+}
+
+// MakeOrderedAccountsIter implements trackerdb.Catchpoint
+func (c *sqlCatchpoint) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter {
+ return MakeOrderedAccountsIter(c.e, accountCount)
+}
+
+type sqlBatchScope struct {
+ tx *sql.Tx
+ committed bool
+ trackerdb.Writer
+}
+
+func (bs *sqlBatchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ return db.ResetTransactionWarnDeadline(ctx, bs.tx, deadline)
+}
+
+func (bs *sqlBatchScope) Close() error {
+ if !bs.committed {
+ return bs.tx.Rollback()
+ }
+ return nil
+}
+
+func (bs *sqlBatchScope) Commit() error {
+ err := bs.tx.Commit()
+ if err != nil {
+ return err
+ }
+ bs.committed = true
+ return nil
+}
+
+type sqlSnapshotScope struct {
+ tx *sql.Tx
+ trackerdb.Reader
+}
+
+func (ss sqlSnapshotScope) Close() error {
+ return ss.tx.Rollback()
+}
+
+type sqlTransactionScope struct {
+ tx *sql.Tx
+ committed bool
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+func (txs *sqlTransactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ return RunMigrations(ctx, txs.tx, params, log, targetVersion)
+}
+
+func (txs *sqlTransactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ return db.ResetTransactionWarnDeadline(ctx, txs.tx, deadline)
+}
+
+func (txs *sqlTransactionScope) Close() error {
+ if !txs.committed {
+ return txs.tx.Rollback()
+ }
+ return nil
+}
+
+func (txs *sqlTransactionScope) Commit() error {
+ err := txs.tx.Commit()
+ if err != nil {
+ return err
+ }
+ txs.committed = true
+ return nil
+}
diff --git a/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go b/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go
deleted file mode 100644
index 74327be2bc..0000000000
--- a/ledger/store/trackerdb/sqlitedriver/store_sqlite_impl.go
+++ /dev/null
@@ -1,288 +0,0 @@
-// Copyright (C) 2019-2023 Algorand, Inc.
-// This file is part of go-algorand
-//
-// go-algorand is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// go-algorand is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with go-algorand. If not, see .
-
-package sqlitedriver
-
-import (
- "context"
- "database/sql"
- "os"
- "testing"
- "time"
-
- "github.com/algorand/go-algorand/config"
- "github.com/algorand/go-algorand/data/basics"
- "github.com/algorand/go-algorand/ledger/store/trackerdb"
- "github.com/algorand/go-algorand/logging"
- "github.com/algorand/go-algorand/protocol"
- "github.com/algorand/go-algorand/util/db"
-)
-
-type trackerSQLStore struct {
- // expose the internals for now so we can slowly change the code depending on them
- pair db.Pair
-}
-
-type sqlBatchScope struct {
- tx *sql.Tx
-}
-
-type sqlSnapshotScope struct {
- tx *sql.Tx
-}
-
-type sqlTransactionScope struct {
- tx *sql.Tx
-}
-
-// OpenTrackerSQLStore opens the sqlite database store
-func OpenTrackerSQLStore(dbFilename string, dbMem bool) (store *trackerSQLStore, err error) {
- db, err := db.OpenPair(dbFilename, dbMem)
- if err != nil {
- return
- }
-
- return &trackerSQLStore{db}, nil
-}
-
-// CreateTrackerSQLStore crates a tracker SQL db from sql db handle.
-func CreateTrackerSQLStore(pair db.Pair) *trackerSQLStore {
- return &trackerSQLStore{pair}
-}
-
-// SetLogger sets the Logger, mainly for unit test quietness
-func (s *trackerSQLStore) SetLogger(log logging.Logger) {
- s.pair.Rdb.SetLogger(log)
- s.pair.Wdb.SetLogger(log)
-}
-
-func (s *trackerSQLStore) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) {
- return s.pair.Wdb.SetSynchronousMode(ctx, mode, fullfsync)
-}
-
-func (s *trackerSQLStore) IsSharedCacheConnection() bool {
- return s.pair.Wdb.IsSharedCacheConnection()
-}
-
-func (s *trackerSQLStore) Batch(fn trackerdb.BatchFn) (err error) {
- return s.BatchContext(context.Background(), fn)
-}
-
-func (s *trackerSQLStore) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) {
- return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
- return fn(ctx, sqlBatchScope{tx})
- })
-}
-
-func (s *trackerSQLStore) Snapshot(fn trackerdb.SnapshotFn) (err error) {
- return s.SnapshotContext(context.Background(), fn)
-}
-
-func (s *trackerSQLStore) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) {
- return s.pair.Rdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
- return fn(ctx, sqlSnapshotScope{tx})
- })
-}
-
-func (s *trackerSQLStore) Transaction(fn trackerdb.TransactionFn) (err error) {
- return s.TransactionContext(context.Background(), fn)
-}
-
-func (s *trackerSQLStore) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) {
- return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
- return fn(ctx, sqlTransactionScope{tx})
- })
-}
-
-func (s *trackerSQLStore) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) {
- return AccountsInitDbQueries(s.pair.Rdb.Handle)
-}
-
-func (s *trackerSQLStore) MakeOnlineAccountsOptimizedReader() (trackerdb.OnlineAccountsReader, error) {
- return OnlineAccountsInitDbQueries(s.pair.Rdb.Handle)
-}
-
-func (s *trackerSQLStore) MakeCatchpointReaderWriter() (trackerdb.CatchpointReaderWriter, error) {
- w := NewCatchpointSQLReaderWriter(s.pair.Wdb.Handle)
- return w, nil
-}
-
-// TODO: rename: this is a sqlite specific name, this could also be used to trigger compact on KV stores.
-// it seems to only be used during a v2 migration
-func (s *trackerSQLStore) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) {
- _, err = s.pair.Wdb.Vacuum(ctx)
- return
-}
-
-func (s *trackerSQLStore) CleanupTest(dbName string, inMemory bool) {
- s.pair.Close()
- if !inMemory {
- os.Remove(dbName)
- }
-}
-
-func (s *trackerSQLStore) ResetToV6Test(ctx context.Context) error {
- var resetExprs = []string{
- `DROP TABLE IF EXISTS onlineaccounts`,
- `DROP TABLE IF EXISTS txtail`,
- `DROP TABLE IF EXISTS onlineroundparamstail`,
- `DROP TABLE IF EXISTS catchpointfirststageinfo`,
- }
-
- return s.pair.Wdb.AtomicContext(ctx, func(ctx context.Context, tx *sql.Tx) error {
- for _, stmt := range resetExprs {
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
- }
- return nil
- })
-}
-
-func (s *trackerSQLStore) Close() {
- s.pair.Close()
-}
-
-// Testing returns this scope, exposed as an interface with test functions
-func (txs sqlTransactionScope) Testing() trackerdb.TestTransactionScope {
- return txs
-}
-
-func (txs sqlTransactionScope) MakeCatchpointReaderWriter() (trackerdb.CatchpointReaderWriter, error) {
- return NewCatchpointSQLReaderWriter(txs.tx), nil
-}
-
-func (txs sqlTransactionScope) MakeAccountsReaderWriter() (trackerdb.AccountsReaderWriter, error) {
- return NewAccountsSQLReaderWriter(txs.tx), nil
-}
-
-// implements Testing interface
-func (txs sqlTransactionScope) MakeAccountsOptimizedReader() (trackerdb.AccountsReader, error) {
- return AccountsInitDbQueries(txs.tx)
-}
-
-func (txs sqlTransactionScope) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) {
- return MakeAccountsSQLWriter(txs.tx, hasAccounts, hasResources, hasKvPairs, hasCreatables)
-}
-
-func (txs sqlTransactionScope) MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (w trackerdb.OnlineAccountsWriter, err error) {
- return MakeOnlineAccountsSQLWriter(txs.tx, hasAccounts)
-}
-
-// implements Testing interface
-func (txs sqlTransactionScope) MakeOnlineAccountsOptimizedReader() (r trackerdb.OnlineAccountsReader, err error) {
- return OnlineAccountsInitDbQueries(txs.tx)
-}
-
-func (txs sqlTransactionScope) MakeMerkleCommitter(staging bool) (trackerdb.MerkleCommitter, error) {
- return MakeMerkleCommitter(txs.tx, staging)
-}
-
-func (txs sqlTransactionScope) MakeOrderedAccountsIter(accountCount int) trackerdb.OrderedAccountsIter {
- return MakeOrderedAccountsIter(txs.tx, accountCount)
-}
-
-func (txs sqlTransactionScope) MakeKVsIter(ctx context.Context) (trackerdb.KVsIter, error) {
- return MakeKVsIter(ctx, txs.tx)
-}
-
-func (txs sqlTransactionScope) MakeEncodedAccoutsBatchIter() trackerdb.EncodedAccountsBatchIter {
- return MakeEncodedAccoutsBatchIter(txs.tx)
-}
-
-func (txs sqlTransactionScope) MakeSpVerificationCtxReaderWriter() trackerdb.SpVerificationCtxReaderWriter {
- return makeStateProofVerificationReaderWriter(txs.tx, txs.tx)
-}
-
-func (txs sqlTransactionScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
- return RunMigrations(ctx, txs.tx, params, log, targetVersion)
-}
-
-func (txs sqlTransactionScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
- return db.ResetTransactionWarnDeadline(ctx, txs.tx, deadline)
-}
-
-// implements Testing interface
-func (txs sqlTransactionScope) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
- return AccountsInitTest(tb, txs.tx, initAccounts, proto)
-}
-
-// implements Testing interface
-func (txs sqlTransactionScope) AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
- return AccountsInitLightTest(tb, txs.tx, initAccounts, proto)
-}
-
-// Testing returns this scope, exposed as an interface with test functions
-func (bs sqlBatchScope) Testing() trackerdb.TestBatchScope {
- return bs
-}
-
-func (bs sqlBatchScope) MakeCatchpointWriter() (trackerdb.CatchpointWriter, error) {
- return NewCatchpointSQLReaderWriter(bs.tx), nil
-}
-
-func (bs sqlBatchScope) MakeAccountsWriter() (trackerdb.AccountsWriterExt, error) {
- return NewAccountsSQLReaderWriter(bs.tx), nil
-}
-
-func (bs sqlBatchScope) MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (trackerdb.AccountsWriter, error) {
- return MakeAccountsSQLWriter(bs.tx, hasAccounts, hasResources, hasKvPairs, hasCreatables)
-}
-
-// implements Testing interface
-func (bs sqlBatchScope) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
- return RunMigrations(ctx, bs.tx, params, log, targetVersion)
-}
-
-func (bs sqlBatchScope) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
- return db.ResetTransactionWarnDeadline(ctx, bs.tx, deadline)
-}
-
-// implements Testing interface
-func (bs sqlBatchScope) AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
- return AccountsInitTest(tb, bs.tx, initAccounts, proto)
-}
-
-// implements Testing interface
-func (bs sqlBatchScope) ModifyAcctBaseTest() error {
- return modifyAcctBaseTest(bs.tx)
-}
-
-// implements Testing interface
-func (bs sqlBatchScope) AccountsUpdateSchemaTest(ctx context.Context) (err error) {
- return AccountsUpdateSchemaTest(ctx, bs.tx)
-}
-
-func (bs sqlBatchScope) MakeSpVerificationCtxWriter() trackerdb.SpVerificationCtxWriter {
- return makeStateProofVerificationWriter(bs.tx)
-}
-
-func (ss sqlSnapshotScope) MakeAccountsReader() (trackerdb.AccountsReaderExt, error) {
- return NewAccountsSQLReaderWriter(ss.tx), nil
-}
-
-func (ss sqlSnapshotScope) MakeCatchpointReader() (trackerdb.CatchpointReader, error) {
- return NewCatchpointSQLReaderWriter(ss.tx), nil
-}
-
-func (ss sqlSnapshotScope) MakeCatchpointPendingHashesIterator(hashCount int) trackerdb.CatchpointPendingHashesIter {
- return MakeCatchpointPendingHashesIterator(hashCount, ss.tx)
-}
-
-func (ss sqlSnapshotScope) MakeSpVerificationCtxReader() trackerdb.SpVerificationCtxReader {
- return makeStateProofVerificationReader(ss.tx)
-}
diff --git a/ledger/store/trackerdb/sqlitedriver/testing.go b/ledger/store/trackerdb/sqlitedriver/testing.go
index 8d0a61afd7..a7d32bf0ca 100644
--- a/ledger/store/trackerdb/sqlitedriver/testing.go
+++ b/ledger/store/trackerdb/sqlitedriver/testing.go
@@ -18,7 +18,6 @@ package sqlitedriver
import (
"context"
- "database/sql"
"fmt"
"strings"
"testing"
@@ -33,59 +32,55 @@ import (
"github.com/stretchr/testify/require"
)
-// DbOpenTrackerTest opens a sqlite db file for testing purposes.
-func DbOpenTrackerTest(t testing.TB, inMemory bool) (trackerdb.TrackerStore, string) {
- fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64())
+// OpenForTesting opens a sqlite db file for testing purposes.
+// It set the logging to the test logger and uses a tmp directory associated to the test for the db.
+// The test tmp direction is automatically cleaned up by the golang test framework.
+func OpenForTesting(t testing.TB, inMemory bool) (trackerdb.Store, string) {
+ fn := fmt.Sprintf("%s/%s.%d", t.TempDir(), strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64())
- dbs, err := db.OpenPair(fn, inMemory)
+ store, err := Open(fn, inMemory, logging.TestingLog(t))
require.NoErrorf(t, err, "Filename : %s\nInMemory: %v", fn, inMemory)
- return &trackerSQLStore{dbs}, fn
-}
-
-// SetDbTrackerTestLogging sets a testing logger on a database.
-func SetDbTrackerTestLogging(t testing.TB, dbs trackerdb.TrackerStore) {
- dblogger := logging.TestingLog(t)
- dbs.SetLogger(dblogger)
+ return store, fn
}
// AccountsInitLightTest initializes an empty database for testing without the extra methods being called.
// implements Testing interface, test function only
-func AccountsInitLightTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
- newDB, err := accountsInit(tx, initAccounts, proto)
+func AccountsInitLightTest(tb testing.TB, e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
+ newDB, err := accountsInit(e, initAccounts, proto)
require.NoError(tb, err)
return newDB, err
}
// modifyAcctBaseTest tweaks the database to move backards.
// implements Testing interface, test function only
-func modifyAcctBaseTest(tx *sql.Tx) error {
- _, err := tx.Exec("update acctrounds set rnd = 1 WHERE id='acctbase' ")
+func modifyAcctBaseTest(e db.Executable) error {
+ _, err := e.Exec("update acctrounds set rnd = 1 WHERE id='acctbase' ")
return err
}
// AccountsInitTest initializes an empty database for testing.
// implements Testing interface, test function only
-func AccountsInitTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
- newDB, err := accountsInit(tx, initAccounts, config.Consensus[proto])
+func AccountsInitTest(tb testing.TB, e db.Executable, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool) {
+ newDB, err := accountsInit(e, initAccounts, config.Consensus[proto])
require.NoError(tb, err)
- err = accountsAddNormalizedBalance(tx, config.Consensus[proto])
+ err = accountsAddNormalizedBalance(e, config.Consensus[proto])
require.NoError(tb, err)
- err = accountsCreateResourceTable(context.Background(), tx)
+ err = accountsCreateResourceTable(context.Background(), e)
require.NoError(tb, err)
- err = performResourceTableMigration(context.Background(), tx, nil)
+ err = performResourceTableMigration(context.Background(), e, nil)
require.NoError(tb, err)
- err = accountsCreateOnlineAccountsTable(context.Background(), tx)
+ err = accountsCreateOnlineAccountsTable(context.Background(), e)
require.NoError(tb, err)
- err = accountsCreateTxTailTable(context.Background(), tx)
+ err = accountsCreateTxTailTable(context.Background(), e)
require.NoError(tb, err)
- err = performOnlineAccountsTableMigration(context.Background(), tx, nil, nil)
+ err = performOnlineAccountsTableMigration(context.Background(), e, nil, nil)
require.NoError(tb, err)
// since this is a test that starts from genesis, there is no tail that needs to be migrated.
@@ -94,41 +89,41 @@ func AccountsInitTest(tb testing.TB, tx *sql.Tx, initAccounts map[basics.Address
err = performTxTailTableMigration(context.Background(), nil, db.Accessor{})
require.NoError(tb, err)
- err = accountsCreateOnlineRoundParamsTable(context.Background(), tx)
+ err = accountsCreateOnlineRoundParamsTable(context.Background(), e)
require.NoError(tb, err)
- err = performOnlineRoundParamsTailMigration(context.Background(), tx, db.Accessor{}, true, proto)
+ err = performOnlineRoundParamsTailMigration(context.Background(), e, db.Accessor{}, true, proto)
require.NoError(tb, err)
- err = accountsCreateBoxTable(context.Background(), tx)
+ err = accountsCreateBoxTable(context.Background(), e)
require.NoError(tb, err)
- err = performKVStoreNullBlobConversion(context.Background(), tx)
+ err = performKVStoreNullBlobConversion(context.Background(), e)
require.NoError(tb, err)
return newDB
}
// AccountsUpdateSchemaTest adds some empty tables for tests to work with a "v6" store.
-func AccountsUpdateSchemaTest(ctx context.Context, tx *sql.Tx) (err error) {
- if err := accountsCreateOnlineAccountsTable(ctx, tx); err != nil {
+func AccountsUpdateSchemaTest(ctx context.Context, e db.Executable) (err error) {
+ if err := accountsCreateOnlineAccountsTable(ctx, e); err != nil {
return err
}
- if err := accountsCreateTxTailTable(ctx, tx); err != nil {
+ if err := accountsCreateTxTailTable(ctx, e); err != nil {
return err
}
- if err := accountsCreateOnlineRoundParamsTable(ctx, tx); err != nil {
+ if err := accountsCreateOnlineRoundParamsTable(ctx, e); err != nil {
return err
}
- if err := accountsCreateCatchpointFirstStageInfoTable(ctx, tx); err != nil {
+ if err := accountsCreateCatchpointFirstStageInfoTable(ctx, e); err != nil {
return err
}
// this line creates kvstore table, even if it is not required in accountDBVersion 6 -> 7
// or in later version where we need kvstore table, some tests will fail
- if err := accountsCreateBoxTable(ctx, tx); err != nil {
+ if err := accountsCreateBoxTable(ctx, e); err != nil {
return err
}
- if err := createStateProofVerificationTable(ctx, tx); err != nil {
+ if err := createStateProofVerificationTable(ctx, e); err != nil {
return err
}
return nil
diff --git a/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go b/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go
index a5c74274c6..3082887816 100644
--- a/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go
+++ b/ledger/store/trackerdb/sqlitedriver/trackerdbV2.go
@@ -29,6 +29,7 @@ import (
"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/crypto/merkletrie"
"github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util/db"
@@ -50,9 +51,9 @@ type trackerDBSchemaInitializer struct {
// RunMigrations initializes the accounts DB if needed and return current account round.
// as part of the initialization, it tests the current database schema version, and perform upgrade
// procedures to bring it up to the database schema supported by the binary.
-func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+func RunMigrations(ctx context.Context, e db.Executable, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
// check current database version.
- dbVersion, err := db.GetUserVersion(ctx, tx)
+ dbVersion, err := db.GetUserVersion(ctx, e)
if err != nil {
return trackerdb.InitParams{}, fmt.Errorf("trackerDBInitialize unable to read database schema version : %v", err)
}
@@ -78,61 +79,61 @@ func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log
// perform the initialization/upgrade
switch tu.version() {
case 0:
- err = tu.upgradeDatabaseSchema0(ctx, tx)
+ err = tu.upgradeDatabaseSchema0(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 0 : %v", err)
return
}
case 1:
- err = tu.upgradeDatabaseSchema1(ctx, tx)
+ err = tu.upgradeDatabaseSchema1(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 1 : %v", err)
return
}
case 2:
- err = tu.upgradeDatabaseSchema2(ctx, tx)
+ err = tu.upgradeDatabaseSchema2(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 2 : %v", err)
return
}
case 3:
- err = tu.upgradeDatabaseSchema3(ctx, tx)
+ err = tu.upgradeDatabaseSchema3(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 3 : %v", err)
return
}
case 4:
- err = tu.upgradeDatabaseSchema4(ctx, tx)
+ err = tu.upgradeDatabaseSchema4(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 4 : %v", err)
return
}
case 5:
- err = tu.upgradeDatabaseSchema5(ctx, tx)
+ err = tu.upgradeDatabaseSchema5(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 5 : %v", err)
return
}
case 6:
- err = tu.upgradeDatabaseSchema6(ctx, tx)
+ err = tu.upgradeDatabaseSchema6(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 6 : %v", err)
return
}
case 7:
- err = tu.upgradeDatabaseSchema7(ctx, tx)
+ err = tu.upgradeDatabaseSchema7(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 7 : %v", err)
return
}
case 8:
- err = tu.upgradeDatabaseSchema8(ctx, tx)
+ err = tu.upgradeDatabaseSchema8(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 8 : %v", err)
return
}
case 9:
- err = tu.upgradeDatabaseSchema9(ctx, tx)
+ err = tu.upgradeDatabaseSchema9(ctx, e)
if err != nil {
tu.log.Warnf("trackerDBInitialize failed to upgrade accounts database (ledger.tracker.sqlite) from schema 9 : %v", err)
return
@@ -147,10 +148,10 @@ func RunMigrations(ctx context.Context, tx *sql.Tx, params trackerdb.Params, log
return trackerdb.InitParams{SchemaVersion: tu.schemaVersion, VacuumOnStartup: tu.vacuumOnStartup}, nil
}
-func (tu *trackerDBSchemaInitializer) setVersion(ctx context.Context, tx *sql.Tx, version int32) (err error) {
+func (tu *trackerDBSchemaInitializer) setVersion(ctx context.Context, e db.Executable, version int32) (err error) {
oldVersion := tu.schemaVersion
tu.schemaVersion = version
- _, err = db.SetUserVersion(ctx, tx, tu.schemaVersion)
+ _, err = db.SetUserVersion(ctx, e, tu.schemaVersion)
if err != nil {
return fmt.Errorf("trackerDBInitialize unable to update database schema version from %d to %d: %v", oldVersion, version, err)
}
@@ -180,13 +181,13 @@ func (tu trackerDBSchemaInitializer) version() int32 {
// The accountbase would get initialized with the au.initAccounts
// The accounttotals would get initialized to align with the initialization account added to accountbase
// The acctrounds would get updated to indicate that the balance matches round 0
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context, tx *sql.Tx) (err error) {
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context, e db.Executable) (err error) {
tu.log.Infof("upgradeDatabaseSchema0 initializing schema")
- tu.newDatabase, err = accountsInit(tx, tu.InitAccounts, config.Consensus[tu.InitProto])
+ tu.newDatabase, err = accountsInit(e, tu.InitAccounts, config.Consensus[tu.InitProto])
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema0 unable to initialize schema : %v", err)
}
- return tu.setVersion(ctx, tx, 1)
+ return tu.setVersion(ctx, e, 1)
}
// upgradeDatabaseSchema1 upgrades the database schema from version 1 to version 2
@@ -204,7 +205,7 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema0(ctx context.Context
//
// This upgrade doesn't change any of the actual database schema ( i.e. tables, indexes ) but rather just performing
// a functional update to it's content.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context, tx *sql.Tx) (err error) {
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context, e db.Executable) (err error) {
var modifiedAccounts uint
if tu.newDatabase {
goto schemaUpdateComplete
@@ -212,14 +213,13 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context
// update accounts encoding.
tu.log.Infof("upgradeDatabaseSchema1 verifying accounts data encoding")
- modifiedAccounts, err = reencodeAccounts(ctx, tx)
+ modifiedAccounts, err = reencodeAccounts(ctx, e)
if err != nil {
return err
}
if modifiedAccounts > 0 {
- crw := NewCatchpointSQLReaderWriter(tx)
- arw := NewAccountsSQLReaderWriter(tx)
+ arw := NewAccountsSQLReaderWriter(e)
tu.log.Infof("upgradeDatabaseSchema1 reencoded %d accounts", modifiedAccounts)
@@ -230,26 +230,14 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema1(ctx context.Context
return fmt.Errorf("upgradeDatabaseSchema1 unable to reset account hashes : %v", err)
}
- tu.log.Infof("upgradeDatabaseSchema1 preparing queries")
- tu.log.Infof("upgradeDatabaseSchema1 resetting prior catchpoints")
- // delete the last catchpoint label if we have any.
- err = crw.WriteCatchpointStateString(ctx, trackerdb.CatchpointStateLastCatchpoint, "")
- if err != nil {
- return fmt.Errorf("upgradeDatabaseSchema1 unable to clear prior catchpoint : %v", err)
- }
-
- tu.log.Infof("upgradeDatabaseSchema1 deleting stored catchpoints")
- // delete catchpoints.
- err = crw.DeleteStoredCatchpoints(ctx, tu.Params.DbPathPrefix)
- if err != nil {
- return fmt.Errorf("upgradeDatabaseSchema1 unable to delete stored catchpoints : %v", err)
- }
+ // we no longer cleanup catchpoints here
+ // Note: we no longer support cleanly upgrading from version 1.
} else {
tu.log.Infof("upgradeDatabaseSchema1 found that no accounts needed to be reencoded")
}
schemaUpdateComplete:
- return tu.setVersion(ctx, tx, 2)
+ return tu.setVersion(ctx, e, 2)
}
// upgradeDatabaseSchema2 upgrades the database schema from version 2 to version 3
@@ -257,30 +245,30 @@ schemaUpdateComplete:
// This upgrade only enables the database vacuuming which will take place once the upgrade process is complete.
// If the user has already specified the OptimizeAccountsDatabaseOnStartup flag in the configuration file, this
// step becomes a no-op.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema2(ctx context.Context, tx *sql.Tx) (err error) {
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema2(ctx context.Context, e db.Executable) (err error) {
if !tu.newDatabase {
tu.vacuumOnStartup = true
}
// update version
- return tu.setVersion(ctx, tx, 3)
+ return tu.setVersion(ctx, e, 3)
}
// upgradeDatabaseSchema3 upgrades the database schema from version 3 to version 4,
// adding the normalizedonlinebalance column to the accountbase table.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema3(ctx context.Context, tx *sql.Tx) (err error) {
- err = accountsAddNormalizedBalance(tx, config.Consensus[tu.InitProto])
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema3(ctx context.Context, e db.Executable) (err error) {
+ err = accountsAddNormalizedBalance(e, config.Consensus[tu.InitProto])
if err != nil {
return err
}
// update version
- return tu.setVersion(ctx, tx, 4)
+ return tu.setVersion(ctx, e, 4)
}
// upgradeDatabaseSchema4 does not change the schema but migrates data:
// remove empty AccountData entries from accountbase table
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context, tx *sql.Tx) (err error) {
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context, e db.Executable) (err error) {
var numDeleted int64
var addresses []basics.Address
@@ -288,13 +276,13 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context
goto done
}
- numDeleted, addresses, err = removeEmptyAccountData(tx, tu.CatchpointEnabled)
+ numDeleted, addresses, err = removeEmptyAccountData(e, tu.CatchpointEnabled)
if err != nil {
return err
}
if tu.CatchpointEnabled && len(addresses) > 0 {
- mc, err := MakeMerkleCommitter(tx, false)
+ mc, err := MakeMerkleCommitter(e, false)
if err != nil {
// at this point record deleted and DB is pruned for account data
// if hash deletion fails just log it and do not abort startup
@@ -332,15 +320,15 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema4(ctx context.Context
done:
tu.log.Infof("upgradeDatabaseSchema4: deleted %d rows", numDeleted)
- return tu.setVersion(ctx, tx, 5)
+ return tu.setVersion(ctx, e, 5)
}
// upgradeDatabaseSchema5 upgrades the database schema from version 5 to version 6,
// adding the resources table and clearing empty catchpoint directories.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context, tx *sql.Tx) (err error) {
- arw := NewAccountsSQLReaderWriter(tx)
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context, e db.Executable) (err error) {
+ arw := NewAccountsSQLReaderWriter(e)
- err = accountsCreateResourceTable(ctx, tx)
+ err = accountsCreateResourceTable(ctx, e)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema5 unable to create resources table : %v", err)
}
@@ -360,7 +348,7 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context
tu.log.Infof("upgradeDatabaseSchema5 upgraded %d out of %d accounts [ %3.1f%% ]", processed, total, float64(processed)*100.0/float64(total))
}
- err = performResourceTableMigration(ctx, tx, migrationProcessLog)
+ err = performResourceTableMigration(ctx, e, migrationProcessLog)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema5 unable to complete data migration : %v", err)
}
@@ -372,46 +360,66 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema5(ctx context.Context
}
// update version
- return tu.setVersion(ctx, tx, 6)
+ return tu.setVersion(ctx, e, 6)
}
-func (tu *trackerDBSchemaInitializer) deleteUnfinishedCatchpoint(ctx context.Context, tx *sql.Tx) error {
- cts := NewCatchpointSQLReaderWriter(tx)
- // Delete an unfinished catchpoint if there is one.
- round, err := cts.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingCatchpoint)
+func (tu *trackerDBSchemaInitializer) deleteUnfinishedCatchpoint(ctx context.Context, e db.Executable) error {
+ // this method comes from a time when catchpoint internal state was stored alongside the ledger data.
+
+ // Note: this is duplicate legacy** code from catchpointdb.ReadCatchpointStateUint64
+ var round uint64
+ err := db.Retry(func() (err error) {
+ var v sql.NullInt64
+ err = e.QueryRowContext(ctx, "SELECT intval FROM catchpointstate WHERE id=?", catchpointdb.CatchpointStateWritingCatchpoint).Scan(&v)
+ if err == sql.ErrNoRows {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if v.Valid {
+ round = uint64(v.Int64)
+ }
+ return nil
+ })
if err != nil {
return err
}
- if round == 0 {
- return nil
+
+ relCatchpointFilePath := filepath.Join(catchpointdb.CatchpointDirName, catchpointdb.MakeCatchpointFilePath(basics.Round(round)))
+ err = catchpointdb.RemoveSingleCatchpointFileFromDisk(tu.DbPathPrefix, relCatchpointFilePath)
+ if err != nil {
+ return err
}
- relCatchpointFilePath := filepath.Join(
- trackerdb.CatchpointDirName,
- trackerdb.MakeCatchpointFilePath(basics.Round(round)))
- err = trackerdb.RemoveSingleCatchpointFileFromDisk(tu.DbPathPrefix, relCatchpointFilePath)
+ // Note: this is duplicate legacy** code from catchpointdb.WriteCatchpointStateUint64
+ _, err = e.ExecContext(ctx, "DELETE FROM catchpointstate WHERE id=?", catchpointdb.CatchpointStateWritingCatchpoint)
+ if err != nil {
+ return err
+ }
+ _, err = e.ExecContext(ctx, "INSERT OR REPLACE INTO catchpointstate(id, intval) VALUES(?, ?)", catchpointdb.CatchpointStateWritingCatchpoint, 0)
if err != nil {
return err
}
- return cts.WriteCatchpointStateUint64(ctx, trackerdb.CatchpointStateWritingCatchpoint, 0)
+ return nil
}
// upgradeDatabaseSchema6 upgrades the database schema from version 6 to version 7,
// adding a new onlineaccounts table
// TODO: onlineaccounts: upgrade as needed after switching to the final table version
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context, tx *sql.Tx) (err error) {
- err = accountsCreateOnlineAccountsTable(ctx, tx)
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context, e db.Executable) (err error) {
+ err = accountsCreateOnlineAccountsTable(ctx, e)
if err != nil {
return err
}
- err = accountsCreateTxTailTable(ctx, tx)
+ err = accountsCreateTxTailTable(ctx, e)
if err != nil {
return err
}
- err = accountsCreateOnlineRoundParamsTable(ctx, tx)
+ err = accountsCreateOnlineRoundParamsTable(ctx, e)
if err != nil {
return err
}
@@ -426,54 +434,54 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema6(ctx context.Context
lastProgressInfoMsg = time.Now()
tu.log.Infof("upgradeDatabaseSchema6 upgraded %d out of %d accounts [ %3.1f%% ]", processed, total, float64(processed)*100.0/float64(total))
}
- err = performOnlineAccountsTableMigration(ctx, tx, migrationProcessLog, tu.log)
+ err = performOnlineAccountsTableMigration(ctx, e, migrationProcessLog, tu.log)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema6 unable to complete online account data migration : %w", err)
}
if !tu.newDatabase {
- err = performTxTailTableMigration(ctx, tx, tu.BlockDb.Rdb)
+ err = performTxTailTableMigration(ctx, e, tu.BlockDb.Rdb)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema6 unable to complete transaction tail data migration : %w", err)
}
}
- err = performOnlineRoundParamsTailMigration(ctx, tx, tu.BlockDb.Rdb, tu.newDatabase, tu.InitProto)
+ err = performOnlineRoundParamsTailMigration(ctx, e, tu.BlockDb.Rdb, tu.newDatabase, tu.InitProto)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema6 unable to complete online round params data migration : %w", err)
}
- err = tu.deleteUnfinishedCatchpoint(ctx, tx)
+ err = tu.deleteUnfinishedCatchpoint(ctx, e)
if err != nil {
return err
}
- err = accountsCreateCatchpointFirstStageInfoTable(ctx, tx)
+ err = accountsCreateCatchpointFirstStageInfoTable(ctx, e)
if err != nil {
return err
}
- err = accountsCreateUnfinishedCatchpointsTable(ctx, tx)
+ err = accountsCreateUnfinishedCatchpointsTable(ctx, e)
if err != nil {
return err
}
// update version
- return tu.setVersion(ctx, tx, 7)
+ return tu.setVersion(ctx, e, 7)
}
// upgradeDatabaseSchema7 upgrades the database schema from version 7 to version 8.
// adding the kvstore table for box feature support.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema7(ctx context.Context, tx *sql.Tx) (err error) {
- err = accountsCreateBoxTable(ctx, tx)
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema7(ctx context.Context, e db.Executable) (err error) {
+ err = accountsCreateBoxTable(ctx, e)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema7 unable to create kvstore through createTables : %v", err)
}
- return tu.setVersion(ctx, tx, 8)
+ return tu.setVersion(ctx, e, 8)
}
// upgradeDatabaseSchema8 upgrades the database schema from version 8 to version 9,
// forcing a rebuild of the accounthashes table on betanet nodes. Otherwise it has no effect.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context, tx *sql.Tx) (err error) {
- arw := NewAccountsSQLReaderWriter(tx)
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context, e db.Executable) (err error) {
+ arw := NewAccountsSQLReaderWriter(e)
betanetGenesisHash, _ := crypto.DigestFromString("TBMBVTC7W24RJNNUZCF7LWZD2NMESGZEQSMPG5XQD7JY4O7JKVWQ")
if tu.GenesisHash == betanetGenesisHash && !tu.FromCatchpoint {
// reset hash round to 0, forcing catchpointTracker.initializeHashes to rebuild accounthashes
@@ -482,39 +490,39 @@ func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema8(ctx context.Context
return fmt.Errorf("upgradeDatabaseSchema8 unable to reset acctrounds table 'hashbase' round : %v", err)
}
}
- return tu.setVersion(ctx, tx, 9)
+ return tu.setVersion(ctx, e, 9)
}
// upgradeDatabaseSchema9 upgrades the database schema from version 9 to version 10,
// adding a new stateproofverification table,
// scrubbing out all nil values from kvstore table and replace with empty byte slice.
-func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema9(ctx context.Context, tx *sql.Tx) (err error) {
- err = createStateProofVerificationTable(ctx, tx)
+func (tu *trackerDBSchemaInitializer) upgradeDatabaseSchema9(ctx context.Context, e db.Executable) (err error) {
+ err = createStateProofVerificationTable(ctx, e)
if err != nil {
return err
}
- err = performKVStoreNullBlobConversion(ctx, tx)
+ err = performKVStoreNullBlobConversion(ctx, e)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema9 unable to replace kvstore nil entries with empty byte slices : %v", err)
}
- err = convertOnlineRoundParamsTail(ctx, tx)
+ err = convertOnlineRoundParamsTail(ctx, e)
if err != nil {
return fmt.Errorf("upgradeDatabaseSchema10 unable to convert onlineroundparamstail: %v", err)
}
// update version
- return tu.setVersion(ctx, tx, 10)
+ return tu.setVersion(ctx, e, 10)
}
func removeEmptyDirsOnSchemaUpgrade(dbDirectory string) (err error) {
- catchpointRootDir := filepath.Join(dbDirectory, trackerdb.CatchpointDirName)
+ catchpointRootDir := filepath.Join(dbDirectory, catchpointdb.CatchpointDirName)
if _, err := os.Stat(catchpointRootDir); os.IsNotExist(err) {
return nil
}
for {
- emptyDirs, err := trackerdb.GetEmptyDirs(catchpointRootDir)
+ emptyDirs, err := catchpointdb.GetEmptyDirs(catchpointRootDir)
if err != nil {
return err
}
diff --git a/ledger/store/trackerdb/store.go b/ledger/store/trackerdb/store.go
index 17e0e720aa..7001dab5bf 100644
--- a/ledger/store/trackerdb/store.go
+++ b/ledger/store/trackerdb/store.go
@@ -20,77 +20,152 @@ import (
"context"
"time"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/encoded"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/util/db"
)
-// BatchScope is the write scope to the store.
-type BatchScope interface {
- MakeCatchpointWriter() (CatchpointWriter, error)
- MakeAccountsWriter() (AccountsWriterExt, error)
- MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (AccountsWriter, error)
- ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error)
- Testing() TestBatchScope
- MakeSpVerificationCtxWriter() SpVerificationCtxWriter
+// Store is the interface for the tracker db.
+type Store interface {
+ ReaderWriter
+ // settings
+ SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error)
+ IsSharedCacheConnection() bool
+ // batch support
+ Batch(fn BatchFn) (err error)
+ BatchContext(ctx context.Context, fn BatchFn) (err error)
+ BeginBatch(ctx context.Context) (Batch, error)
+ // snapshot support
+ Snapshot(fn SnapshotFn) (err error)
+ SnapshotContext(ctx context.Context, fn SnapshotFn) (err error)
+ BeginSnapshot(ctx context.Context) (Snapshot, error)
+ // transaction support
+ Transaction(fn TransactionFn) (err error)
+ TransactionContext(ctx context.Context, fn TransactionFn) (err error)
+ BeginTransaction(ctx context.Context) (Transaction, error)
+ // maintenance
+ Vacuum(ctx context.Context) (stats db.VacuumStats, err error)
+ // testing
+ ResetToV6Test(ctx context.Context) error
+ // cleanup
+ Close()
}
-// SnapshotScope is the read scope to the store.
-type SnapshotScope interface {
+// Reader is the interface for the trackerdb read operations.
+type Reader interface {
MakeAccountsReader() (AccountsReaderExt, error)
- MakeCatchpointReader() (CatchpointReader, error)
- MakeCatchpointPendingHashesIterator(hashCount int) CatchpointPendingHashesIter
-
+ MakeAccountsOptimizedReader() (AccountsReader, error)
+ MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error)
MakeSpVerificationCtxReader() SpVerificationCtxReader
}
-// TransactionScope is the read/write scope to the store.
-type TransactionScope interface {
- MakeCatchpointReaderWriter() (CatchpointReaderWriter, error)
- MakeAccountsReaderWriter() (AccountsReaderWriter, error)
+// Writer is the interface for the trackerdb write operations.
+type Writer interface {
+ // trackerdb
+ MakeAccountsWriter() (AccountsWriterExt, error)
MakeAccountsOptimizedWriter(hasAccounts, hasResources, hasKvPairs, hasCreatables bool) (AccountsWriter, error)
- MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (w OnlineAccountsWriter, err error)
- MakeMerkleCommitter(staging bool) (MerkleCommitter, error)
+ MakeOnlineAccountsOptimizedWriter(hasAccounts bool) (OnlineAccountsWriter, error)
+ MakeSpVerificationCtxWriter() SpVerificationCtxWriter
+ // testing
+ Testing() WriterTestExt
+}
+
+// Catchpoint is currently holding most of the methods related to catchpoint.
+//
+// TODO: we still need to do a refactoring pass on catchpoint
+//
+// there are two distinct set of methods present:
+// - read/write ops for managing catchpoint data
+// - read/write internal catchup durable state (not sure if this should not be hidden)
+// - read ops on trackerdb to support building catchpoints
+// - write op for applying catchpoint to trackerdb
+// we should split these two sets of methods into two separate interfaces
+type Catchpoint interface {
+ // reader (used to generate a catchpoint)
+ MakeCatchpointPendingHashesIterator(hashCount int) CatchpointPendingHashesIter
MakeOrderedAccountsIter(accountCount int) OrderedAccountsIter
MakeKVsIter(ctx context.Context) (KVsIter, error)
MakeEncodedAccoutsBatchIter() EncodedAccountsBatchIter
- RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error)
- ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error)
- Testing() TestTransactionScope
- MakeSpVerificationCtxReaderWriter() SpVerificationCtxReaderWriter
+ // writer (used to apply a catchpoint)
+ MakeCatchpointWriter() (CatchpointApply, error)
+ // reader/writer
+ MakeMerkleCommitter(staging bool) (MerkleCommitter, error)
}
-// BatchFn is the callback lambda used in `Batch`.
-type BatchFn func(ctx context.Context, tx BatchScope) error
-
-// SnapshotFn is the callback lambda used in `Snapshot`.
-type SnapshotFn func(ctx context.Context, tx SnapshotScope) error
+type CatchpointApply interface {
+ Reset(ctx context.Context, newCatchup bool) error
+ Write(ctx context.Context, payload CatchpointPayload) (CatchpointReport, error)
+ Apply(ctx context.Context, balancesRound basics.Round, merkleRootRound basics.Round) error
+}
-// TransactionFn is the callback lambda used in `Transaction`.
-type TransactionFn func(ctx context.Context, tx TransactionScope) error
+type CatchpointPayload struct {
+ Accounts []NormalizedAccountBalance
+ KVRecords []encoded.KVRecordV6
+}
-// TrackerStore is the interface for the tracker db.
-type TrackerStore interface {
- SetLogger(log logging.Logger)
- SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error)
- IsSharedCacheConnection() bool
+type CatchpointReport struct {
+ BalancesWriteDuration time.Duration
+ CreatablesWriteDuration time.Duration
+ HashesWriteDuration time.Duration
+ KVWriteDuration time.Duration
+}
- Batch(fn BatchFn) (err error)
- BatchContext(ctx context.Context, fn BatchFn) (err error)
+// ReaderWriter is the interface for the trackerdb read/write operations.
+//
+// Some of the operatiosn available here might not be present in neither the Reader nor the Writer interfaces.
+// This is because some operations might require to be able to read and write at the same time.
+type ReaderWriter interface {
+ Reader
+ Writer
+ // init
+ RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error)
+ // Note: at the moment, catchpoint methods are only accesible via reader/writer
+ Catchpoint
+}
- Snapshot(fn SnapshotFn) (err error)
- SnapshotContext(ctx context.Context, fn SnapshotFn) (err error)
+// BatchScope is an atomic write-only scope to the store.
+type BatchScope interface {
+ Writer
+ ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error)
+}
- Transaction(fn TransactionFn) (err error)
- TransactionContext(ctx context.Context, fn TransactionFn) (err error)
+// Batch is an atomic write-only accecssor to the store.
+type Batch interface {
+ BatchScope
+ Commit() error
+ Close() error
+}
- MakeAccountsOptimizedReader() (AccountsReader, error)
- MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error)
+// SnapshotScope is an atomic read-only scope to the store.
+type SnapshotScope interface {
+ Reader
+}
- MakeCatchpointReaderWriter() (CatchpointReaderWriter, error)
+// Snapshot is an atomic read-only accecssor to the store.
+type Snapshot interface {
+ SnapshotScope
+ Close() error
+}
- Vacuum(ctx context.Context) (stats db.VacuumStats, err error)
- Close()
- CleanupTest(dbName string, inMemory bool)
+// TransactionScope is an atomic read/write scope to the store.
+type TransactionScope interface {
+ ReaderWriter
+ ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error)
+}
- ResetToV6Test(ctx context.Context) error
+// Transaction is an atomic read/write accecssor to the store.
+type Transaction interface {
+ TransactionScope
+ Commit() error
+ Close() error
}
+
+// BatchFn is the callback lambda used in `Batch`.
+type BatchFn func(ctx context.Context, tx BatchScope) error
+
+// SnapshotFn is the callback lambda used in `Snapshot`.
+type SnapshotFn func(ctx context.Context, tx SnapshotScope) error
+
+// TransactionFn is the callback lambda used in `Transaction`.
+type TransactionFn func(ctx context.Context, tx TransactionScope) error
diff --git a/ledger/store/trackerdb/testdb/testdb.go b/ledger/store/trackerdb/testdb/testdb.go
new file mode 100644
index 0000000000..ae85edca78
--- /dev/null
+++ b/ledger/store/trackerdb/testdb/testdb.go
@@ -0,0 +1,34 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testdb
+
+import (
+ "testing"
+
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/dualdriver"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
+)
+
+// OpenForTesting will create a testing store to be used on tests outside of the trackerdb package.
+func OpenForTesting(t testing.TB) trackerdb.Store {
+ primaryDB, _ := sqlitedriver.OpenForTesting(t, true)
+ secondaryDB := pebbledbdriver.OpenForTesting(t, true)
+
+ return dualdriver.MakeStore(primaryDB, secondaryDB)
+}
diff --git a/ledger/store/trackerdb/testinterface.go b/ledger/store/trackerdb/testinterface.go
index 961857e874..a406375a2f 100644
--- a/ledger/store/trackerdb/testinterface.go
+++ b/ledger/store/trackerdb/testinterface.go
@@ -23,41 +23,26 @@ import (
"github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger/ledgercore"
- "github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
)
// testinterface.go contains interface extensions specific to testing
// testing interfaces should be made accessible by calling the Testing() method
// on the related interface. Example:
-// testTx := tx.Testing()
+// testTx := tx.Testing()
// these can also be inlined:
-// tx.Testing.AccountsInitTest(...)
-
-// TestBatchScope is an interface to extend BatchScope with test-only methods
-type TestBatchScope interface {
- BatchScope
+// tx.Testing.AccountsInitTest(...)
+// WriterTestExt is an interface to extend Writer with test-only methods
+type WriterTestExt interface {
AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool)
+ AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error)
AccountsUpdateSchemaTest(ctx context.Context) (err error)
- RunMigrations(ctx context.Context, params Params, log logging.Logger, targetVersion int32) (mgr InitParams, err error)
ModifyAcctBaseTest() error
}
-// TestTransactionScope is an interface to extend TransactionScope with test-only methods
-type TestTransactionScope interface {
- TransactionScope
-
- MakeAccountsOptimizedReader() (AccountsReader, error)
- MakeOnlineAccountsOptimizedReader() (OnlineAccountsReader, error)
- AccountsInitTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto protocol.ConsensusVersion) (newDatabase bool)
- AccountsInitLightTest(tb testing.TB, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error)
-}
-
-// TestAccountsReaderExt is an interface to extend AccountsReaderExt with test-only methods
-type TestAccountsReaderExt interface {
- AccountsReaderExt
-
+// AccountsReaderTestExt is an interface to extend AccountsReaderExt with test-only methods
+type AccountsReaderTestExt interface {
AccountsAllTest() (bals map[basics.Address]basics.AccountData, err error)
CheckCreatablesTest(t *testing.T, iteration int, expectedDbImage map[basics.CreatableIndex]ledgercore.ModifiedCreatable)
}
diff --git a/ledger/store/trackerdb/testsuite/README.md b/ledger/store/trackerdb/testsuite/README.md
new file mode 100644
index 0000000000..15270cfde9
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/README.md
@@ -0,0 +1,29 @@
+!! **DO NOT IMPORT THIS MODULE** !!
+
+---
+
+This is the `generickv` tests implemented _outside_ the package.
+This is done to avoid cirucular dependencies on some of the testing wizardry taking place.
+Namely, making the tests polymorphic on each database implementation,
+so we can reuse the test suite against multiple backends.
+
+# Adding tests to the suite
+
+1. Use the following signature on your tests:
+
+```go
+func CustomTestDoingSomething(t *customT) {
+ // your test..
+}
+```
+
+The `customT` type behaves just like `testing.T` but has some extras.
+
+2. Register your test with the suite in the `init()` function in your test file.
+
+```go
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("something", CustomTestDoingSomething)
+}
+```
\ No newline at end of file
diff --git a/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go b/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go
new file mode 100644
index 0000000000..3c17c96b8e
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/accounts_ext_kv_test.go
@@ -0,0 +1,300 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/data/transactions"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("global-round-update", CustomTestRoundUpdate)
+ registerTest("global-totals", CustomTestTotals)
+ registerTest("txtail-update", CustomTestTxTail)
+ registerTest("online_accounts-round_params-update", CustomTestOnlineAccountParams)
+ registerTest("accounts-lookup_by_rowid", CustomTestAccountLookupByRowID)
+ registerTest("resources-lookup_by_rowid", CustomTestResourceLookupByRowID)
+}
+
+func CustomTestRoundUpdate(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // update the round
+ err = aw.UpdateAccountsRound(basics.Round(1))
+ require.NoError(t, err)
+
+ // read the round
+ rnd, err := ar.AccountsRound()
+ require.NoError(t, err)
+ require.Equal(t, basics.Round(1), rnd)
+}
+
+func CustomTestTotals(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ totals := ledgercore.AccountTotals{
+ Online: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 42}},
+ Offline: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 1000}},
+ NotParticipating: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 8}},
+ RewardsLevel: 9000,
+ }
+
+ // update the totals
+ err = aw.AccountsPutTotals(totals, false)
+ require.NoError(t, err)
+
+ // read the totals
+ readTotals, err := ar.AccountsTotals(context.Background(), false)
+ require.NoError(t, err)
+ require.Equal(t, totals, readTotals)
+
+ // generate some staging values
+ stagingTotals := ledgercore.AccountTotals{
+ Online: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 1}},
+ Offline: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 2}},
+ NotParticipating: ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 3}},
+ RewardsLevel: 4,
+ }
+
+ // update the (staging) totals
+ err = aw.AccountsPutTotals(stagingTotals, true)
+ require.NoError(t, err)
+
+ // read the totals
+ readTotals, err = ar.AccountsTotals(context.Background(), true)
+ require.NoError(t, err)
+ require.Equal(t, stagingTotals, readTotals)
+
+ // double check the live data is still there
+ readTotals, err = ar.AccountsTotals(context.Background(), false)
+ require.NoError(t, err)
+ require.Equal(t, totals, readTotals)
+}
+
+func CustomTestTxTail(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ baseRound := basics.Round(0)
+ roundData := []*trackerdb.TxTailRound{
+ {
+ TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-0")))},
+ },
+ {
+ TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-1")))},
+ },
+ {
+ TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-2")))},
+ },
+ }
+ // TODO: remove this conversion once we change the API to take the actual types
+ var rawRoundData [][]byte
+ for _, tail := range roundData {
+ raw := protocol.Encode(tail)
+ rawRoundData = append(rawRoundData, raw)
+ }
+
+ // write TxTail's
+ err = aw.TxtailNewRound(context.Background(), baseRound, rawRoundData, baseRound)
+ require.NoError(t, err)
+
+ // load TxTail's (error, must be the latest round)
+ _, _, _, err = ar.LoadTxTail(context.Background(), basics.Round(1))
+ require.Error(t, err)
+
+ // load TxTail's
+ txtails, hashes, readBaseRound, err := ar.LoadTxTail(context.Background(), basics.Round(2))
+ require.NoError(t, err)
+ require.Len(t, txtails, 3) // assert boundries
+ require.Equal(t, roundData[0], txtails[0]) // assert ordering
+ require.Len(t, hashes, 3)
+ require.Equal(t, basics.Round(0), readBaseRound)
+
+ // generate some more test data
+ roundData = []*trackerdb.TxTailRound{
+ {
+ TxnIDs: []transactions.Txid{transactions.Txid(crypto.Hash([]byte("tx-3")))},
+ },
+ }
+ // reset data
+ rawRoundData = make([][]byte, 0)
+ // TODO: remove this conversion once we change the API to take the actual types
+ for _, tail := range roundData {
+ raw := protocol.Encode(tail)
+ rawRoundData = append(rawRoundData, raw)
+ }
+ // write TxTail's (delete everything before round 2)
+ err = aw.TxtailNewRound(context.Background(), basics.Round(3), rawRoundData, basics.Round(2))
+ require.NoError(t, err)
+
+ // load TxTail's
+ txtails, hashes, readBaseRound, err = ar.LoadTxTail(context.Background(), basics.Round(3))
+ require.NoError(t, err)
+ require.Len(t, txtails, 2)
+ require.Len(t, hashes, 2)
+ require.Equal(t, basics.Round(2), readBaseRound)
+}
+
+func CustomTestOnlineAccountParams(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ startRound := basics.Round(0)
+ roundParams := []ledgercore.OnlineRoundParamsData{
+ {OnlineSupply: 100},
+ {OnlineSupply: 42},
+ {OnlineSupply: 9000},
+ }
+
+ // clean up the db before starting with the test
+ // Note: some engines might start with some data built-in data for round 0
+ err = aw.AccountsPruneOnlineRoundParams(basics.Round(42))
+ require.NoError(t, err)
+
+ // write round params
+ err = aw.AccountsPutOnlineRoundParams(roundParams, startRound)
+ require.NoError(t, err)
+
+ // read round params
+ readParams, endRound, err := ar.AccountsOnlineRoundParams()
+ require.NoError(t, err)
+ require.Len(t, readParams, 3) // assert boundries
+ require.Equal(t, roundParams[0], readParams[0]) // assert ordering
+ require.Equal(t, basics.Round(2), endRound) // check round
+
+ // prune params
+ err = aw.AccountsPruneOnlineRoundParams(basics.Round(1))
+ require.NoError(t, err)
+
+ // read round params (again, after prunning)
+ readParams, endRound, err = ar.AccountsOnlineRoundParams()
+ require.NoError(t, err)
+ require.Len(t, readParams, 2) // assert boundries
+ require.Equal(t, roundParams[1], readParams[0]) // assert ordering, and first item
+ require.Equal(t, basics.Round(2), endRound) // check round
+}
+
+func CustomTestAccountLookupByRowID(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA := trackerdb.BaseAccountData{
+ RewardsBase: 1000,
+ }
+ normBalanceA := dataA.NormalizedOnlineBalance(t.proto)
+ refA, err := aow.InsertAccount(addrA, normBalanceA, dataA)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // non-existing account
+ _, err = ar.LookupAccountRowID(RandomAddress())
+ require.Error(t, err)
+ require.Equal(t, err, trackerdb.ErrNotFound)
+
+ // read account
+ ref, err := ar.LookupAccountRowID(addrA)
+ require.NoError(t, err)
+ require.Equal(t, refA, ref)
+}
+
+func CustomTestResourceLookupByRowID(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+
+ // generate some test data
+ resDataA0 := trackerdb.MakeResourcesData(0)
+ resDataA0.SetAssetParams(basics.AssetParams{
+ Total: 100,
+ UnitName: "t",
+ AssetName: "test-asset",
+ Manager: addrA,
+ Reserve: addrA,
+ Freeze: addrA,
+ Clawback: addrA,
+ URL: "http://127.0.0.1/8000",
+ }, true)
+ resDataA0.SetAssetHolding(basics.AssetHolding{Amount: 10})
+ aidxResA0 := basics.CreatableIndex(0)
+ _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // non-existing resource
+ _, err = ar.LookupResourceDataByAddrID(refAccA, basics.CreatableIndex(100))
+ require.Error(t, err)
+ require.Equal(t, err, trackerdb.ErrNotFound)
+
+ // read resource
+ data, err := ar.LookupResourceDataByAddrID(refAccA, aidxResA0)
+ require.NoError(t, err)
+ // parse the raw data
+ var res trackerdb.ResourcesData
+ err = protocol.Decode(data, &res)
+ require.NoError(t, err)
+ // assert that we got the resource
+ require.Equal(t, resDataA0, res)
+
+ // read resource on nil account
+ _, err = ar.LookupResourceDataByAddrID(nil, basics.CreatableIndex(100))
+ require.Error(t, err)
+ require.Equal(t, err, trackerdb.ErrNotFound)
+}
diff --git a/ledger/store/trackerdb/testsuite/accounts_kv_test.go b/ledger/store/trackerdb/testsuite/accounts_kv_test.go
new file mode 100644
index 0000000000..0b8e74a715
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/accounts_kv_test.go
@@ -0,0 +1,480 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("accounts-crud", CustomTestAccountsCrud)
+ registerTest("resources-crud", CustomTestResourcesCrud)
+ registerTest("resources-query-all", CustomTestResourcesQueryAll)
+ registerTest("kv-crud", CustomTestAppKVCrud)
+ registerTest("creatables-crud", CustomTestCreatablesCrud)
+ registerTest("creatables-query-all", CustomTestCreatablesQueryAll)
+}
+
+func CustomTestAccountsCrud(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA := trackerdb.BaseAccountData{
+ RewardsBase: 1000,
+ }
+
+ // insert the account
+ normBalanceA := dataA.NormalizedOnlineBalance(t.proto)
+ refA, err := aow.InsertAccount(addrA, normBalanceA, dataA)
+ require.NoError(t, err)
+
+ // read the account
+ padA, err := aor.LookupAccount(addrA)
+ require.NoError(t, err)
+ require.Equal(t, addrA, padA.Addr) // addr is present and correct
+ require.Equal(t, refA, padA.Ref) // same ref as when we inserted it
+ require.Equal(t, dataA, padA.AccountData) // same data
+ require.Equal(t, expectedRound, padA.Round) // db round
+
+ // read the accounts "ref"
+ readRefA, err := ar.LookupAccountRowID(addrA)
+ require.NoError(t, err)
+ require.Equal(t, refA, readRefA) // same ref as when we inserted it
+
+ // update the account
+ dataA.RewardsBase = 98287
+ normBalanceA = dataA.NormalizedOnlineBalance(t.proto)
+ _, err = aow.UpdateAccount(refA, normBalanceA, dataA)
+ require.NoError(t, err)
+
+ // read updated account
+ padA, err = aor.LookupAccount(addrA)
+ require.NoError(t, err)
+ require.Equal(t, dataA, padA.AccountData) // same updated data
+
+ // delete account
+ _, err = aow.DeleteAccount(refA)
+ require.NoError(t, err)
+
+ // read deleted account
+ // Note: this is a bit counter-intuitive but lookup returns a value
+ // even when the account doesnt exist.
+ padA, err = aor.LookupAccount(addrA)
+ require.NoError(t, err)
+ require.Equal(t, addrA, padA.Addr) // the addr is there
+ require.Empty(t, padA.AccountData) // no data
+ require.Equal(t, expectedRound, padA.Round) // db round (this is present even if record does not exist)
+}
+
+func CustomTestResourcesCrud(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ //
+ // pre-fill the db with an account for testing
+ //
+
+ // account
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // generate some test data
+ resDataA0 := trackerdb.MakeResourcesData(0)
+ resDataA0.SetAssetParams(basics.AssetParams{
+ Total: 100,
+ UnitName: "t",
+ AssetName: "test-asset",
+ Manager: addrA,
+ Reserve: addrA,
+ Freeze: addrA,
+ Clawback: addrA,
+ URL: "http://127.0.0.1/8000",
+ }, true)
+ resDataA0.SetAssetHolding(basics.AssetHolding{Amount: 10})
+ aidxResA0 := basics.CreatableIndex(0)
+
+ // insert the resource
+ refResA0, err := aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+ require.NotNil(t, refResA0)
+
+ // read the resource
+ prdA0, err := aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable)
+ require.NoError(t, err)
+ require.Equal(t, aidxResA0, prdA0.Aidx) // aidx is present and correct
+ require.Equal(t, refAccA, prdA0.AcctRef) // acctRef is present and correct
+ require.Equal(t, resDataA0, prdA0.Data) // same data
+ require.Equal(t, expectedRound, prdA0.Round) // db round
+
+ // update the resource
+ resDataA0.Amount = 900
+ _, err = aow.UpdateResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+
+ // read updated resource
+ prdA0, err = aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable)
+ require.NoError(t, err)
+ require.Equal(t, resDataA0, prdA0.Data) // same updated data
+
+ // delete resource
+ _, err = aow.DeleteResource(refAccA, aidxResA0)
+ require.NoError(t, err)
+
+ // read deleted resource
+ // Note: this is a bit counter-intuitive but lookup returns a value
+ // even when the account doesnt exist.
+ prdA0, err = aor.LookupResources(addrA, aidxResA0, basics.AssetCreatable)
+ require.NoError(t, err)
+ require.Equal(t, aidxResA0, prdA0.Aidx) // the aidx is there
+ require.Nil(t, prdA0.AcctRef) // the account ref is not present
+ require.Equal(t, trackerdb.MakeResourcesData(0), prdA0.Data) // rnd 0, clean data
+ require.Equal(t, expectedRound, prdA0.Round) // db round (this is present even if record does not exist)
+}
+
+func CustomTestResourcesQueryAll(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, false)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ //
+ // pre-fill the db with an account for testing
+ //
+
+ // account A
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+
+ // resource A-0
+ resDataA0 := trackerdb.ResourcesData{}
+ aidxResA0 := basics.CreatableIndex(0)
+ _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+
+ // resource A-1
+ resDataA1 := trackerdb.ResourcesData{}
+ aidxResA1 := basics.CreatableIndex(1)
+ _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ prs, rnd, err := aor.LookupAllResources(addrA)
+ require.NoError(t, err)
+ require.Equal(t, aidxResA0, prs[0].Aidx)
+ require.Equal(t, aidxResA1, prs[1].Aidx)
+ require.Equal(t, expectedRound, prs[0].Round) // db round (inside resources)
+ require.Equal(t, expectedRound, rnd) // db round (from the return)
+}
+
+func CustomTestAppKVCrud(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, true, false)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ //
+ // pre-fill the db with an account for testing
+ //
+
+ // account
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+ // resource
+ resDataA0 := trackerdb.ResourcesData{}
+ aidxResA0 := basics.CreatableIndex(0)
+ refResA0, err := aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+ require.NotNil(t, refResA0)
+
+ //
+ // test
+ //
+
+ // insert the kv
+ kvKey := "foobar-mykey"
+ kvValue := []byte("1234")
+ err = aow.UpsertKvPair(kvKey, kvValue)
+ require.NoError(t, err)
+
+ // read the kv
+ pv1, err := aor.LookupKeyValue(kvKey)
+ require.NoError(t, err)
+ require.Equal(t, kvValue, pv1.Value) // same data
+ require.Equal(t, expectedRound, pv1.Round) // db round
+
+ // update the kv
+ kvValue = []byte("777")
+ err = aow.UpsertKvPair(kvKey, kvValue)
+ require.NoError(t, err)
+
+ // read updated kv
+ pv1, err = aor.LookupKeyValue(kvKey)
+ require.NoError(t, err)
+ require.Equal(t, kvValue, pv1.Value) // same data
+
+ // delete the kv
+ err = aow.DeleteKvPair(kvKey)
+ require.NoError(t, err)
+
+ // read deleted kv
+ require.NoError(t, err)
+
+ // read deleted kv
+ // Note: this is a bit counter-intuitive but lookup returns a value
+ // even when the record doesn't exist.
+ pv1, err = aor.LookupKeyValue(kvKey)
+ require.NoError(t, err)
+ require.Equal(t, expectedRound, pv1.Round) // db round (this is present even if record does not exist)
+}
+
+func CustomTestCreatablesCrud(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, true)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ //
+ // pre-fill the db with an account for testing
+ //
+
+ // account A
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+
+ // resource A-0
+ resDataA0 := trackerdb.ResourcesData{}
+ aidxResA0 := basics.CreatableIndex(0)
+ _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+
+ // resource A-1
+ resDataA1 := trackerdb.ResourcesData{}
+ aidxResA1 := basics.CreatableIndex(1)
+ _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // insert creator for A0
+ resA0ctype := basics.AssetCreatable
+ cRefA0, err := aow.InsertCreatable(aidxResA0, resA0ctype, addrA[:])
+ require.NoError(t, err)
+ require.NotNil(t, cRefA0)
+
+ // insert creator for A1
+ resA1ctype := basics.AppCreatable
+ cRefA1, err := aow.InsertCreatable(aidxResA1, resA1ctype, addrA[:])
+ require.NoError(t, err)
+ require.NotNil(t, cRefA1)
+
+ // lookup creator (correct ctype)
+ addr, ok, rnd, err := aor.LookupCreator(aidxResA0, basics.AssetCreatable)
+ require.NoError(t, err)
+ require.True(t, ok) // ok=true when it works
+ require.Equal(t, addrA, addr) // correct owner
+ require.Equal(t, expectedRound, rnd) // db round
+
+ // lookup creator (invalid ctype)
+ _, ok, rnd, err = aor.LookupCreator(aidxResA0, basics.AppCreatable)
+ require.NoError(t, err)
+ require.False(t, ok) // ok=false when its doesnt match
+ require.Equal(t, expectedRound, rnd) // db round (this is present even if record does not exist)
+
+ // lookup creator (unknown index)
+ _, ok, rnd, err = aor.LookupCreator(basics.CreatableIndex(999), basics.AppCreatable)
+ require.NoError(t, err)
+ require.False(t, ok) // ok=false when it doesn't exist
+ require.Equal(t, expectedRound, rnd) // db round (this is present even if record does not exist)
+
+}
+
+func CustomTestCreatablesQueryAll(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, true, false, true)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ //
+ // pre-fill the db with an account for testing
+ //
+
+ // account A
+ addrA := RandomAddress()
+ accDataA := trackerdb.BaseAccountData{RewardsBase: 1000}
+ refAccA, err := aow.InsertAccount(addrA, accDataA.NormalizedOnlineBalance(t.proto), accDataA)
+ require.NoError(t, err)
+
+ // resource A-0
+ resDataA0 := trackerdb.ResourcesData{}
+ aidxResA0 := basics.CreatableIndex(0)
+ _, err = aow.InsertResource(refAccA, aidxResA0, resDataA0)
+ require.NoError(t, err)
+
+ // resource A-1
+ resDataA1 := trackerdb.ResourcesData{}
+ aidxResA1 := basics.CreatableIndex(1)
+ _, err = aow.InsertResource(refAccA, aidxResA1, resDataA1)
+ require.NoError(t, err)
+
+ // resource A-2
+ resDataA2 := trackerdb.ResourcesData{}
+ aidxResA2 := basics.CreatableIndex(2)
+ _, err = aow.InsertResource(refAccA, aidxResA2, resDataA2)
+ require.NoError(t, err)
+
+ // creator for A0
+ resA0ctype := basics.AssetCreatable
+ cRefA0, err := aow.InsertCreatable(aidxResA0, resA0ctype, addrA[:])
+ require.NoError(t, err)
+ require.NotNil(t, cRefA0)
+
+ // creator for A1
+ resA1ctype := basics.AppCreatable
+ cRefA1, err := aow.InsertCreatable(aidxResA1, resA1ctype, addrA[:])
+ require.NoError(t, err)
+ require.NotNil(t, cRefA1)
+
+ // creator for A2
+ resA2ctype := basics.AppCreatable
+ cRefA2, err := aow.InsertCreatable(aidxResA2, resA2ctype, addrA[:])
+ require.NoError(t, err)
+ require.NotNil(t, cRefA2)
+
+ //
+ // test
+ //
+
+ // filter by type
+ cls, rnd, err := aor.ListCreatables(basics.CreatableIndex(99), 5, basics.AssetCreatable)
+ require.NoError(t, err)
+ require.Len(t, cls, 1) // only one asset
+ require.Equal(t, aidxResA0, cls[0].Index) // resource A-0
+ require.Equal(t, expectedRound, rnd) // db round
+
+ // with multiple results
+ cls, rnd, err = aor.ListCreatables(basics.CreatableIndex(99), 5, basics.AppCreatable)
+ require.NoError(t, err)
+ require.Len(t, cls, 2) // two apps
+ require.Equal(t, aidxResA2, cls[0].Index) // resource A-2
+ require.Equal(t, aidxResA1, cls[1].Index) // resource A-1
+ require.Equal(t, expectedRound, rnd) // db round
+
+ // limit results
+ cls, rnd, err = aor.ListCreatables(basics.CreatableIndex(99), 1, basics.AppCreatable)
+ require.NoError(t, err)
+ require.Len(t, cls, 1) // two apps
+ require.Equal(t, aidxResA2, cls[0].Index) // resource A-2
+ require.Equal(t, expectedRound, rnd) // db round
+
+ // filter maxId
+ cls, rnd, err = aor.ListCreatables(aidxResA2, 10, basics.AppCreatable)
+ require.NoError(t, err)
+ require.Len(t, cls, 2) // only one app since to that cidx
+ require.Equal(t, aidxResA2, cls[0].Index) // resource A-2 (checks for order too)
+ require.Equal(t, aidxResA1, cls[1].Index) // resource A-1
+ require.Equal(t, expectedRound, rnd) // db round
+
+}
diff --git a/ledger/store/trackerdb/testsuite/dbsemantics_test.go b/ledger/store/trackerdb/testsuite/dbsemantics_test.go
new file mode 100644
index 0000000000..b155b918a9
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/dbsemantics_test.go
@@ -0,0 +1,82 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("db-semantics-transaction", CustomTestTransaction)
+}
+
+// This test will ensure that transaction semantics carry the same meaning across all engine implementations.
+func CustomTestTransaction(t *customT) {
+ aow, err := t.db.MakeAccountsOptimizedWriter(true, false, false, false)
+ require.NoError(t, err)
+
+ aor, err := t.db.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA := trackerdb.BaseAccountData{
+ RewardsBase: 1000,
+ }
+
+ // insert the account
+ normBalanceA := dataA.NormalizedOnlineBalance(t.proto)
+ refA, err := aow.InsertAccount(addrA, normBalanceA, dataA)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ err = t.db.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
+ // create a scoped writer
+ aow, err := tx.MakeAccountsOptimizedWriter(true, false, false, false)
+ require.NoError(t, err)
+
+ // create a scoped reader
+ aor, err := tx.MakeAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ // read an account
+ padA, err := aor.LookupAccount(addrA)
+ require.NoError(t, err)
+ require.Equal(t, refA, padA.Ref) // same ref as when we inserted it
+
+ // update the account
+ dataA.RewardsBase = 98287
+ normBalanceA = dataA.NormalizedOnlineBalance(t.proto)
+ _, err = aow.UpdateAccount(refA, normBalanceA, dataA)
+ require.NoError(t, err)
+
+ return nil
+ })
+ require.NoError(t, err)
+
+ // read the updated record outside the transaction to make sure it was commited
+ padA, err := aor.LookupAccount(addrA)
+ require.NoError(t, err)
+ require.Equal(t, uint64(98287), padA.AccountData.RewardsBase) // same updated data
+}
diff --git a/ledger/store/trackerdb/testsuite/dual_test.go b/ledger/store/trackerdb/testsuite/dual_test.go
new file mode 100644
index 0000000000..68648b5f2b
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/dual_test.go
@@ -0,0 +1,36 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/testdb"
+)
+
+func TestDualEngines(t *testing.T) {
+ dbFactory := func(proto config.ConsensusParams) dbForTests {
+ db := testdb.OpenForTesting(t)
+ seedDb(t, db)
+
+ return db
+ }
+
+ // run the suite
+ runGenericTestsWithDB(t, dbFactory)
+}
diff --git a/ledger/store/trackerdb/testsuite/migration_test.go b/ledger/store/trackerdb/testsuite/migration_test.go
new file mode 100644
index 0000000000..10d9c74737
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/migration_test.go
@@ -0,0 +1,140 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("db-migration-check-basic", CustomTestChecBasicMigration)
+ // Disabled since it's technically broken the way its written.
+ // registerTest("db-migration-check-with-accounts", CustomTestCheckMigrationWithAccounts)
+}
+
+func CustomTestChecBasicMigration(t *customT) {
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // check round
+ round, err := ar.AccountsRound()
+ require.NoError(t, err)
+ require.Equal(t, basics.Round(0), round) // initialized to round 0
+
+ // check account totals
+ totals, err := ar.AccountsTotals(context.Background(), false)
+ require.NoError(t, err)
+ require.Equal(t, uint64(0), totals.RewardsLevel)
+ require.Equal(t, ledgercore.AlgoCount{}, totals.Online)
+ require.Equal(t, ledgercore.AlgoCount{}, totals.Offline)
+ require.Equal(t, ledgercore.AlgoCount{}, totals.NotParticipating)
+
+ // check tx-tails
+ txTailData, hashes, baseRound, err := ar.LoadTxTail(context.Background(), basics.Round(0))
+ require.NoError(t, err)
+ require.Len(t, txTailData, 0) // no data
+ require.Len(t, hashes, 0) // no data
+ require.Equal(t, basics.Round(1), baseRound) // (the impls return +1 at the end)
+
+ // check online accounts
+ oas, err := ar.OnlineAccountsAll(99)
+ require.NoError(t, err)
+ require.Len(t, oas, 0)
+
+ // check online round params
+ oparams, endRound, err := ar.AccountsOnlineRoundParams()
+ require.NoError(t, err)
+ require.Len(t, oparams, 1)
+ require.Equal(t, basics.Round(0), endRound)
+ require.Equal(t, uint64(0), oparams[0].OnlineSupply)
+ require.Equal(t, uint64(0), oparams[0].RewardsLevel)
+ require.Equal(t, protocol.ConsensusCurrentVersion, oparams[0].CurrentProtocol)
+}
+
+func makeAccountData(status basics.Status, algos basics.MicroAlgos) basics.AccountData {
+ ad := basics.AccountData{Status: status, MicroAlgos: algos}
+ if status == basics.Online {
+ ad.VoteFirstValid = 1
+ ad.VoteLastValid = 100_000
+ }
+ return ad
+}
+
+func CustomTestCheckMigrationWithAccounts(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // reset
+ aw.AccountsReset(context.Background())
+
+ initAccounts := make(map[basics.Address]basics.AccountData)
+
+ addrA := basics.Address(crypto.Hash([]byte("a")))
+ initAccounts[addrA] = makeAccountData(basics.Online, basics.MicroAlgos{Raw: 100})
+
+ addrB := basics.Address(crypto.Hash([]byte("b")))
+ initAccounts[addrB] = makeAccountData(basics.Online, basics.MicroAlgos{Raw: 42})
+
+ addrC := basics.Address(crypto.Hash([]byte("c")))
+ initAccounts[addrC] = makeAccountData(basics.Offline, basics.MicroAlgos{Raw: 30})
+
+ addrD := basics.Address(crypto.Hash([]byte("d")))
+ initAccounts[addrD] = makeAccountData(basics.NotParticipating, basics.MicroAlgos{Raw: 7})
+
+ params := trackerdb.Params{
+ InitProto: protocol.ConsensusCurrentVersion,
+ InitAccounts: initAccounts,
+ }
+
+ // re-run migrations
+ _, err = t.db.RunMigrations(context.Background(), params, logging.TestingLog(t), trackerdb.AccountDBVersion)
+ require.NoError(t, err)
+
+ // check account totals
+ totals, err := ar.AccountsTotals(context.Background(), false)
+ require.NoError(t, err)
+ require.Equal(t, uint64(0), totals.RewardsLevel)
+ require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 142}}, totals.Online)
+ require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 30}}, totals.Offline)
+ require.Equal(t, ledgercore.AlgoCount{Money: basics.MicroAlgos{Raw: 7}}, totals.NotParticipating)
+
+ // check online accounts
+ oas, err := ar.OnlineAccountsAll(99)
+ require.NoError(t, err)
+ require.Len(t, oas, 2)
+
+ // check online round params
+ oparams, endRound, err := ar.AccountsOnlineRoundParams()
+ require.NoError(t, err)
+ require.Len(t, oparams, 1)
+ require.Equal(t, basics.Round(0), endRound)
+ require.Equal(t, uint64(142), oparams[0].OnlineSupply)
+ require.Equal(t, uint64(0), oparams[0].RewardsLevel)
+ require.Equal(t, protocol.ConsensusCurrentVersion, oparams[0].CurrentProtocol)
+}
diff --git a/ledger/store/trackerdb/testsuite/mockdb_test.go b/ledger/store/trackerdb/testsuite/mockdb_test.go
new file mode 100644
index 0000000000..cc9397f2b4
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/mockdb_test.go
@@ -0,0 +1,34 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+)
+
+func TestMockDB(t *testing.T) {
+ dbFactory := func(proto config.ConsensusParams) dbForTests {
+ db := makeMockDB(proto)
+
+ seedDb(t, db)
+
+ return db
+ }
+ runGenericTestsWithDB(t, dbFactory)
+}
diff --git a/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go b/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go
new file mode 100644
index 0000000000..ed444794da
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/onlineaccounts_kv_test.go
@@ -0,0 +1,554 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("online-accounts-write-read", CustomTestOnlineAccountsWriteRead)
+ registerTest("online-accounts-all", CustomTestOnlineAccountsAll)
+ registerTest("online-accounts-top", CustomTestAccountsOnlineTop)
+ registerTest("online-accounts-get-by-addr", CustomTestLookupOnlineAccountDataByAddress)
+ registerTest("online-accounts-history", CustomTestOnlineAccountHistory)
+ registerTest("online-accounts-totals", CustomTestOnlineAccountTotals)
+ registerTest("online-accounts-delete", CustomTestOnlineAccountsDelete)
+ registerTest("online-accounts-expired", CustomTestAccountsOnlineExpired)
+}
+
+func CustomTestOnlineAccountsWriteRead(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ oar, err := t.db.MakeOnlineAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ updRoundA := uint64(400)
+ lastValidA := uint64(500)
+ dataA := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA := dataA.NormalizedOnlineBalance(t.proto)
+
+ // write
+ refA, err := oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA)
+ require.NoError(t, err)
+
+ // read
+ poA, err := oar.LookupOnline(addrA, basics.Round(updRoundA))
+ require.NoError(t, err)
+ require.Equal(t, addrA, poA.Addr)
+ require.Equal(t, refA, poA.Ref)
+ require.Equal(t, dataA, poA.AccountData)
+ require.Equal(t, basics.Round(updRoundA), poA.UpdRound) // check the "update round" was read
+ require.Equal(t, expectedRound, poA.Round)
+
+ // write a new version
+ dataA.MicroAlgos = basics.MicroAlgos{Raw: uint64(321)}
+ normalizedBalA = dataA.NormalizedOnlineBalance(t.proto)
+ updRoundA = uint64(450)
+ _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA)
+ require.NoError(t, err)
+
+ // read (latest)
+ poA, err = oar.LookupOnline(addrA, basics.Round(500))
+ require.NoError(t, err)
+ require.Equal(t, dataA, poA.AccountData) // check the data is from the new version
+ require.Equal(t, basics.Round(updRoundA), poA.UpdRound) // check the "update round"
+
+ // read (original)
+ poA, err = oar.LookupOnline(addrA, basics.Round(405))
+ require.NoError(t, err)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(100)}, poA.AccountData.MicroAlgos) // check the data is from the new version
+ require.Equal(t, basics.Round(400), poA.UpdRound) // check the "update round"
+
+ // read (at upd round)
+ poA, err = oar.LookupOnline(addrA, basics.Round(450))
+ require.NoError(t, err)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(321)}, poA.AccountData.MicroAlgos) // check the data is from the new version
+ require.Equal(t, basics.Round(450), poA.UpdRound) // check the "update round"
+}
+
+func CustomTestOnlineAccountHistory(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ oar, err := t.db.MakeOnlineAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA1 := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(20)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto)
+
+ refA1, err := oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(2), uint64(2))
+ require.NoError(t, err)
+
+ // generate some test data
+ dataA2 := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto)
+
+ refA2, err := oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(3), uint64(3))
+ require.NoError(t, err)
+
+ // generate some test data
+ addrB := RandomAddress()
+ dataB1 := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(75)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto)
+
+ refB1, err := oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(3), uint64(3))
+ require.NoError(t, err)
+
+ //
+ // the test
+ //
+
+ resultsA, rnd, err := oar.LookupOnlineHistory(addrA)
+ require.NoError(t, err)
+ require.Equal(t, expectedRound, rnd) // check the db round
+ require.Len(t, resultsA, 2)
+ require.Equal(t, basics.Round(2), resultsA[0].UpdRound) // check ordering
+ require.Equal(t, basics.Round(3), resultsA[1].UpdRound) // check ordering
+ // check item fields
+ require.Empty(t, resultsA[0].Round) // check the db round is not set
+ require.Equal(t, addrA, resultsA[0].Addr) // check addr
+ require.Equal(t, refA1, resultsA[0].Ref) // check ref
+ require.Equal(t, dataA1, resultsA[0].AccountData) // check data
+ // check ref is valid on all
+ require.Equal(t, refA2, resultsA[1].Ref) // check ref
+
+ // check for B
+ resultsB, _, err := oar.LookupOnlineHistory(addrB)
+ require.NoError(t, err)
+ require.Len(t, resultsB, 1)
+ require.Equal(t, addrB, resultsB[0].Addr) // check addr
+ require.Equal(t, refB1, resultsB[0].Ref) // check ref
+}
+
+func CustomTestOnlineAccountsAll(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ voteLastValid := uint64(0)
+
+ // generate some test data
+ addrA := basics.Address(crypto.Hash([]byte("a")))
+ dataA0 := trackerdb.BaseOnlineAccountData{
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(200)},
+ }
+ _, err = oaw.InsertOnlineAccount(addrA, dataA0.NormalizedOnlineBalance(t.proto), dataA0, 0, voteLastValid)
+ require.NoError(t, err)
+
+ dataA1 := trackerdb.BaseOnlineAccountData{
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(250)},
+ }
+ _, err = oaw.InsertOnlineAccount(addrA, dataA1.NormalizedOnlineBalance(t.proto), dataA1, 1, voteLastValid)
+ require.NoError(t, err)
+
+ addrB := basics.Address(crypto.Hash([]byte("b")))
+ dataB := trackerdb.BaseOnlineAccountData{
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ }
+ _, err = oaw.InsertOnlineAccount(addrB, dataB.NormalizedOnlineBalance(t.proto), dataB, 0, voteLastValid)
+ require.NoError(t, err)
+
+ addrC := basics.Address(crypto.Hash([]byte("c")))
+ dataC := trackerdb.BaseOnlineAccountData{
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(30)},
+ }
+ _, err = oaw.InsertOnlineAccount(addrC, dataC.NormalizedOnlineBalance(t.proto), dataC, 0, voteLastValid)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // read all accounts (with max accounts)
+ poA, err := ar.OnlineAccountsAll(2)
+ require.NoError(t, err)
+ require.Len(t, poA, 3) // account A has 2 records + 1 record from account B
+
+ require.Equal(t, addrA, poA[0].Addr)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(200)}, poA[0].AccountData.MicroAlgos)
+ require.Equal(t, basics.Round(0), poA[0].UpdRound)
+
+ require.Equal(t, addrA, poA[1].Addr)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(250)}, poA[1].AccountData.MicroAlgos)
+ require.Equal(t, basics.Round(1), poA[1].UpdRound)
+
+ require.Equal(t, addrB, poA[2].Addr)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(100)}, poA[2].AccountData.MicroAlgos)
+ require.Equal(t, basics.Round(0), poA[2].UpdRound)
+}
+
+func CustomTestAccountsOnlineTop(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ var testData []basics.Address
+ updRound := uint64(0)
+ for i := 0; i < 10; i++ {
+ addr := RandomAddress()
+ microAlgos := basics.MicroAlgos{Raw: uint64(10 + i*100)}
+ rewardBase := uint64(200 + i)
+ lastValid := uint64(500 + i)
+ data := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: microAlgos,
+ RewardsBase: rewardBase,
+ }
+ normalizedBal := data.NormalizedOnlineBalance(t.proto)
+
+ // write
+ _, err := oaw.InsertOnlineAccount(addr, normalizedBal, data, updRound, lastValid)
+ require.NoError(t, err)
+
+ testData = append(testData, addr)
+ }
+
+ // read (all)
+ poA, err := ar.AccountsOnlineTop(basics.Round(0), 0, 10, t.proto)
+ require.NoError(t, err)
+ require.Contains(t, poA, testData[9]) // most money
+ require.Contains(t, poA, testData[0]) // least money
+
+ // read (just a few)
+ poA, err = ar.AccountsOnlineTop(basics.Round(0), 1, 2, t.proto)
+ require.NoError(t, err)
+ require.Len(t, poA, 2)
+ require.Contains(t, poA, testData[8]) // (second most money, we skipped 1)
+ require.Contains(t, poA, testData[7]) // (third, we only have 2 items)
+}
+
+func CustomTestLookupOnlineAccountDataByAddress(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ updRoundA := uint64(400)
+ lastValidA := uint64(500)
+ dataA := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA := dataA.NormalizedOnlineBalance(t.proto)
+
+ refA, err := oaw.InsertOnlineAccount(addrA, normalizedBalA, dataA, updRoundA, lastValidA)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // check non-existing account
+ nonExistingAddr := RandomAddress()
+ _, _, err = ar.LookupOnlineAccountDataByAddress(nonExistingAddr)
+ require.Error(t, err)
+ require.Equal(t, trackerdb.ErrNotFound, err) // check the error type
+
+ // read existing addr
+ readRef, readData, err := ar.LookupOnlineAccountDataByAddress(addrA)
+ require.NoError(t, err)
+ require.Equal(t, refA, readRef) // check ref is the same
+ // the method returns raw bytes, parse them
+ var badA trackerdb.BaseOnlineAccountData
+ err = protocol.Decode(readData, &badA)
+ require.NoError(t, err)
+ require.Equal(t, dataA, badA)
+}
+
+func CustomTestOnlineAccountTotals(t *customT) {
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ oaor, err := t.db.MakeOnlineAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ roundParams := []ledgercore.OnlineRoundParamsData{
+ {OnlineSupply: 100},
+ {OnlineSupply: 42},
+ {OnlineSupply: 9000},
+ }
+ err = aw.AccountsPutOnlineRoundParams(roundParams, basics.Round(3))
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // lookup totals
+ totals, err := oaor.LookupOnlineTotalsHistory(basics.Round(4))
+ require.NoError(t, err)
+ require.Equal(t, basics.MicroAlgos{Raw: uint64(42)}, totals)
+
+ // lookup not found
+ _, err = oaor.LookupOnlineTotalsHistory(basics.Round(121))
+ require.Error(t, err)
+ require.Equal(t, trackerdb.ErrNotFound, err)
+}
+
+func CustomTestOnlineAccountsDelete(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ oar, err := t.db.MakeOnlineAccountsOptimizedReader()
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ aw, err := t.db.MakeAccountsWriter()
+ require.NoError(t, err)
+
+ // set round to 3
+ // Note: this will be used to check that we read the round
+ expectedRound := basics.Round(3)
+ err = aw.UpdateAccountsRound(expectedRound)
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA1 := trackerdb.BaseOnlineAccountData{
+ // some value so its NOT empty
+ BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(20)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(0), uint64(2))
+ require.NoError(t, err)
+
+ // generate some test data
+ dataA2 := trackerdb.BaseOnlineAccountData{
+ // some value so its NOT empty
+ BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(1), uint64(3))
+ require.NoError(t, err)
+
+ // generate some test data
+ addrB := RandomAddress()
+ dataB1 := trackerdb.BaseOnlineAccountData{
+ // some value so its NOT empty
+ BaseVotingData: trackerdb.BaseVotingData{VoteKeyDilution: 1},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(75)},
+ RewardsBase: uint64(200),
+ }
+ normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(2), uint64(3))
+ require.NoError(t, err)
+
+ // timeline
+ // round 0: A touched [0]
+ // round 1: A touched [1,0]
+ // round 2: B touched [2] + A remains [1,0]
+
+ //
+ // the test
+ //
+
+ // delete before 0 (no changes)
+ err = aw.OnlineAccountsDelete(basics.Round(0))
+ require.NoError(t, err)
+
+ // check they are all there
+ oas, err := ar.AccountsOnlineTop(basics.Round(0), 0, 10, t.proto)
+ require.NoError(t, err)
+ require.Len(t, oas, 1)
+ require.Equal(t, oas[addrA].MicroAlgos, basics.MicroAlgos{Raw: uint64(20)}) // check item
+ // read the accounts directly
+ poaA, err := oar.LookupOnline(addrA, basics.Round(0))
+ require.NoError(t, err)
+ require.NotNil(t, poaA.Ref) // A was found
+
+ // delete before round 1
+ err = aw.OnlineAccountsDelete(basics.Round(1))
+ require.NoError(t, err)
+
+ // check they are all there
+ oas, err = ar.AccountsOnlineTop(basics.Round(1), 0, 10, t.proto)
+ require.NoError(t, err)
+ require.Len(t, oas, 1)
+ require.Equal(t, oas[addrA].MicroAlgos, basics.MicroAlgos{Raw: uint64(100)}) // check item
+ // read the accounts directly
+ poaA, err = oar.LookupOnline(addrA, basics.Round(1))
+ require.NoError(t, err)
+ require.NotNil(t, poaA.Ref) // A was found
+
+ // delete before round 2
+ err = aw.OnlineAccountsDelete(basics.Round(2))
+ require.NoError(t, err)
+
+ // check they are all there
+ oas, err = ar.AccountsOnlineTop(basics.Round(2), 0, 10, t.proto)
+ require.NoError(t, err)
+ require.Len(t, oas, 2)
+ require.Equal(t, oas[addrB].MicroAlgos, basics.MicroAlgos{Raw: uint64(75)}) // check item
+ // read the accounts directly
+ poaA, err = oar.LookupOnline(addrA, basics.Round(2))
+ require.NoError(t, err)
+ require.NotNil(t, poaA.Ref) // A is still found, the latest record is kept
+ require.Equal(t, basics.Round(1), poaA.UpdRound) // the latest we find is at 1
+ poaB, err := oar.LookupOnline(addrB, basics.Round(2))
+ require.NoError(t, err)
+ require.NotNil(t, poaB.Ref) // B was found
+}
+
+func CustomTestAccountsOnlineExpired(t *customT) {
+ oaw, err := t.db.MakeOnlineAccountsOptimizedWriter(true)
+ require.NoError(t, err)
+
+ ar, err := t.db.MakeAccountsReader()
+ require.NoError(t, err)
+
+ // generate some test data
+ addrA := RandomAddress()
+ dataA1 := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(2)},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(20)},
+ RewardsBase: uint64(0),
+ }
+ normalizedBalA1 := dataA1.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA1, dataA1, uint64(0), uint64(2))
+ require.NoError(t, err)
+
+ // generate some test data
+ dataA2 := trackerdb.BaseOnlineAccountData{
+ BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(5)},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(100)},
+ RewardsBase: uint64(0),
+ }
+ normalizedBalA2 := dataA2.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrA, normalizedBalA2, dataA2, uint64(1), uint64(5))
+ require.NoError(t, err)
+
+ // generate some test data
+ addrB := RandomAddress()
+ dataB1 := trackerdb.BaseOnlineAccountData{
+ // some value so its NOT empty
+ BaseVotingData: trackerdb.BaseVotingData{VoteLastValid: basics.Round(7)},
+ MicroAlgos: basics.MicroAlgos{Raw: uint64(75)},
+ RewardsBase: uint64(0),
+ }
+ normalizedBalB1 := dataB1.NormalizedOnlineBalance(t.proto)
+
+ _, err = oaw.InsertOnlineAccount(addrB, normalizedBalB1, dataB1, uint64(2), uint64(7))
+ require.NoError(t, err)
+
+ // timeline
+ // round 0: A touched [0] // A expires at 2
+ // round 1: A touched [1,0] // A expires at 5
+ // round 2: B touched [2] + A remains [1,0] // A expires at 5, B expires at 7
+
+ //
+ // the test
+ //
+
+ // read (none)
+ expAccts, err := ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(0), t.proto, 0)
+ require.NoError(t, err)
+ require.Empty(t, expAccts)
+
+ // read (at acct round, voteRnd > lastValid)
+ expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(4), t.proto, 0)
+ require.NoError(t, err)
+ require.Len(t, expAccts, 1)
+ require.Equal(t, expAccts[addrA].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(20)}) // check item
+
+ // read (at acct round, voteRnd = lastValid)
+ expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(2), t.proto, 0)
+ require.NoError(t, err)
+ require.Empty(t, expAccts)
+
+ // read (at acct round, voteRnd < lastValid)
+ expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(0), basics.Round(1), t.proto, 0)
+ require.NoError(t, err)
+ require.Empty(t, expAccts)
+
+ // read (take latest exp value)
+ expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(1), basics.Round(4), t.proto, 0)
+ require.NoError(t, err)
+ require.Len(t, expAccts, 0)
+
+ // read (all)
+ expAccts, err = ar.ExpiredOnlineAccountsForRound(basics.Round(3), basics.Round(20), t.proto, 0)
+ require.Len(t, expAccts, 2)
+ require.Equal(t, expAccts[addrA].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(100)}) // check item
+ require.Equal(t, expAccts[addrB].MicroAlgosWithRewards, basics.MicroAlgos{Raw: uint64(75)}) // check item
+}
diff --git a/ledger/store/trackerdb/testsuite/pebbledb_test.go b/ledger/store/trackerdb/testsuite/pebbledb_test.go
new file mode 100644
index 0000000000..56f614b7e9
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/pebbledb_test.go
@@ -0,0 +1,44 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPebbleDB(t *testing.T) {
+ dbFactory := func(proto config.ConsensusParams) dbForTests {
+ // create a tmp dir for the db, the testing runtime will clean it up automatically
+ dir := fmt.Sprintf("%s/db", t.TempDir())
+
+ db, err := pebbledbdriver.Open(dir, false, proto, logging.TestingLog(t))
+ require.NoError(t, err)
+
+ seedDb(t, db)
+
+ return db
+ }
+
+ // run the suite
+ runGenericTestsWithDB(t, dbFactory)
+}
diff --git a/ledger/store/trackerdb/testsuite/sqlitedb_test.go b/ledger/store/trackerdb/testsuite/sqlitedb_test.go
new file mode 100644
index 0000000000..c4a3d1a7b8
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/sqlitedb_test.go
@@ -0,0 +1,43 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSqliteDB(t *testing.T) {
+ dbFactory := func(config.ConsensusParams) dbForTests {
+ // create a tmp dir for the db, the testing runtime will clean it up automatically
+ fn := fmt.Sprintf("%s/tracker-db.sqlite", t.TempDir())
+ db, err := sqlitedriver.Open(fn, false, logging.TestingLog(t))
+ require.NoError(t, err)
+
+ seedDb(t, db)
+
+ return db
+ }
+
+ // run the suite
+ runGenericTestsWithDB(t, dbFactory)
+}
diff --git a/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go b/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go
new file mode 100644
index 0000000000..bc42141735
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/stateproofs_kv_test.go
@@ -0,0 +1,130 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+import (
+ "context"
+
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/stretchr/testify/require"
+)
+
+func init() {
+ // register tests that will run on each KV implementation
+ registerTest("stateproofs-crud", CustomTestStateproofsReadWrite)
+ registerTest("stateproofs-query-all", CustomTestStateproofsQueryAll)
+}
+
+func CustomTestStateproofsReadWrite(t *customT) {
+ spw := t.db.MakeSpVerificationCtxWriter()
+ spr := t.db.MakeSpVerificationCtxReader()
+
+ //
+ // test
+ //
+
+ // store no items
+ err := spw.StoreSPContexts(context.Background(), []*ledgercore.StateProofVerificationContext{})
+ require.NoError(t, err)
+
+ // store some items
+ vcs := []*ledgercore.StateProofVerificationContext{
+ {
+ LastAttestedRound: basics.Round(0),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 42},
+ },
+ {
+ LastAttestedRound: basics.Round(1),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 100},
+ },
+ {
+ LastAttestedRound: basics.Round(2),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 200},
+ },
+ }
+ err = spw.StoreSPContexts(context.Background(), vcs)
+ require.NoError(t, err)
+
+ // read non-existing item
+ vc, err := spr.LookupSPContext(basics.Round(9000))
+ require.Error(t, err)
+ require.Equal(t, trackerdb.ErrNotFound, err)
+
+ // read back a single item
+ vc, err = spr.LookupSPContext(basics.Round(0))
+ require.NoError(t, err)
+ require.Equal(t, basics.Round(0), vc.LastAttestedRound) // check round is set
+ require.Equal(t, basics.MicroAlgos{Raw: 42}, vc.OnlineTotalWeight) // check payload is read
+
+ // delete some items
+ err = spw.DeleteOldSPContexts(context.Background(), basics.Round(1))
+ require.NoError(t, err)
+
+ // read delete items
+ vc, err = spr.LookupSPContext(basics.Round(0))
+ require.Error(t, err)
+ require.Equal(t, trackerdb.ErrNotFound, err)
+
+ // read back remaining items
+ vc, err = spr.LookupSPContext(basics.Round(1))
+ require.NoError(t, err)
+ require.Equal(t, basics.Round(1), vc.LastAttestedRound) // check round is set
+ require.Equal(t, basics.MicroAlgos{Raw: 100}, vc.OnlineTotalWeight) // check payload is read
+
+ // read back remaining items
+ vc, err = spr.LookupSPContext(basics.Round(2))
+ require.NoError(t, err)
+ require.Equal(t, basics.Round(2), vc.LastAttestedRound) // check round is set
+ require.Equal(t, basics.MicroAlgos{Raw: 200}, vc.OnlineTotalWeight) // check payload is read
+}
+
+func CustomTestStateproofsQueryAll(t *customT) {
+ spw := t.db.MakeSpVerificationCtxWriter()
+ spr := t.db.MakeSpVerificationCtxReader()
+
+ // prepare the test with some data
+ // store some items
+ vcs := []*ledgercore.StateProofVerificationContext{
+ {
+ LastAttestedRound: basics.Round(0),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 42},
+ },
+ {
+ LastAttestedRound: basics.Round(1),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 100},
+ },
+ {
+ LastAttestedRound: basics.Round(2),
+ OnlineTotalWeight: basics.MicroAlgos{Raw: 200},
+ },
+ }
+ err := spw.StoreSPContexts(context.Background(), vcs)
+ require.NoError(t, err)
+
+ //
+ // test
+ //
+
+ // read all data
+ result, err := spr.GetAllSPContexts(context.Background())
+ require.NoError(t, err)
+ require.Len(t, result, 3) // check all items are present
+ require.Equal(t, basics.Round(0), result[0].LastAttestedRound) // check first item
+ require.Equal(t, basics.Round(2), result[2].LastAttestedRound) // check last item
+}
diff --git a/ledger/store/trackerdb/testsuite/utils_test.go b/ledger/store/trackerdb/testsuite/utils_test.go
new file mode 100644
index 0000000000..342b8a5b89
--- /dev/null
+++ b/ledger/store/trackerdb/testsuite/utils_test.go
@@ -0,0 +1,440 @@
+// Copyright (C) 2019-2023 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand. If not, see .
+
+package testsuite
+
+// A collection of utility functions and types to write the tests in this module.
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/algorand/go-algorand/config"
+ "github.com/algorand/go-algorand/crypto"
+ "github.com/algorand/go-algorand/data/basics"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb"
+ "github.com/algorand/go-algorand/ledger/store/trackerdb/generickv"
+ "github.com/algorand/go-algorand/logging"
+ "github.com/algorand/go-algorand/protocol"
+ "github.com/algorand/go-algorand/test/partitiontest"
+ "github.com/algorand/go-algorand/util/db"
+ "github.com/stretchr/testify/require"
+)
+
+type customT struct {
+ db dbForTests
+ proto config.ConsensusParams
+ *testing.T
+}
+
+type dbForTests interface {
+ // generickv.KvWrite
+ // generickv.KvRead
+ trackerdb.Store
+}
+
+type genericTestEntry struct {
+ name string
+ f func(*customT)
+}
+
+// list of tests to be run on each KV DB implementation
+var genericTests []genericTestEntry
+
+// registerTest registers the given test with the suite
+func registerTest(name string, f func(*customT)) {
+ genericTests = append(genericTests, genericTestEntry{name, f})
+}
+
+// runGenericTestsWithDB runs a generic set of tests on the given database
+func runGenericTestsWithDB(t *testing.T, dbFactory func(config.ConsensusParams) (db dbForTests)) {
+ proto := config.Consensus[protocol.ConsensusCurrentVersion]
+ for _, entry := range genericTests {
+ // run each test defined in the suite using the Golang subtest
+ t.Run(entry.name, func(t *testing.T) {
+ partitiontest.PartitionTest(t)
+ // instantiate a new db for each test
+ entry.f(&customT{dbFactory(proto), proto, t})
+ })
+ }
+}
+
+func seedDb(t *testing.T, db dbForTests) {
+ params := trackerdb.Params{InitProto: protocol.ConsensusCurrentVersion}
+ _, err := db.RunMigrations(context.Background(), params, logging.TestingLog(t), trackerdb.AccountDBVersion)
+ require.NoError(t, err)
+}
+
+// RandomAddress generates a random address
+//
+// TODO: this method is defined in ledgertesting, should be moved up to basics so it can be used in more places
+func RandomAddress() basics.Address {
+ var addr basics.Address
+ crypto.RandBytes(addr[:])
+ return addr
+}
+
+type mockDB struct {
+ kvs kvstore
+ proto config.ConsensusParams
+ // use the generickv implementations
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+}
+
+func makeMockDB(proto config.ConsensusParams) trackerdb.Store {
+ kvs := kvstore{data: make(map[string][]byte)}
+ var db trackerdb.Store
+ db = &mockDB{
+ kvs,
+ proto,
+ generickv.MakeReader(&kvs, proto),
+ generickv.MakeWriter(db, &kvs, &kvs),
+ generickv.MakeCatchpoint(),
+ }
+ return db
+}
+
+func (db *mockDB) SetSynchronousMode(ctx context.Context, mode db.SynchronousMode, fullfsync bool) (err error) {
+ // TODO
+ return nil
+}
+
+func (db *mockDB) IsSharedCacheConnection() bool {
+ return false
+}
+
+// RunMigrations implements trackerdb.Store
+func (db *mockDB) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ // create a anonym struct that impls the interface for the migration runner
+ aux := struct {
+ *mockDB
+ *kvstore
+ }{db, &db.kvs}
+ return generickv.RunMigrations(ctx, aux, params, targetVersion)
+}
+
+// Batch implements trackerdb.Store
+func (db *mockDB) Batch(fn trackerdb.BatchFn) (err error) {
+ return db.BatchContext(context.Background(), fn)
+}
+
+// BatchContext implements trackerdb.Store
+func (db *mockDB) BatchContext(ctx context.Context, fn trackerdb.BatchFn) (err error) {
+ handle, err := db.BeginBatch(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the batch
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ // commit the batch
+ err = handle.Commit()
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginBatch implements trackerdb.Store
+func (db *mockDB) BeginBatch(ctx context.Context) (trackerdb.Batch, error) {
+ scope := mockBatch{db}
+ return &struct {
+ mockBatch
+ trackerdb.Writer
+ }{scope, generickv.MakeWriter(db, &scope, &db.kvs)}, nil
+}
+
+// Snapshot implements trackerdb.Store
+func (db *mockDB) Snapshot(fn trackerdb.SnapshotFn) (err error) {
+ return db.SnapshotContext(context.Background(), fn)
+}
+
+// SnapshotContext implements trackerdb.Store
+func (db *mockDB) SnapshotContext(ctx context.Context, fn trackerdb.SnapshotFn) (err error) {
+ handle, err := db.BeginSnapshot(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the snapshot
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginSnapshot implements trackerdb.Store
+func (db *mockDB) BeginSnapshot(ctx context.Context) (trackerdb.Snapshot, error) {
+ scope := mockSnapshot{db}
+ return &struct {
+ mockSnapshot
+ trackerdb.Reader
+ }{scope, generickv.MakeReader(&scope, db.proto)}, nil
+}
+
+// Transaction implements trackerdb.Store
+func (db *mockDB) Transaction(fn trackerdb.TransactionFn) (err error) {
+ return db.TransactionContext(context.Background(), fn)
+}
+
+// TransactionContext implements trackerdb.Store
+func (db *mockDB) TransactionContext(ctx context.Context, fn trackerdb.TransactionFn) (err error) {
+ handle, err := db.BeginTransaction(ctx)
+ if err != nil {
+ return
+ }
+ defer handle.Close()
+
+ // run the transaction
+ err = fn(ctx, handle)
+ if err != nil {
+ return
+ }
+
+ // commit the transaction
+ err = handle.Commit()
+ if err != nil {
+ return
+ }
+
+ return err
+}
+
+// BeginTransaction implements trackerdb.Store
+func (db *mockDB) BeginTransaction(ctx context.Context) (trackerdb.Transaction, error) {
+ scope := mockTransaction{db, db.proto}
+
+ return &struct {
+ mockTransaction
+ trackerdb.Reader
+ trackerdb.Writer
+ trackerdb.Catchpoint
+ }{scope, generickv.MakeReader(&scope, db.proto), generickv.MakeWriter(db, &scope, &scope), generickv.MakeCatchpoint()}, nil
+}
+
+func (db *mockDB) Vacuum(ctx context.Context) (stats db.VacuumStats, err error) {
+ // TODO
+ return stats, nil
+}
+
+func (db *mockDB) ResetToV6Test(ctx context.Context) error {
+ // TODO
+ return nil
+}
+
+func (db *mockDB) Close() {
+ // TODO
+}
+
+type kvstore struct {
+ data map[string][]byte
+}
+
+func (kvs *kvstore) Set(key, value []byte) error {
+ kvs.data[string(key)] = value
+ return nil
+}
+
+func (kvs *kvstore) Get(key []byte) (data []byte, closer io.Closer, err error) {
+ data, ok := kvs.data[string(key)]
+ if !ok {
+ err = trackerdb.ErrNotFound
+ return
+ }
+ return data, io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (kvs *kvstore) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ //
+ var keys []string
+
+ slow := string(low)
+ shigh := string(high)
+
+ for k := range kvs.data {
+ if k > slow && k < shigh {
+ keys = append(keys, k)
+ }
+ }
+
+ sort.Strings(keys)
+ if reverse {
+ for i, j := 0, len(keys)-1; i < j; i, j = i+1, j-1 {
+ keys[i], keys[j] = keys[j], keys[i]
+ }
+ }
+
+ return &mockIter{kvs, keys, -1}
+}
+
+func (kvs *kvstore) Delete(key []byte) error {
+ delete(kvs.data, string(key))
+ return nil
+}
+
+func (kvs *kvstore) DeleteRange(start, end []byte) error {
+ var toDelete []string
+ for k := range kvs.data {
+ if k > string(start) && k < string(end) {
+ toDelete = append(toDelete, k)
+ }
+ }
+ for i := range toDelete {
+ delete(kvs.data, toDelete[i])
+ }
+ return nil
+}
+
+type mockSnapshot struct {
+ db *mockDB
+}
+
+func (ss mockSnapshot) Get(key []byte) (value []byte, closer io.Closer, err error) {
+ return ss.db.kvs.Get(key)
+}
+
+func (ss mockSnapshot) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ return ss.db.kvs.NewIter(low, high, reverse)
+}
+func (ss mockSnapshot) Close() error {
+ return nil
+}
+
+type mockTransaction struct {
+ db *mockDB
+ proto config.ConsensusParams
+}
+
+func (txs mockTransaction) RunMigrations(ctx context.Context, params trackerdb.Params, log logging.Logger, targetVersion int32) (mgr trackerdb.InitParams, err error) {
+ // create a anonym struct that impls the interface for the migration runner
+ aux := struct {
+ *mockDB
+ *kvstore
+ }{txs.db, &txs.db.kvs}
+ return generickv.RunMigrations(ctx, aux, params, targetVersion)
+}
+
+func (txs mockTransaction) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ return time.Now(), nil
+}
+
+func (txs mockTransaction) Close() error {
+ return nil
+}
+
+func (txs mockTransaction) Commit() error {
+ return nil
+}
+
+func (txs mockTransaction) Set(key, value []byte) error {
+ return txs.db.kvs.Set(key, value)
+}
+
+func (txs mockTransaction) Get(key []byte) (value []byte, closer io.Closer, err error) {
+ return txs.db.kvs.Get(key)
+}
+
+func (txs mockTransaction) NewIter(low, high []byte, reverse bool) generickv.KvIter {
+ return txs.db.kvs.NewIter(low, high, reverse)
+}
+
+func (txs mockTransaction) Delete(key []byte) error {
+ return txs.db.kvs.Delete(key)
+}
+
+func (txs mockTransaction) DeleteRange(start, end []byte) error {
+ return txs.db.kvs.DeleteRange(start, end)
+}
+
+type mockBatch struct {
+ db *mockDB
+}
+
+func (bs mockBatch) ResetTransactionWarnDeadline(ctx context.Context, deadline time.Time) (prevDeadline time.Time, err error) {
+ return time.Now(), nil
+}
+
+func (bs mockBatch) Close() error {
+ return nil
+}
+
+func (bs mockBatch) Commit() error {
+ return nil
+}
+
+func (bs mockBatch) Set(key, value []byte) error {
+ return bs.db.kvs.Set(key, value)
+}
+
+func (bs mockBatch) Delete(key []byte) error {
+ return bs.db.kvs.Delete(key)
+}
+
+func (bs mockBatch) DeleteRange(start, end []byte) error {
+ return bs.db.kvs.DeleteRange(start, end)
+}
+
+type mockIter struct {
+ kvs *kvstore
+ keys []string
+ curr int
+}
+
+func (iter *mockIter) Next() bool {
+ if iter.curr < len(iter.keys)-1 {
+ iter.curr++
+ return true
+ }
+ iter.curr = -1
+ return false
+}
+
+func (iter *mockIter) Key() []byte {
+ return []byte(iter.keys[iter.curr])
+}
+
+func (iter *mockIter) KeySlice() generickv.Slice {
+ return nil
+}
+
+func (iter *mockIter) Value() ([]byte, error) {
+ return iter.kvs.data[iter.keys[iter.curr]], nil
+}
+
+func (iter *mockIter) ValueSlice() (generickv.Slice, error) {
+ return nil, nil
+}
+
+func (iter *mockIter) Valid() bool {
+ return iter.curr != -1
+}
+
+func (iter *mockIter) Close() {}
diff --git a/ledger/tracker.go b/ledger/tracker.go
index 39ea9d4b18..be672cb9f8 100644
--- a/ledger/tracker.go
+++ b/ledger/tracker.go
@@ -30,6 +30,7 @@ import (
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger/eval"
"github.com/algorand/go-algorand/ledger/ledgercore"
+ "github.com/algorand/go-algorand/ledger/store/catchpointdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/logging/telemetryspec"
@@ -134,7 +135,8 @@ type ledgerTracker interface {
// ledgerForTracker defines the part of the ledger that a tracker can
// access. This is particularly useful for testing trackers in isolation.
type ledgerForTracker interface {
- trackerDB() trackerdb.TrackerStore
+ trackerDB() trackerdb.Store
+ catchpointDB() catchpointdb.Store
blockDB() db.Pair
trackerLog() logging.Logger
trackerEvalVerified(bookkeeping.Block, eval.LedgerForEvaluator) (ledgercore.StateDelta, error)
@@ -174,7 +176,7 @@ type trackerRegistry struct {
// cached to avoid SQL queries.
dbRound basics.Round
- dbs trackerdb.TrackerStore
+ dbs trackerdb.Store
log logging.Logger
// the synchronous mode that would be used for the account database.
@@ -554,7 +556,7 @@ func (tr *trackerRegistry) commitRound(dcc *deferredCommitContext) error {
start := time.Now()
ledgerCommitroundCount.Inc(nil)
err = tr.dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -566,7 +568,7 @@ func (tr *trackerRegistry) commitRound(dcc *deferredCommitContext) error {
}
}
- return arw.UpdateAccountsRound(dbRound + basics.Round(offset))
+ return aw.UpdateAccountsRound(dbRound + basics.Round(offset))
})
ledgerCommitroundMicros.AddMicrosecondsSince(start, nil)
diff --git a/ledger/trackerdb.go b/ledger/trackerdb.go
index 8955a27c0f..bf23075813 100644
--- a/ledger/trackerdb.go
+++ b/ledger/trackerdb.go
@@ -38,44 +38,57 @@ func trackerDBInitialize(l ledgerForTracker, catchpointEnabled bool, dbPathPrefi
return
}
- err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
- arw, err := tx.MakeAccountsReaderWriter()
- if err != nil {
- return err
- }
+ tp := trackerdb.Params{
+ InitAccounts: l.GenesisAccounts(),
+ InitProto: l.GenesisProtoVersion(),
+ GenesisHash: l.GenesisHash(),
+ FromCatchpoint: false,
+ CatchpointEnabled: catchpointEnabled,
+ DbPathPrefix: dbPathPrefix,
+ BlockDb: bdbs,
+ }
- tp := trackerdb.Params{
- InitAccounts: l.GenesisAccounts(),
- InitProto: l.GenesisProtoVersion(),
- GenesisHash: l.GenesisHash(),
- FromCatchpoint: false,
- CatchpointEnabled: catchpointEnabled,
- DbPathPrefix: dbPathPrefix,
- BlockDb: bdbs,
- }
- var err0 error
- mgr, err0 = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion)
- if err0 != nil {
- return err0
- }
- lastBalancesRound, err := arw.AccountsRound()
- if err != nil {
- return err
- }
- // Check for blocks DB and tracker DB un-sync
- if lastBalancesRound > lastestBlockRound {
- log.Warnf("trackerDBInitialize: resetting accounts DB (on round %v, but blocks DB's latest is %v)", lastBalancesRound, lastestBlockRound)
- err0 = arw.AccountsReset(ctx)
- if err0 != nil {
- return err0
+ // run migrations
+ mgr, err = dbs.RunMigrations(context.Background(), tp, log, trackerdb.AccountDBVersion)
+ if err != nil {
+ return
+ }
+
+ // create reader for db
+ ar, err := dbs.MakeAccountsReader()
+ if err != nil {
+ return
+ }
+
+ // check current round
+ lastBalancesRound, err := ar.AccountsRound()
+ if err != nil {
+ return
+ }
+
+ // Check for blocks DB and tracker DB un-sync
+ if lastBalancesRound > lastestBlockRound {
+ log.Warnf("trackerDBInitialize: resetting accounts DB (on round %v, but blocks DB's latest is %v)", lastBalancesRound, lastestBlockRound)
+ err = dbs.Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) error {
+ var aw trackerdb.AccountsWriterExt
+ aw, err = tx.MakeAccountsWriter()
+ if err != nil {
+ return err
+ }
+ err = aw.AccountsReset(ctx)
+ if err != nil {
+ return err
}
- mgr, err0 = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion)
- if err0 != nil {
- return err0
+ mgr, err = tx.RunMigrations(ctx, tp, log, trackerdb.AccountDBVersion)
+ if err != nil {
+ return err
}
+ return nil
+ })
+ if err != nil {
+ return
}
- return nil
- })
+ }
return
}
diff --git a/ledger/txtail.go b/ledger/txtail.go
index a86a8af5bc..7d71ea27ef 100644
--- a/ledger/txtail.go
+++ b/ledger/txtail.go
@@ -273,7 +273,7 @@ func (t *txTail) prepareCommit(dcc *deferredCommitContext) (err error) {
}
func (t *txTail) commitRound(ctx context.Context, tx trackerdb.TransactionScope, dcc *deferredCommitContext) error {
- arw, err := tx.MakeAccountsReaderWriter()
+ aw, err := tx.MakeAccountsWriter()
if err != nil {
return err
}
@@ -282,7 +282,7 @@ func (t *txTail) commitRound(ctx context.Context, tx trackerdb.TransactionScope,
// the formula is similar to the committedUpTo: rnd + 1 - retain size
forgetBeforeRound := (dcc.newBase() + 1).SubSaturate(basics.Round(dcc.txTailRetainSize))
baseRound := dcc.oldBase + 1
- if err := arw.TxtailNewRound(ctx, baseRound, dcc.txTailDeltas, forgetBeforeRound); err != nil {
+ if err := aw.TxtailNewRound(ctx, baseRound, dcc.txTailDeltas, forgetBeforeRound); err != nil {
return fmt.Errorf("txTail: unable to persist new round %d : %w", baseRound, err)
}
return nil
diff --git a/ledger/txtail_test.go b/ledger/txtail_test.go
index a21af51208..ed75b591c9 100644
--- a/ledger/txtail_test.go
+++ b/ledger/txtail_test.go
@@ -150,7 +150,7 @@ func (t *txTailTestLedger) initialize(ts *testing.T, protoVersion protocol.Conse
// create a corresponding blockdb.
inMemory := true
t.blockDBs, _ = storetesting.DbOpenTest(ts, inMemory)
- t.trackerDBs, _ = sqlitedriver.DbOpenTrackerTest(ts, inMemory)
+ t.trackerDBs, _ = sqlitedriver.OpenForTesting(ts, inMemory)
t.protoVersion = protoVersion
err := t.trackerDBs.Batch(func(transactionCtx context.Context, tx trackerdb.BatchScope) (err error) {
diff --git a/test/testdata/configs/config-v28.json b/test/testdata/configs/config-v28.json
new file mode 100644
index 0000000000..5ab6675299
--- /dev/null
+++ b/test/testdata/configs/config-v28.json
@@ -0,0 +1,118 @@
+{
+ "Version": 28,
+ "AccountUpdatesStatsInterval": 5000000000,
+ "AccountsRebuildSynchronousMode": 1,
+ "AgreementIncomingBundlesQueueLength": 15,
+ "AgreementIncomingProposalsQueueLength": 50,
+ "AgreementIncomingVotesQueueLength": 20000,
+ "AnnounceParticipationKey": true,
+ "Archival": false,
+ "BaseLoggerDebugLevel": 4,
+ "BlockServiceCustomFallbackEndpoints": "",
+ "BroadcastConnectionsLimit": -1,
+ "CadaverDirectory": "",
+ "CadaverSizeTarget": 0,
+ "CatchpointFileHistoryLength": 365,
+ "CatchpointInterval": 10000,
+ "CatchpointTracking": 0,
+ "CatchupBlockDownloadRetryAttempts": 1000,
+ "CatchupBlockValidateMode": 0,
+ "CatchupFailurePeerRefreshRate": 10,
+ "CatchupGossipBlockFetchTimeoutSec": 4,
+ "CatchupHTTPBlockFetchTimeoutSec": 4,
+ "CatchupLedgerDownloadRetryAttempts": 50,
+ "CatchupParallelBlocks": 16,
+ "ConnectionsRateLimitingCount": 60,
+ "ConnectionsRateLimitingWindowSeconds": 1,
+ "DNSBootstrapID": ".algorand.network",
+ "DNSSecurityFlags": 1,
+ "DeadlockDetection": 0,
+ "DeadlockDetectionThreshold": 30,
+ "DisableLedgerLRUCache": false,
+ "DisableLocalhostConnectionRateLimit": true,
+ "DisableNetworking": false,
+ "DisableOutgoingConnectionThrottling": false,
+ "EnableAccountUpdatesStats": false,
+ "EnableAgreementReporting": false,
+ "EnableAgreementTimeMetrics": false,
+ "EnableAssembleStats": false,
+ "EnableBlockService": false,
+ "EnableBlockServiceFallbackToArchiver": true,
+ "EnableCatchupFromArchiveServers": false,
+ "EnableDeveloperAPI": false,
+ "EnableExperimentalAPI": false,
+ "EnableGossipBlockService": true,
+ "EnableIncomingMessageFilter": false,
+ "EnableLedgerService": false,
+ "EnableMetricReporting": false,
+ "EnableOutgoingNetworkMessageFiltering": true,
+ "EnablePingHandler": true,
+ "EnableProcessBlockStats": false,
+ "EnableProfiler": false,
+ "EnableRequestLogger": false,
+ "EnableRuntimeMetrics": false,
+ "EnableTopAccountsReporting": false,
+ "EnableTxBacklogRateLimiting": false,
+ "EnableUsageLog": false,
+ "EnableVerbosedTransactionSyncLogging": false,
+ "EndpointAddress": "127.0.0.1:0",
+ "FallbackDNSResolverAddress": "",
+ "ForceFetchTransactions": false,
+ "ForceRelayMessages": false,
+ "GossipFanout": 4,
+ "HeartbeatUpdateInterval": 600,
+ "IncomingConnectionsLimit": 2400,
+ "IncomingMessageFilterBucketCount": 5,
+ "IncomingMessageFilterBucketSize": 512,
+ "IsIndexerActive": false,
+ "LedgerSynchronousMode": 2,
+ "LogArchiveMaxAge": "",
+ "LogArchiveName": "node.archive.log",
+ "LogSizeLimit": 1073741824,
+ "MaxAPIBoxPerApplication": 100000,
+ "MaxAPIResourcesPerAccount": 100000,
+ "MaxAcctLookback": 4,
+ "MaxCatchpointDownloadDuration": 7200000000000,
+ "MaxConnectionsPerIP": 15,
+ "MinCatchpointFileDownloadBytesPerSecond": 20480,
+ "NetAddress": "",
+ "NetworkMessageTraceServer": "",
+ "NetworkProtocolVersion": "",
+ "NodeExporterListenAddress": ":9100",
+ "NodeExporterPath": "./node_exporter",
+ "OptimizeAccountsDatabaseOnStartup": false,
+ "OutgoingMessageFilterBucketCount": 3,
+ "OutgoingMessageFilterBucketSize": 128,
+ "ParticipationKeysRefreshInterval": 60000000000,
+ "PeerConnectionsUpdateInterval": 3600,
+ "PeerPingPeriodSeconds": 0,
+ "PriorityPeers": {},
+ "ProposalAssemblyTime": 500000000,
+ "PublicAddress": "",
+ "ReconnectTime": 60000000000,
+ "ReservedFDs": 256,
+ "RestConnectionsHardLimit": 2048,
+ "RestConnectionsSoftLimit": 1024,
+ "RestReadTimeoutSeconds": 15,
+ "RestWriteTimeoutSeconds": 120,
+ "RunHosted": false,
+ "StorageEngine": "sqlite",
+ "SuggestedFeeBlockHistory": 3,
+ "SuggestedFeeSlidingWindowSize": 50,
+ "TLSCertFile": "",
+ "TLSKeyFile": "",
+ "TelemetryToLog": true,
+ "TransactionSyncDataExchangeRate": 0,
+ "TransactionSyncSignificantMessageThreshold": 0,
+ "TxBacklogReservedCapacityPerPeer": 20,
+ "TxBacklogServiceRateWindowSeconds": 10,
+ "TxBacklogSize": 26000,
+ "TxIncomingFilteringFlags": 1,
+ "TxPoolExponentialIncreaseFactor": 2,
+ "TxPoolSize": 75000,
+ "TxSyncIntervalSeconds": 60,
+ "TxSyncServeResponseSize": 1000000,
+ "TxSyncTimeoutSeconds": 30,
+ "UseXForwardedForAddressField": "",
+ "VerifiedTranscationsCacheSize": 150000
+}
diff --git a/tools/block-generator/go.mod b/tools/block-generator/go.mod
index 306627f71b..ea785629bf 100644
--- a/tools/block-generator/go.mod
+++ b/tools/block-generator/go.mod
@@ -23,28 +23,47 @@ require (
github.com/algorand/oapi-codegen v1.12.0-algorand.0 // indirect
github.com/algorand/websocket v1.4.6 // indirect
github.com/aws/aws-sdk-go v1.33.0 // indirect
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.1.2 // indirect
+ github.com/cockroachdb/errors v1.8.1 // indirect
+ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+ github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 // indirect
+ github.com/cockroachdb/redact v1.0.8 // indirect
+ github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/consensys/gnark-crypto v0.7.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 // indirect
github.com/dchest/siphash v1.2.1 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
+ github.com/klauspost/compress v1.11.13 // indirect
+ github.com/kr/pretty v0.2.1 // indirect
+ github.com/kr/text v0.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-sqlite3 v1.10.0 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.41 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/olivere/elastic v6.2.14+incompatible // indirect
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/prometheus/client_golang v1.12.0 // indirect
+ github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect
+ github.com/prometheus/common v0.32.1 // indirect
+ github.com/prometheus/procfs v0.7.3 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/crypto v0.1.0 // indirect
+ golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
+ google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 // indirect
)
diff --git a/tools/block-generator/go.sum b/tools/block-generator/go.sum
index bd2732c8fc..cfaa581bca 100644
--- a/tools/block-generator/go.sum
+++ b/tools/block-generator/go.sum
@@ -13,20 +13,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
-cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
-cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
-cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -35,7 +21,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -46,17 +31,22 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
+github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
+github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
+github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k=
github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g=
github.com/algorand/falcon v0.0.0-20220727072124-02a2a64c4414 h1:nwYN+GQ7Z5OOfZwqBO1ma7DSlP7S1YrKWICOyjkwqrc=
@@ -67,59 +57,52 @@ github.com/algorand/go-deadlock v0.2.2 h1:L7AKATSUCzoeVuOgpTipfCEjdUu5ECmlje8R7l
github.com/algorand/go-deadlock v0.2.2/go.mod h1:Hat1OXKqKNUcN/iv74FjGhF4hsOE2l7gOgQ9ZVIq6Fk=
github.com/algorand/go-sumhash v0.1.0 h1:b/QRhyLuF//vOcicBIxBXYW8bERNoeLxieht/dUYpVg=
github.com/algorand/go-sumhash v0.1.0/go.mod h1:OOe7jdDWUhLkuP1XytkK5gnLu9entAviN5DfDZh6XAc=
-github.com/algorand/graphtrace v0.1.0/go.mod h1:HscLQrzBdH1BH+5oehs3ICd8SYcXvnSL9BjfTu8WHCc=
github.com/algorand/msgp v1.1.53 h1:D6HKLyvLE6ltfsf8Apsrc+kqYb/CcOZEAfh1DpkPrNg=
github.com/algorand/msgp v1.1.53/go.mod h1:5K3d58/poT5fPmtiwuQft6GjgSrVEM46KoXdLrID8ZU=
github.com/algorand/oapi-codegen v1.12.0-algorand.0 h1:W9PvED+wAJc+9EeXPONnA+0zE9UhynEqoDs4OgAxKhk=
github.com/algorand/oapi-codegen v1.12.0-algorand.0/go.mod h1:tIWJ9K/qrLDVDt5A1p82UmxZIEGxv2X+uoujdhEAL48=
github.com/algorand/websocket v1.4.6 h1:I0kV4EYwatuUrKtNiwzYYgojgwh6pksDmlqntKG2Woc=
github.com/algorand/websocket v1.4.6/go.mod h1:HJmdGzFtnlUQ4nTzZP6WrT29oGYf1t6Ybi64vROcT+M=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.33.0 h1:Bq5Y6VTLbfnJp1IV8EL/qUU5qO1DYHda/zis/sqevkY=
github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e h1:CHPYEbz71w8DqJ7DRIq+MXyCQsdibK08vdcQTY4ufas=
-github.com/chrismcguire/gobberish v0.0.0-20150821175641-1d8adb509a0e/go.mod h1:6Xhs0ZlsRjXLIiSMLKafbZxML/j30pg9Z1priLuha5s=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/consensys/bavard v0.1.10/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
+github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
+github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
+github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
+github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
+github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4 h1:Mp4A1rMoAE45WdoVI7a7jwjiIj8eZEXkl3aYbdMJACs=
+github.com/cockroachdb/pebble v0.0.0-20230123220951-b418e86f4cd4/go.mod h1:rWEpkT1ud5qGG2m1HqTBRWbLgi+oodC4+BWC46uVPjw=
+github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
+github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
+github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/consensys/gnark-crypto v0.7.0 h1:rwdy8+ssmLYRqKp+ryRRgQJl/rCq2uv+n83cOydm5UE=
github.com/consensys/gnark-crypto v0.7.0/go.mod h1:KPSuJzyxkJA8xZ/+CV47tyqkr9MmpZA3PXivK4VPrVg=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/daixiang0/gci v0.3.2/go.mod h1:jaASoJmv/ykO9dAAPy31iJnreV19248qKDdVWf3QgC4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -127,57 +110,54 @@ github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC
github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
github.com/dchest/siphash v1.2.1 h1:4cLinnzVJDKxTCl9B01807Yiy+W7ZzVHj/KIroQRvT4=
github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
-github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
-github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
+github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
-github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
+github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
-github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
-github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc=
+github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
+github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
+github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
-github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
-github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -185,8 +165,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -202,12 +180,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -217,17 +194,13 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -235,138 +208,96 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
-github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
-github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
-github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
-github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
-github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
-github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
+github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
+github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
+github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI=
+github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
-github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
+github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
+github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
+github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
+github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo=
-github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
+github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
+github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
-github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
-github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
-github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
-github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
-github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY=
-github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
-github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/matryer/moq v0.2.7/go.mod h1:kITsx543GOENm48TUAQyJ9+SAvFSr7iGQXPoth/VUBk=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg=
+github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
-github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
@@ -375,120 +306,121 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM=
+github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/olivere/elastic v6.2.14+incompatible h1:k+KadwNP/dkXE0/eu+T6otk1+5fe0tEpPyQJ4XVm5i8=
github.com/olivere/elastic v6.2.14+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg=
+github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y=
+github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
-github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
-github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
-github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
-github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM=
-github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
-github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
-github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -501,6 +433,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -513,8 +447,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -523,21 +455,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -546,7 +473,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -557,26 +484,15 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
-golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -584,18 +500,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -607,14 +512,12 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -624,18 +527,19 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -645,74 +549,44 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -720,7 +594,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -749,25 +622,12 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -785,29 +645,13 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
-google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
-google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
-google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
-google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -831,46 +675,13 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
-google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -883,22 +694,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -911,28 +706,27 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 h1:q/fZgS8MMadqFFGa8WL4Oyz+TmjiZfi8UrzWhTl8d5w=
gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009/go.mod h1:O0bY1e/dSoxMYZYTHP0SWKxG5EWLEvKR9/cOjWPPMKU=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -942,7 +736,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+pgregory.net/rapid v0.4.8 h1:d+5SGZWUbJPbl3ss6tmPFqnNeQR6VDOFly+eTjwPiEw=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
diff --git a/tools/debug/transplanter/main.go b/tools/debug/transplanter/main.go
index 86528254e8..8120f8107a 100644
--- a/tools/debug/transplanter/main.go
+++ b/tools/debug/transplanter/main.go
@@ -212,18 +212,18 @@ func transcribeSnappyLog(filePath string, output chan txGroupItem, wg *sync.Wait
}
defer file.Close()
- var headerDecoder headerDecoder
+ var hd headerDecoder
if strings.Contains(path.Base(filePath), "_v2") {
- headerDecoder = decoderV2{}
+ hd = decoderV2{}
} else {
- headerDecoder = decoderV1{}
+ hd = decoderV1{}
}
snappyReader := snappy.NewReader(file)
var n int
for {
- headers, lenMsg, err := headerDecoder.decodeHeader(snappyReader)
+ headers, lenMsg, err := hd.decodeHeader(snappyReader)
if err == io.EOF {
break
} else if err != nil {
@@ -395,7 +395,7 @@ func main() {
os.Exit(1)
}
syncRound := uint64(*roundStart) - cfg.MaxAcctLookback + 1
- err := followerNode.SetSyncRound(syncRound)
+ err = followerNode.SetSyncRound(syncRound)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot configure catchup: %v", err)
os.Exit(1)