From ad0d8652b7b65783f8f56d5bc4677ec29d7243fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 29 Jun 2020 15:12:35 +0100 Subject: [PATCH 1/7] add Anton's first badger implementation Taken from https://gist.github.com/melekes/85b8c07dd917828dc2ac6696129b73f2 as of June 2020, plus 'go mod tidy' to add the dependencies. --- badger_db.go | 316 +++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 63 ++++++++-- 3 files changed, 372 insertions(+), 9 deletions(-) create mode 100644 badger_db.go diff --git a/badger_db.go b/badger_db.go new file mode 100644 index 000000000..21bd56f5a --- /dev/null +++ b/badger_db.go @@ -0,0 +1,316 @@ +package db + +import ( + "bufio" + "io" + "os" + "sync" + + "github.com/dgraph-io/badger/v2" +) + +func init() { + registerDBCreator(BadgerDBBackend, badgerDBCreator, true) +} + +type Options badger.Options + +func badgerDBCreator(dbName, dir string) (DB, error) { + return NewBadgerDB(dbName, dir) +} + +var ( + _KB = int64(1024) + _MB = 1024 * _KB + _GB = 1024 * _MB +) + +// NewBadgerDB creates a Badger key-value store backed to the +// directory dir supplied. If dir does not exist, we create it. +func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { + // BadgerDB doesn't expose a way for us to + // create a DB with the user's supplied name. + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + opts := badger.DefaultOptions + // // Arbitrary size given that at Tendermint + // // we'll need huge KeyValue stores. + opts.ValueLogFileSize = 1 * _GB + // opts.SyncWrites = false + opts.Dir = dir + opts.ValueDir = dir + + return NewBadgerDBWithOptions(opts) +} + +// NewBadgerDBWithOptions creates a BadgerDB key value store +// gives the flexibility of initializing a database with the +// respective options. +func NewBadgerDBWithOptions(opts badger.Options) (*BadgerDB, error) { + db, err := badger.Open(opts) + if err != nil { + return nil, err + } + return &BadgerDB{db: db}, nil +} + +type BadgerDB struct { + db *badger.DB +} + +var _ DB = (*BadgerDB)(nil) + +func (b *BadgerDB) Get(key []byte) []byte { + var val []byte + err := b.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err != nil { + return err + } + val, err = item.Value() + if err != nil { + return err + } + return nil + }) + if err != nil { + // Unfortunate that Get can't return errors + // TODO: Propose allowing DB's Get to return errors too. + panic(err) + } + // var valueSave []byte + // err := valueItem.Value(func(origValue []byte) error { + // // TODO: Decide if we should just assign valueSave to origValue + // // since here we aren't dealing with iterators directly. + // valueSave = make([]byte, len(origValue)) + // copy(valueSave, origValue) + // return nil + // }) + // if err != nil { + // // TODO: ditto:: Propose allowing DB's Get to return errors too. + // panic(err) + // } + return val +} + +func (b *BadgerDB) Has(key []byte) bool { + var found bool + err := b.db.View(func(txn *badger.Txn) error { + _, err := txn.Get(key) + if err != nil && err != badger.ErrKeyNotFound { + return err + } + found = (err != badger.ErrKeyNotFound) + return nil + }) + if err != nil { + // Unfortunate that Get can't return errors + // TODO: Propose allowing DB's Get to return errors too. + panic(err) + } + return found +} + +func (b *BadgerDB) Set(key, value []byte) { + err := b.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, value) + }) + if err != nil { + panic(err) + } +} + +func (b *BadgerDB) SetSync(key, value []byte) { + err := b.db.Update(func(txn *badger.Txn) error { + return txn.Set(key, value) + }) + if err != nil { + panic(err) + } +} + +func (b *BadgerDB) Delete(key []byte) { + err := b.db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) + if err != nil { + panic(err) + } +} + +func (b *BadgerDB) DeleteSync(key []byte) { + err := b.db.Update(func(txn *badger.Txn) error { + return txn.Delete(key) + }) + if err != nil { + panic(err) + } +} + +func (b *BadgerDB) Close() { + if err := b.db.Close(); err != nil { + panic(err) + } +} + +func (b *BadgerDB) Fprint(w io.Writer) { + // bIter := b.Iterator() + // defer bIter.Release() + + // var bw *bufio.Writer + // if bbw, ok := w.(*bufio.Writer); ok { + // bw = bbw + // } else { + // bw = bufio.NewWriter(w) + // } + // defer bw.Flush() + + // i := uint64(0) + // for bIter.rewind(); bIter.valid(); bIter.Next() { + // k, v := bIter.kv() + // fmt.Fprintf(bw, "[%X]:\t[%X]\n", k, v) + // i += 1 + // if i%1024 == 0 { + // bw.Flush() + // i = 0 + // } + // } +} + +func (b *BadgerDB) Print() { + bw := bufio.NewWriter(os.Stdout) + b.Fprint(bw) +} + +func (b *BadgerDB) Iterator(start, end []byte) Iterator { + // dbIter := b.db.NewIterator(badger.IteratorOptions{ + // PrefetchValues: true, + + // // Arbitrary PrefetchSize + // PrefetchSize: 10, + // }) + // // Ensure that we are always at the zeroth item + // dbIter.Rewind() + return nil +} + +func (b *BadgerDB) ReverseIterator(start, end []byte) Iterator { + return nil +} + +func (b *BadgerDB) IteratorPrefix(prefix []byte) Iterator { + return b.Iterator(prefix, nil) +} + +func (b *BadgerDB) Stats() map[string]string { + return nil +} + +func (b *BadgerDB) NewBatch() Batch { + return &badgerDBBatch{db: b} +} + +var _ Batch = (*badgerDBBatch)(nil) + +type badgerDBBatch struct { + entriesMu sync.Mutex + entries []*badger.Entry + + db *BadgerDB +} + +func (bb *badgerDBBatch) Set(key, value []byte) { + bb.entriesMu.Lock() + bb.entries = append(bb.entries, &badger.Entry{ + Key: key, + Value: value, + }) + bb.entriesMu.Unlock() +} + +// Unfortunately Badger doesn't have a batch delete +// The closest that we can do is do a delete from the DB. +// Hesitant to do DeleteAsync because that changes the +// expected ordering +func (bb *badgerDBBatch) Delete(key []byte) { + // bb.db.Delete(key) +} + +// Write commits all batch sets to the DB +func (bb *badgerDBBatch) Write() { + bb.entriesMu.Lock() + entries := bb.entries + bb.entries = nil + bb.entriesMu.Unlock() + + if len(entries) == 0 { + return + } + + err := bb.db.db.Update(func(txn *badger.Txn) error { + for _, e := range entries { + if err := txn.SetEntry(e); err != nil { + return err + } + } + return nil + }) + if err != nil { + panic(err) + } + // var buf *bytes.Buffer // It'll be lazily allocated when needed + // for i, entry := range entries { + // if err := entry.Error; err != nil { + // if buf == nil { + // buf = new(bytes.Buffer) + // } + // fmt.Fprintf(buf, "#%d: entry err: %v\n", i, err) + // } + // } + // if buf != nil { + // panic(string(buf.Bytes())) + // } +} + +func (bb *badgerDBBatch) WriteSync() { + bb.entriesMu.Lock() + entries := bb.entries + bb.entries = nil + bb.entriesMu.Unlock() + + if len(entries) == 0 { + return + } + + err := bb.db.db.Update(func(txn *badger.Txn) error { + for _, e := range entries { + if err := txn.SetEntry(e); err != nil { + return err + } + } + return nil + }) + if err != nil { + panic(err) + } + + // var buf *bytes.Buffer // It'll be lazily allocated when needed + // for i, entry := range entries { + // if err := entry.Error; err != nil { + // if buf == nil { + // buf = new(bytes.Buffer) + // } + // fmt.Fprintf(buf, "#%d: entry err: %v\n", i, err) + // } + // } + // if buf != nil { + // panic(string(buf.Bytes())) + // } +} + +type badgerDBIterator struct { + mu sync.RWMutex + + iter *badger.Iterator +} diff --git a/go.mod b/go.mod index 3a1805d34..8eca74163 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/tendermint/tm-db go 1.12 require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/badger/v2 v2.0.3 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect diff --git a/go.sum b/go.sum index 563b6c954..78bcfdbdf 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,31 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= +github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= +github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -24,6 +43,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= @@ -34,37 +54,60 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= -github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -75,6 +118,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -83,8 +127,10 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= @@ -96,6 +142,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -106,12 +153,12 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= From a5af3dd9a342dad94d3f1475a4c67e80e62dd219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 29 Jun 2020 15:37:20 +0100 Subject: [PATCH 2/7] update to the latest tm-db interface Mainly, many now return errors, so we can get rid of all panics. Also add the missing BackendType. --- badger_db.go | 112 +++++++++++++++++---------------------------------- db.go | 2 + 2 files changed, 39 insertions(+), 75 deletions(-) diff --git a/badger_db.go b/badger_db.go index 21bd56f5a..8c81cf72b 100644 --- a/badger_db.go +++ b/badger_db.go @@ -61,40 +61,20 @@ type BadgerDB struct { var _ DB = (*BadgerDB)(nil) -func (b *BadgerDB) Get(key []byte) []byte { +func (b *BadgerDB) Get(key []byte) ([]byte, error) { var val []byte err := b.db.View(func(txn *badger.Txn) error { item, err := txn.Get(key) if err != nil { return err } - val, err = item.Value() - if err != nil { - return err - } - return nil + val, err = item.ValueCopy(nil) + return err }) - if err != nil { - // Unfortunate that Get can't return errors - // TODO: Propose allowing DB's Get to return errors too. - panic(err) - } - // var valueSave []byte - // err := valueItem.Value(func(origValue []byte) error { - // // TODO: Decide if we should just assign valueSave to origValue - // // since here we aren't dealing with iterators directly. - // valueSave = make([]byte, len(origValue)) - // copy(valueSave, origValue) - // return nil - // }) - // if err != nil { - // // TODO: ditto:: Propose allowing DB's Get to return errors too. - // panic(err) - // } - return val + return val, err } -func (b *BadgerDB) Has(key []byte) bool { +func (b *BadgerDB) Has(key []byte) (bool, error) { var found bool err := b.db.View(func(txn *badger.Txn) error { _, err := txn.Get(key) @@ -104,54 +84,35 @@ func (b *BadgerDB) Has(key []byte) bool { found = (err != badger.ErrKeyNotFound) return nil }) - if err != nil { - // Unfortunate that Get can't return errors - // TODO: Propose allowing DB's Get to return errors too. - panic(err) - } - return found + return found, err } -func (b *BadgerDB) Set(key, value []byte) { - err := b.db.Update(func(txn *badger.Txn) error { +func (b *BadgerDB) Set(key, value []byte) error { + return b.db.Update(func(txn *badger.Txn) error { return txn.Set(key, value) }) - if err != nil { - panic(err) - } } -func (b *BadgerDB) SetSync(key, value []byte) { - err := b.db.Update(func(txn *badger.Txn) error { +func (b *BadgerDB) SetSync(key, value []byte) error { + return b.db.Update(func(txn *badger.Txn) error { return txn.Set(key, value) }) - if err != nil { - panic(err) - } } -func (b *BadgerDB) Delete(key []byte) { - err := b.db.Update(func(txn *badger.Txn) error { +func (b *BadgerDB) Delete(key []byte) error { + return b.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) }) - if err != nil { - panic(err) - } } -func (b *BadgerDB) DeleteSync(key []byte) { - err := b.db.Update(func(txn *badger.Txn) error { +func (b *BadgerDB) DeleteSync(key []byte) error { + return b.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) }) - if err != nil { - panic(err) - } } -func (b *BadgerDB) Close() { - if err := b.db.Close(); err != nil { - panic(err) - } +func (b *BadgerDB) Close() error { + return b.db.Close() } func (b *BadgerDB) Fprint(w io.Writer) { @@ -178,12 +139,13 @@ func (b *BadgerDB) Fprint(w io.Writer) { // } } -func (b *BadgerDB) Print() { +func (b *BadgerDB) Print() error { bw := bufio.NewWriter(os.Stdout) b.Fprint(bw) + return nil } -func (b *BadgerDB) Iterator(start, end []byte) Iterator { +func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) { // dbIter := b.db.NewIterator(badger.IteratorOptions{ // PrefetchValues: true, @@ -192,14 +154,14 @@ func (b *BadgerDB) Iterator(start, end []byte) Iterator { // }) // // Ensure that we are always at the zeroth item // dbIter.Rewind() - return nil + return nil, nil } -func (b *BadgerDB) ReverseIterator(start, end []byte) Iterator { - return nil +func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { + return nil, nil } -func (b *BadgerDB) IteratorPrefix(prefix []byte) Iterator { +func (b *BadgerDB) IteratorPrefix(prefix []byte) (Iterator, error) { return b.Iterator(prefix, nil) } @@ -220,35 +182,37 @@ type badgerDBBatch struct { db *BadgerDB } -func (bb *badgerDBBatch) Set(key, value []byte) { +func (bb *badgerDBBatch) Set(key, value []byte) error { bb.entriesMu.Lock() bb.entries = append(bb.entries, &badger.Entry{ Key: key, Value: value, }) bb.entriesMu.Unlock() + return nil } // Unfortunately Badger doesn't have a batch delete // The closest that we can do is do a delete from the DB. // Hesitant to do DeleteAsync because that changes the // expected ordering -func (bb *badgerDBBatch) Delete(key []byte) { +func (bb *badgerDBBatch) Delete(key []byte) error { // bb.db.Delete(key) + return nil } // Write commits all batch sets to the DB -func (bb *badgerDBBatch) Write() { +func (bb *badgerDBBatch) Write() error { bb.entriesMu.Lock() entries := bb.entries bb.entries = nil bb.entriesMu.Unlock() if len(entries) == 0 { - return + return nil } - err := bb.db.db.Update(func(txn *badger.Txn) error { + return bb.db.db.Update(func(txn *badger.Txn) error { for _, e := range entries { if err := txn.SetEntry(e); err != nil { return err @@ -256,9 +220,6 @@ func (bb *badgerDBBatch) Write() { } return nil }) - if err != nil { - panic(err) - } // var buf *bytes.Buffer // It'll be lazily allocated when needed // for i, entry := range entries { // if err := entry.Error; err != nil { @@ -273,17 +234,17 @@ func (bb *badgerDBBatch) Write() { // } } -func (bb *badgerDBBatch) WriteSync() { +func (bb *badgerDBBatch) WriteSync() error { bb.entriesMu.Lock() entries := bb.entries bb.entries = nil bb.entriesMu.Unlock() if len(entries) == 0 { - return + return nil } - err := bb.db.db.Update(func(txn *badger.Txn) error { + return bb.db.db.Update(func(txn *badger.Txn) error { for _, e := range entries { if err := txn.SetEntry(e); err != nil { return err @@ -291,9 +252,6 @@ func (bb *badgerDBBatch) WriteSync() { } return nil }) - if err != nil { - panic(err) - } // var buf *bytes.Buffer // It'll be lazily allocated when needed // for i, entry := range entries { @@ -309,6 +267,10 @@ func (bb *badgerDBBatch) WriteSync() { // } } +func (bb *badgerDBBatch) Close() error { + return nil // TODO +} + type badgerDBIterator struct { mu sync.RWMutex diff --git a/db.go b/db.go index bc8b4a08d..a453a38ce 100644 --- a/db.go +++ b/db.go @@ -33,6 +33,8 @@ const ( // - requires gcc // - use rocksdb build tag (go build -tags rocksdb) RocksDBBackend BackendType = "rocksdb" + + BadgerDBBackend BackendType = "badger" ) type dbCreator func(name string, dir string) (DB, error) From 3a21ca94d34e3e494a076f3c31df532f06b3158f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 29 Jun 2020 16:07:51 +0100 Subject: [PATCH 3/7] remove unnecessary code The commented out code might be useful in the future, but it's in the git history anyway. --- badger_db.go | 95 +++++----------------------------------------------- 1 file changed, 8 insertions(+), 87 deletions(-) diff --git a/badger_db.go b/badger_db.go index 8c81cf72b..7a8016d8e 100644 --- a/badger_db.go +++ b/badger_db.go @@ -1,46 +1,30 @@ package db import ( - "bufio" - "io" "os" + "path/filepath" "sync" "github.com/dgraph-io/badger/v2" ) -func init() { - registerDBCreator(BadgerDBBackend, badgerDBCreator, true) -} - -type Options badger.Options +func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) } func badgerDBCreator(dbName, dir string) (DB, error) { return NewBadgerDB(dbName, dir) } -var ( - _KB = int64(1024) - _MB = 1024 * _KB - _GB = 1024 * _MB -) - // NewBadgerDB creates a Badger key-value store backed to the // directory dir supplied. If dir does not exist, we create it. func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { - // BadgerDB doesn't expose a way for us to - // create a DB with the user's supplied name. - if err := os.MkdirAll(dir, 0755); err != nil { + // Since Badger doesn't support database names, we join both to obtain + // the final directory to use for the database. + path := filepath.Join(dir, dbName) + + if err := os.MkdirAll(path, 0755); err != nil { return nil, err } - opts := badger.DefaultOptions - // // Arbitrary size given that at Tendermint - // // we'll need huge KeyValue stores. - opts.ValueLogFileSize = 1 * _GB - // opts.SyncWrites = false - opts.Dir = dir - opts.ValueDir = dir - + opts := badger.DefaultOptions(path) return NewBadgerDBWithOptions(opts) } @@ -115,45 +99,11 @@ func (b *BadgerDB) Close() error { return b.db.Close() } -func (b *BadgerDB) Fprint(w io.Writer) { - // bIter := b.Iterator() - // defer bIter.Release() - - // var bw *bufio.Writer - // if bbw, ok := w.(*bufio.Writer); ok { - // bw = bbw - // } else { - // bw = bufio.NewWriter(w) - // } - // defer bw.Flush() - - // i := uint64(0) - // for bIter.rewind(); bIter.valid(); bIter.Next() { - // k, v := bIter.kv() - // fmt.Fprintf(bw, "[%X]:\t[%X]\n", k, v) - // i += 1 - // if i%1024 == 0 { - // bw.Flush() - // i = 0 - // } - // } -} - func (b *BadgerDB) Print() error { - bw := bufio.NewWriter(os.Stdout) - b.Fprint(bw) return nil } func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) { - // dbIter := b.db.NewIterator(badger.IteratorOptions{ - // PrefetchValues: true, - - // // Arbitrary PrefetchSize - // PrefetchSize: 10, - // }) - // // Ensure that we are always at the zeroth item - // dbIter.Rewind() return nil, nil } @@ -161,10 +111,6 @@ func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { return nil, nil } -func (b *BadgerDB) IteratorPrefix(prefix []byte) (Iterator, error) { - return b.Iterator(prefix, nil) -} - func (b *BadgerDB) Stats() map[string]string { return nil } @@ -220,18 +166,6 @@ func (bb *badgerDBBatch) Write() error { } return nil }) - // var buf *bytes.Buffer // It'll be lazily allocated when needed - // for i, entry := range entries { - // if err := entry.Error; err != nil { - // if buf == nil { - // buf = new(bytes.Buffer) - // } - // fmt.Fprintf(buf, "#%d: entry err: %v\n", i, err) - // } - // } - // if buf != nil { - // panic(string(buf.Bytes())) - // } } func (bb *badgerDBBatch) WriteSync() error { @@ -252,19 +186,6 @@ func (bb *badgerDBBatch) WriteSync() error { } return nil }) - - // var buf *bytes.Buffer // It'll be lazily allocated when needed - // for i, entry := range entries { - // if err := entry.Error; err != nil { - // if buf == nil { - // buf = new(bytes.Buffer) - // } - // fmt.Fprintf(buf, "#%d: entry err: %v\n", i, err) - // } - // } - // if buf != nil { - // panic(string(buf.Bytes())) - // } } func (bb *badgerDBBatch) Close() error { From 5c75ddf0aae752555d3b09e2f54391c3da0d54b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 29 Jun 2020 16:26:46 +0100 Subject: [PATCH 4/7] implement the badgerdb iterator Passing all the relevant tests. --- backend_test.go | 16 ++++++ badger_db.go | 127 ++++++++++++++++++++++++++++++++++++++++++++++-- db.go | 2 +- 3 files changed, 140 insertions(+), 5 deletions(-) diff --git a/backend_test.go b/backend_test.go index cf6307586..63630beca 100644 --- a/backend_test.go +++ b/backend_test.go @@ -300,6 +300,22 @@ func testDBIterator(t *testing.T, backend BackendType) { require.NoError(t, err) verifyIterator(t, ritr, []int64(nil), "reverse iterator from 2 (ex) to 4") + + // Ensure that the iterators don't panic with an empty database. + dir2, err := ioutil.TempDir("", "tm-db-test") + require.NoError(t, err) + db2, err := NewDB(name, backend, dir2) + require.NoError(t, err) + defer cleanupDBDir(dir2, name) + + itr, err = db2.Iterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, itr, nil, "forward iterator with empty db") + + ritr, err = db2.ReverseIterator(nil, nil) + require.NoError(t, err) + verifyIterator(t, ritr, nil, "reverse iterator with empty db") + } func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) { diff --git a/badger_db.go b/badger_db.go index 7a8016d8e..ce9294c39 100644 --- a/badger_db.go +++ b/badger_db.go @@ -1,6 +1,7 @@ package db import ( + "bytes" "os" "path/filepath" "sync" @@ -46,19 +47,30 @@ type BadgerDB struct { var _ DB = (*BadgerDB)(nil) func (b *BadgerDB) Get(key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } var val []byte err := b.db.View(func(txn *badger.Txn) error { item, err := txn.Get(key) - if err != nil { + if err == badger.ErrKeyNotFound { + return nil + } else if err != nil { return err } val, err = item.ValueCopy(nil) + if err == nil && val == nil { + val = []byte{} + } return err }) return val, err } func (b *BadgerDB) Has(key []byte) (bool, error) { + if len(key) == 0 { + return false, errKeyEmpty + } var found bool err := b.db.View(func(txn *badger.Txn) error { _, err := txn.Get(key) @@ -72,24 +84,42 @@ func (b *BadgerDB) Has(key []byte) (bool, error) { } func (b *BadgerDB) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } return b.db.Update(func(txn *badger.Txn) error { return txn.Set(key, value) }) } func (b *BadgerDB) SetSync(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } return b.db.Update(func(txn *badger.Txn) error { return txn.Set(key, value) }) } func (b *BadgerDB) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } return b.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) }) } func (b *BadgerDB) DeleteSync(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } return b.db.Update(func(txn *badger.Txn) error { return txn.Delete(key) }) @@ -103,12 +133,38 @@ func (b *BadgerDB) Print() error { return nil } +func (b *BadgerDB) iteratorOpts(start, end []byte, opts badger.IteratorOptions) (*badgerDBIterator, error) { + if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { + return nil, errKeyEmpty + } + txn := b.db.NewTransaction(false) + iter := txn.NewIterator(opts) + iter.Rewind() + iter.Seek(start) + if opts.Reverse && iter.Valid() && bytes.Equal(iter.Item().Key(), start) { + // If we're going in reverse, our starting point was "end", + // which is exclusive. + iter.Next() + } + return &badgerDBIterator{ + reverse: opts.Reverse, + start: start, + end: end, + + txn: txn, + iter: iter, + }, nil +} + func (b *BadgerDB) Iterator(start, end []byte) (Iterator, error) { - return nil, nil + opts := badger.DefaultIteratorOptions + return b.iteratorOpts(start, end, opts) } func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { - return nil, nil + opts := badger.DefaultIteratorOptions + opts.Reverse = true + return b.iteratorOpts(end, start, opts) } func (b *BadgerDB) Stats() map[string]string { @@ -129,6 +185,12 @@ type badgerDBBatch struct { } func (bb *badgerDBBatch) Set(key, value []byte) error { + if len(key) == 0 { + return errKeyEmpty + } + if value == nil { + return errValueNil + } bb.entriesMu.Lock() bb.entries = append(bb.entries, &badger.Entry{ Key: key, @@ -143,6 +205,9 @@ func (bb *badgerDBBatch) Set(key, value []byte) error { // Hesitant to do DeleteAsync because that changes the // expected ordering func (bb *badgerDBBatch) Delete(key []byte) error { + if len(key) == 0 { + return errKeyEmpty + } // bb.db.Delete(key) return nil } @@ -193,7 +258,61 @@ func (bb *badgerDBBatch) Close() error { } type badgerDBIterator struct { - mu sync.RWMutex + reverse bool + start, end []byte + txn *badger.Txn iter *badger.Iterator + + lastErr error +} + +func (i *badgerDBIterator) Close() error { + i.iter.Close() + i.txn.Discard() + return nil +} + +func (i *badgerDBIterator) Domain() (start, end []byte) { return i.start, i.end } +func (i *badgerDBIterator) Error() error { return i.lastErr } + +func (i *badgerDBIterator) Next() { + if !i.Valid() { + panic("iterator is invalid") + } + i.iter.Next() +} + +func (i *badgerDBIterator) Valid() bool { + if !i.iter.Valid() { + return false + } + if len(i.end) > 0 { + key := i.iter.Item().Key() + if c := bytes.Compare(key, i.end); (!i.reverse && c >= 0) || (i.reverse && c < 0) { + // We're at the end key, or past the end. + return false + } + } + return true +} + +func (i *badgerDBIterator) Key() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + // Note that we don't use KeyCopy, so this is only valid until the next + // call to Next. + return i.iter.Item().KeyCopy(nil) +} + +func (i *badgerDBIterator) Value() []byte { + if !i.Valid() { + panic("iterator is invalid") + } + val, err := i.iter.Item().ValueCopy(nil) + if err != nil { + i.lastErr = err + } + return val } diff --git a/db.go b/db.go index a453a38ce..4d518c0b0 100644 --- a/db.go +++ b/db.go @@ -34,7 +34,7 @@ const ( // - use rocksdb build tag (go build -tags rocksdb) RocksDBBackend BackendType = "rocksdb" - BadgerDBBackend BackendType = "badger" + BadgerDBBackend BackendType = "badgerdb" ) type dbCreator func(name string, dir string) (DB, error) From 0e1ad4e06f547497346fd2933d2723fd56f64136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 30 Jun 2020 18:48:06 +0100 Subject: [PATCH 5/7] implement badger write batches --- badger_db.go | 124 ++++++++++++++++++++------------------------------- 1 file changed, 49 insertions(+), 75 deletions(-) diff --git a/badger_db.go b/badger_db.go index ce9294c39..9ec0cae90 100644 --- a/badger_db.go +++ b/badger_db.go @@ -2,9 +2,9 @@ package db import ( "bytes" + "fmt" "os" "path/filepath" - "sync" "github.com/dgraph-io/badger/v2" ) @@ -16,7 +16,7 @@ func badgerDBCreator(dbName, dir string) (DB, error) { } // NewBadgerDB creates a Badger key-value store backed to the -// directory dir supplied. If dir does not exist, we create it. +// directory dir supplied. If dir does not exist, it will be created. func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { // Since Badger doesn't support database names, we join both to obtain // the final directory to use for the database. @@ -26,6 +26,8 @@ func NewBadgerDB(dbName, dir string) (*BadgerDB, error) { return nil, err } opts := badger.DefaultOptions(path) + opts.SyncWrites = false // note that we have Sync methods + opts.Logger = nil // badger is too chatty by default return NewBadgerDBWithOptions(opts) } @@ -95,16 +97,15 @@ func (b *BadgerDB) Set(key, value []byte) error { }) } -func (b *BadgerDB) SetSync(key, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil +func withSync(db *badger.DB, err error) error { + if err != nil { + return err } - return b.db.Update(func(txn *badger.Txn) error { - return txn.Set(key, value) - }) + return db.Sync() +} + +func (b *BadgerDB) SetSync(key, value []byte) error { + return withSync(b.db, b.Set(key, value)) } func (b *BadgerDB) Delete(key []byte) error { @@ -117,12 +118,7 @@ func (b *BadgerDB) Delete(key []byte) error { } func (b *BadgerDB) DeleteSync(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - return b.db.Update(func(txn *badger.Txn) error { - return txn.Delete(key) - }) + return withSync(b.db, b.Delete(key)) } func (b *BadgerDB) Close() error { @@ -172,89 +168,67 @@ func (b *BadgerDB) Stats() map[string]string { } func (b *BadgerDB) NewBatch() Batch { - return &badgerDBBatch{db: b} + wb := &badgerDBBatch{ + db: b.db, + wb: b.db.NewWriteBatch(), + firstFlush: make(chan struct{}, 1), + } + wb.firstFlush <- struct{}{} + return wb } var _ Batch = (*badgerDBBatch)(nil) type badgerDBBatch struct { - entriesMu sync.Mutex - entries []*badger.Entry - - db *BadgerDB + db *badger.DB + wb *badger.WriteBatch + + // Calling db.Flush twice panics, so we must keep track of whether we've + // flushed already on our own. If Write can receive from the firstFlush + // channel, then it's the first and only Flush call we should do. + // + // Upstream bug report: + // https://github.com/dgraph-io/badger/issues/1394 + firstFlush chan struct{} } -func (bb *badgerDBBatch) Set(key, value []byte) error { +func (b *badgerDBBatch) Set(key, value []byte) error { if len(key) == 0 { return errKeyEmpty } if value == nil { return errValueNil } - bb.entriesMu.Lock() - bb.entries = append(bb.entries, &badger.Entry{ - Key: key, - Value: value, - }) - bb.entriesMu.Unlock() - return nil + return b.wb.Set(key, value) } -// Unfortunately Badger doesn't have a batch delete -// The closest that we can do is do a delete from the DB. -// Hesitant to do DeleteAsync because that changes the -// expected ordering -func (bb *badgerDBBatch) Delete(key []byte) error { +func (b *badgerDBBatch) Delete(key []byte) error { if len(key) == 0 { return errKeyEmpty } - // bb.db.Delete(key) - return nil + return b.wb.Delete(key) } -// Write commits all batch sets to the DB -func (bb *badgerDBBatch) Write() error { - bb.entriesMu.Lock() - entries := bb.entries - bb.entries = nil - bb.entriesMu.Unlock() - - if len(entries) == 0 { - return nil +func (b *badgerDBBatch) Write() error { + select { + case <-b.firstFlush: + return b.wb.Flush() + default: + return fmt.Errorf("batch already flushed") } - - return bb.db.db.Update(func(txn *badger.Txn) error { - for _, e := range entries { - if err := txn.SetEntry(e); err != nil { - return err - } - } - return nil - }) } -func (bb *badgerDBBatch) WriteSync() error { - bb.entriesMu.Lock() - entries := bb.entries - bb.entries = nil - bb.entriesMu.Unlock() - - if len(entries) == 0 { - return nil - } - - return bb.db.db.Update(func(txn *badger.Txn) error { - for _, e := range entries { - if err := txn.SetEntry(e); err != nil { - return err - } - } - return nil - }) +func (b *badgerDBBatch) WriteSync() error { + return withSync(b.db, b.Write()) } -func (bb *badgerDBBatch) Close() error { - return nil // TODO +func (b *badgerDBBatch) Close() error { + select { + case <-b.firstFlush: // a Flush after Cancel panics too + default: + } + b.wb.Cancel() + return nil } type badgerDBIterator struct { From b21fcd5efbc549cefb219e875de902457fb77fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Wed, 1 Jul 2020 11:39:04 +0100 Subject: [PATCH 6/7] disable the interfacer linter It's been deprecated for years and for good reason. The warning it was showing in this branch wasn't useful. --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 4feb4c55d..0eb19c0cd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -21,7 +21,7 @@ linters: - gosimple - govet - ineffassign - - interfacer + # - interfacer - lll - misspell - maligned From 78b63183d6514a00cfbc99de032282a79c610355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 9 Jul 2020 11:07:15 +0100 Subject: [PATCH 7/7] add README and CHANGELOG entries --- CHANGELOG.md | 2 ++ README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9573e8edc..bc5ff9a55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- [\#115](https://github.com/tendermint/tm-db/pull/115) Add a `BadgerDB` backend (@mvdan) + ## 0.6.0 **2020-06-24** diff --git a/README.md b/README.md index 77bee382c..6b7215fd3 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Go 1.13+ - **[RocksDB](https://github.com/tecbot/gorocksdb) [experimental]:** A [Go wrapper](https://github.com/tecbot/gorocksdb) around [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees for on-disk storage, but is optimized for fast storage media such as SSDs and memory. Supports atomic transactions, but not full ACID transactions. +- **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A key-value database written as a pure-Go alternative to others like LevelDB and RocksDB. Makes use of multiple goroutines for performance, and includes advanced features such as transactions, write batches, compression, and more. + ## Meta-databases - **PrefixDB [stable]:** A database which wraps another database and uses a static prefix for all keys. This allows multiple logical databases to be stored in a common underlying databases by using different namespaces. Used by the Cosmos SDK to give different modules their own namespaced database in a single application database.