diff --git a/common/errors.go b/common/errors.go index 90af1cee..d4f1a469 100644 --- a/common/errors.go +++ b/common/errors.go @@ -28,16 +28,20 @@ const ( ) var ( - // ErrWrongSigner returns if it's a wrong signer + // ErrWrongSigner is returned if it's a wrong signer ErrWrongSigner = errors.New("wrong signer") - // ErrInconsistentRoot returns if the block and dump states have different root + // ErrInconsistentRoot is returned if the block and dump states have different root ErrInconsistentRoot = errors.New("inconsistent root") - // ErrInconsistentStates returns if the number of blocks, dumps or recipents are different + // ErrInconsistentStates is returned if the number of blocks, dumps or receipts are different ErrInconsistentStates = errors.New("inconsistent states") // ErrInvalidTD is returned when a block has invalid TD ErrInvalidTD = errors.New("invalid TD") - // ErrInvalidReceiptLog returns if it's a invalid receipt log + // ErrInvalidReceiptLog is returned if it's a invalid receipt log ErrInvalidReceiptLog = errors.New("invalid receipt log") + // ErrHasPrevBalance is returned if an account has a previous balance when it's a new subscription + ErrHasPrevBalance = errors.New("missing previous balance") + // ErrMissingPrevBalance is returned if an account is missing previous balance when it's an old subscription + ErrMissingPrevBalance = errors.New("missing previous balance") ) // DuplicateError checks whether it's a duplicate key error diff --git a/example/main.go b/example/main.go index 1853e172..9a5efd7b 100644 --- a/example/main.go +++ b/example/main.go @@ -17,10 +17,11 @@ package main import ( - "context" "fmt" + "github.com/ethereum/go-ethereum/common" - "github.com/getamis/eth-indexer/store" + "github.com/getamis/eth-indexer/model" + "github.com/getamis/eth-indexer/store/account" "github.com/getamis/sirius/database" gormFactory "github.com/getamis/sirius/database/gorm" "github.com/getamis/sirius/database/mysql" @@ -35,7 +36,12 @@ func main() { ), ) addr := common.HexToAddress("0x756f45e3fa69347a9a973a725e3c98bc4db0b5a0") - manager := store.NewServiceManager(db) - balance, blockNumber, _ := manager.GetBalance(context.Background(), addr, -1) - fmt.Println(balance, blockNumber) + store := account.NewWithDB(db) + + account, err := store.FindAccount(model.ETHAddress, addr) + if err != nil { + fmt.Printf("Failed to find account: %v\n", err) + } else { + fmt.Printf("Find account, block_number: %v, balance: %v, \n", account.Balance, account.BlockNumber) + } } diff --git a/store/account/account.go b/store/account/account.go index 0bbe7c21..70aeeff4 100644 --- a/store/account/account.go +++ b/store/account/account.go @@ -17,6 +17,8 @@ package account import ( + "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/getamis/eth-indexer/model" "github.com/jinzhu/gorm" @@ -39,6 +41,7 @@ type Store interface { // Accounts InsertAccount(account *model.Account) error FindAccount(contractAddress common.Address, address common.Address, blockNr ...int64) (result *model.Account, err error) + FindLatestAccounts(contractAddress common.Address, addrs [][]byte) (result []*model.Account, err error) DeleteAccounts(contractAddress common.Address, from, to int64) error // Transfer events @@ -144,6 +147,29 @@ func (t *store) FindAccount(contractAddress common.Address, address common.Addre return } +func (t *store) FindLatestAccounts(contractAddress common.Address, addrs [][]byte) (result []*model.Account, err error) { + if len(addrs) == 0 { + return []*model.Account{}, nil + } + + acct := model.Account{ + ContractAddress: contractAddress.Bytes(), + } + // The following query does not work because the select fields needs to also be in group by fields (ONLY_FULL_GROUP_BY mode) + // "select address, balance, MAX(block_number) as block_number from %s where address in (?) group by address" + // and the following query + // "select address, balance, MAX(block_number) as block_number from %s where address in (?) group by (address, balance)" + // is not what we want, because (address, balance) isn't unique + query := fmt.Sprintf( + "select * from %s as t1, (select address, MAX(block_number) as block_number from %s where address in (?) group by address) as t2 where t1.address = t2.address and t1.block_number = t2.block_number", + acct.TableName(), acct.TableName()) + err = t.db.Raw(query, addrs).Scan(&result).Error + if err != nil { + return + } + return +} + func (t *store) DeleteAccounts(contractAddress common.Address, from, to int64) error { return t.db.Delete(model.Account{ ContractAddress: contractAddress.Bytes(), diff --git a/store/account/account_test.go b/store/account/account_test.go index 170fb535..51dead9e 100644 --- a/store/account/account_test.go +++ b/store/account/account_test.go @@ -19,6 +19,7 @@ package account import ( "os" "reflect" + "strconv" "testing" gethCommon "github.com/ethereum/go-ethereum/common" @@ -38,11 +39,15 @@ func makeERC20(hexAddr string) *model.ERC20 { } func makeAccount(contractAddress []byte, blockNum int64, hexAddr string) *model.Account { + return makeAccountWithBalance(contractAddress, blockNum, hexAddr, "987654321098765432109876543210") +} + +func makeAccountWithBalance(contractAddress []byte, blockNum int64, hexAddr, balance string) *model.Account { return &model.Account{ ContractAddress: contractAddress, BlockNumber: blockNum, Address: common.HexToBytes(hexAddr), - Balance: "987654321098765432109876543210", + Balance: balance, } } @@ -210,6 +215,79 @@ var _ = Describe("Account Database Test", func() { }) }) + Context("FindLatestAccounts()", func() { + It("finds the eth records with highest block numbers", func() { + store := NewWithDB(db) + hexAddr0 := "0xF287a379e6caCa6732E50b88D23c290aA990A892" // does not exist in DB + hexAddr1 := "0xA287a379e6caCa6732E50b88D23c290aA990A892" + hexAddr2 := "0xC487a379e6caCa6732E50b88D23c290aA990A892" + hexAddr3 := "0xD487a379e6caCa6732E50b88D23c290aA990A892" + + blockNumber := int64(1000300) + var expected []*model.Account + for _, hexAddr := range []string{hexAddr1, hexAddr2, hexAddr3} { + var acct *model.Account + for i := 0; i < 3; i++ { + acct = makeAccountWithBalance(model.ETHBytes, blockNumber+int64(i), hexAddr, strconv.FormatInt(blockNumber, 10)) + err := store.InsertAccount(acct) + Expect(err).Should(Succeed()) + } + // the last one is with the highest block number + expected = append(expected, acct) + blockNumber++ + } + + addrs := [][]byte{common.HexToBytes(hexAddr0), common.HexToBytes(hexAddr1), common.HexToBytes(hexAddr2), common.HexToBytes(hexAddr3), common.HexToBytes(hexAddr3)} + // should return accounts at latest block number + accounts, err := store.FindLatestAccounts(model.ETHAddress, addrs) + Expect(err).Should(Succeed()) + Expect(len(accounts)).Should(Equal(3)) + for i, acct := range accounts { + acct.ContractAddress = model.ETHBytes + Expect(acct).Should(Equal(expected[i])) + } + }) + + It("finds the erc20 records with highest block numbers", func() { + store := NewWithDB(db) + + // Insert code to create table + hexAddr := "0xB287a379e6caCa6732E50b88D23c290aA990A892" + tokenAddr := gethCommon.HexToAddress(hexAddr) + erc20 := makeERC20(hexAddr) + err := store.InsertERC20(erc20) + Expect(err).Should(Succeed()) + + hexAddr0 := "0xF287a379e6caCa6732E50b88D23c290aA990A892" // does not exist in DB + hexAddr1 := "0xA287a379e6caCa6732E50b88D23c290aA990A892" + hexAddr2 := "0xC487a379e6caCa6732E50b88D23c290aA990A892" + hexAddr3 := "0xD487a379e6caCa6732E50b88D23c290aA990A892" + + blockNumber := int64(1000300) + var expected []*model.Account + for _, hexAddr := range []string{hexAddr1, hexAddr2, hexAddr3} { + var acct *model.Account + for i := 0; i < 3; i++ { + acct = makeAccountWithBalance(erc20.Address, blockNumber+int64(i), hexAddr, strconv.FormatInt(blockNumber, 10)) + err := store.InsertAccount(acct) + Expect(err).Should(Succeed()) + } + // the last one is with the highest block number + expected = append(expected, acct) + blockNumber++ + } + addrs := [][]byte{common.HexToBytes(hexAddr0), common.HexToBytes(hexAddr1), common.HexToBytes(hexAddr2), common.HexToBytes(hexAddr3), common.HexToBytes(hexAddr3)} + // should return accounts at latest block number + accounts, err := store.FindLatestAccounts(tokenAddr, addrs) + Expect(err).Should(Succeed()) + Expect(len(accounts)).Should(Equal(3)) + for i, acct := range accounts { + acct.ContractAddress = erc20.Address + Expect(acct).Should(Equal(expected[i])) + } + }) + }) + Context("DeleteAccounts()", func() { It("deletes eth account states from a block number", func() { store := NewWithDB(db) diff --git a/store/account/mocks/Store.go b/store/account/mocks/Store.go index c09f8b2e..f028aa7c 100644 --- a/store/account/mocks/Store.go +++ b/store/account/mocks/Store.go @@ -128,6 +128,29 @@ func (_m *Store) FindERC20Storage(address common.Address, key common.Hash, block return r0, r1 } +// FindLatestAccounts provides a mock function with given fields: contractAddress, addrs +func (_m *Store) FindLatestAccounts(contractAddress common.Address, addrs [][]byte) ([]*model.Account, error) { + ret := _m.Called(contractAddress, addrs) + + var r0 []*model.Account + if rf, ok := ret.Get(0).(func(common.Address, [][]byte) []*model.Account); ok { + r0 = rf(contractAddress, addrs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Account) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(common.Address, [][]byte) error); ok { + r1 = rf(contractAddress, addrs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // FindTransfer provides a mock function with given fields: contractAddress, address, blockNr func (_m *Store) FindTransfer(contractAddress common.Address, address common.Address, blockNr ...int64) (*model.Transfer, error) { _va := make([]interface{}, len(blockNr)) diff --git a/store/balance.go b/store/balance.go deleted file mode 100644 index 2748f5e6..00000000 --- a/store/balance.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "errors" - "math/big" - - gethCommon "github.com/ethereum/go-ethereum/common" - "github.com/getamis/eth-indexer/common" - "github.com/getamis/eth-indexer/model" - "github.com/getamis/sirius/log" -) - -var ErrInvalidBalance = errors.New("invalid balance") - -func (srv *serviceManager) GetBalance(ctx context.Context, address gethCommon.Address, blockNr int64) (balance *big.Int, blockNumber *big.Int, err error) { - logger := log.New("addr", address.Hex(), "number", blockNr) - // Find header - var hdr *model.Header - if common.IsLatestBlock(blockNr) { - hdr, err = srv.FindLatestBlock() - } else { - hdr, err = srv.FindBlockByNumber(blockNr) - } - if err != nil { - logger.Error("Failed to find header for block", "err", err) - return nil, nil, err - } - blockNumber = big.NewInt(hdr.Number) - - // Find account - account, err := srv.FindAccount(model.ETHAddress, address, hdr.Number) - if err != nil { - logger.Error("Failed to find account", "err", err) - return nil, nil, err - } - var ok bool - balance, ok = new(big.Int).SetString(account.Balance, 10) - if !ok { - logger.Error("Failed to covert balance", "balance", account.Balance) - return nil, nil, ErrInvalidBalance - } - - return -} diff --git a/store/balance_erc20.go b/store/balance_erc20.go deleted file mode 100644 index 4c4327f6..00000000 --- a/store/balance_erc20.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "errors" - "math/big" - - ethCommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/getamis/eth-indexer/common" - "github.com/getamis/eth-indexer/model" - "github.com/getamis/eth-indexer/store/account" - "github.com/getamis/sirius/log" - "github.com/jinzhu/gorm" - "github.com/shopspring/decimal" -) - -var ( - // ErrNotSelf retruns if the address is not my contraact address - ErrNotSelf = errors.New("not self address") -) - -// Implement vm.ContractRef -type contractAccount struct { - address ethCommon.Address -} - -func (account *contractAccount) ReturnGas(*big.Int, *big.Int) {} -func (account *contractAccount) Address() ethCommon.Address { return account.address } -func (account *contractAccount) Value() *big.Int { return ethCommon.Big0 } -func (account *contractAccount) SetCode(ethCommon.Hash, []byte) {} -func (account *contractAccount) ForEachStorage(callback func(key, value ethCommon.Hash) bool) {} - -// Implement vm.StateDB. In current version, we only read the states in the given account (contract). -type contractDB struct { - blockNumber int64 - code *model.ERC20 - account *model.Account - accountStore account.Store - err error -} - -func (contractDB) CreateAccount(addr ethCommon.Address) {} -func (contractDB) SubBalance(addr ethCommon.Address, balance *big.Int) {} -func (contractDB) AddBalance(addr ethCommon.Address, balance *big.Int) {} -func (contractDB) SetNonce(addr ethCommon.Address, nonce uint64) {} -func (contractDB) SetCode(addr ethCommon.Address, codes []byte) {} -func (contractDB) SetState(addr ethCommon.Address, hash1 ethCommon.Hash, hash2 ethCommon.Hash) {} -func (contractDB) Suicide(addr ethCommon.Address) bool { return false } -func (contractDB) HasSuicided(addr ethCommon.Address) bool { return false } -func (contractDB) RevertToSnapshot(snap int) {} -func (contractDB) Snapshot() int { return 0 } -func (contractDB) AddLog(*types.Log) {} -func (contractDB) AddPreimage(hash ethCommon.Hash, images []byte) {} -func (contractDB) ForEachStorage(addr ethCommon.Address, f func(ethCommon.Hash, ethCommon.Hash) bool) { -} -func (contractDB) AddRefund(fund uint64) {} -func (contractDB) GetRefund() uint64 { return 0 } -func (contractDB) AddTransferLog(*types.TransferLog) {} - -// self checks whether the address is the contract address. -func (db *contractDB) self(addr ethCommon.Address) bool { - return addr == ethCommon.BytesToAddress(db.account.Address) -} - -// mustBeSelf checks whether the address is the contract address. If not, set error to ErrNotSelf -func (db *contractDB) mustBeSelf(addr ethCommon.Address) (result bool) { - defer func() { - if !result { - db.err = ErrNotSelf - } - }() - return db.self(addr) -} -func (db contractDB) Exist(addr ethCommon.Address) bool { - return db.self(addr) -} -func (db contractDB) Empty(addr ethCommon.Address) bool { - return !db.self(addr) -} -func (db *contractDB) GetBalance(addr ethCommon.Address) *big.Int { - if db.mustBeSelf(addr) { - v, ok := new(big.Int).SetString(db.account.Balance, 10) - if ok { - return v - } - return ethCommon.Big0 - - } - return ethCommon.Big0 -} -func (db *contractDB) GetNonce(addr ethCommon.Address) uint64 { - return 0 -} -func (db *contractDB) GetCodeHash(addr ethCommon.Address) ethCommon.Hash { - if db.mustBeSelf(addr) { - return crypto.Keccak256Hash(db.code.Code) - } - return ethCommon.Hash{} -} -func (db *contractDB) GetCode(addr ethCommon.Address) []byte { - if db.mustBeSelf(addr) { - return db.code.Code - } - return []byte{} -} -func (db *contractDB) GetCodeSize(addr ethCommon.Address) int { - if db.mustBeSelf(addr) { - return len(db.GetCode(addr)) - } - return 0 -} -func (db *contractDB) GetState(addr ethCommon.Address, key ethCommon.Hash) ethCommon.Hash { - if db.mustBeSelf(addr) { - s, err := db.accountStore.FindERC20Storage(addr, key, db.blockNumber) - if err != nil { - // not found error means there is no storage at this block number - if err != gorm.ErrRecordNotFound { - db.err = err - } - return ethCommon.Hash{} - } - return ethCommon.BytesToHash(s.Value) - } - return ethCommon.Hash{} -} - -func (srv *serviceManager) GetERC20Balance(ctx context.Context, contractAddress, address ethCommon.Address, blockNr int64) (*decimal.Decimal, *big.Int, error) { - logger := log.New("contractAddr", contractAddress.Hex(), "addr", address.Hex(), "number", blockNr) - // Find contract code - erc20, err := srv.FindERC20(contractAddress) - if err != nil { - logger.Error("Failed to find contract code", "err", err) - return nil, nil, err - } - - // Find header - var hdr *model.Header - if common.IsLatestBlock(blockNr) { - hdr, err = srv.FindLatestBlock() - } else { - hdr, err = srv.FindBlockByNumber(blockNr) - } - if err != nil { - logger.Error("Failed to find header for block", "err", err) - return nil, nil, err - } - blockNumber := big.NewInt(hdr.Number) - - // Find contract account - account, err := srv.FindAccount(model.ETHAddress, contractAddress, hdr.Number) - if err != nil { - logger.Error("Failed to find contract", "err", err) - return nil, nil, err - } - - // Get balance from contract - db := &contractDB{ - blockNumber: blockNumber.Int64(), - code: erc20, - account: account, - accountStore: srv.accountStore, - } - balance, err := BalanceOf(db, contractAddress, address) - if err != nil { - logger.Error("Failed to get balance", "err", err) - return nil, nil, err - } - if db.err != nil { - logger.Error("Failed to get balance due to state db error", "err", db.err) - return nil, nil, db.err - } - - // Consider decimals - result := decimal.NewFromBigInt(balance, -int32(erc20.Decimals)) - return &result, blockNumber, nil -} diff --git a/store/balance_erc20_test.go b/store/balance_erc20_test.go deleted file mode 100644 index 304e2449..00000000 --- a/store/balance_erc20_test.go +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "errors" - "math" - "math/big" - "math/rand" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/params" - indexerCommon "github.com/getamis/eth-indexer/common" - "github.com/getamis/eth-indexer/contracts" - "github.com/getamis/eth-indexer/contracts/backends" - "github.com/getamis/eth-indexer/model" - accountMock "github.com/getamis/eth-indexer/store/account/mocks" - hdrMock "github.com/getamis/eth-indexer/store/block_header/mocks" - "github.com/jinzhu/gorm" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/stretchr/testify/mock" -) - -var _ = Describe("DB ERC 20 Test", func() { - var auth *bind.TransactOpts - var contract *contracts.MithrilToken - var contractAddr common.Address - var sim *backends.SimulatedBackend - var db *contractDB - var mockAccountStore *accountMock.Store - var storages map[string]*model.ERC20Storage - var fundedAddress common.Address - var fundedBalance *big.Int - - BeforeEach(func() { - mockAccountStore = new(accountMock.Store) - - // pre-defined account - key, _ := crypto.GenerateKey() - auth = bind.NewKeyedTransactor(key) - - alloc := make(core.GenesisAlloc) - alloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(100000000000000)} - sim = backends.NewSimulatedBackend(alloc) - - // Deploy Mithril token contract - var err error - contractAddr, _, contract, err = contracts.DeployMithrilToken(auth, sim) - Expect(contract).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - sim.Commit() - - By("init token supply") - tx, err := contract.Init(auth, big.NewInt(math.MaxInt64), auth.From, auth.From) - type account struct { - address common.Address - balance *big.Int - } - Expect(tx).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - sim.Commit() - - By("fund some token to an address") - fundedAddress = common.HexToAddress(getFakeAddress()) - fundedBalance = big.NewInt(int64(rand.Uint32())) - _, err = contract.Transfer(auth, fundedAddress, fundedBalance) - Expect(err).Should(BeNil()) - sim.Commit() - - By("get dirty storage") - now := sim.Blockchain().CurrentBlock().NumberU64() - dump, err := eth.GetDirtyStorage(params.AllEthashProtocolChanges, sim.Blockchain(), now) - Expect(err).Should(BeNil()) - - By("find the contract code and data") - blockNumber := int64(sim.Blockchain().CurrentBlock().NumberU64()) - var code *model.ERC20 - storages = make(map[string]*model.ERC20Storage) - for addrStr, account := range dump.Accounts { - if contractAddr == common.HexToAddress(addrStr) { - c, err := sim.CodeAt(context.Background(), contractAddr, nil) - Expect(err).Should(BeNil()) - - code = &model.ERC20{ - Address: contractAddr.Bytes(), - Code: c, - } - - for k, v := range account.Storage { - storages[k] = &model.ERC20Storage{ - BlockNumber: blockNumber, - Address: contractAddr.Bytes(), - Key: common.Hex2Bytes(k), - Value: common.Hex2Bytes(v), - } - } - break - } - } - Expect(code).ShouldNot(BeNil()) - - db = &contractDB{ - blockNumber: blockNumber, - code: code, - accountStore: mockAccountStore, - account: &model.Account{ - Address: contractAddr.Bytes(), - Balance: "0", - }, - } - }) - AfterEach(func() { - mockAccountStore.AssertExpectations(GinkgoT()) - }) - - Context("Contract DB", func() { - notSelfAddr := common.HexToAddress(getFakeAddress()) - It("self()", func() { - Expect(db.self(contractAddr)).Should(BeTrue()) - Expect(db.err).Should(BeNil()) - Expect(db.self(notSelfAddr)).Should(BeFalse()) - Expect(db.err).Should(BeNil()) - }) - It("mustBeSelf()", func() { - Expect(db.mustBeSelf(contractAddr)).Should(BeTrue()) - Expect(db.err).Should(BeNil()) - Expect(db.mustBeSelf(notSelfAddr)).Should(BeFalse()) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - It("Exist()", func() { - Expect(db.Exist(contractAddr)).Should(BeTrue()) - Expect(db.err).Should(BeNil()) - Expect(db.Exist(notSelfAddr)).Should(BeFalse()) - Expect(db.err).Should(BeNil()) - }) - It("Empty()", func() { - Expect(db.Empty(contractAddr)).Should(BeFalse()) - Expect(db.err).Should(BeNil()) - Expect(db.Empty(notSelfAddr)).Should(BeTrue()) - Expect(db.err).Should(BeNil()) - }) - It("GetBalance()", func() { - // Currently, we cannot send ether to Mirthril contract because its contract implementation. - // The balance of contract is always zero. - balance, ok := new(big.Int).SetString(db.account.Balance, 10) - Expect(ok).Should(BeTrue()) - Expect(db.GetBalance(contractAddr)).Should(Equal(balance)) - Expect(db.err).Should(BeNil()) - Expect(db.GetBalance(notSelfAddr).Int64()).Should(BeZero()) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - It("GetNonce()", func() { - Expect(db.GetNonce(contractAddr)).Should(BeZero()) - Expect(db.err).Should(BeNil()) - }) - It("GetCodeHash()", func() { - Expect(db.GetCodeHash(contractAddr)).Should(Equal(crypto.Keccak256Hash(db.code.Code))) - Expect(db.err).Should(BeNil()) - Expect(db.GetCodeHash(notSelfAddr)).Should(Equal(common.Hash{})) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - It("GetCode()", func() { - Expect(db.GetCode(contractAddr)).Should(Equal(db.code.Code)) - Expect(db.err).Should(BeNil()) - Expect(db.GetCode(notSelfAddr)).Should(Equal([]byte{})) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - It("GetCodeSize()", func() { - Expect(db.GetCodeSize(contractAddr)).Should(Equal(len(db.GetCode(contractAddr)))) - Expect(db.err).Should(BeNil()) - Expect(db.GetCodeSize(notSelfAddr)).Should(BeZero()) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - It("GetState()", func() { - randomHash := common.HexToHash(getRandomString(64)) - mockFindREC20Storage(mockAccountStore, storages) - Expect(db.GetState(contractAddr, randomHash)).Should(Equal(common.Hash{})) - Expect(db.err).Should(BeNil()) - Expect(db.GetState(notSelfAddr, randomHash)).Should(Equal(common.Hash{})) - Expect(db.err).Should(Equal(ErrNotSelf)) - }) - }) - - Context("GetERC20Balance()", func() { - var mockAccountStore *accountMock.Store - var mockHdrStore *hdrMock.Store - var manager *serviceManager - blockNumber := int64(10) - header := &model.Header{ - Number: 100, - } - - BeforeEach(func() { - mockAccountStore = new(accountMock.Store) - mockHdrStore = new(hdrMock.Store) - manager = &serviceManager{ - accountStore: mockAccountStore, - blockHeaderStore: mockHdrStore, - } - }) - - Context("with valid parameters", func() { - Context("latest block", func() { - It("funded address", func() { - mockFindREC20Storage(mockAccountStore, storages) - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindLatestBlock").Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(db.account, nil).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, -1) - Expect(err).Should(BeNil()) - Expect(expBalance.String()).Should(Equal(fundedBalance.String())) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - It("non-funded address", func() { - otherAddr := common.HexToAddress(getFakeAddress()) - mockFindREC20Storage(mockAccountStore, storages) - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindLatestBlock").Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(db.account, nil).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, otherAddr, -1) - Expect(err).Should(BeNil()) - Expect(expBalance.IntPart()).Should(BeZero()) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - }) - Context("non latest block", func() { - It("funded address", func() { - mockFindREC20Storage(mockAccountStore, storages) - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(db.account, nil).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, blockNumber) - Expect(err).Should(BeNil()) - Expect(expBalance.String()).Should(Equal(fundedBalance.String())) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - It("non-funded address", func() { - mockFindREC20Storage(mockAccountStore, storages) - otherAddr := common.HexToAddress(getFakeAddress()) - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(db.account, nil).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, otherAddr, blockNumber) - Expect(err).Should(BeNil()) - Expect(expBalance.IntPart()).Should(BeZero()) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - }) - }) - - Context("with invalid parameters", func() { - unknownErr := errors.New("unknown error") - It("failed to execute state db", func() { - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(db.account, nil).Once() - mockAccountStore.On("FindERC20Storage", mock.AnythingOfType("common.Address"), mock.AnythingOfType("common.Hash"), mock.AnythingOfType("int64")).Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find contract address", func() { - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, contractAddr, header.Number).Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find state block", func() { - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find latest state block", func() { - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, nil).Once() - mockHdrStore.On("FindLatestBlock").Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, -1) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find state block", func() { - mockAccountStore.On("FindERC20", contractAddr).Return(db.code, unknownErr).Once() - expBalance, expNumber, err := manager.GetERC20Balance(context.Background(), contractAddr, fundedAddress, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - }) - }) -}) - -func mockFindREC20Storage(mockAccountStore *accountMock.Store, storages map[string]*model.ERC20Storage) { - mockAccountStore.On("FindERC20Storage", mock.AnythingOfType("common.Address"), mock.AnythingOfType("common.Hash"), mock.AnythingOfType("int64")).Return( - func(address common.Address, key common.Hash, blockNr int64) *model.ERC20Storage { - v, ok := storages[indexerCommon.HashHex(key)] - if ok { - return v - } - return nil - }, func(address common.Address, key common.Hash, blockNr int64) error { - _, ok := storages[indexerCommon.HashHex(key)] - if ok { - return nil - } - return gorm.ErrRecordNotFound - }).Once() -} diff --git a/store/balance_test.go b/store/balance_test.go deleted file mode 100644 index bba2febd..00000000 --- a/store/balance_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/getamis/eth-indexer/model" - acctMock "github.com/getamis/eth-indexer/store/account/mocks" - hdrMock "github.com/getamis/eth-indexer/store/block_header/mocks" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("DB Eth Balance Test", func() { - var mockAccountStore *acctMock.Store - var mockHdrStore *hdrMock.Store - var manager *serviceManager - var addr common.Address - blockNumber := int64(10) - header := &model.Header{ - Number: 100, - } - - BeforeEach(func() { - mockAccountStore = new(acctMock.Store) - mockHdrStore = new(hdrMock.Store) - manager = &serviceManager{ - accountStore: mockAccountStore, - blockHeaderStore: mockHdrStore, - } - addr = common.HexToAddress(getFakeAddress()) - }) - - AfterEach(func() { - mockAccountStore.AssertExpectations(GinkgoT()) - }) - - Context("with valid parameters", func() { - account := &model.Account{ - Address: addr.Bytes(), - Balance: "1000", - } - accountBalance, _ := new(big.Int).SetString(account.Balance, 10) - It("latest block", func() { - mockHdrStore.On("FindLatestBlock").Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, addr, header.Number).Return(account, nil).Once() - expBalance, expNumber, err := manager.GetBalance(context.Background(), addr, -1) - Expect(err).Should(BeNil()) - Expect(expBalance).Should(Equal(accountBalance)) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - It("certain block", func() { - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, addr, header.Number).Return(account, nil).Once() - expBalance, expNumber, err := manager.GetBalance(context.Background(), addr, blockNumber) - Expect(err).Should(BeNil()) - Expect(expBalance).Should(Equal(accountBalance)) - Expect(expNumber.Int64()).Should(Equal(header.Number)) - }) - }) - - Context("with invalid parameters", func() { - unknownErr := errors.New("unknown error") - It("failed to find state block", func() { - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetBalance(context.Background(), addr, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find latest state block", func() { - mockHdrStore.On("FindLatestBlock").Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetBalance(context.Background(), addr, -1) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - It("failed to find account", func() { - mockHdrStore.On("FindBlockByNumber", blockNumber).Return(header, nil).Once() - mockAccountStore.On("FindAccount", model.ETHAddress, addr, header.Number).Return(nil, unknownErr).Once() - expBalance, expNumber, err := manager.GetBalance(context.Background(), addr, blockNumber) - Expect(err).Should(Equal(unknownErr)) - Expect(expBalance).Should(BeNil()) - Expect(expNumber).Should(BeNil()) - }) - }) -}) diff --git a/store/contract_call.go b/store/contract_call.go deleted file mode 100644 index 86076413..00000000 --- a/store/contract_call.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "math" - "math/big" - "strings" - - "github.com/ethereum/go-ethereum/accounts/abi" - ethCommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/params" - "github.com/getamis/eth-indexer/contracts" - "github.com/getamis/sirius/log" -) - -// Call calls the specific contract method call in given state -func Call(db vm.StateDB, contractABI string, contractAddress ethCommon.Address, method string, result interface{}, inputs ...interface{}) error { - // Construct EVM and contract - contract := vm.NewContract(&contractAccount{}, &contractAccount{ - address: contractAddress, - }, ethCommon.Big0, math.MaxUint64) - contract.SetCallCode(&contractAddress, db.GetCodeHash(contractAddress), db.GetCode(contractAddress)) - evm := vm.NewEVM(vm.Context{}, db, params.MainnetChainConfig, vm.Config{}) - inter := vm.NewInterpreter(evm, vm.Config{}) - - // Create new call message - parsed, err := abi.JSON(strings.NewReader(contractABI)) - data, err := parsed.Pack(method, inputs...) - if err != nil { - log.Error("Failed to parse balanceOf method", "err", err) - return err - } - - // Run contract - ret, err := inter.Run(contract, data) - if err != nil { - log.Error("Failed to run contract", "err", err) - return err - } - - // Unpack result into result - return parsed.Unpack(result, method, ret) -} - -// BalanceOf returns the amount of ERC20 token at the given state db -func BalanceOf(db vm.StateDB, contractAddress ethCommon.Address, address ethCommon.Address) (*big.Int, error) { - result := new(*big.Int) - err := Call(db, contracts.ERC20TokenABI, contractAddress, "balanceOf", result, address) - if err != nil { - return nil, err - } - return *result, nil -} diff --git a/store/contract_call_test.go b/store/contract_call_test.go deleted file mode 100644 index bc3ffb5c..00000000 --- a/store/contract_call_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "fmt" - "math" - "math/big" - "math/rand" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/params" - "github.com/getamis/eth-indexer/contracts" - "github.com/getamis/eth-indexer/contracts/backends" - "github.com/getamis/eth-indexer/model" - accountMocks "github.com/getamis/eth-indexer/store/account/mocks" - "github.com/stretchr/testify/mock" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Call Test", func() { - var auth *bind.TransactOpts - var contract *contracts.MithrilToken - var contractAddr common.Address - var sim *backends.SimulatedBackend - var mockAccountStore *accountMocks.Store - BeforeEach(func() { - mockAccountStore = new(accountMocks.Store) - // pre-defined account - key, _ := crypto.GenerateKey() - auth = bind.NewKeyedTransactor(key) - - alloc := make(core.GenesisAlloc) - alloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(100000000)} - sim = backends.NewSimulatedBackend(alloc) - - // Deploy Mithril token contract - var err error - contractAddr, _, contract, err = contracts.DeployMithrilToken(auth, sim) - Expect(contract).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - sim.Commit() - }) - - AfterEach(func() { - mockAccountStore.AssertExpectations(GinkgoT()) - }) - - It("BalanceOf", func() { - By("init token supply") - tx, err := contract.Init(auth, big.NewInt(math.MaxInt64), auth.From, auth.From) - type account struct { - address common.Address - balance *big.Int - dirtyStateDB map[string]state.DirtyDumpAccount - } - Expect(tx).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - sim.Commit() - - By("transfer token to accounts") - accounts := make(map[uint64]*account) - for i := 0; i < 100; i++ { - acc := &account{ - address: common.HexToAddress(getFakeAddress()), - balance: big.NewInt(int64(rand.Uint32())), - } - tx, err := contract.Transfer(auth, acc.address, acc.balance) - Expect(tx).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - accounts[sim.Blockchain().CurrentBlock().NumberU64()+1] = acc - sim.Commit() - } - - By("get current state db") - stateDB, err := sim.Blockchain().State() - Expect(stateDB).ShouldNot(BeNil()) - - By("ensure all account token balance are expected") - for _, account := range accounts { - result, err := BalanceOf(stateDB, contractAddr, account.address) - Expect(err).Should(BeNil()) - Expect(account.balance).Should(Equal(result)) - } - - By("get dirty storage") - for blockNumber, account := range accounts { - dump, err := eth.GetDirtyStorage(params.AllEthashProtocolChanges, sim.Blockchain(), blockNumber) - account.dirtyStateDB = dump.Accounts - Expect(err).Should(BeNil()) - accounts[blockNumber] = account - } - - By("find the contract code") - code, err := sim.CodeAt(context.Background(), contractAddr, nil) - Expect(code).ShouldNot(BeNil()) - Expect(err).Should(BeNil()) - - contractCode := &model.ERC20{ - Address: contractAddr.Bytes(), - Code: code, - } - - By("mock account store") - mockAccountStore.On("FindERC20Storage", mock.AnythingOfType("common.Address"), mock.AnythingOfType("common.Hash"), mock.AnythingOfType("int64")).Return(func(address common.Address, key common.Hash, blockNr int64) *model.ERC20Storage { - s, _ := accounts[uint64(blockNr)].dirtyStateDB[common.Bytes2Hex(address.Bytes())] - kayHash := common.Bytes2Hex(key.Bytes()) - value, _ := s.Storage[kayHash] - return &model.ERC20Storage{ - Address: address.Bytes(), - BlockNumber: blockNr, - Key: key.Bytes(), - Value: common.Hex2Bytes(value), - } - }, nil) - - By("ensure all account token balance are expected based on contract code and data") - for blockNumber, account := range accounts { - db := &contractDB{ - blockNumber: int64(blockNumber), - code: contractCode, - accountStore: mockAccountStore, - account: &model.Account{ - Address: contractAddr.Bytes(), - }, - } - result, err := BalanceOf(db, contractAddr, account.address) - Expect(err).Should(BeNil()) - Expect(db.err).Should(BeNil()) - Expect(account.balance).Should(Equal(result)) - } - }) -}) - -// ---------------------------------------------------------------------------- -var letters = []rune("abcdef0123456789") - -func getRandomString(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} - -func getFakeAddress() string { - return fmt.Sprintf("0x%s", getRandomString(40)) -} diff --git a/store/mocks/ServiceManager.go b/store/mocks/ServiceManager.go index 9abfcd7c..57ff8290 100644 --- a/store/mocks/ServiceManager.go +++ b/store/mocks/ServiceManager.go @@ -1,4 +1,4 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. +// Code generated by mockery v1.0.0 package mocks import big "math/big" @@ -14,17 +14,26 @@ type ServiceManager struct { } // AddSubscriptions provides a mock function with given fields: group, addrs -func (_m *ServiceManager) AddSubscriptions(group int64, addrs []common.Address) error { +func (_m *ServiceManager) AddSubscriptions(group int64, addrs []common.Address) ([]common.Address, error) { ret := _m.Called(group, addrs) - var r0 error - if rf, ok := ret.Get(0).(func(int64, []common.Address) error); ok { + var r0 []common.Address + if rf, ok := ret.Get(0).(func(int64, []common.Address) []common.Address); ok { r0 = rf(group, addrs) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.Address) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64, []common.Address) error); ok { + r1 = rf(group, addrs) + } else { + r1 = ret.Error(1) } - return r0 + return r0, r1 } // FindBlockByHash provides a mock function with given fields: hash diff --git a/store/service_store.go b/store/service_store.go deleted file mode 100644 index 77257785..00000000 --- a/store/service_store.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 The eth-indexer Authors -// This file is part of the eth-indexer library. -// -// The eth-indexer library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The eth-indexer library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the eth-indexer library. If not, see . - -package store - -import ( - "context" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/jinzhu/gorm" - "github.com/shopspring/decimal" - - "github.com/getamis/eth-indexer/model" - accStore "github.com/getamis/eth-indexer/store/account" - bhStore "github.com/getamis/eth-indexer/store/block_header" - subStore "github.com/getamis/eth-indexer/store/subscription" - txStore "github.com/getamis/eth-indexer/store/transaction" -) - -//go:generate mockery -name ServiceManager - -// ServiceManager is a wrapper interface that serves data for RPC services. -type ServiceManager interface { - // Block header store - FindBlockByNumber(blockNumber int64) (result *model.Header, err error) - FindBlockByHash(hash []byte) (result *model.Header, err error) - FindLatestBlock() (result *model.Header, err error) - - // Transaction store - FindTransaction(hash []byte) (result *model.Transaction, err error) - FindTransactionsByBlockHash(blockHash []byte) (result []*model.Transaction, err error) - - // Account store - FindERC20(address common.Address) (result *model.ERC20, err error) - - // Subscription store - AddSubscriptions(group int64, addrs []common.Address) error - GetSubscriptions(group int64, page, limit uint64) (result []*model.Subscription, total uint64, err error) - - // GetBalance returns the amount of wei for the given address in the state of the - // given block number. If blockNr < 0, the given block is the latest block. - // Noted that the return block number may be different from the input one because - // we don't have state in the input one. - GetBalance(ctx context.Context, address common.Address, blockNr int64) (balance *big.Int, blockNumber *big.Int, err error) - - // GetERC20Balance returns the amount of ERC20 token for the given address in the state of the - // given block number. If blockNr < 0, the given block is the latest block. - // Noted that the return block number may be different from the input one because - // we don't have state in the input one. - GetERC20Balance(ctx context.Context, contractAddress, address common.Address, blockNr int64) (*decimal.Decimal, *big.Int, error) - - // Subscriptions store - FindTotalBalance(blockNumber int64, token common.Address, group int64) (result *model.TotalBalance, err error) -} - -type accountStore = accStore.Store -type blockHeaderStore = bhStore.Store -type transactionStore = txStore.Store -type subscriptionStore = subStore.Store - -type serviceManager struct { - accountStore - blockHeaderStore - transactionStore - subscriptionStore -} - -// NewServiceManager news a service manager to serve data for RPC services. -func NewServiceManager(db *gorm.DB) ServiceManager { - return &serviceManager{ - accountStore: accStore.NewWithDB(db), - blockHeaderStore: bhStore.NewWithDB(db), - transactionStore: txStore.NewWithDB(db), - subscriptionStore: subStore.NewWithDB(db), - } -} - -func (srv *serviceManager) AddSubscriptions(group int64, addrs []common.Address) (err error) { - if len(addrs) == 0 { - return nil - } - subs := make([]*model.Subscription, len(addrs)) - for i, addr := range addrs { - subs[i] = &model.Subscription{ - Group: group, - Address: addr.Bytes(), - } - } - return srv.subscriptionStore.BatchInsert(subs) -} - -func (srv *serviceManager) GetSubscriptions(group int64, page, limit uint64) (result []*model.Subscription, total uint64, err error) { - return srv.subscriptionStore.FindByGroup(group, &model.QueryParameters{ - Page: page, - Limit: limit, - OrderBy: "created_at", - Order: "asc", - }) -} diff --git a/store/subscription/subscription.go b/store/subscription/subscription.go index aee5b003..b5199a64 100644 --- a/store/subscription/subscription.go +++ b/store/subscription/subscription.go @@ -21,12 +21,17 @@ import ( "github.com/getamis/sirius/log" "github.com/jinzhu/gorm" + idxCommon "github.com/getamis/eth-indexer/common" "github.com/getamis/eth-indexer/model" ) +const ( + ErrCodeDuplicateKey uint16 = 1062 +) + //go:generate mockery -name Store type Store interface { - BatchInsert(subs []*model.Subscription) error + BatchInsert(subs []*model.Subscription) ([]common.Address, error) BatchUpdateBlockNumber(blockNumber int64, addrs [][]byte) error Find(blockNumber int64) (result []*model.Subscription, err error) // FindOldSubscriptions find old subscriptions by addresses @@ -50,7 +55,7 @@ func NewWithDB(db *gorm.DB) Store { } } -func (t *store) BatchInsert(subs []*model.Subscription) (err error) { +func (t *store) BatchInsert(subs []*model.Subscription) (duplicated []common.Address, err error) { dbTx := t.db.Begin() defer func() { if err != nil { @@ -63,12 +68,16 @@ func (t *store) BatchInsert(subs []*model.Subscription) (err error) { } }() for _, sub := range subs { - err := dbTx.Create(sub).Error - if err != nil { - return err + createErr := dbTx.Create(sub).Error + if createErr != nil { + if idxCommon.DuplicateError(createErr) { + duplicated = append(duplicated, common.BytesToAddress(sub.Address)) + } else { + return nil, createErr + } } } - return nil + return duplicated, nil } func (t *store) BatchUpdateBlockNumber(blockNumber int64, addrs [][]byte) error { diff --git a/store/subscription/subscription_test.go b/store/subscription/subscription_test.go index 67f862bd..359843eb 100644 --- a/store/subscription/subscription_test.go +++ b/store/subscription/subscription_test.go @@ -67,12 +67,14 @@ var _ = Describe("Database Test", func() { } By("insert new subscription") - err := store.BatchInsert([]*model.Subscription{data1}) + duplicated, err := store.BatchInsert([]*model.Subscription{data1}) Expect(err).Should(Succeed()) + Expect(len(duplicated)).Should(Equal(0)) - By("failed to subscription again") - err = store.BatchInsert([]*model.Subscription{data1}) - Expect(err).ShouldNot(BeNil()) + By("duplicated should be 1") + duplicated, err = store.BatchInsert([]*model.Subscription{data1}) + Expect(err).Should(Succeed()) + Expect(len(duplicated)).Should(Equal(1)) data2 := &model.Subscription{ BlockNumber: 100, @@ -80,7 +82,8 @@ var _ = Describe("Database Test", func() { } By("insert another new subscription") - err = store.BatchInsert([]*model.Subscription{data2}) + duplicated, err = store.BatchInsert([]*model.Subscription{data2}) + Expect(len(duplicated)).Should(Equal(0)) Expect(err).Should(Succeed()) }) @@ -103,7 +106,7 @@ var _ = Describe("Database Test", func() { } By("insert three new subscriptions") data := []*model.Subscription{data1, data2, data3} - err := store.BatchInsert(data) + _, err := store.BatchInsert(data) Expect(err).Should(Succeed()) res, err := store.Find(data1.BlockNumber) @@ -139,7 +142,7 @@ var _ = Describe("Database Test", func() { } By("insert three new subscriptions") data := []*model.Subscription{data1, data2, data3} - err := store.BatchInsert(data) + _, err := store.BatchInsert(data) Expect(err).Should(Succeed()) res, err := store.FindOldSubscriptions([][]byte{ @@ -206,7 +209,7 @@ var _ = Describe("Database Test", func() { } By("Should be successful to insert", func() { - err := store.BatchInsert(subs) + _, err := store.BatchInsert(subs) Expect(err).Should(Succeed()) }) @@ -269,7 +272,7 @@ var _ = Describe("Database Test", func() { } By("insert three new subscriptions") data := []*model.Subscription{data1, data2, data3} - err := store.BatchInsert(data) + _, err := store.BatchInsert(data) Expect(err).Should(Succeed()) res, err := store.Find(0) diff --git a/store/transfer_processor.go b/store/transfer_processor.go index 553e3a3a..e0c2684e 100644 --- a/store/transfer_processor.go +++ b/store/transfer_processor.go @@ -21,9 +21,10 @@ import ( "context" "math/big" - "github.com/ethereum/go-ethereum/common" + gethCommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/getamis/eth-indexer/client" + "github.com/getamis/eth-indexer/common" "github.com/getamis/eth-indexer/model" "github.com/getamis/eth-indexer/store/account" "github.com/getamis/eth-indexer/store/subscription" @@ -34,7 +35,7 @@ type transferProcessor struct { logger log.Logger blockNumber int64 // tokenList includes ETH and erc20 tokens - tokenList map[common.Address]struct{} + tokenList map[gethCommon.Address]struct{} subStore subscription.Store accountStore account.Store balancer client.Balancer @@ -51,10 +52,10 @@ func newTransferProcessor(blockNumber int64, subStore subscription.Store, accountStore account.Store, balancer client.Balancer) *transferProcessor { - tokenList := make(map[common.Address]struct{}, len(erc20List)+1) + tokenList := make(map[gethCommon.Address]struct{}, len(erc20List)+1) tokenList[model.ETHAddress] = struct{}{} for addr := range erc20List { - tokenList[common.HexToAddress(addr)] = struct{}{} + tokenList[gethCommon.HexToAddress(addr)] = struct{}{} } return &transferProcessor{ logger: log.New("number", blockNumber), @@ -70,19 +71,19 @@ func newTransferProcessor(blockNumber int64, func (s *transferProcessor) process(ctx context.Context, events []*model.Transfer) (err error) { // Update total balance for new subscriptions, map[group][token]balance - totalBalances := make(map[int64]map[common.Address]*big.Int) + totalBalances := make(map[int64]map[gethCommon.Address]*big.Int) totalFees := make(map[int64]*big.Int) // Collect modified addresses - seenAddrs := make(map[common.Address]struct{}) + seenAddrs := make(map[gethCommon.Address]struct{}) var addrs [][]byte for _, e := range events { - fromAddr := common.BytesToAddress(e.From) + fromAddr := gethCommon.BytesToAddress(e.From) if _, ok := seenAddrs[fromAddr]; !ok { seenAddrs[fromAddr] = struct{}{} addrs = append(addrs, e.From) } - toAddr := common.BytesToAddress(e.To) + toAddr := gethCommon.BytesToAddress(e.To) if _, ok := seenAddrs[toAddr]; !ok { seenAddrs[toAddr] = struct{}{} addrs = append(addrs, e.To) @@ -96,17 +97,17 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe return err } - contractsAddrs := make(map[common.Address]map[common.Address]struct{}) - newSubs := make(map[common.Address]*model.Subscription) + contractsAddrs := make(map[gethCommon.Address]map[gethCommon.Address]struct{}) + newSubs := make(map[gethCommon.Address]*model.Subscription) var newAddrs [][]byte for _, sub := range newSubResults { - newAddr := common.BytesToAddress(sub.Address) + newAddr := gethCommon.BytesToAddress(sub.Address) newAddrs = append(newAddrs, sub.Address) newSubs[newAddr] = sub // Make sure to collect ETH/ERC20 balances for the new subscriptions too. for token := range s.tokenList { if contractsAddrs[token] == nil { - contractsAddrs[token] = make(map[common.Address]struct{}) + contractsAddrs[token] = make(map[gethCommon.Address]struct{}) } contractsAddrs[token][newAddr] = struct{}{} } @@ -127,52 +128,44 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe } // Calculate tx fee - fees := make(map[common.Hash]*big.Int) + fees := make(map[gethCommon.Hash]*big.Int) // Assume the tx and receipt are in the same order for i, tx := range s.txs { r := s.receipts[i] price, _ := new(big.Int).SetString(tx.GasPrice, 10) - fees[common.BytesToHash(tx.Hash)] = new(big.Int).Mul(price, big.NewInt(int64(r.GasUsed))) + fees[gethCommon.BytesToHash(tx.Hash)] = new(big.Int).Mul(price, big.NewInt(int64(r.GasUsed))) } // Construct a set of subscription for membership testing - allSubs := make(map[common.Address]*model.Subscription) + allSubs := make(map[gethCommon.Address]*model.Subscription) for _, sub := range subs { - allSubs[common.BytesToAddress(sub.Address)] = sub + allSubs[gethCommon.BytesToAddress(sub.Address)] = sub } // Insert events if it's a subscribed account - addrDiff := make(map[common.Address]map[common.Address]*big.Int) - feeDiff := make(map[common.Address]*big.Int) + feeDiff := make(map[gethCommon.Address]*big.Int) for _, e := range events { - _, hasFrom := allSubs[common.BytesToAddress(e.From)] - _, hasTo := allSubs[common.BytesToAddress(e.To)] + _, hasFrom := allSubs[gethCommon.BytesToAddress(e.From)] + _, hasTo := allSubs[gethCommon.BytesToAddress(e.To)] if !hasFrom && !hasTo { continue } err := s.accountStore.InsertTransfer(e) if err != nil { - s.logger.Error("Failed to insert ERC20 transfer event", "value", e.Value, "from", common.Bytes2Hex(e.From), "to", common.Bytes2Hex(e.To), "err", err) + s.logger.Error("Failed to insert ERC20 transfer event", "value", e.Value, "from", common.BytesToHex(e.From), "to", common.BytesToHex(e.To), "err", err) return err } - contractAddr := common.BytesToAddress(e.Address) - if addrDiff[contractAddr] == nil { - addrDiff[contractAddr] = make(map[common.Address]*big.Int) - contractsAddrs[contractAddr] = make(map[common.Address]struct{}) + contractAddr := gethCommon.BytesToAddress(e.Address) + if contractsAddrs[contractAddr] == nil { + contractsAddrs[contractAddr] = make(map[gethCommon.Address]struct{}) } - d, _ := new(big.Int).SetString(e.Value, 10) if hasFrom { - from := common.BytesToAddress(e.From) - if addrDiff[contractAddr][from] == nil { - addrDiff[contractAddr][from] = new(big.Int).Neg(d) - contractsAddrs[contractAddr][from] = struct{}{} - } else { - addrDiff[contractAddr][from] = new(big.Int).Add(addrDiff[contractAddr][from], new(big.Int).Neg(d)) - } + from := gethCommon.BytesToAddress(e.From) + contractsAddrs[contractAddr][from] = struct{}{} // Add fee if it's a ETH event - if f, ok := fees[common.BytesToHash(e.TxHash)]; ok && bytes.Equal(e.Address, model.ETHBytes) { + if f, ok := fees[gethCommon.BytesToHash(e.TxHash)]; ok && bytes.Equal(e.Address, model.ETHBytes) { if feeDiff[from] == nil { feeDiff[from] = new(big.Int).Set(f) } else { @@ -181,13 +174,8 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe } } if hasTo { - to := common.BytesToAddress(e.To) - if addrDiff[contractAddr][to] == nil { - addrDiff[contractAddr][to] = d - contractsAddrs[contractAddr][to] = struct{}{} - } else { - addrDiff[contractAddr][to] = new(big.Int).Add(addrDiff[contractAddr][to], d) - } + to := gethCommon.BytesToAddress(e.To) + contractsAddrs[contractAddr][to] = struct{}{} } } @@ -198,8 +186,16 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe return err } - // Insert balance if it's a subscribed account + // Insert balance and calculate diff to total balances + addrDiff := make(map[gethCommon.Address]map[gethCommon.Address]*big.Int) + allAddrs := append(addrs, newAddrs...) for contractAddr, addrs := range results { + // Get last recorded balance for these accounts + latestBalances, err := s.getLatestBalances(contractAddr, allAddrs) + if err != nil { + s.logger.Error("Failed to get previous balances", "contractAddr", contractAddr.Hex(), "len", len(allAddrs), "err", err) + return err + } for addr, balance := range addrs { b := &model.Account{ ContractAddress: contractAddr.Bytes(), @@ -213,15 +209,30 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe return err } + if addrDiff[contractAddr] == nil { + addrDiff[contractAddr] = make(map[gethCommon.Address]*big.Int) + } + acct := latestBalances[addr] + diff := new(big.Int) + // If addr is a new subscription, add its balance to addrDiff for totalBalances. - // Note that we overwrite the original value if it exists, because the diff to - // totalBalances for a new subscription is its current balance. if newSubs[addr] != nil { - if addrDiff[contractAddr] == nil { - addrDiff[contractAddr] = make(map[common.Address]*big.Int) + // double check we don't have its previous balance + if acct != nil { + s.logger.Error("New subscription had previous balance", "block", acct.BlockNumber, "addr", addr.Hex(), "balance", acct.Balance) + return common.ErrHasPrevBalance } - addrDiff[contractAddr][addr] = balance + diff = balance + } else { + // make sure we have an old balance + if acct == nil { + s.logger.Error("Old subscription missing previous balance", "contractAddr", contractAddr.Hex(), "addr", addr.Hex()) + return common.ErrMissingPrevBalance + } + prevBalance, _ := new(big.Int).SetString(acct.Balance, 10) + diff.Sub(balance, prevBalance) } + addrDiff[contractAddr][addr] = diff } } @@ -243,7 +254,7 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe // Init total balance for the group if totalBalances[sub.Group] == nil { - totalBalances[sub.Group] = make(map[common.Address]*big.Int) + totalBalances[sub.Group] = make(map[gethCommon.Address]*big.Int) } tb, ok := totalBalances[sub.Group][token] if !ok { @@ -264,11 +275,6 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe } else { totalFees[sub.Group] = new(big.Int).Add(f, totalFees[sub.Group]) } - - // Subtract tx fee from total balance if it's not a new subscriptions. - if newSubs[addr] == nil { - totalBalances[sub.Group][token] = new(big.Int).Sub(totalBalances[sub.Group][token], f) - } } } } @@ -294,3 +300,16 @@ func (s *transferProcessor) process(ctx context.Context, events []*model.Transfe } return nil } + +// Get last recorded balance data for these accounts +func (s *transferProcessor) getLatestBalances(contractAddr gethCommon.Address, addrs [][]byte) (map[gethCommon.Address]*model.Account, error) { + balances, err := s.accountStore.FindLatestAccounts(contractAddr, addrs) + if err != nil { + return nil, err + } + lastBalances := make(map[gethCommon.Address]*model.Account) + for _, acct := range balances { + lastBalances[gethCommon.BytesToAddress(acct.Address)] = acct + } + return lastBalances, nil +} diff --git a/store/transfer_processor_test.go b/store/transfer_processor_test.go index 58f9fb5b..a6abe51f 100644 --- a/store/transfer_processor_test.go +++ b/store/transfer_processor_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/getamis/eth-indexer/client/mocks" "github.com/getamis/eth-indexer/model" + "github.com/getamis/eth-indexer/store/account" subsStore "github.com/getamis/eth-indexer/store/subscription" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -88,8 +89,9 @@ var _ = Describe("Subscription Test", func() { } // Insert subscription subStore := subsStore.NewWithDB(db) - err := subStore.BatchInsert(subs) + duplicated, err := subStore.BatchInsert(subs) Expect(err).Should(BeNil()) + Expect(len(duplicated)).Should(Equal(0)) // Insert ERC20 total balance err = subStore.InsertTotalBalance(&model.TotalBalance{ @@ -260,6 +262,24 @@ var _ = Describe("Subscription Test", func() { err = manager.InsertERC20(erc20) Expect(err).Should(BeNil()) + acctStore := account.NewWithDB(db) + // Insert previous ERC20 balance for the old subscriptions + err = acctStore.InsertAccount(&model.Account{ + ContractAddress: erc20.Address, + BlockNumber: 99, + Address: subs[0].Address, + Balance: "2000", + }) + Expect(err).Should(BeNil()) + // Insert previous ether balance for the old subscriptions + err = acctStore.InsertAccount(&model.Account{ + ContractAddress: model.ETHBytes, + BlockNumber: 99, + Address: subs[0].Address, + Balance: "1000", + }) + Expect(err).Should(BeNil()) + err = manager.Init(mockBalancer) Expect(err).Should(BeNil())