-
Notifications
You must be signed in to change notification settings - Fork 130
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(primitives/state-machine)
TrieBackend implementation (#4318)
- Loading branch information
Showing
25 changed files
with
5,217 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright 2024 ChainSafe Systems (ON) | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| package costlru | ||
|
|
||
| import ( | ||
| "math" | ||
| "time" | ||
|
|
||
| "github.com/elastic/go-freelru" | ||
| ) | ||
|
|
||
| // LRU is a cost based LRU wrapped around [freelru.LRU]. | ||
| type LRU[K comparable, V any] struct { | ||
| currentCost uint | ||
| maxCost uint | ||
| costFunc func(K, V) uint32 | ||
| onEvictCallback freelru.OnEvictCallback[K, V] | ||
| *freelru.LRU[K, V] | ||
| } | ||
|
|
||
| // Costructor for [LRU]. | ||
| func New[K comparable, V any]( | ||
| maxCost uint, hash freelru.HashKeyCallback[K], costFunc func(K, V) uint32, | ||
| ) (*LRU[K, V], error) { | ||
| var capacity = uint32(math.MaxUint32) | ||
| if maxCost < math.MaxUint32 { | ||
| capacity = uint32(maxCost) | ||
| } | ||
| lru, err := freelru.New[K, V](capacity, hash) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| l := LRU[K, V]{ | ||
| maxCost: maxCost, | ||
| costFunc: costFunc, | ||
| LRU: lru, | ||
| } | ||
| lru.SetOnEvict(l.evictCallback) | ||
|
|
||
| return &l, nil | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) costRemove(key K, value V) (cost uint32, removed bool, canAdd bool) { | ||
| if l.LRU.Contains(key) { | ||
| oldVal, ok := l.LRU.Peek(key) | ||
| if !ok { | ||
| panic("should be in lru") | ||
| } | ||
| cost := l.costFunc(key, oldVal) | ||
| l.currentCost -= uint(cost) | ||
| } | ||
| cost = l.costFunc(key, value) | ||
| if uint(cost) > l.maxCost { | ||
| return cost, removed, false | ||
| } | ||
| for uint(cost)+l.currentCost > l.maxCost { | ||
| _, _, removed = l.LRU.RemoveOldest() | ||
| if !removed { | ||
| panic("huh?") | ||
| } | ||
| } | ||
| return cost, removed, true | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) Add(key K, value V) (added bool, evicted bool) { | ||
| cost, removed, canAdd := l.costRemove(key, value) | ||
| l.currentCost += uint(cost) | ||
| evicted = l.LRU.Add(key, value) | ||
| return canAdd, evicted || removed | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) AddWithLifetime(key K, value V, lifetime time.Duration) (added bool, evicted bool) { | ||
| cost, removed, canAdd := l.costRemove(key, value) | ||
| l.currentCost += uint(cost) | ||
| evicted = l.LRU.AddWithLifetime(key, value, lifetime) | ||
| return canAdd, evicted || removed | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) evictCallback(key K, value V) { | ||
| cost := l.costFunc(key, value) | ||
| if uint(cost) <= l.currentCost { | ||
| l.currentCost -= uint(cost) | ||
| } else { | ||
| l.currentCost = 0 | ||
| } | ||
| if l.onEvictCallback != nil { | ||
| l.onEvictCallback(key, value) | ||
| } | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) SetOnEvict(onEvict freelru.OnEvictCallback[K, V]) { | ||
| l.onEvictCallback = onEvict | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) Cost() uint { | ||
| return l.currentCost | ||
| } | ||
|
|
||
| func (l *LRU[K, V]) MaxCost() uint { | ||
| return l.maxCost | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,188 @@ | ||
| // Copyright 2024 ChainSafe Systems (ON) | ||
| // SPDX-License-Identifier: LGPL-3.0-only | ||
|
|
||
| package costlru | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/ChainSafe/gossamer/internal/primitives/core/hash" | ||
| "github.com/ChainSafe/gossamer/internal/primitives/runtime" | ||
| "github.com/dolthub/maphash" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| type Hasher struct { | ||
| maphash.Hasher[ValueCacheKeyHash[hash.H256]] | ||
| } | ||
|
|
||
| func (h Hasher) Hash(key ValueCacheKeyHash[hash.H256]) uint32 { | ||
| return uint32(h.Hasher.Hash(key)) | ||
| } | ||
|
|
||
| type ValueCacheKeyHash[H runtime.Hash] struct { | ||
| StorageRoot H | ||
| StorageKey string | ||
| } | ||
|
|
||
| func TestLRU(t *testing.T) { | ||
| hasher := Hasher{maphash.NewHasher[ValueCacheKeyHash[hash.H256]]()} | ||
|
|
||
| var costFunc = func(key ValueCacheKeyHash[hash.H256], val []byte) uint32 { | ||
| keyCost := uint32(len(key.StorageKey)) | ||
| return keyCost + uint32(len(val)) | ||
| } | ||
|
|
||
| maxNum := uint(1024) | ||
| maxSize := uint(costFunc(ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: hash.NewRandomH256(), | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| }, []byte{1})) * maxNum | ||
|
|
||
| l, err := New[ValueCacheKeyHash[hash.H256], []byte](maxSize, hasher.Hash, costFunc) | ||
| require.NoError(t, err) | ||
|
|
||
| someRoot := hash.NewRandomH256() | ||
| allKeys := make([]ValueCacheKeyHash[hash.H256], 0) | ||
| for i := 0; i < int(maxNum)*2; i++ { | ||
| hash := ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: someRoot, | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| } | ||
| allKeys = append(allKeys, hash) | ||
| added, evicted := l.Add(hash, []byte{uint8(i)}) | ||
| require.True(t, added) | ||
| if i >= int(maxNum) { | ||
| require.True(t, evicted) | ||
| } else { | ||
| require.False(t, evicted) | ||
| } | ||
| } | ||
|
|
||
| require.Equal(t, maxSize, l.currentCost) | ||
|
|
||
| for i, key := range l.Keys() { | ||
| require.Equal(t, allKeys[int(maxNum)+i], key) | ||
| } | ||
| } | ||
|
|
||
| func TestLRU_EvictCallback(t *testing.T) { | ||
| hasher := Hasher{maphash.NewHasher[ValueCacheKeyHash[hash.H256]]()} | ||
|
|
||
| var costFunc = func(key ValueCacheKeyHash[hash.H256], val []byte) uint32 { | ||
| keyCost := uint32(len(key.StorageKey)) | ||
| return keyCost + uint32(len(val)) | ||
| } | ||
|
|
||
| maxNum := uint(1024) | ||
| maxSize := uint(costFunc(ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: hash.NewRandomH256(), | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| }, []byte{1})) * maxNum | ||
|
|
||
| evictCount := 0 | ||
| l, err := New[ValueCacheKeyHash[hash.H256], []byte](maxSize, hasher.Hash, costFunc) | ||
| require.NoError(t, err) | ||
| l.SetOnEvict(func(vckh ValueCacheKeyHash[hash.H256], b []byte) { | ||
| evictCount++ | ||
| }) | ||
|
|
||
| someRoot := hash.NewRandomH256() | ||
| allKeys := make([]ValueCacheKeyHash[hash.H256], 0) | ||
| for i := 0; i < int(maxNum)*2; i++ { | ||
| hash := ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: someRoot, | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| } | ||
| allKeys = append(allKeys, hash) | ||
| added, evicted := l.Add(hash, []byte{uint8(i)}) | ||
| require.True(t, added) | ||
| if i >= int(maxNum) { | ||
| require.True(t, evicted) | ||
| } else { | ||
| require.False(t, evicted) | ||
| } | ||
| } | ||
|
|
||
| require.Equal(t, maxSize, l.currentCost) | ||
|
|
||
| for i, key := range l.Keys() { | ||
| require.Equal(t, allKeys[int(maxNum)+i], key) | ||
| } | ||
|
|
||
| require.Equal(t, int(maxNum), evictCount) | ||
| } | ||
|
|
||
| func TestLRU_Purge(t *testing.T) { | ||
| hasher := Hasher{maphash.NewHasher[ValueCacheKeyHash[hash.H256]]()} | ||
|
|
||
| var costFunc = func(key ValueCacheKeyHash[hash.H256], val []byte) uint32 { | ||
| keyCost := uint32(len(key.StorageKey)) | ||
| return keyCost + uint32(len(val)) | ||
| } | ||
|
|
||
| maxNum := uint(3) | ||
| maxSize := uint(costFunc(ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: hash.NewRandomH256(), | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| }, []byte{1})) * maxNum | ||
|
|
||
| l, err := New[ValueCacheKeyHash[hash.H256], []byte](maxSize, hasher.Hash, costFunc) | ||
| require.NoError(t, err) | ||
|
|
||
| someRoot := hash.NewRandomH256() | ||
| allKeys := make([]ValueCacheKeyHash[hash.H256], 0) | ||
| for i := 0; i < int(maxNum)*2; i++ { | ||
| hash := ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: someRoot, | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| } | ||
| allKeys = append(allKeys, hash) | ||
| added, evicted := l.Add(hash, []byte{uint8(i)}) | ||
| require.True(t, added) | ||
| if i >= int(maxNum) { | ||
| require.True(t, evicted) | ||
| } else { | ||
| require.False(t, evicted) | ||
| } | ||
| } | ||
|
|
||
| require.Equal(t, maxSize, l.currentCost) | ||
|
|
||
| for i, key := range l.Keys() { | ||
| require.Equal(t, allKeys[int(maxNum)+i], key) | ||
| } | ||
|
|
||
| l.Purge() | ||
| require.Equal(t, 0, l.Len()) | ||
| require.Equal(t, uint(0), l.currentCost) | ||
| } | ||
|
|
||
| func TestLRU_Same_Entries(t *testing.T) { | ||
| hasher := Hasher{maphash.NewHasher[ValueCacheKeyHash[hash.H256]]()} | ||
|
|
||
| var costFunc = func(key ValueCacheKeyHash[hash.H256], val []byte) uint32 { | ||
| keyCost := uint32(len(key.StorageKey)) | ||
| return keyCost + uint32(len(val)) | ||
| } | ||
|
|
||
| someKey := ValueCacheKeyHash[hash.H256]{ | ||
| StorageRoot: hash.NewRandomH256(), | ||
| StorageKey: string(hash.NewRandomH256()), | ||
| } | ||
|
|
||
| maxNum := uint(5) | ||
| maxSize := uint(costFunc(someKey, []byte{1})) * maxNum | ||
|
|
||
| l, err := New[ValueCacheKeyHash[hash.H256], []byte](maxSize, hasher.Hash, costFunc) | ||
| require.NoError(t, err) | ||
|
|
||
| for i := 0; i < int(maxNum); i++ { | ||
| added, evicted := l.Add(someKey, []byte{1}) | ||
| require.True(t, added) | ||
| require.False(t, evicted) | ||
| } | ||
| require.Equal(t, 1, l.Len()) | ||
| require.Equal(t, int(l.Cost()), int(costFunc(someKey, []byte{1}))) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.