From 252cb11ea740bb212ddf3c837e564594f2037fc9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:06:07 +0700 Subject: [PATCH 1/6] save --- erigon-lib/state/aggregator.go | 16 +++++++++------- erigon-lib/state/domain.go | 9 --------- erigon-lib/state/domain_shared.go | 2 +- erigon-lib/state/inverted_index.go | 14 +++++--------- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 2e43acb31c9..6c861e8c432 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -914,7 +914,7 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du "until commit", time.Until(started.Add(timeout)).String(), "pruneLimit", pruneLimit, "aggregatedStep", ac.StepsInFiles(kv.StateDomains...), - "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx), + "stepsRangeInDB", ac.stepsRangeInDBAsStr(tx), "pruned", fullStat.String(), ) case <-ctx.Done(): @@ -924,13 +924,15 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du } } -func (a *Aggregator) StepsRangeInDBAsStr(tx kv.Tx) string { +func (at *AggregatorRoTx) stepsRangeInDBAsStr(tx kv.Tx) string { steps := make([]string, 0, kv.DomainLen+4) - for _, d := range a.d { - steps = append(steps, d.stepsRangeInDBAsStr(tx)) + for _, dt := range at.d { + a1, a2 := dt.ht.iit.stepsRangeInDB(tx) + steps = append(steps, fmt.Sprintf("%s:%.1f", dt.d.filenameBase, a2-a1)) } - for _, ii := range a.iis { - steps = append(steps, ii.stepsRangeInDBAsStr(tx)) + for _, iit := range at.iis { + a1, a2 := iit.stepsRangeInDB(tx) + steps = append(steps, fmt.Sprintf("%s:%.1f", iit.ii.filenameBase, a2-a1)) } return strings.Join(steps, ", ") } @@ -1065,7 +1067,7 @@ func (ac *AggregatorRoTx) Prune(ctx context.Context, tx kv.RwTx, limit uint64, l } //ac.a.logger.Info("aggregator prune", "step", step, // "txn_range", fmt.Sprintf("[%d,%d)", txFrom, txTo), "limit", limit, - // /*"stepsLimit", limit/ac.a.aggregationStep,*/ "stepsRangeInDB", ac.a.StepsRangeInDBAsStr(tx)) + // /*"stepsLimit", limit/ac.a.aggregationStep,*/ "stepsRangeInDB", ac.a.stepsRangeInDBAsStr(tx)) aggStat := newAggregatorPruneStat() for id, d := range ac.d { var err error diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 270071d876a..5f2a92aaf64 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -2017,15 +2017,6 @@ func (sr *SegStreamReader) Next() (k, v []byte, err error) { return k, v, nil } -func (d *Domain) stepsRangeInDBAsStr(tx kv.Tx) string { - a1, a2 := d.History.InvertedIndex.stepsRangeInDB(tx) - //ad1, ad2 := d.stepsRangeInDB(tx) - //if ad2-ad1 < 0 { - // fmt.Printf("aaa: %f, %f\n", ad1, ad2) - //} - return fmt.Sprintf("%s:%.1f", d.filenameBase, a2-a1) -} - func (dt *DomainRoTx) Files() (res []string) { for _, item := range dt.files { if item.src.decompressor != nil { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 42c2fd0b062..219d3ac1bec 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -187,7 +187,7 @@ func (sd *SharedDomains) Unwind(ctx context.Context, rwTx kv.RwTx, blockUnwindTo defer logEvery.Stop() sd.logger.Info("aggregator unwind", "step", step, "txUnwindTo", txUnwindTo, "stepsRangeInDB", sd.aggTx.a.StepsRangeInDBAsStr(rwTx)) - //fmt.Printf("aggregator unwind step %d txUnwindTo %d stepsRangeInDB %s\n", step, txUnwindTo, sd.aggTx.a.StepsRangeInDBAsStr(rwTx)) + //fmt.Printf("aggregator unwind step %d txUnwindTo %d stepsRangeInDB %s\n", step, txUnwindTo, sd.aggTx.a.stepsRangeInDBAsStr(rwTx)) sf := time.Now() defer mxUnwindSharedTook.ObserveDuration(sf) diff --git a/erigon-lib/state/inverted_index.go b/erigon-lib/state/inverted_index.go index 592f1665726..78d499058df 100644 --- a/erigon-lib/state/inverted_index.go +++ b/erigon-lib/state/inverted_index.go @@ -1222,18 +1222,14 @@ func (ii *InvertedIndex) integrateDirtyFiles(sf InvertedFiles, txNumFrom, txNumT ii.dirtyFiles.Set(fi) } -func (ii *InvertedIndex) stepsRangeInDBAsStr(tx kv.Tx) string { - a1, a2 := ii.stepsRangeInDB(tx) - return fmt.Sprintf("%s: %.1f", ii.filenameBase, a2-a1) -} -func (ii *InvertedIndex) stepsRangeInDB(tx kv.Tx) (from, to float64) { - fst, _ := kv.FirstKey(tx, ii.keysTable) +func (iit *InvertedIndexRoTx) stepsRangeInDB(tx kv.Tx) (from, to float64) { + fst, _ := kv.FirstKey(tx, iit.ii.keysTable) if len(fst) > 0 { - from = float64(binary.BigEndian.Uint64(fst)) / float64(ii.aggregationStep) + from = float64(binary.BigEndian.Uint64(fst)) / float64(iit.ii.aggregationStep) } - lst, _ := kv.LastKey(tx, ii.keysTable) + lst, _ := kv.LastKey(tx, iit.ii.keysTable) if len(lst) > 0 { - to = float64(binary.BigEndian.Uint64(lst)) / float64(ii.aggregationStep) + to = float64(binary.BigEndian.Uint64(lst)) / float64(iit.ii.aggregationStep) } if to == 0 { to = from From 288c1b169794bd39c5a09e84919d8d2bb8c78624 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:08:22 +0700 Subject: [PATCH 2/6] save --- erigon-lib/state/domain_shared.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 219d3ac1bec..ecf53e6f889 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -186,8 +186,8 @@ func (sd *SharedDomains) Unwind(ctx context.Context, rwTx kv.RwTx, blockUnwindTo logEvery := time.NewTicker(30 * time.Second) defer logEvery.Stop() sd.logger.Info("aggregator unwind", "step", step, - "txUnwindTo", txUnwindTo, "stepsRangeInDB", sd.aggTx.a.StepsRangeInDBAsStr(rwTx)) - //fmt.Printf("aggregator unwind step %d txUnwindTo %d stepsRangeInDB %s\n", step, txUnwindTo, sd.aggTx.a.stepsRangeInDBAsStr(rwTx)) + "txUnwindTo", txUnwindTo) + //fmt.Printf("aggregator unwind step %d txUnwindTo %d\n", step, txUnwindTo) sf := time.Now() defer mxUnwindSharedTook.ObserveDuration(sf) From eebc67223fe122690486f22569e7004196f2337b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:09:38 +0700 Subject: [PATCH 3/6] save --- erigon-lib/state/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 6c861e8c432..291b0c8707f 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -925,7 +925,7 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du } func (at *AggregatorRoTx) stepsRangeInDBAsStr(tx kv.Tx) string { - steps := make([]string, 0, kv.DomainLen+4) + steps := make([]string, 0, len(at.d)+len(at.iis)) for _, dt := range at.d { a1, a2 := dt.ht.iit.stepsRangeInDB(tx) steps = append(steps, fmt.Sprintf("%s:%.1f", dt.d.filenameBase, a2-a1)) From b4357036916d6d4fdb330fa35b5f39743968d826 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:13:02 +0700 Subject: [PATCH 4/6] save --- erigon-lib/state/aggregator_bench_test.go | 10 ++++------ erigon-lib/state/aggregator_test.go | 7 +++---- erigon-lib/state/archive_test.go | 9 ++++----- erigon-lib/state/bpstree_bench_test.go | 3 +-- erigon-lib/state/btree_index_test.go | 9 ++++----- erigon-lib/state/inverted_index_test.go | 4 ++-- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/erigon-lib/state/aggregator_bench_test.go b/erigon-lib/state/aggregator_bench_test.go index 03a46b0aa92..24ee3871c78 100644 --- a/erigon-lib/state/aggregator_bench_test.go +++ b/erigon-lib/state/aggregator_bench_test.go @@ -22,20 +22,18 @@ import ( "flag" "fmt" "os" - "path" "path/filepath" "testing" "time" - "github.com/erigontech/erigon-lib/common/datadir" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/stretchr/testify/require" "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/mdbx" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon-lib/seg" ) @@ -142,7 +140,7 @@ func Benchmark_BtreeIndex_Search(b *testing.B) { defer os.RemoveAll(tmp) dataPath := "../../data/storage.256-288.kv" - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") comp := seg.CompressKeys | seg.CompressVals buildBtreeIndex(b, dataPath, indexPath, comp, 1, logger, true) @@ -174,7 +172,7 @@ func benchInitBtreeIndex(b *testing.B, M uint64, compression seg.FileCompression b.Cleanup(func() { os.RemoveAll(tmp) }) dataPath := generateKV(b, tmp, 52, 10, 1000000, logger, 0) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bt") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bt") buildBtreeIndex(b, dataPath, indexPath, compression, 1, logger, true) diff --git a/erigon-lib/state/aggregator_test.go b/erigon-lib/state/aggregator_test.go index f120961bad7..10fe639777b 100644 --- a/erigon-lib/state/aggregator_test.go +++ b/erigon-lib/state/aggregator_test.go @@ -22,11 +22,9 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "github.com/erigontech/erigon-lib/types/accounts" "math" "math/rand" "os" - "path" "path/filepath" "strings" "sync/atomic" @@ -49,6 +47,7 @@ import ( "github.com/erigontech/erigon-lib/kv/stream" "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/seg" + "github.com/erigontech/erigon-lib/types/accounts" "github.com/holiman/uint256" "github.com/stretchr/testify/require" ) @@ -1069,7 +1068,7 @@ func generateKV(tb testing.TB, tmp string, keySize, valueSize, keyCount int, log rnd := newRnd(0) values := make([]byte, valueSize) - dataPath := path.Join(tmp, fmt.Sprintf("%dk.kv", keyCount/1000)) + dataPath := filepath.Join(tmp, fmt.Sprintf("%dk.kv", keyCount/1000)) comp, err := seg.NewCompressor(context.Background(), "cmp", dataPath, tmp, seg.DefaultCfg, log.LvlDebug, logger) require.NoError(tb, err) @@ -1118,7 +1117,7 @@ func generateKV(tb testing.TB, tmp string, keySize, valueSize, keyCount int, log compPath := decomp.FilePath() ps := background.NewProgressSet() - IndexFile := path.Join(tmp, fmt.Sprintf("%dk.bt", keyCount/1000)) + IndexFile := filepath.Join(tmp, fmt.Sprintf("%dk.bt", keyCount/1000)) err = BuildBtreeIndexWithDecompressor(IndexFile, decomp, compressFlags, ps, tb.TempDir(), 777, logger, true) require.NoError(tb, err) diff --git a/erigon-lib/state/archive_test.go b/erigon-lib/state/archive_test.go index 559ceceeead..99e7775d9aa 100644 --- a/erigon-lib/state/archive_test.go +++ b/erigon-lib/state/archive_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "context" - "path" "path/filepath" "sort" "testing" @@ -87,7 +86,7 @@ func TestArchiveWriter(t *testing.T) { writeLatest(t, w, td) w.Close() - decomp, err := seg.NewDecompressor(path.Join(tmp, "uncompressed")) + decomp, err := seg.NewDecompressor(filepath.Join(tmp, "uncompressed")) require.NoError(t, err) defer decomp.Close() @@ -102,7 +101,7 @@ func TestArchiveWriter(t *testing.T) { writeLatest(t, w, td) w.Close() - decomp, err := seg.NewDecompressor(path.Join(tmp, "compressed")) + decomp, err := seg.NewDecompressor(filepath.Join(tmp, "compressed")) require.NoError(t, err) defer decomp.Close() ds := (datasize.B * datasize.ByteSize(decomp.Size())).HR() @@ -117,7 +116,7 @@ func TestArchiveWriter(t *testing.T) { writeLatest(t, w, td) w.Close() - decomp, err := seg.NewDecompressor(path.Join(tmp, "compressed-keys")) + decomp, err := seg.NewDecompressor(filepath.Join(tmp, "compressed-keys")) require.NoError(t, err) defer decomp.Close() ds := (datasize.B * datasize.ByteSize(decomp.Size())).HR() @@ -132,7 +131,7 @@ func TestArchiveWriter(t *testing.T) { writeLatest(t, w, td) w.Close() - decomp, err := seg.NewDecompressor(path.Join(tmp, "compressed-vals")) + decomp, err := seg.NewDecompressor(filepath.Join(tmp, "compressed-vals")) require.NoError(t, err) defer decomp.Close() ds := (datasize.B * datasize.ByteSize(decomp.Size())).HR() diff --git a/erigon-lib/state/bpstree_bench_test.go b/erigon-lib/state/bpstree_bench_test.go index c0ac5a3f2a6..0f4908dc3f1 100644 --- a/erigon-lib/state/bpstree_bench_test.go +++ b/erigon-lib/state/bpstree_bench_test.go @@ -1,7 +1,6 @@ package state import ( - "path" "path/filepath" "testing" @@ -20,7 +19,7 @@ func BenchmarkBpsTreeSeek(t *testing.B) { dataPath := generateKV(t, tmp, 52, 180, keyCount, logger, 0) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") buildBtreeIndex(t, dataPath, indexPath, compressFlags, 1, logger, true) kv, bt, err := OpenBtreeIndexAndDataFile(indexPath, dataPath, uint64(M), compressFlags, false) diff --git a/erigon-lib/state/btree_index_test.go b/erigon-lib/state/btree_index_test.go index c916cc3366e..4d92157b2eb 100644 --- a/erigon-lib/state/btree_index_test.go +++ b/erigon-lib/state/btree_index_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "fmt" - "path" "path/filepath" "testing" @@ -83,7 +82,7 @@ func Test_BtreeIndex_Seek(t *testing.T) { t.Run("empty index", func(t *testing.T) { dataPath := generateKV(t, tmp, 52, 180, 0, logger, 0) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") buildBtreeIndex(t, dataPath, indexPath, compressFlags, 1, logger, true) kv, bt, err := OpenBtreeIndexAndDataFile(indexPath, dataPath, uint64(M), compressFlags, false) @@ -94,7 +93,7 @@ func Test_BtreeIndex_Seek(t *testing.T) { }) dataPath := generateKV(t, tmp, 52, 180, keyCount, logger, 0) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") buildBtreeIndex(t, dataPath, indexPath, compressFlags, 1, logger, true) kv, bt, err := OpenBtreeIndexAndDataFile(indexPath, dataPath, uint64(M), compressFlags, false) @@ -169,7 +168,7 @@ func Test_BtreeIndex_Build(t *testing.T) { keys, err := pivotKeysFromKV(dataPath) require.NoError(t, err) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") buildBtreeIndex(t, dataPath, indexPath, compressFlags, 1, logger, true) require.NoError(t, err) @@ -222,7 +221,7 @@ func Test_BtreeIndex_Seek2(t *testing.T) { compressFlags := seg.CompressKeys | seg.CompressVals dataPath := generateKV(t, tmp, 52, 48, keyCount, logger, compressFlags) - indexPath := path.Join(tmp, filepath.Base(dataPath)+".bti") + indexPath := filepath.Join(tmp, filepath.Base(dataPath)+".bti") buildBtreeIndex(t, dataPath, indexPath, compressFlags, 1, logger, true) kv, bt, err := OpenBtreeIndexAndDataFile(indexPath, dataPath, uint64(M), compressFlags, false) diff --git a/erigon-lib/state/inverted_index_test.go b/erigon-lib/state/inverted_index_test.go index e173e4f7df4..83b5248b066 100644 --- a/erigon-lib/state/inverted_index_test.go +++ b/erigon-lib/state/inverted_index_test.go @@ -286,7 +286,7 @@ func TestInvIndexAfterPrune(t *testing.T) { ic.Close() err = db.Update(ctx, func(tx kv.RwTx) error { - from, to := ii.stepsRangeInDB(tx) + from, to := ic.stepsRangeInDB(tx) require.Equal(t, "0.1", fmt.Sprintf("%.1f", from)) require.Equal(t, "0.4", fmt.Sprintf("%.1f", to)) @@ -315,7 +315,7 @@ func TestInvIndexAfterPrune(t *testing.T) { require.Nil(t, k, table) } - from, to := ii.stepsRangeInDB(tx) + from, to := ic.stepsRangeInDB(tx) require.Equal(t, float64(0), from) require.Equal(t, float64(0), to) } From 4ebd51acd5e311a96bd625d1876110e193fc9fb9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:14:29 +0700 Subject: [PATCH 5/6] save --- erigon-lib/state/domain.go | 4 ++++ erigon-lib/state/domain_test.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/erigon-lib/state/domain.go b/erigon-lib/state/domain.go index 5f2a92aaf64..0e7ae9ef348 100644 --- a/erigon-lib/state/domain.go +++ b/erigon-lib/state/domain.go @@ -2017,6 +2017,10 @@ func (sr *SegStreamReader) Next() (k, v []byte, err error) { return k, v, nil } +func (dt *DomainRoTx) stepsRangeInDB(tx kv.Tx) (from, to float64) { + return dt.ht.iit.stepsRangeInDB(tx) +} + func (dt *DomainRoTx) Files() (res []string) { for _, item := range dt.files { if item.src.decompressor != nil { diff --git a/erigon-lib/state/domain_test.go b/erigon-lib/state/domain_test.go index d7832c2375c..fbb322c791a 100644 --- a/erigon-lib/state/domain_test.go +++ b/erigon-lib/state/domain_test.go @@ -919,7 +919,7 @@ func TestDomain_PruneOnWrite(t *testing.T) { require.EqualValues(t, v[:], storedV, label) } - from, to := d.stepsRangeInDB(tx) + from, to := dc.stepsRangeInDB(tx) require.Equal(t, 3, int(from)) require.Equal(t, 4, int(to)) From 9a1f55119b3dad22fc95326042413f8d42c24255 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sun, 2 Mar 2025 17:15:03 +0700 Subject: [PATCH 6/6] save --- erigon-lib/state/aggregator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 291b0c8707f..22d658031ee 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -927,7 +927,7 @@ func (ac *AggregatorRoTx) PruneSmallBatches(ctx context.Context, timeout time.Du func (at *AggregatorRoTx) stepsRangeInDBAsStr(tx kv.Tx) string { steps := make([]string, 0, len(at.d)+len(at.iis)) for _, dt := range at.d { - a1, a2 := dt.ht.iit.stepsRangeInDB(tx) + a1, a2 := dt.stepsRangeInDB(tx) steps = append(steps, fmt.Sprintf("%s:%.1f", dt.d.filenameBase, a2-a1)) } for _, iit := range at.iis {