use count instead of full list of accounts

Signed-off-by: Jason Frame <jason.frame@consensys.net>
hyperledger · Sep 21, 2023 · 8f75da4 · 8f75da4
1 parent 1c948ff
commit 8f75da4
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 32 deletions.
diff --git a/...in/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java b/...in/java/org/hyperledger/besu/ethereum/bonsai/storage/BonsaiWorldStateKeyValueStorage.java
@@ -14,9 +14,9 @@
  */
 package org.hyperledger.besu.ethereum.bonsai.storage;
 
-import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_HASH_BY_CODE_HASH;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_HASH_COUNT;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE_BY_HASH;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
 
@@ -25,7 +25,6 @@
 import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy;
 import org.hyperledger.besu.ethereum.bonsai.storage.flat.FullFlatDbReaderStrategy;
 import org.hyperledger.besu.ethereum.bonsai.storage.flat.PartialFlatDbReaderStrategy;
-import org.hyperledger.besu.ethereum.rlp.RLP;
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
 import org.hyperledger.besu.ethereum.trie.MerkleTrie;
@@ -41,18 +40,17 @@
 import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
 import org.hyperledger.besu.util.Subscribers;
 
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Optional;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
-import com.google.common.collect.Lists;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.slf4j.Logger;
@@ -154,11 +152,27 @@ public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
     }
   }
 
-  public List<Bytes32> getAccountsForCodeHash(final Bytes32 codeHash) {
+  // TODO JF move these somewhere else
+  protected byte[] longToByteArray(final long l) {
+    final ByteBuffer buf =
+        ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN);
+    buf.putLong(l);
+    return buf.array();
+  }
+
+  protected long longFromByteArray(final byte[] a) {
+    final ByteBuffer buf =
+        ByteBuffer.allocate(Long.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN);
+    buf.put(a);
+    buf.flip();
+    return buf.getLong();
+  }
+
+  public long getCodeHashCount(final Bytes32 codeHash) {
     return composedWorldStateStorage
-        .get(ACCOUNT_HASH_BY_CODE_HASH, codeHash.toArrayUnsafe())
-        .map(bytes -> RLP.input(Bytes.of(bytes)).readList(in -> Bytes32.wrap(in.readBytes32(), 0)))
-        .orElse(Lists.newArrayList());
+        .get(CODE_HASH_COUNT, codeHash.toArrayUnsafe())
+        .map(this::longFromByteArray)
+        .orElse(0L);
   }
 
   public Optional<Bytes> getAccount(final Hash accountHash) {
@@ -379,17 +393,14 @@ public Updater(
 
     @Override
     public BonsaiUpdater removeCode(final Hash accountHash, final Hash codeHash) {
-      // TODO JF this is almost duplicated with the update code accounts block in putCode
-      final List<Bytes32> codeAccounts = worldStateStorage.getAccountsForCodeHash(codeHash);
-      final Set<Bytes32> updatedCodeAccounts = new HashSet<>(codeAccounts);
-      updatedCodeAccounts.remove(accountHash);
-
-      final Bytes encodedAddresses =
-          RLP.encode(o -> o.writeList(updatedCodeAccounts, (val, out) -> out.writeBytes(val)));
+      final long codeHashCount = worldStateStorage.getCodeHashCount(codeHash);
+      final long updatedCodeHashCount = codeHashCount > 0 ? codeHashCount - 1 : 0;
       composedWorldStateTransaction.put(
-          ACCOUNT_HASH_BY_CODE_HASH, codeHash.toArray(), encodedAddresses.toArrayUnsafe());
+          CODE_HASH_COUNT,
+          codeHash.toArray(),
+          worldStateStorage.longToByteArray(updatedCodeHashCount));
 
-      if (updatedCodeAccounts.isEmpty()) {
+      if (updatedCodeHashCount <= 0) {
         composedWorldStateTransaction.remove(CODE_STORAGE_BY_HASH, codeHash.toArrayUnsafe());
       }
       return this;
@@ -402,15 +413,14 @@ public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, fin
         return this;
       }
 
-      final List<Bytes32> codeAccounts = worldStateStorage.getAccountsForCodeHash(codeHash);
-      final Set<Bytes32> updatedCodeAccounts = new HashSet<>(codeAccounts);
-      updatedCodeAccounts.add(accountHash);
-      final Bytes encodedAddresses =
-          RLP.encode(o -> o.writeList(updatedCodeAccounts, (val, out) -> out.writeBytes(val)));
+      final long codeHashCount = worldStateStorage.getCodeHashCount(codeHash);
+      final long updatedCodeHashCount = codeHashCount < 0 ? 1 : codeHashCount + 1;
       composedWorldStateTransaction.put(
-          ACCOUNT_HASH_BY_CODE_HASH, codeHash.toArray(), encodedAddresses.toArrayUnsafe());
+          CODE_HASH_COUNT,
+          codeHash.toArray(),
+          worldStateStorage.longToByteArray(updatedCodeHashCount));
 
-      if (codeAccounts.isEmpty()) {
+      if (codeHashCount <= 0) {
         composedWorldStateTransaction.put(
             CODE_STORAGE_BY_HASH, codeHash.toArrayUnsafe(), code.toArrayUnsafe());
       }

diff --git a/...m/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java b/...m/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/worldview/BonsaiWorldState.java
@@ -248,8 +248,10 @@ private void updateCode(
               worldStateUpdater.getCodeToUpdate().entrySet()) {
             final Bytes updatedCode = codeUpdate.getValue().getUpdated();
             final Hash accountHash = codeUpdate.getKey().addressHash();
-            if (updatedCode == null || updatedCode.isEmpty()) {
-              final Hash priorCodeHash = Hash.hash(codeUpdate.getValue().getPrior());
+            final Bytes priorCode = codeUpdate.getValue().getPrior();
+            if ((updatedCode == null || updatedCode.isEmpty())
+                && !(priorCode == null || priorCode.isEmpty())) {
+              final Hash priorCodeHash = Hash.hash(priorCode);
               bonsaiUpdater.removeCode(accountHash, priorCodeHash);
             } else {
               final Hash codeHash = Hash.hash(codeUpdate.getValue().getUpdated());

diff --git a/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/...c/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java
@@ -43,29 +43,34 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   CHAIN_PRUNER_STATE(new byte[] {18}),
 
   CODE_STORAGE_BY_HASH(new byte[] {19}),
-  ACCOUNT_HASH_BY_CODE_HASH(new byte[] {20});
+  CODE_HASH_COUNT(new byte[] {20}, new int[] {2});
 
   private final byte[] id;
   private final int[] versionList;
   private final boolean containsStaticData;
+  private final boolean usesUint64MergeOperator;
 
   KeyValueSegmentIdentifier(final byte[] id) {
     this(id, new int[] {0, 1, 2});
   }
 
   KeyValueSegmentIdentifier(final byte[] id, final boolean containsStaticData) {
-    this(id, new int[] {0, 1, 2}, containsStaticData);
+    this(id, new int[] {0, 1, 2}, containsStaticData, false);
   }
 
   KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) {
-    this(id, versionList, false);
+    this(id, versionList, false, false);
   }
 
   KeyValueSegmentIdentifier(
-      final byte[] id, final int[] versionList, final boolean containsStaticData) {
+      final byte[] id,
+      final int[] versionList,
+      final boolean containsStaticData,
+      final boolean usesUint64MergeOperator) {
     this.id = id;
     this.versionList = versionList;
     this.containsStaticData = containsStaticData;
+    this.usesUint64MergeOperator = usesUint64MergeOperator;
   }
 
   @Override
@@ -83,6 +88,11 @@ public boolean containsStaticData() {
     return containsStaticData;
   }
 
+  @Override
+  public boolean useUInt64MergeOperator() {
+    return usesUint64MergeOperator;
+  }
+
   @Override
   public boolean includeInDatabaseVersion(final int version) {
     return Arrays.contains(versionList, version);

diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle
@@ -69,7 +69,7 @@ Calculated : ${currentHash}
 tasks.register('checkAPIChanges', FileStateChecker) {
   description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
   files = sourceSets.main.allJava.files
-  knownHash = 'tpSnjt4HgqSiOTJhBbYdB0r1nFX4QZbicjfloI71Wf0='
+  knownHash = 'cgGISLm+mC7ySmcFqR55eIU1YNl6ZycZDIEOUkIm+F0='
 }
 check.dependsOn('checkAPIChanges')
 

diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java
@@ -56,4 +56,13 @@ default boolean includeInDatabaseVersion(final int version) {
    * @return true if the segment contains only static data
    */
   boolean containsStaticData();
+
+  /**
+   * Define this if the segment will store only uint64 values and wants to use the merge operator.
+   *
+   * @return true if the segment wants to use the uint64 merge operator
+   */
+  default boolean useUInt64MergeOperator() {
+    return false;
+  }
 }
diff --git a/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/...ledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java
@@ -61,6 +61,7 @@
 import org.rocksdb.Statistics;
 import org.rocksdb.Status;
 import org.rocksdb.TransactionDBOptions;
+import org.rocksdb.UInt64AddOperator;
 import org.rocksdb.WriteOptions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -227,6 +228,10 @@ private ColumnFamilyDescriptor createColumnDescriptor(final SegmentIdentifier se
           .setMinBlobSize(100)
           .setBlobCompressionType(CompressionType.LZ4_COMPRESSION);
     }
+    // TODO JF specify a merge operator name instead to make this more generic
+    if (segment.useUInt64MergeOperator()) {
+      options.setMergeOperator(new UInt64AddOperator());
+    }
 
     return new ColumnFamilyDescriptor(segment.getId(), options);
   }