Skip to content

Commit

Permalink
Bonsai code storage by hash (hyperledger#5889)
Browse files Browse the repository at this point in the history
Add a storage mode for Bonsai that can store the code by hash instead of account hash

Signed-off-by: Jason Frame <jason.frame@consensys.net>
  • Loading branch information
jframe authored Jan 31, 2024
1 parent bc36d2c commit 9c02518
Show file tree
Hide file tree
Showing 20 changed files with 600 additions and 67 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.hyperledger.besu.cli.options.stable;

import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_LIMIT_TRIE_LOGS_ENABLED;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;
import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.Unstable.MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT;
Expand Down Expand Up @@ -85,6 +86,14 @@ public static class Unstable {
description =
"The max number of blocks to load and prune trie logs for at startup. (default: ${DEFAULT-VALUE})")
private int bonsaiTrieLogPruningWindowSize = DEFAULT_BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE;

@CommandLine.Option(
hidden = true,
names = {"--Xbonsai-code-using-code-hash-enabled"},
arity = "1",
description =
"Enables code storage using code hash instead of by account hash. (default: ${DEFAULT-VALUE})")
private boolean bonsaiCodeUsingCodeHashEnabled = DEFAULT_BONSAI_CODE_USING_CODE_HASH_ENABLED;
}
/**
* Create data storage options.
Expand Down Expand Up @@ -138,6 +147,8 @@ static DataStorageOptions fromConfig(final DataStorageConfiguration domainObject
domainObject.getUnstable().getBonsaiLimitTrieLogsEnabled();
dataStorageOptions.unstableOptions.bonsaiTrieLogPruningWindowSize =
domainObject.getUnstable().getBonsaiTrieLogPruningWindowSize();
dataStorageOptions.unstableOptions.bonsaiCodeUsingCodeHashEnabled =
domainObject.getUnstable().getBonsaiCodeStoredByCodeHashEnabled();

return dataStorageOptions;
}
Expand All @@ -151,6 +162,7 @@ public DataStorageConfiguration toDomainObject() {
ImmutableDataStorageConfiguration.Unstable.builder()
.bonsaiLimitTrieLogsEnabled(unstableOptions.bonsaiLimitTrieLogsEnabled)
.bonsaiTrieLogPruningWindowSize(unstableOptions.bonsaiTrieLogPruningWindowSize)
.bonsaiCodeStoredByCodeHashEnabled(unstableOptions.bonsaiCodeUsingCodeHashEnabled)
.build())
.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,28 @@ public void bonsaiTrieLogRetentionLimitShouldBeAboveMinimum() {
"511");
}

@Test
public void bonsaiCodeUsingCodeHashEnabledCanBeEnabled() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(
dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled())
.isEqualTo(true),
"--Xbonsai-code-using-code-hash-enabled",
"true");
}

@Test
public void bonsaiCodeUsingCodeHashEnabledCanBeDisabled() {
internalTestSuccess(
dataStorageConfiguration ->
assertThat(
dataStorageConfiguration.getUnstable().getBonsaiCodeStoredByCodeHashEnabled())
.isEqualTo(false),
"--Xbonsai-code-using-code-hash-enabled",
"false");
}

@Override
protected DataStorageConfiguration createDefaultDomainObject() {
return DataStorageConfiguration.DEFAULT_CONFIG;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public Optional<Bytes> getAccount(final Hash accountHash) {
}

@Override
public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
public Optional<Bytes> getCode(final Hash codeHash, final Hash accountHash) {
return isClosedGet() ? Optional.empty() : super.getCode(codeHash, accountHash);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public FlatDbMode getFlatDbMode() {
}

@Override
public Optional<Bytes> getCode(final Bytes32 codeHash, final Hash accountHash) {
public Optional<Bytes> getCode(final Hash codeHash, final Hash accountHash) {
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
} else {
Expand Down Expand Up @@ -323,7 +323,7 @@ public FlatDbStrategy getFlatDbStrategy() {
}

public interface BonsaiUpdater extends WorldStateStorage.Updater {
BonsaiUpdater removeCode(final Hash accountHash);
BonsaiUpdater removeCode(final Hash accountHash, final Hash codeHash);

BonsaiUpdater removeAccountInfoState(final Hash accountHash);

Expand Down Expand Up @@ -356,14 +356,14 @@ public Updater(
}

@Override
public BonsaiUpdater removeCode(final Hash accountHash) {
flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash);
public BonsaiUpdater removeCode(final Hash accountHash, final Hash codeHash) {
flatDbStrategy.removeFlatCode(composedWorldStateTransaction, accountHash, codeHash);
return this;
}

@Override
public BonsaiUpdater putCode(final Hash accountHash, final Bytes32 codeHash, final Bytes code) {
if (code.size() == 0) {
public BonsaiUpdater putCode(final Hash accountHash, final Hash codeHash, final Bytes code) {
if (code.isEmpty()) {
// Don't save empty values
return this;
}
Expand All @@ -379,7 +379,7 @@ public BonsaiUpdater removeAccountInfoState(final Hash accountHash) {

@Override
public BonsaiUpdater putAccountInfoState(final Hash accountHash, final Bytes accountValue) {
if (accountValue.size() == 0) {
if (accountValue.isEmpty()) {
// Don't save empty values
return this;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/

package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat;

import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;

import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;

import java.util.Optional;

import org.apache.tuweni.bytes.Bytes;

public class AccountHashCodeStorageStrategy implements CodeStorageStrategy {
@Override
public Optional<Bytes> getFlatCode(
final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) {
return storage
.get(CODE_STORAGE, accountHash.toArrayUnsafe())
.map(Bytes::wrap)
.filter(b -> Hash.hash(b).equals(codeHash));
}

@Override
public void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash,
final Bytes code) {
transaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe());
}

@Override
public void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash) {
transaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/

package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat;

import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;

import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;

import java.util.Optional;

import org.apache.tuweni.bytes.Bytes;

public class CodeHashCodeStorageStrategy implements CodeStorageStrategy {
static final Bytes CODE_PREFIX = Bytes.of(1);

@Override
public Optional<Bytes> getFlatCode(
final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) {
return storage.get(CODE_STORAGE, prefixKey(CODE_PREFIX, codeHash)).map(Bytes::wrap);
}

@Override
public void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash,
final Bytes code) {
transaction.put(CODE_STORAGE, prefixKey(CODE_PREFIX, codeHash), code.toArrayUnsafe());
}

@Override
public void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash) {
// TODO JF Part of #5388 add reference counting so that code can be removed
}

private byte[] prefixKey(final Bytes prefix, final Bytes key) {
return Bytes.concatenate(prefix, key).toArrayUnsafe();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/

package org.hyperledger.besu.ethereum.trie.bonsai.storage.flat;

import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;

import java.util.Optional;

import org.apache.tuweni.bytes.Bytes;

public interface CodeStorageStrategy {

Optional<Bytes> getFlatCode(
final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage);

void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash,
final Bytes code);

void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash);
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,12 @@ public abstract class FlatDbStrategy {

protected final Counter getStorageValueCounter;
protected final Counter getStorageValueFlatDatabaseCounter;
protected final CodeStorageStrategy codeStorageStrategy;

public FlatDbStrategy(final MetricsSystem metricsSystem) {
public FlatDbStrategy(
final MetricsSystem metricsSystem, final CodeStorageStrategy codeStorageStrategy) {
this.metricsSystem = metricsSystem;
this.codeStorageStrategy = codeStorageStrategy;

getAccountCounter =
metricsSystem.createCounter(
Expand Down Expand Up @@ -107,14 +110,11 @@ public abstract Optional<Bytes> getFlatStorageValueByStorageSlotKey(
* Retrieves the code data for the given code hash and account hash.
*/
public Optional<Bytes> getFlatCode(
final Bytes32 codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) {
final Hash codeHash, final Hash accountHash, final SegmentedKeyValueStorage storage) {
if (codeHash.equals(Hash.EMPTY)) {
return Optional.of(Bytes.EMPTY);
} else {
return storage
.get(CODE_STORAGE, accountHash.toArrayUnsafe())
.map(Bytes::wrap)
.filter(b -> Hash.hash(b).equals(codeHash));
return codeStorageStrategy.getFlatCode(codeHash, accountHash, storage);
}
}

Expand Down Expand Up @@ -162,8 +162,10 @@ public void removeFlatAccountStorageValueByStorageSlotHash(
* Removes code for the given account hash.
*/
public void removeFlatCode(
final SegmentedKeyValueStorageTransaction transaction, final Hash accountHash) {
transaction.remove(CODE_STORAGE, accountHash.toArrayUnsafe());
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Hash codeHash) {
codeStorageStrategy.removeFlatCode(transaction, accountHash, codeHash);
}

/*
Expand All @@ -172,9 +174,9 @@ public void removeFlatCode(
public void putFlatCode(
final SegmentedKeyValueStorageTransaction transaction,
final Hash accountHash,
final Bytes32 codeHash,
final Hash codeHash,
final Bytes code) {
transaction.put(CODE_STORAGE, accountHash.toArrayUnsafe(), code.toArrayUnsafe());
codeStorageStrategy.putFlatCode(transaction, accountHash, codeHash, code);
}

public void clearAll(final SegmentedKeyValueStorage storage) {
Expand Down
Loading

0 comments on commit 9c02518

Please sign in to comment.