Merge branch 'main' into eip-6110

Signed-off-by: Navie Chan <naviechan@gmail.com>
hyperledger · Feb 19, 2023 · 1c8de5a · 1c8de5a
2 parents d57d70b + ab576d6
commit 1c8de5a
Show file tree

Hide file tree

Showing 536 changed files with 10,926 additions and 11,547 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,29 +1,29 @@
 ---
 version: 2.1
 orbs:
-  win: circleci/windows@2.2.0
+  win: circleci/windows@5.0
 
 executors:
   besu_executor_small:
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: small
     working_directory: ~/project
     environment:
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2
 
   besu_executor_med: # 2cpu, 4G ram
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: medium
     working_directory: ~/project
     environment:
       architecture: "amd64"
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2
 
   besu_arm64_executor_med: # 2cpu, 8G ram
-    machine:
-      image: ubuntu-2004:202101-01
+    machine: #https://circleci.com/developer/machine/image/ubuntu-2204
+      image: ubuntu-2204:2022.10.2
     resource_class: arm.medium
     working_directory: ~/project
     environment:
@@ -32,15 +32,15 @@ executors:
 
   besu_executor_xl: # 8cpu, 16G ram
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: xlarge
     working_directory: ~/project
     environment:
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=4
 
   xl_machine_executor:
-    machine:
-      image: ubuntu-2004:202201-02 #Ubuntu 20.04, Docker v20.10.12, Docker Compose v1.29.2, Google Cloud SDK updates
+    machine: #https://circleci.com/developer/machine/image/ubuntu-2204
+      image: ubuntu-2204:2022.10.2
     resource_class: xlarge
 
   trivy_executor:
@@ -60,6 +60,7 @@ commands:
             sudo apt-get update
             sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools
             sudo service haveged restart
+            java --version
       - restore_gradle_cache
   restore_gradle_cache:
     description: "Restore Gradle cache"
@@ -225,8 +226,8 @@ jobs:
       - attach_workspace:
           at: ~/project
       - run:
-          name: AcceptanceTests
-          no_output_timeout: 30m
+          name: AcceptanceTests (Mainnet)
+          no_output_timeout: 20m
           command: |
             CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
               | sed 's@.*/src/test/java/@@' \
@@ -235,7 +236,29 @@ jobs:
               | circleci tests split --split-by=timings --timings-type=classname)
             # Format the arguments to "./gradlew test"
             GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTest $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestMainnet $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsNonMainnet:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestNonMainnet $GRADLE_ARGS
       - capture_test_results
       - capture_test_logs
 
@@ -247,13 +270,13 @@ jobs:
           at: ~/project
       - setup_remote_docker
       - run:
-          name: hadoLint_openjdk_11
+          name: hadoLint_openjdk_17
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
       - run:
-          name: hadoLint_openjdk_11_debug
+          name: hadoLint_openjdk_17_debug
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11-debug/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
       - run:
           name: hadoLint_openjdk_latest
           command: |
@@ -280,9 +303,9 @@ jobs:
       - attach_workspace:
           at: ~/project
       - run:
-          name: hadoLint_openjdk_11
+          name: hadoLint_openjdk_17
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
       - run:
           name: hadoLint_openjdk_latest
           command: |
@@ -291,6 +314,11 @@ jobs:
           name: hadoLint_graalvm
           command: |
             docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
+      - run:
+          name: Java_17
+          command: |
+            sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless
+            sudo update-java-alternatives -a
       - run:
           name: build image
           command: |
@@ -332,6 +360,11 @@ jobs:
       - prepare
       - attach_workspace:
           at: ~/project
+      - run:
+          name: Java_17
+          command: |
+            sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless
+            sudo update-java-alternatives -a
       - run:
           name: Publish Docker
           command: |
@@ -439,3 +472,4 @@ workflows:
     jobs:
       - assemble
       - dockerScan
+      - acceptanceTestsNonMainnet
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -12,6 +12,10 @@
 - [ ] I thought about documentation and added the `doc-change-required` label to this PR if
     [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).
 
+## Acceptance Tests (Non Mainnet)
+
+- [ ] I have considered running `./gradlew acceptanceTestNonMainnet` locally if my PR affects non-mainnet modules.
+
 ## Changelog
 
 - [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -16,7 +16,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: adopt
-          java-version: 11
+          java-version: 17
           cache: gradle
       - name: spotless
         run: ./gradlew --no-daemon --parallel clean spotlessCheck

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -41,6 +41,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    - name: Set up Java
+      uses: actions/setup-java@v2
+      with:
+        distribution: adopt
+        java-version: 17
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,6 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '17'
       - name: Login to DockerHub
         run: echo '${{ secrets.DOCKER_PASSWORD_RW }}' | docker login -u '${{ secrets.DOCKER_USER_RW }}' --password-stdin
       - name: Setup Gradle

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,32 +1,38 @@
 # Changelog
 
-## 23.1.1
+## 23.1.0-RC2
 
 ### Breaking Changes
-- Add a new CLI option to limit the number of requests in a single RPC batch request. Default=1 [#4965](https://github.com/hyperledger/besu/pull/4965)
-
-- Changed JsonRpc http service to return the error -32602 (Invalid params) with a 200 http status code
+- Change JsonRpc http service to return the error -32602 (Invalid params) with a 200 http status code
+- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
+- PKCS11 with nss module (PKCS11 based HSM can be used in DevP2P TLS and QBFT PKI) does not work with RSA keys 
+in Java 17. SoftHSM is tested manually and working. (Other PKCS11 HSM are not tested). The relevant unit and acceptance 
+tests are updated to use EC private keys instead of RSA keys.
+- Change eth_feeHistory parameter `blockCount` to accept hexadecimal string (was accepting plain integer) [#5047](https://github.com/hyperledger/besu/pull/5047)
 
 ### Additions and Improvements
+- Default rpc batch request to 1000 [#5104](https://github.com/hyperledger/besu/pull/5104)
+- Add a new CLI option to limit the number of requests in a single RPC batch request. [#4965](https://github.com/hyperledger/besu/pull/4965)
+- Support for new DATAHASH opcode as part of EIP-4844 [#4823](https://github.com/hyperledger/besu/issues/4823)
+- Send only hash announcement for blob transaction type [#4940](https://github.com/hyperledger/besu/pull/4940)
+- Add `excess_data_gas` field to block header [#4958](https://github.com/hyperledger/besu/pull/4958)
+- Add `max_fee_per_data_gas` field to transaction [#4970](https://github.com/hyperledger/besu/pull/4970)
 - Added option to evm CLI tool to allow code execution at specific forks [#4913](https://github.com/hyperledger/besu/pull/4913)
 - Improve get account performance by using the world state updater cache [#4897](https://github.com/hyperledger/besu/pull/4897)
 - Add new KZG precompile and option to override the trusted setup being used [#4822](https://github.com/hyperledger/besu/issues/4822)
 - Add implementation for eth_createAccessList RPC method [#4942](https://github.com/hyperledger/besu/pull/4942)
-
-### Bug Fixes
-
-## 23.1.0
-
-### Breaking Changes
-
-### Additions and Improvements
-- Support for new DATAHASH opcode as part of EIP-4844 [#4823](https://github.com/hyperledger/besu/issues/4823)
-- Send only hash announcement for blob transaction type [#4940](https://github.com/hyperledger/besu/pull/4940)
-- Add `excess_data_gas` field to block header [#4958](https://github.com/hyperledger/besu/pull/4958)
+- Add implementation for engine_exchangeCapabilities [#4997](https://github.com/hyperledger/besu/pull/4997)
+- Add implementation for engine_getPayloadBodiesByRangeV1 and engine_getPayloadBodiesByHashV1 [#4980](https://github.com/hyperledger/besu/pull/4980)
+- Updated reference tests to v11.3 [#4996](https://github.com/hyperledger/besu/pull/4996)
+- Add DebugGetRawBlock and DebugGetRawHeader RPC methods [#5011](https://github.com/hyperledger/besu/pull/5011)
+- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
+- Add worldstate auto-heal mechanism [#5059](https://github.com/hyperledger/besu/pull/5059)
+- Support for EIP-4895 - Withdrawals for Shanghai fork
+- If a PoS block creation repetition takes less than a configurable duration, then waits before next repetition [#5048](https://github.com/hyperledger/besu/pull/5048)
 
 ### Bug Fixes
 - Mitigation fix for stale bonsai code storage leading to log rolling issues on contract recreates [#4906](https://github.com/hyperledger/besu/pull/4906)
-
+- Ensure latest cached layered worldstate is subscribed to storage, fix problem with RPC calls using 'latest' [#5076](https://github.com/hyperledger/besu/pull/5076)
 
 ## 23.1.0-RC1
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -31,6 +31,7 @@
 | Danno Ferrin     | shemnon          | shemnon          |
 | Simon Dudley     | siladu           | siladu           |
 | Usman Saleem     | usmansaleem      | usmansaleem      |
+| Zhenyang Shi     | wcgcyx           | wcgcyx           |
 
 
 ## Emeritus Maintainers

diff --git a/README.md b/README.md
@@ -1,7 +1,5 @@
 # Besu Ethereum Client
  [![CircleCI](https://circleci.com/gh/hyperledger/besu/tree/main.svg?style=svg)](https://circleci.com/gh/hyperledger/besu/tree/main)
- [![Total alerts](https://img.shields.io/lgtm/alerts/g/hyperledger/besu.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/hyperledger/besu/alerts/)
- [![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/hyperledger/besu.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/hyperledger/besu/context:java)
  [![Documentation Status](https://readthedocs.org/projects/hyperledger-besu/badge/?version=latest)](https://besu.hyperledger.org/en/latest/?badge=latest)
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3174/badge)](https://bestpractices.coreinfrastructure.org/projects/3174)
  [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/hyperledger/besu/blob/main/LICENSE)

diff --git a/...tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java b/...tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java
@@ -201,6 +201,7 @@ protected void failed(final Throwable e, final Description description) {
           LOG.error(
               "==========================================================================================");
           LOG.error("Test failed. Reported Throwable at the point of failure:", e);
+          LOG.error(e.getMessage());
         }
 
         @Override

diff --git a/...yperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java b/...yperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java
@@ -16,6 +16,9 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_JKS;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_PKCS11;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_PKCS12;
 
 import org.hyperledger.besu.cli.config.NetworkName;
 import org.hyperledger.besu.crypto.KeyPair;
@@ -34,7 +37,6 @@
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration;
-import org.hyperledger.besu.pki.keystore.KeyStoreWrapper;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.pki.PKCS11Utils;
 
@@ -366,38 +368,38 @@ public BesuNodeConfigurationBuilder p2pTLSEnabled(final String name, final Strin
     final TLSConfiguration.Builder builder = TLSConfiguration.Builder.tlsConfiguration();
     try {
       final String nsspin = "/pki-certs/%s/nsspin.txt";
-      final String truststore = "/pki-certs/%s/truststore.jks";
-      final String crl = "/pki-certs/%s/crl.pem";
+      final String truststore = "/pki-certs/%s/truststore.p12";
+      final String crl = "/pki-certs/crl/crl.pem";
       switch (type) {
-        case KeyStoreWrapper.KEYSTORE_TYPE_JKS:
+        case KEYSTORE_TYPE_JKS:
           builder
               .withKeyStoreType(type)
-              .withKeyStorePath(toPath(String.format("/pki-certs/%s/keystore.jks", name)))
+              .withKeyStorePath(toPath(String.format("/pki-certs/%s/%<s.jks", name)))
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withTrustStoreType(type)
+              .withTrustStoreType(KEYSTORE_TYPE_PKCS12)
               .withTrustStorePath(toPath(String.format(truststore, name)))
               .withTrustStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withTrustStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
-        case KeyStoreWrapper.KEYSTORE_TYPE_PKCS12:
+        case KEYSTORE_TYPE_PKCS12:
           builder
               .withKeyStoreType(type)
-              .withKeyStorePath(toPath(String.format("/pki-certs/%s/keys.p12", name)))
+              .withKeyStorePath(toPath(String.format("/pki-certs/%s/%<s.p12", name)))
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withTrustStoreType(KeyStoreWrapper.KEYSTORE_TYPE_JKS)
+              .withTrustStoreType(KEYSTORE_TYPE_PKCS12)
               .withTrustStorePath(toPath(String.format(truststore, name)))
               .withTrustStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withTrustStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
-        case KeyStoreWrapper.KEYSTORE_TYPE_PKCS11:
+        case KEYSTORE_TYPE_PKCS11:
           builder
               .withKeyStoreType(type)
               .withKeyStorePath(
@@ -406,7 +408,7 @@ public BesuNodeConfigurationBuilder p2pTLSEnabled(final String name, final Strin
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
       }
     } catch (Exception e) {