Merge branch 'kaleido-noderunner-dev-cherry-pick-479530dbc' of github…

….com:kaleido-io/besu into kaleido-noderunner-dev

Signed-off-by: Chengxuan Xing <[email protected]>

.circleci/config.yml

@@ -240,7 +240,7 @@ jobs:
       - capture_test_results
       - capture_test_logs
 
-  acceptanceTestsNonMainnet:
+  acceptanceTestsCliqueBft:
     parallelism: 6
     executor: xl_machine_executor
     steps:
@@ -258,7 +258,51 @@ jobs:
               | circleci tests split --split-by=timings --timings-type=classname)
             # Format the arguments to "./gradlew test"
             GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTestNonMainnet $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestCliqueBft $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsPrivacy:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestPrivacy $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsPermissioning:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestPermissioning $GRADLE_ARGS
       - capture_test_results
       - capture_test_logs
 
@@ -401,6 +445,15 @@ workflows:
       - acceptanceTests:
           requires:
             - assemble
+      - acceptanceTestsCliqueBft:
+          requires:
+            - assemble
+      - acceptanceTestsPrivacy:
+          requires:
+            - assemble
+      - acceptanceTestsPermissioning:
+          requires:
+            - assemble
       - buildDocker:
           requires:
             - assemble
@@ -472,4 +525,3 @@ workflows:
     jobs:
       - assemble
       - dockerScan
-      - acceptanceTestsNonMainnet

.github/workflows/pr-checklist-on-open.yml

@@ -16,5 +16,5 @@ jobs:
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I have considered running `./gradlew acceptanceTestNonMainnet` locally if my PR affects non-mainnet modules.\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests'
+              body: '- [ ] I thought about documentation and added the `doc-change-required` label to this PR if [updates are required](https://wiki.hyperledger.org/display/BESU/Documentation).\n- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).\n- [ ] If my PR includes database changes (e.g. KeyValueSegmentIdentifier) I have thought about compatibility and performed forwards and backwards compatibility tests'
             })

CHANGELOG.md

@@ -58,6 +58,7 @@ and in case a rollback is needed, before installing a previous version, the migr
 
 ### Breaking Changes
 - Add request content length limit for the JSON-RPC API (5MB) [#5467](https://github.com/hyperledger/besu/pull/5467)
+- `min-block-occupancy-ratio` options is now ignored on PoS networks [#5491](https://github.com/hyperledger/besu/pull/5491)
 
 ### Additions and Improvements
 - Set the retention policy for RocksDB log files to maintain only the logs from the last week [#5428](https://github.com/hyperledger/besu/pull/5428)
@@ -69,13 +70,17 @@ and in case a rollback is needed, before installing a previous version, the migr
 - Early access - layered transaction pool implementation [#5290](https://github.com/hyperledger/besu/pull/5290)
 - New RPC method `debug_getRawReceipts` [#5476](https://github.com/hyperledger/besu/pull/5476)
 - Add TrieLogFactory plugin support [#5440](https://github.com/hyperledger/besu/pull/5440)
+- Ignore `min-block-occupancy-ratio` option when on PoS networks, since in some cases, it prevents to have full blocks even if enough transactions are present [#5491](https://github.com/hyperledger/besu/pull/5491) 
 
 ### Bug Fixes
 - Fix eth_feeHistory response for the case in which blockCount is higher than highestBlock requested. [#5397](https://github.com/hyperledger/besu/pull/5397)
 - Fix Besu Docker image failing to start due to NoClassDefFoundError with org.xerial.snappy.Snappy library. [#5462](https://github.com/hyperledger/besu/pull/5462)
 
 ### Download Links
 
+https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.4.1/besu-23.4.1.tar.gz / sha256: 49d3a7a069cae307497093d834f873ce7804a46dd59207d5e8321459532d318e
+https://hyperledger.jfrog.io/hyperledger/besu-binaries/besu/23.4.1/besu-23.4.1.zip / sha256: 1d82ed83a816968aa9366d9310b275ca6438100f5d3eb1ec03d3474b2a5f5e76
+
 ## 23.4.0
 
 ### Breaking Changes

acceptance-tests/dsl/build.gradle

@@ -36,9 +36,9 @@ dependencies {
   implementation 'io.vertx:vertx-core'
   implementation 'junit:junit'
   implementation 'io.opentelemetry:opentelemetry-api'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-io'
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-io'
+  implementation 'io.tmio:tuweni-units'
   implementation 'org.assertj:assertj-core'
   implementation 'org.awaitility:awaitility'
   implementation 'org.java-websocket:Java-WebSocket'

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -386,7 +386,7 @@ public String getHostName() {
     return LOCALHOST;
   }
 
-  private NodeRequests nodeRequests() {
+  public NodeRequests nodeRequests() {
     Optional<WebSocketService> websocketService = Optional.empty();
     if (nodeRequests == null) {
       final Web3jService web3jService;

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -45,7 +45,9 @@
 import org.hyperledger.besu.plugin.services.PicoCLIOptions;
 import org.hyperledger.besu.plugin.services.SecurityModuleService;
 import org.hyperledger.besu.plugin.services.StorageService;
+import org.hyperledger.besu.plugin.services.TransactionSelectionService;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
+import org.hyperledger.besu.plugin.services.txselection.TransactionSelectorFactory;
 import org.hyperledger.besu.services.BesuConfigurationImpl;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -54,6 +56,7 @@
 import org.hyperledger.besu.services.RpcEndpointServiceImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
+import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -63,6 +66,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
@@ -91,14 +95,22 @@ private BesuPluginContextImpl buildPluginContext(
     besuPluginContext.addService(StorageService.class, storageService);
     besuPluginContext.addService(SecurityModuleService.class, securityModuleService);
     besuPluginContext.addService(PicoCLIOptions.class, new PicoCLIOptionsImpl(commandLine));
-
-    final Path pluginsPath = node.homeDirectory().resolve("plugins");
-    final File pluginsDirFile = pluginsPath.toFile();
-    if (!pluginsDirFile.isDirectory()) {
-      pluginsDirFile.mkdirs();
-      pluginsDirFile.deleteOnExit();
+    besuPluginContext.addService(
+        TransactionSelectionService.class, new TransactionSelectionServiceImpl());
+
+    final Path pluginsPath;
+    final String pluginDir = System.getProperty("besu.plugins.dir");
+    if (pluginDir == null || pluginDir.isEmpty()) {
+      pluginsPath = node.homeDirectory().resolve("plugins");
+      final File pluginsDirFile = pluginsPath.toFile();
+      if (!pluginsDirFile.isDirectory()) {
+        pluginsDirFile.mkdirs();
+        pluginsDirFile.deleteOnExit();
+      }
+      System.setProperty("besu.plugins.dir", pluginsPath.toString());
+    } else {
+      pluginsPath = Path.of(pluginDir);
     }
-    System.setProperty("besu.plugins.dir", pluginsPath.toString());
     besuPluginContext.registerPlugins(pluginsPath);
 
     commandLine.parseArgs(node.getConfiguration().getExtraCLIOptions().toArray(new String[0]));
@@ -169,6 +181,9 @@ public void startNode(final BesuNode node) {
 
     final int maxPeers = 25;
 
+    final Optional<TransactionSelectorFactory> transactionSelectorFactory =
+        getTransactionSelectorFactory(besuPluginContext);
+
     builder
         .synchronizerConfiguration(new SynchronizerConfiguration.Builder().build())
         .dataDirectory(node.homeDirectory())
@@ -190,7 +205,8 @@ public void startNode(final BesuNode node) {
         .lowerBoundPeers(maxPeers)
         .maxRemotelyInitiatedPeers(15)
         .networkConfiguration(node.getNetworkingConfiguration())
-        .randomPeerPriority(false);
+        .randomPeerPriority(false)
+        .transactionSelectorFactory(transactionSelectorFactory);
 
     node.getGenesisConfig()
         .map(GenesisConfigFile::fromConfig)
@@ -299,4 +315,11 @@ public void startConsoleCapture() {
   public String getConsoleContents() {
     throw new RuntimeException("Console contents can only be captured in process execution");
   }
+
+  private Optional<TransactionSelectorFactory> getTransactionSelectorFactory(
+      final BesuPluginContextImpl besuPluginContext) {
+    final Optional<TransactionSelectionService> txSelectionService =
+        besuPluginContext.getService(TransactionSelectionService.class);
+    return txSelectionService.isPresent() ? txSelectionService.get().get() : Optional.empty();
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -107,10 +107,33 @@ public BesuNode createMinerNode(
     return create(config);
   }
 
+  public BesuNode createMinerNodeWithExtraCliOptions(
+      final String name,
+      final UnaryOperator<BesuNodeConfigurationBuilder> configModifier,
+      final List<String> extraCliOptions)
+      throws IOException {
+    BesuNodeConfigurationBuilder builder =
+        new BesuNodeConfigurationBuilder()
+            .name(name)
+            .miningEnabled()
+            .jsonRpcEnabled()
+            .webSocketEnabled()
+            .extraCLIOptions(extraCliOptions);
+    builder = configModifier.apply(builder);
+    final BesuNodeConfiguration config = builder.build();
+
+    return create(config);
+  }
+
   public BesuNode createMinerNode(final String name) throws IOException {
     return createMinerNode(name, UnaryOperator.identity());
   }
 
+  public BesuNode createMinerNodeWithExtraCliOptions(
+      final String name, final List<String> extraCliOptions) throws IOException {
+    return createMinerNodeWithExtraCliOptions(name, UnaryOperator.identity(), extraCliOptions);
+  }
+
   public BesuNode createMinerNodeWithRevertReasonEnabled(final String name) throws IOException {
     return createMinerNode(name, BesuNodeConfigurationBuilder::revertReasonEnabled);
   }

acceptance-tests/tests/build.gradle

@@ -76,7 +76,7 @@ dependencies {
   testImplementation 'junit:junit'
   testImplementation 'org.apache.commons:commons-compress'
   testImplementation 'org.apache.logging.log4j:log4j-core'
-  testImplementation 'org.apache.tuweni:tuweni-crypto'
+  testImplementation 'io.tmio:tuweni-crypto'
   testImplementation 'org.assertj:assertj-core'
   testImplementation 'org.awaitility:awaitility'
   testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -155,11 +155,36 @@ task acceptanceTestMainnet(type: Test) {
   doFirst { mkdir "${buildDir}/jvmErrorLogs" }
 }
 
-task acceptanceTestNonMainnet(type: Test) {
+task acceptanceTestCliqueBft(type: Test) {
   inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
   include '**/bft/**'
   include '**/clique/**'
-  include '**/permissioning/**'
+
+  useJUnitPlatform {}
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs Clique and BFT Besu acceptance tests.'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+
+task acceptanceTestPrivacy(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
   include '**/privacy/**'
 
   useJUnitPlatform {}
@@ -169,7 +194,34 @@ task acceptanceTestNonMainnet(type: Test) {
   systemProperty 'acctests.runBesuAsProcess', 'true'
   systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
   mustRunAfter rootProject.subprojects*.test
-  description = 'Runs NON-MAINNET Besu acceptance tests for permissioning and privacy (excluding mainnet features).'
+  description = 'Runs Privacy Besu acceptance tests.'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+
+task acceptanceTestPermissioning(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
+  include '**/permissioning/**'
+
+  useJUnitPlatform {}
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs Permissioning Besu acceptance tests.'
   group = 'verification'
 
   jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineAcceptanceTest.java → acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/jsonrpc/ExecutionEngineParisAcceptanceTest.java

@@ -24,13 +24,13 @@
 import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
-public class ExecutionEngineAcceptanceTest extends AbstractJsonRpcTest {
-  private static final String GENESIS_FILE = "/jsonrpc/engine/genesis.json";
-  private static final String TEST_CASE_PATH = "/jsonrpc/engine/test-cases/";
+public class ExecutionEngineParisAcceptanceTest extends AbstractJsonRpcTest {
+  private static final String GENESIS_FILE = "/jsonrpc/engine/paris/genesis.json";
+  private static final String TEST_CASE_PATH = "/jsonrpc/engine/paris/test-cases/";
 
   private static JsonRpcTestsContext testsContext;
 
-  public ExecutionEngineAcceptanceTest(final String ignored, final URI testCaseFileURI) {
+  public ExecutionEngineParisAcceptanceTest(final String ignored, final URI testCaseFileURI) {
     super(ignored, testsContext, testCaseFileURI);
   }