From 9fbe4fbbb2e82671ddc8b7fe127b6d8706b2fc7c Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Sun, 14 Jan 2024 22:50:45 +0100 Subject: [PATCH 001/155] Upgrade Guava dependency (#6396) * Bump com.google.guava:guava to 33.0.0 Signed-off-by: Fabio Di Fabio * Update pending tx estimated memory size after Guava update Signed-off-by: Fabio Di Fabio * Rebuilt Gradle validation metadata Signed-off-by: Fabio Di Fabio * Update CHANGELOG Signed-off-by: Fabio Di Fabio --------- Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + ...ingTransactionEstimatedMemorySizeTest.java | 2 +- gradle/verification-metadata.xml | 96 +++++++++---------- gradle/versions.gradle | 2 +- 4 files changed, 47 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abcc2ab3c85..bb8d3cb9bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) +- Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java index c9c81543a63..20b94f5bf8c 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/PendingTransactionEstimatedMemorySizeTest.java @@ -58,7 +58,7 @@ public class PendingTransactionEstimatedMemorySizeTest extends BaseTransactionPo private static final Set> SHARED_CLASSES = Set.of(SignatureAlgorithm.class, TransactionType.class); private static final Set COMMON_CONSTANT_FIELD_PATHS = - Set.of(".value.ctor", ".hashNoSignature"); + Set.of(".value.ctor", ".hashNoSignature", ".signature.encoded.delegate"); private static final Set EIP1559_EIP4844_CONSTANT_FIELD_PATHS = Sets.union(COMMON_CONSTANT_FIELD_PATHS, Set.of(".gasPrice")); private static final Set FRONTIER_ACCESS_LIST_CONSTANT_FIELD_PATHS = diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 30d55eb6840..58a562fff88 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -760,14 +760,6 @@ - - - - - - - - @@ -777,9 +769,6 @@ - - - @@ -792,6 +781,14 @@ + + + + + + + + @@ -808,11 +805,6 @@ - - - - - @@ -828,6 +820,11 @@ + + + + + @@ -920,12 +917,12 @@ - - - + + + - - + + @@ -944,16 +941,22 @@ + + + + + + + + + + + - - - - - @@ -964,6 +967,11 @@ + + + + + @@ -998,14 +1006,6 @@ - - - - - - - - @@ -3492,22 +3492,6 @@ - - - - - - - - - - - - - - - - @@ -3523,8 +3507,16 @@ - - + + + + + + + + + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b497109d782..c27af91f09b 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -46,7 +46,7 @@ dependencyManagement { entry 'error_prone_test_helpers' } - dependency 'com.google.guava:guava:31.1-jre' + dependency 'com.google.guava:guava:33.0.0-jre' dependency 'com.graphql-java:graphql-java:21.3' From 39d7042ca3f41091a7d24d12bfca6c85fe5b4dd8 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Mon, 15 Jan 2024 10:09:37 +1000 Subject: [PATCH 002/155] log a different warning if pruning is enabled with BONSAI (#6401) Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 2 +- .../org/hyperledger/besu/cli/BesuCommand.java | 12 ++++++++++-- .../org/hyperledger/besu/cli/BesuCommandTest.java | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8d3cb9bff..13f26a25781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,7 @@ ### Breaking Changes ### Deprecations -- Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) +- Forest pruning (`pruning-enabled` option) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230) ### Additions and Improvements - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 9842d52f875..d2fb5243313 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -148,6 +148,7 @@ import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder; import org.hyperledger.besu.ethereum.trie.forest.pruner.PrunerConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract; import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract; import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract; @@ -2063,8 +2064,15 @@ && isOptionSet(commandLine, "--sync-min-peers")) { } if (isPruningEnabled()) { - logger.warn( - "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + if (dataStorageOptions + .toDomainObject() + .getDataStorageFormat() + .equals(DataStorageFormat.BONSAI)) { + logger.warn("Forest pruning is ignored with Bonsai data storage format."); + } else { + logger.warn( + "Forest pruning is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format."); + } } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index ba0e5b2754a..dc56c34845d 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3840,8 +3840,8 @@ public void pruningParametersAreCaptured() throws Exception { } @Test - public void pruningLogsDeprecationWarning() { - parseCommand("--pruning-enabled"); + public void pruningLogsDeprecationWarningWithForest() { + parseCommand("--pruning-enabled", "--data-storage-format=FOREST"); verify(mockControllerBuilder).isPruningEnabled(true); @@ -3854,6 +3854,17 @@ public void pruningLogsDeprecationWarning() { + " To save disk space consider switching to Bonsai data storage format.")); } + @Test + public void pruningLogsIgnoredWarningWithBonsai() { + parseCommand("--pruning-enabled", "--data-storage-format=BONSAI"); + + verify(mockControllerBuilder).isPruningEnabled(true); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); + verify(mockLogger).warn(contains("Forest pruning is ignored with Bonsai data storage format.")); + } + @Test public void devModeOptionMustBeUsed() throws Exception { parseCommand("--network", "dev"); From 724e3d040e74204c9ace7844a0651cbdebd671f9 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Tue, 16 Jan 2024 00:27:57 +0100 Subject: [PATCH 003/155] Upgrade Mockito (#6397) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../ethereum/api/handlers/HandlerFactory.java | 4 +- .../internal/methods/DebugTraceBlock.java | 23 ++-- .../JsonRpcHttpServiceHostAllowlistTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceLoginTest.java | 62 ++++----- .../JsonRpcHttpServiceRpcApisTest.java | 129 +++++++++--------- .../api/jsonrpc/JsonRpcHttpServiceTest.java | 86 ++++++------ .../jsonrpc/JsonRpcHttpServiceTestBase.java | 88 +++++++----- .../JsonRpcHttpServiceTlsClientAuthTest.java | 64 +++++---- ...RpcHttpServiceTlsMisconfigurationTest.java | 64 +++++---- .../jsonrpc/JsonRpcHttpServiceTlsTest.java | 64 +++++---- .../DebugStandardTraceBlockToFileTest.java | 28 ++-- .../internal/methods/DebugTraceBlockTest.java | 45 +++--- .../backwardsync/BackwardSyncAlgSpec.java | 4 +- .../rlpx/connections/netty/DeFramerTest.java | 13 +- .../evm/operations/ChainIdOperationTest.java | 3 +- gradle/verification-metadata.xml | 46 +++---- gradle/versions.gradle | 2 +- 18 files changed, 396 insertions(+), 394 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13f26a25781..d520eba5425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377) - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) +- Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java index 2d3639d52f6..03465f70f21 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/HandlerFactory.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import io.opentelemetry.api.trace.Tracer; @@ -35,7 +36,8 @@ public static Handler timeout( assert methods != null && globalOptions != null; return TimeoutHandler.handler( Optional.of(globalOptions), - methods.keySet().stream().collect(Collectors.toMap(String::new, ignored -> globalOptions))); + methods.keySet().stream() + .collect(Collectors.toMap(Function.identity(), ignored -> globalOptions))); } public static Handler authentication( diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java index 270f176c7a8..6dab0436a84 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java @@ -46,15 +46,15 @@ public class DebugTraceBlock implements JsonRpcMethod { private static final Logger LOG = LoggerFactory.getLogger(DebugTraceBlock.class); private final Supplier blockTracerSupplier; private final BlockHeaderFunctions blockHeaderFunctions; - private final BlockchainQueries blockchain; + private final BlockchainQueries blockchainQueries; public DebugTraceBlock( final Supplier blockTracerSupplier, final BlockHeaderFunctions blockHeaderFunctions, - final BlockchainQueries blockchain) { + final BlockchainQueries blockchainQueries) { this.blockTracerSupplier = blockTracerSupplier; this.blockHeaderFunctions = blockHeaderFunctions; - this.blockchain = blockchain; + this.blockchainQueries = blockchainQueries; } @Override @@ -79,18 +79,17 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { .map(TransactionTraceParams::traceOptions) .orElse(TraceOptions.DEFAULT); - if (this.blockchain.blockByHash(block.getHeader().getParentHash()).isPresent()) { + if (this.blockchainQueries.blockByHash(block.getHeader().getParentHash()).isPresent()) { final Collection results = Tracer.processTracing( - blockchain, + blockchainQueries, Optional.of(block.getHeader()), - mutableWorldState -> { - return blockTracerSupplier - .get() - .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) - .map(BlockTrace::getTransactionTraces) - .map(DebugTraceTransactionResult::of); - }) + mutableWorldState -> + blockTracerSupplier + .get() + .trace(mutableWorldState, block, new DebugOperationTracer(traceOptions)) + .map(BlockTrace::getTransactionTraces) + .map(DebugTraceTransactionResult::of)) .orElse(null); return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); } else { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java index da7319282fb..f433bcdc977 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceHostAllowlistTest.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -98,38 +97,37 @@ public void initServerAndClient() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java index 7b9eac24788..aa1b582174f 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceLoginTest.java @@ -19,7 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.util.Lists.list; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -129,37 +128,36 @@ public static void initServerAndClient() throws Exception { new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - genesisConfigOptions, - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig(genesisConfigOptions), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.empty(), - Optional.empty(), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + genesisConfigOptions, + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig(genesisConfigOptions), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.empty(), + Optional.empty(), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(); jwtAuth = service.authenticationService.get().getJwtAuthProvider(); service.start().join(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java index 6870c4fed03..8323f61b2c5 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceRpcApisTest.java @@ -17,7 +17,6 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.config.StubGenesisConfigOptions; @@ -201,37 +200,36 @@ private JsonRpcHttpService createJsonRpcHttpServiceWithRpcApis(final JsonRpcConf supportedCapabilities.add(EthProtocol.ETH63); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - mock(P2PNetwork.class), - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - config.getRpcApis(), - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + mock(P2PNetwork.class), + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + config.getRpcApis(), + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -302,8 +300,7 @@ private JsonRpcHttpService createJsonRpcHttpService( final WebSocketConfiguration webSocketConfiguration, final P2PNetwork p2pNetwork, final MetricsConfiguration metricsConfiguration, - final NatService natService) - throws Exception { + final NatService natService) { final Set supportedCapabilities = new HashSet<>(); supportedCapabilities.add(EthProtocol.ETH62); supportedCapabilities.add(EthProtocol.ETH63); @@ -311,37 +308,36 @@ private JsonRpcHttpService createJsonRpcHttpService( webSocketConfiguration.setPort(0); final Map rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - NETWORK_ID, - new StubGenesisConfigOptions(), - p2pNetwork, - blockchainQueries, - mock(Synchronizer.class), - ProtocolScheduleFixture.MAINNET, - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - jsonRpcConfiguration.getRpcApis(), - mock(PrivacyParameters.class), - jsonRpcConfiguration, - webSocketConfiguration, - metricsConfiguration, - natService, - new HashMap<>(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + NETWORK_ID, + new StubGenesisConfigOptions(), + p2pNetwork, + blockchainQueries, + mock(Synchronizer.class), + ProtocolScheduleFixture.MAINNET, + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + jsonRpcConfiguration.getRpcApis(), + mock(PrivacyParameters.class), + jsonRpcConfiguration, + webSocketConfiguration, + metricsConfiguration, + natService, + new HashMap<>(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); final JsonRpcHttpService jsonRpcHttpService = new JsonRpcHttpService( vertx, @@ -425,8 +421,7 @@ public RequestBody createNetServicesRequestBody() { "{\"jsonrpc\":\"2.0\",\"id\":" + Json.encode(id) + ",\"method\":\"net_services\"}", JSON); } - public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) - throws Exception { + public JsonRpcHttpService getJsonRpcHttpService(final boolean[] enabledNetServices) { JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault(); WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java index f9c9b14ccd0..901f20ff754 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTest.java @@ -17,10 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Address; @@ -1389,65 +1386,68 @@ public void disabledMethod() throws Exception { + "\"}", JSON); - when(rpcMethods.get(any(String.class))).thenReturn(null); - when(rpcMethods.containsKey(any(String.class))).thenReturn(false); + try (var unused = disableRpcMethod(methodName)) { - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; - testHelper.assertValidJsonRpcError( - json, id, expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.METHOD_NOT_ENABLED; + testHelper.assertValidJsonRpcError( + json, id, expectedError.getCode(), expectedError.getMessage()); + } } - - verify(rpcMethods).containsKey(methodName); - verify(rpcMethods).get(methodName); - - reset(rpcMethods); } @Test public void exceptionallyHandleJsonSingleRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { - final RequestBody body = - RequestBody.create("{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"foo\"}", JSON); + final RequestBody body = + RequestBody.create( + "{\"jsonrpc\":\"2.0\",\"id\":\"666\",\"method\":\"" + methodName + "\"}", JSON); - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonObject json = new JsonObject(resp.body().string()); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - json, "666", expectedError.getCode(), expectedError.getMessage()); + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonObject json = new JsonObject(resp.body().string()); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + json, "666", expectedError.getCode(), expectedError.getMessage()); + } } } @Test public void exceptionallyHandleJsonBatchRequest() throws Exception { + final String methodName = "foo"; final JsonRpcMethod jsonRpcMethod = mock(JsonRpcMethod.class); - when(jsonRpcMethod.getName()).thenReturn("foo"); + when(jsonRpcMethod.getName()).thenReturn(methodName); when(jsonRpcMethod.response(any())).thenThrow(new RuntimeException("test exception")); - doReturn(jsonRpcMethod).when(rpcMethods).get("foo"); - final RequestBody body = - RequestBody.create( - "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"foo\"}," - + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", - JSON); - - try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { - assertThat(resp.code()).isEqualTo(200); - final JsonArray array = new JsonArray(resp.body().string()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); - final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; - testHelper.assertValidJsonRpcError( - array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); - testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + try (var unused = addRpcMethod(methodName, jsonRpcMethod)) { + + final RequestBody body = + RequestBody.create( + "[{\"jsonrpc\":\"2.0\",\"id\":\"000\",\"method\":\"web3_clientVersion\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"111\",\"method\":\"" + + methodName + + "\"}," + + "{\"jsonrpc\":\"2.0\",\"id\":\"222\",\"method\":\"net_version\"}]", + JSON); + + try (final Response resp = client.newCall(buildPostRequest(body)).execute()) { + assertThat(resp.code()).isEqualTo(200); + final JsonArray array = new JsonArray(resp.body().string()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(0), "000"); + final RpcErrorType expectedError = RpcErrorType.INTERNAL_ERROR; + testHelper.assertValidJsonRpcError( + array.getJsonObject(1), "111", expectedError.getCode(), expectedError.getMessage()); + testHelper.assertValidJsonRpcResult(array.getJsonObject(2), "222"); + } } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java index 580af7b7467..257dbf16679 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTestBase.java @@ -16,7 +16,6 @@ package org.hyperledger.besu.ethereum.api.jsonrpc; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -72,8 +71,9 @@ public class JsonRpcHttpServiceTestBase { protected final JsonRpcTestHelper testHelper = new JsonRpcTestHelper(); private static final Vertx vertx = Vertx.vertx(); - protected static Map rpcMethods; + private static Map disabledRpcMethods; + private static Set addedRpcMethods; protected static JsonRpcHttpService service; protected static OkHttpClient client; protected static String baseUrl; @@ -106,39 +106,41 @@ public static void initServerAndClient() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), - EvmConfiguration.DEFAULT), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - JSON_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - new HashMap<>(), - folder, - ethPeersMock, - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID), + EvmConfiguration.DEFAULT), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + JSON_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + new HashMap<>(), + folder, + ethPeersMock, + vertx, + mock(ApiConfiguration.class), + Optional.empty()); + disabledRpcMethods = new HashMap<>(); + addedRpcMethods = new HashSet<>(); + service = createJsonRpcHttpService(createLimitedJsonRpcConfig()); service.start().join(); @@ -189,6 +191,22 @@ protected Request buildGetRequest(final String path) { return new Request.Builder().get().url(baseUrl + path).build(); } + protected AutoCloseable disableRpcMethod(final String methodName) { + disabledRpcMethods.put(methodName, rpcMethods.remove(methodName)); + return () -> resetRpcMethods(); + } + + protected AutoCloseable addRpcMethod(final String methodName, final JsonRpcMethod method) { + rpcMethods.put(methodName, method); + addedRpcMethods.add(methodName); + return () -> resetRpcMethods(); + } + + protected void resetRpcMethods() { + disabledRpcMethods.forEach(rpcMethods::put); + addedRpcMethods.forEach(rpcMethods::remove); + } + /** Tears down the HTTP server. */ @AfterAll public static void shutdownServer() { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java index 3a9a5577a1d..70cc3d58f84 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsClientAuthTest.java @@ -21,7 +21,6 @@ import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -112,38 +111,37 @@ public void initServer() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); System.setProperty("javax.net.ssl.trustStore", CLIENT_AS_CA_CERT.getKeyStoreFile().toString()); System.setProperty( diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java index 695dec93269..ee069f03232 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsMisconfigurationTest.java @@ -20,7 +20,6 @@ import static org.hyperledger.besu.ethereum.api.tls.TlsClientAuthConfiguration.Builder.aTlsClientAuthConfiguration; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -100,38 +99,37 @@ public void beforeEach() { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - tempDir.getRoot(), - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + tempDir.getRoot(), + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); } @AfterEach diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java index 587d142b7f8..295c7b91bd7 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/JsonRpcHttpServiceTlsTest.java @@ -20,7 +20,6 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.DEFAULT_RPC_APIS; import static org.hyperledger.besu.ethereum.api.tls.TlsConfiguration.Builder.aTlsConfiguration; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import org.hyperledger.besu.config.StubGenesisConfigOptions; import org.hyperledger.besu.ethereum.ProtocolContext; @@ -101,38 +100,37 @@ public void initServer() throws Exception { supportedCapabilities.add(EthProtocol.ETH63); rpcMethods = - spy( - new JsonRpcMethodsFactory() - .methods( - CLIENT_VERSION, - CHAIN_ID, - new StubGenesisConfigOptions(), - peerDiscoveryMock, - blockchainQueries, - synchronizer, - MainnetProtocolSchedule.fromConfig( - new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), - mock(ProtocolContext.class), - mock(FilterManager.class), - mock(TransactionPool.class), - mock(MiningParameters.class), - mock(PoWMiningCoordinator.class), - new NoOpMetricsSystem(), - supportedCapabilities, - Optional.of(mock(AccountLocalConfigPermissioningController.class)), - Optional.of(mock(NodeLocalConfigPermissioningController.class)), - DEFAULT_RPC_APIS, - mock(PrivacyParameters.class), - mock(JsonRpcConfiguration.class), - mock(WebSocketConfiguration.class), - mock(MetricsConfiguration.class), - natService, - Collections.emptyMap(), - folder, - mock(EthPeers.class), - vertx, - mock(ApiConfiguration.class), - Optional.empty())); + new JsonRpcMethodsFactory() + .methods( + CLIENT_VERSION, + CHAIN_ID, + new StubGenesisConfigOptions(), + peerDiscoveryMock, + blockchainQueries, + synchronizer, + MainnetProtocolSchedule.fromConfig( + new StubGenesisConfigOptions().constantinopleBlock(0).chainId(CHAIN_ID)), + mock(ProtocolContext.class), + mock(FilterManager.class), + mock(TransactionPool.class), + mock(MiningParameters.class), + mock(PoWMiningCoordinator.class), + new NoOpMetricsSystem(), + supportedCapabilities, + Optional.of(mock(AccountLocalConfigPermissioningController.class)), + Optional.of(mock(NodeLocalConfigPermissioningController.class)), + DEFAULT_RPC_APIS, + mock(PrivacyParameters.class), + mock(JsonRpcConfiguration.class), + mock(WebSocketConfiguration.class), + mock(MetricsConfiguration.class), + natService, + Collections.emptyMap(), + folder, + mock(EthPeers.class), + vertx, + mock(ApiConfiguration.class), + Optional.empty()); service = createJsonRpcHttpService(createJsonRpcConfig()); service.start().join(); baseUrl = service.url(); diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java index 42ec4c697f5..2cf7dcbfc27 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugStandardTraceBlockToFileTest.java @@ -18,11 +18,12 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.BlockTracer; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTracer; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; @@ -30,28 +31,23 @@ import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.MutableWorldState; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.function.Function; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Answers; public class DebugStandardTraceBlockToFileTest { // this tempDir is deliberately static @TempDir private static Path folder; - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final TransactionTracer transactionTracer = mock(TransactionTracer.class); private final DebugStandardTraceBlockToFile debugStandardTraceBlockToFile = new DebugStandardTraceBlockToFile(() -> transactionTracer, blockchainQueries, folder); @@ -76,20 +72,26 @@ public void shouldTraceTheTransactionUsingTheTransactionTracer() { new JsonRpcRequestContext( new JsonRpcRequest("2.0", "debug_standardTraceBlockToFile", params)); - final List paths = new ArrayList<>(); - paths.add("path-1"); - - when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + final List paths = List.of("path-1"); when(blockchain.getBlockByHash(block.getHash())).thenReturn(Optional.of(block)); when(blockchain.getBlockHeader(genesis.getHash())).thenReturn(Optional.of(genesis.getHeader())); + when(blockchainQueries.getBlockchain()).thenReturn(blockchain); + + when(blockchainQueries.getAndMapWorldState(any(), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); when(transactionTracer.traceTransactionToFile( any(MutableWorldState.class), eq(block.getHash()), any(), any())) .thenReturn(paths); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugStandardTraceBlockToFile.response(request); - final List result = (ArrayList) response.getResult(); + final List result = (List) response.getResult(); assertThat(result.size()).isEqualTo(1); } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java index 60841c24ef4..5da556182bc 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockTest.java @@ -18,9 +18,8 @@ import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.hyperledger.besu.datatypes.Wei; @@ -35,32 +34,25 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; import org.hyperledger.besu.ethereum.api.query.BlockWithMetadata; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.debug.TraceFrame; import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; -import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Collection; import java.util.Collections; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Function; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; -import org.mockito.Answers; -import org.mockito.Mockito; public class DebugTraceBlockTest { private final BlockTracer blockTracer = mock(BlockTracer.class); - private final WorldStateArchive archive = - mock(WorldStateArchive.class, Answers.RETURNS_DEEP_STUBS); - private final Blockchain blockchain = mock(Blockchain.class); - private final BlockchainQueries blockchainQueries = - spy(new BlockchainQueries(blockchain, archive)); + private final BlockchainQueries blockchainQueries = mock(BlockchainQueries.class); private final DebugTraceBlock debugTraceBlock = new DebugTraceBlock(() -> blockTracer, new MainnetBlockHeaderFunctions(), blockchainQueries); @@ -127,22 +119,25 @@ public void shouldReturnCorrectResponse() { when(transaction2Trace.getResult()).thenReturn(transaction2Result); when(transaction1Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); when(transaction2Result.getOutput()).thenReturn(Bytes.fromHexString("1234")); - when(blockTracer.trace(any(Tracer.TraceableState.class), Mockito.eq(block), any())) + when(blockTracer.trace(any(Tracer.TraceableState.class), eq(block), any())) .thenReturn(Optional.of(blockTrace)); - when(blockchain.getBlockHeader(parentBlock.getHash())) - .thenReturn(Optional.of(parentBlock.getHeader())); - doAnswer( - invocation -> - Optional.of( - new BlockWithMetadata<>( - parentBlock.getHeader(), - Collections.emptyList(), - Collections.emptyList(), - parentBlock.getHeader().getDifficulty(), - parentBlock.calculateSize()))) - .when(blockchainQueries) - .blockByHash(parentBlock.getHash()); + when(blockchainQueries.blockByHash(parentBlock.getHash())) + .thenReturn( + Optional.of( + new BlockWithMetadata<>( + parentBlock.getHeader(), + Collections.emptyList(), + Collections.emptyList(), + parentBlock.getHeader().getDifficulty(), + parentBlock.calculateSize()))); + when(blockchainQueries.getAndMapWorldState(eq(parentBlock.getHash()), any())) + .thenAnswer( + invocationOnMock -> { + Function> mapper = + invocationOnMock.getArgument(1); + return mapper.apply(mock(Tracer.TraceableState.class)); + }); final JsonRpcSuccessResponse response = (JsonRpcSuccessResponse) debugTraceBlock.response(request); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java index 8a1a0a226e8..83355ecc291 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/backwardsync/BackwardSyncAlgSpec.java @@ -165,7 +165,7 @@ public void shouldAwokeWhenTTDReachedAndReady() throws Exception { ttdCaptor.getValue().onTTDReached(true); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); @@ -192,7 +192,7 @@ public void shouldAwokeWhenConditionReachedAndReady() throws Exception { completionCaptor.getValue().onInitialSyncCompleted(); - voidCompletableFuture.get(100, TimeUnit.MILLISECONDS); + voidCompletableFuture.get(200, TimeUnit.MILLISECONDS); assertThat(voidCompletableFuture).isCompleted(); verify(context.getSyncState()).unsubscribeTTDReached(88L); diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java index dfafb002f49..f5030fd9ad6 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/rlpx/connections/netty/DeFramerTest.java @@ -65,6 +65,7 @@ import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelId; import io.netty.channel.ChannelPipeline; @@ -196,7 +197,7 @@ public void decode_handlesHello() throws ExecutionException, InterruptedExceptio assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -204,7 +205,7 @@ public void decode_handlesHello() throws ExecutionException, InterruptedExceptio when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -246,7 +247,7 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() assertThat(peerConnection.getPeer().getEnodeURL()).isEqualTo(expectedEnode); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); // Next message should be pushed out final PingMessage nextMessage = PingMessage.get(); @@ -254,7 +255,7 @@ public void decode_handlesHelloFromPeerWithAdvertisedPortOf0() when(framer.deframe(eq(nextData))) .thenReturn(new RawMessage(nextMessage.getCode(), nextMessage.getData())) .thenReturn(null); - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); deFramer.decode(ctx, nextData, out); assertThat(out.size()).isEqualTo(1); } @@ -292,7 +293,7 @@ public void decode_handlesUnexpectedPeerId() { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test @@ -321,7 +322,7 @@ public void decode_handlesNoSharedCaps() { assertThat(out).isEmpty(); // Next phase of pipeline should be setup - verify(pipeline, times(1)).addLast(any()); + verify(pipeline, times(1)).addLast(any(ChannelHandler[].class)); } @Test diff --git a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java index 7202be03ce3..e8d2c3d02db 100644 --- a/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java +++ b/evm/src/test/java/org/hyperledger/besu/evm/operations/ChainIdOperationTest.java @@ -27,7 +27,6 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -import org.apache.tuweni.units.bigints.UInt256; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -51,7 +50,7 @@ static Iterable params() { void shouldReturnChainId(final String chainIdString, final int expectedGas) { Bytes32 chainId = Bytes32.fromHexString(chainIdString); ChainIdOperation operation = new ChainIdOperation(new ConstantinopleGasCalculator(), chainId); - final ArgumentCaptor arg = ArgumentCaptor.forClass(UInt256.class); + final ArgumentCaptor arg = ArgumentCaptor.forClass(Bytes.class); when(messageFrame.getRemainingGas()).thenReturn(100L); operation.execute(messageFrame, null); Mockito.verify(messageFrame).getRemainingGas(); diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 58a562fff88..906d4a2ee27 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2858,12 +2858,12 @@ - - - + + + - - + + @@ -2874,17 +2874,17 @@ - - - + + + - - + + - - - + + + @@ -5316,20 +5316,20 @@ - - - + + + - - + + - - - + + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index c27af91f09b..747a386cb38 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -192,7 +192,7 @@ dependencyManagement { entry 'org.jupnp' } - dependencySet(group: 'org.mockito', version:'4.11.0') { + dependencySet(group: 'org.mockito', version:'5.8.0') { entry 'mockito-core' entry 'mockito-junit-jupiter' } From 2c1d3d28410d6b0f3f55b67e0f2ad8cb7e111d5b Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Tue, 16 Jan 2024 10:53:41 +1000 Subject: [PATCH 004/155] cliqueBft AT task flakiness - extend no_output_timeout (#6406) * revert machine change and extend timeout instead Signed-off-by: Sally MacFarlane --------- Signed-off-by: Sally MacFarlane --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d08e515a18..065a0280d04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,7 +240,7 @@ jobs: at: ~/project - run: name: AcceptanceTests (Non-Mainnet) - no_output_timeout: 20m + no_output_timeout: 30m command: | ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft - capture_test_results From c316a6dfb5ea354bad50194840362c6622910b0e Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Tue, 16 Jan 2024 13:53:55 +1100 Subject: [PATCH 005/155] Remove deprecated `--privacy-onchain-groups-enabled` option (#6411) Signed-off-by: Gabriel-Trintinalia --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 21 +------- .../util/ConfigOptionSearchAndRunHandler.java | 11 ---- .../hyperledger/besu/cli/BesuCommandTest.java | 51 ------------------- .../src/test/resources/everything_config.toml | 1 - 5 files changed, 3 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d520eba5425..504d120a9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360) +- The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) ### Deprecations diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index d2fb5243313..c51d03d4305 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -22,7 +22,6 @@ import static org.hyperledger.besu.cli.DefaultCommandValues.getDefaultBesuDataPath; import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.cli.util.CommandLineUtils.isOptionSet; import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH; import static org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration.DEFAULT_GRAPHQL_HTTP_PORT; @@ -953,13 +952,6 @@ static class PrivacyOptionGroup { names = {"--privacy-flexible-groups-enabled"}, description = "Enable flexible privacy groups (default: ${DEFAULT-VALUE})") private final Boolean isFlexiblePrivacyGroupsEnabled = false; - - @Option( - hidden = true, - names = {"--privacy-onchain-groups-enabled"}, - description = - "!!DEPRECATED!! Use `--privacy-flexible-groups-enabled` instead. Enable flexible (onchain) privacy groups (default: ${DEFAULT-VALUE})") - private final Boolean isOnchainPrivacyGroupsEnabled = false; } // Metrics Option Group @@ -1716,8 +1708,7 @@ private void validatePluginOptions() { } if (unstablePrivacyPluginOptions.isPrivacyPluginEnabled() - && (privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { + && privacyOptionGroup.isFlexiblePrivacyGroupsEnabled) { throw new ParameterException( commandLine, "Privacy Plugin can not be used with flexible privacy groups"); } @@ -2056,13 +2047,6 @@ && isOptionSet(commandLine, "--sync-min-peers")) { "--security-module=" + DEFAULT_SECURITY_MODULE); } - if (Boolean.TRUE.equals(privacyOptionGroup.isOnchainPrivacyGroupsEnabled)) { - logger.warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - if (isPruningEnabled()) { if (dataStorageOptions .toDomainObject() @@ -2750,8 +2734,7 @@ private PrivacyParameters privacyParameters() { privacyParametersBuilder.setMultiTenancyEnabled( privacyOptionGroup.isPrivacyMultiTenancyEnabled); privacyParametersBuilder.setFlexiblePrivacyGroupsEnabled( - privacyOptionGroup.isFlexiblePrivacyGroupsEnabled - || privacyOptionGroup.isOnchainPrivacyGroupsEnabled); + privacyOptionGroup.isFlexiblePrivacyGroupsEnabled); privacyParametersBuilder.setPrivacyPluginEnabled( unstablePrivacyPluginOptions.isPrivacyPluginEnabled()); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java index 17cf2e649c9..5866d43cde6 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ConfigOptionSearchAndRunHandler.java @@ -55,7 +55,6 @@ public ConfigOptionSearchAndRunHandler( public List handle(final ParseResult parseResult) throws ParameterException { final CommandLine commandLine = parseResult.commandSpec().commandLine(); final Optional configFile = findConfigFile(parseResult, commandLine); - validatePrivacyOptions(parseResult, commandLine); commandLine.setDefaultValueProvider(createDefaultValueProvider(commandLine, configFile)); commandLine.setExecutionStrategy(resultHandler); commandLine.setParameterExceptionHandler(parameterExceptionHandler); @@ -64,16 +63,6 @@ public List handle(final ParseResult parseResult) throws ParameterExcept return new ArrayList<>(); } - private void validatePrivacyOptions( - final ParseResult parseResult, final CommandLine commandLine) { - if (parseResult.hasMatchedOption("--privacy-onchain-groups-enabled") - && parseResult.hasMatchedOption("--privacy-flexible-groups-enabled")) { - throw new ParameterException( - commandLine, - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - } - private Optional findConfigFile( final ParseResult parseResult, final CommandLine commandLine) { if (parseResult.hasMatchedOption("--config-file") diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index dc56c34845d..26296556302 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -29,7 +29,6 @@ import static org.hyperledger.besu.cli.config.NetworkName.MORDOR; import static org.hyperledger.besu.cli.config.NetworkName.SEPOLIA; import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG; -import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ENGINE; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.ETH; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; @@ -1977,16 +1976,6 @@ public void ethStatsContactOptionCannotBeUsedWithoutEthStatsServerProvided() { "The `--ethstats-contact` requires ethstats server URL to be provided. Either remove --ethstats-contact or provide a URL (via --ethstats=nodename:secret@host:port)"); } - @Test - public void privacyOnchainGroupsEnabledCannotBeUsedWithPrivacyFlexibleGroupsEnabled() { - parseCommand("--privacy-onchain-groups-enabled", "--privacy-flexible-groups-enabled"); - Mockito.verifyNoInteractions(mockRunnerBuilder); - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)) - .contains( - "The `--privacy-onchain-groups-enabled` option is deprecated and you should only use `--privacy-flexible-groups-enabled`"); - } - @Test public void parsesValidBonsaiTrieLimitBackLayersOption() { parseCommand("--data-storage-format", "BONSAI", "--bonsai-historical-block-limit", "11"); @@ -4203,46 +4192,6 @@ public void flexiblePrivacyGroupEnabledFlagDefaultValueIsFalse() { assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(false); } - @Test - public void onchainPrivacyGroupEnabledFlagValueIsSet() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - final ArgumentCaptor privacyParametersArgumentCaptor = - ArgumentCaptor.forClass(PrivacyParameters.class); - - verify(mockControllerBuilder).privacyParameters(privacyParametersArgumentCaptor.capture()); - verify(mockControllerBuilder).build(); - - assertThat(commandOutput.toString(UTF_8)).isEmpty(); - assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); - - final PrivacyParameters privacyParameters = privacyParametersArgumentCaptor.getValue(); - assertThat(privacyParameters.isFlexiblePrivacyGroupsEnabled()).isEqualTo(true); - } - - @Test - public void onchainPrivacyGroupEnabledOptionIsDeprecated() { - parseCommand( - "--privacy-enabled", - "--privacy-public-key-file", - ENCLAVE_PUBLIC_KEY_PATH, - "--privacy-onchain-groups-enabled", - "--min-gas-price", - "0"); - - verify(mockLogger) - .warn( - DEPRECATION_WARNING_MSG, - "--privacy-onchain-groups-enabled", - "--privacy-flexible-groups-enabled"); - } - @Test public void flexiblePrivacyGroupEnabledFlagValueIsSet() { parseCommand( diff --git a/besu/src/test/resources/everything_config.toml b/besu/src/test/resources/everything_config.toml index e516060da86..c653f3f60fd 100644 --- a/besu/src/test/resources/everything_config.toml +++ b/besu/src/test/resources/everything_config.toml @@ -169,7 +169,6 @@ privacy-enabled=false privacy-multi-tenancy-enabled=true privacy-marker-transaction-signing-key-file="./signerKey" privacy-enable-database-migration=false -privacy-onchain-groups-enabled=false privacy-flexible-groups-enabled=false # Transaction Pool From 25f8e57a36b673a44953f63cc2e0f400e08be444 Mon Sep 17 00:00:00 2001 From: Fabio Di Fabio Date: Wed, 17 Jan 2024 10:06:06 +0100 Subject: [PATCH 006/155] Upgrade tech.pegasys.discovery:discovery (#6414) Signed-off-by: Fabio Di Fabio --- CHANGELOG.md | 1 + .../internal/ENRResponsePacketDataTest.java | 48 +++++++---- gradle/allowed-licenses.json | 5 ++ gradle/verification-metadata.xml | 86 ++++++++++--------- gradle/versions.gradle | 2 +- 5 files changed, 87 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 504d120a9fe..e09db4de540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) - Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396) - Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397) +- Upgrade `tech.pegasys.discovery:discovery` [#6414](https://github.com/hyperledger/besu/pull/6414) ### Bug fixes - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344) diff --git a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java index 72bb407d162..7ba47bd97cc 100644 --- a/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java +++ b/ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/ENRResponsePacketDataTest.java @@ -20,6 +20,8 @@ import org.hyperledger.besu.ethereum.rlp.RLP; import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.crypto.SECP256K1; import org.apache.tuweni.units.bigints.UInt64; import org.ethereum.beacon.discovery.schema.EnrField; import org.ethereum.beacon.discovery.schema.IdentitySchema; @@ -34,8 +36,10 @@ public void serializeDeserialize() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -48,7 +52,8 @@ public void serializeDeserialize() { new EnrField(EnrField.TCP, 8080), new EnrField(EnrField.TCP_V6, 8080), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -72,8 +77,10 @@ public void readFrom() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -82,7 +89,8 @@ public void readFrom() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -109,8 +117,10 @@ public void writeTo() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -119,7 +129,8 @@ public void writeTo() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); assertThat(nodeRecord.getNodeId()).isEqualTo(nodeId); @@ -144,8 +155,10 @@ public void readFrom_withExtraFields() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); final Bytes nodeId = Bytes.fromHexString("a448f24c6d18e575453db13171562b71999873db5b286df957af199ec94617f7"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -153,7 +166,9 @@ public void readFrom_withExtraFields() { new EnrField(EnrField.ID, IdentitySchema.V4), new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), - new EnrField(EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey)), + new EnrField( + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey)), new EnrField("foo", Bytes.fromHexString("0x1234"))); nodeRecord.sign(privateKey); @@ -181,8 +196,10 @@ public void readFrom_withExtraFields() { @Test public void readFrom_invalidSignature() { final Bytes requestHash = Bytes.fromHexStringLenient("0x1234"); - final Bytes privateKey = - Bytes.fromHexString("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292"); + final SECP256K1.SecretKey privateKey = + SECP256K1.SecretKey.fromBytes( + Bytes32.fromHexString( + "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f292")); NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.createFromValues( @@ -191,7 +208,8 @@ public void readFrom_invalidSignature() { new EnrField(EnrField.IP_V4, Bytes.fromHexString("0x7F000001")), new EnrField(EnrField.UDP, 30303), new EnrField( - EnrField.PKEY_SECP256K1, Functions.derivePublicKeyFromPrivate(privateKey))); + EnrField.PKEY_SECP256K1, + Functions.deriveCompressedPublicKeyFromPrivate(privateKey))); nodeRecord.sign(privateKey); nodeRecord.set(EnrField.UDP, 1234); diff --git a/gradle/allowed-licenses.json b/gradle/allowed-licenses.json index 25270190806..f7a3d8be8d4 100644 --- a/gradle/allowed-licenses.json +++ b/gradle/allowed-licenses.json @@ -56,6 +56,11 @@ "moduleVersion": "1.0.3", "moduleName": "org.reactivestreams:reactive-streams" }, + { + "moduleLicense": "MIT-0", + "moduleVersion": "1.0.4", + "moduleName": "org.reactivestreams:reactive-streams" + }, { "moduleLicense": "Eclipse Public License - v 1.0", "moduleVersion": "4.13.2", diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 906d4a2ee27..eb7949b754a 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2402,15 +2402,15 @@ - - - + + + - - + + - - + + @@ -3359,44 +3359,44 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -5551,6 +5551,14 @@ + + + + + + + + @@ -5958,15 +5966,15 @@ - - - + + + - - + + - - + + diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 747a386cb38..27fd97fb42d 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -231,6 +231,6 @@ dependencyManagement { dependency 'org.yaml:snakeyaml:2.0' - dependency 'tech.pegasys.discovery:discovery:22.2.0' + dependency 'tech.pegasys.discovery:discovery:22.12.0' } } From cca34edfd1226a9485279877e925d8762388a848 Mon Sep 17 00:00:00 2001 From: Gabriel Fukushima Date: Wed, 6 Dec 2023 17:28:33 +1100 Subject: [PATCH 007/155] Run ATs sequentially (#6244) Signed-off-by: Gabriel Fukushima Signed-off-by: Justin Florentine --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 065a0280d04..64a51a17da1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -428,6 +428,7 @@ workflows: - acceptanceTestsPermissioning: requires: - assemble + - acceptanceTestsCliqueBft - buildDocker: requires: - assemble From 318262d101ea54359caee9bdd310c3a096648386 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 20 Nov 2023 13:09:47 -0500 Subject: [PATCH 008/155] saving/restoring classess is waaaaaaay longer than just recompiling Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 91 +++++++++ .github/workflows/checks.yml | 36 ---- .github/workflows/ci.yml | 172 ++++++++++++++++++ .github/workflows/codeql.yml | 13 +- .github/workflows/compile.yml | 63 +++++++ .github/workflows/dco-merge-group.yml | 10 - .github/workflows/dco.yml | 20 -- .github/workflows/docker.yml | 106 +++++++++++ .../workflows/gradle-wrapper-validation.yml | 6 +- .github/workflows/javadoc.yml | 31 ++++ .github/workflows/nightly.yml | 39 ++++ .github/workflows/pr-checklist-on-draft.yml | 21 +++ .github/workflows/reference-tests.yml | 148 +++++++++++++++ .github/workflows/release.yml | 2 +- .github/workflows/repolinter.yml | 1 - .github/workflows/report.yml | 97 ++++++++++ .github/workflows/spotless.yml | 21 +++ 17 files changed, 797 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/acceptance-tests.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/compile.yml delete mode 100644 .github/workflows/dco-merge-group.yml delete mode 100644 .github/workflows/dco.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/javadoc.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/pr-checklist-on-draft.yml create mode 100644 .github/workflows/reference-tests.yml create mode 100644 .github/workflows/report.yml create mode 100644 .github/workflows/spotless.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml new file mode 100644 index 00000000000..ccf371fb808 --- /dev/null +++ b/.github/workflows/acceptance-tests.yml @@ -0,0 +1,91 @@ +name: acceptance-tests +on: + workflow_call: + + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + runner-indexes: + runs-on: ubuntu-22.04 + name: Generate runner indexes + outputs: + json: ${{ steps.generate-index-list.outputs.json }} + steps: + - id: generate-index-list + run: | + MAX_INDEX=$((${{ env.total-runners }}-1)) + INDEX_LIST=$(seq 0 ${MAX_INDEX}) + INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) + echo "::set-output name=json::${INDEX_JSON}" + acceptanceTestEthereum: + runs-on: [ self-hosted, X64, Linux ] + name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" + needs: + - runner-indexes + if: ${{ github.actor != 'dependabot[bot]' }} + strategy: + fail-fast: false + matrix: + runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: install native packages + run: | + sudo apt-get update + sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools + sudo service haveged restart + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: Split tests + id: split-tests + uses: chaosaffe/split-tests@v1-alpha.1 + with: + glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' + split-total: ${{ env.total-runners }} + split-index: ${{ matrix.runner-index }} + + - name: write out test list + run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt + - name: format gradle args + #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, + #then drop file extension, then insert --tests option between each. + run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@@g' -e 's@/@.@g' -e 's/\.java//g' -e 's/\ /\ --tests\ /g' >> gradleArgs.txt + - name: list keystores + run: find . -name "*.jks" + - name: run acceptance tests + run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` + - name: cleanup tempfiles + run: rm testList.txt gradleArgs.txt + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-acceptance-${{ matrix.runner-index }}-test-results + path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' + retention-days: 1 + - name: Acceptance Test Report html + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: html-acceptance-${{ matrix.runner-index }}-test-results + path: 'acceptance-tests/tests/build/reports/tests/acceptanceTest/**' + retention-days: 1 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee5aa159e92..e69de29bb2d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,36 +0,0 @@ -name: checks -on: - push: - branches: [ main ] - pull_request: - workflow_dispatch: - -jobs: - spotless: - runs-on: [besu-research-ubuntu-16] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - name: spotless - run: ./gradlew --no-daemon --parallel clean spotlessCheck - javadoc_17: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java 17 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - cache: gradle - - name: javadoc (JDK 17) - run: ./gradlew --no-daemon clean javadoc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..539ab8c2ad6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,172 @@ +name: ci +on: + workflow_call: + + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + javadoc_17: + runs-on: [ self-hosted ] + needs: compile + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java 17 + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: javadoc (JDK 17) + run: ./gradlew --no-daemon javadoc + assemble: + runs-on: [self-hosted, Linux, X64] + needs: compile + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: Assemble + uses: gradle/gradle-build-action@v2 + with: + arguments: assemble -Dorg.gradle.parallel=true + - uses: actions/upload-artifact@v3.1.0 + with: + name: besu-zip + path: build/distributions/*.zip + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{env.runner_uid}} + testWindows: + runs-on: windows-2022 + needs: assemble + timeout-minutes: 10 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: Download zip + uses: actions/download-artifact@v3 + with: + name: besu-zip + - name: test Besu + run: | + unzip besu-*.zip -d besu-tmp + cd besu-tmp + mv besu-* ../besu + cd .. + besu\bin\besu.bat --help + besu\bin\besu.bat --version + runner-indexes: + runs-on: ubuntu-22.04 + name: Generate runner indexes + outputs: + json: ${{ steps.generate-index-list.outputs.json }} + steps: + - id: generate-index-list + run: | + MAX_INDEX=$((${{ env.total-runners }}-1)) + INDEX_LIST=$(seq 0 ${MAX_INDEX}) + INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) + echo "::set-output name=json::${INDEX_JSON}" + run-tests: + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 60 + name: "Run test suite" + needs: + - compile + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: test + uses: gradle/gradle-build-action@v2 + with: + arguments: test + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-test-results + path: '**/build/test-results/test/TEST-*.xml' + retention-days: 1 + - name: Upload Test Report html + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-test-results-html + path: '**/build/reports/*' + retention-days: 1 + integration-test: + runs-on: [self-hosted, Linux, X64] + needs: compile + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: test + uses: gradle/gradle-build-action@v2 + with: + arguments: integrationTest + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-integration-test-results + path: '**/build/test-results/integrationTest/TEST-*.xml' + retention-days: 1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 590227e98df..d42aa6f368f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,15 +12,7 @@ name: "CodeQL" on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - paths-ignore: - - '**/*.json' - - '**/*.md' - - '**/*.properties' - - '**/*.txt' + workflow_call: jobs: analyze: name: Analyze @@ -61,7 +53,6 @@ jobs: # Autobuild failed (OOM) # Hence, supply memory args for gradle build - run: | - JAVA_OPTS="-Xmx1000M" ./gradlew --no-scan compileJava - + JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml new file mode 100644 index 00000000000..a53212cdd34 --- /dev/null +++ b/.github/workflows/compile.yml @@ -0,0 +1,63 @@ +name: compile +on: + push: + branches: [ main ] + pull_request: + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + compile: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Compile + uses: gradle/gradle-build-action@v2 + with: + gradle-home-cache-cleanup: true + cache-read-only: false + arguments: compileJava compileTestJava -Dorg.gradle.parallel=true + compileJmh: + runs-on: ubuntu-22.04 + needs: compile + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: compileJmh + uses: gradle/gradle-build-action@v2 + with: + arguments: compileJmh + + javadoc_17: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java 17 + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: javadoc (JDK 17) + run: ./gradlew --no-daemon javadoc diff --git a/.github/workflows/dco-merge-group.yml b/.github/workflows/dco-merge-group.yml deleted file mode 100644 index fee29b6c5d7..00000000000 --- a/.github/workflows/dco-merge-group.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: dco -on: - merge_group: - -jobs: - dco: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents" diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index 5fa9931c77a..00000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: dco -on: - pull_request: - workflow_dispatch: - -jobs: - dco: - runs-on: [besu-research-ubuntu-8] - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - run: echo "This DCO job runs on pull_request event and workflow_dispatch" - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..2f4747f90b8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,106 @@ +name: "Container Images" +on: + workflow_call: + + +jobs: + buildX64Docker: + runs-on: [self-hosted, X64, Linux] + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: hadoLint_openjdk_11 + run: docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile + - name: hadoLint_openjdk_11_debug + run: docker run --rm -i hadolint/hadolint < docker/openjdk-11-debug/Dockerfile + - name: hadoLint_openjdk_latest + run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile + - name: hadoLint_graalvm + run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + - name: distDocker + uses: gradle/gradle-build-action@v2 + with: + arguments: distDocker + - name: test image + run: | + mkdir -p docker/reports + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-amd64 -o ./docker/tests/goss-linux-amd64 + - name: distDocker + uses: gradle/gradle-build-action@v2 + env: + architecture: amd64 + with: + arguments: testDocker + - name: Login to DockerHub + if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} + - name: publish + if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} + run: ./gradlew --no-daemon "-Pbranch=${{ github.head_ref || github.ref_name }}" dockerUpload + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + buildArm64Docker: + runs-on: [self-hosted, ARM64, Linux] + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: zulu + architecture: arm64 + java-version: 17 + cache: gradle + - name: Docker login + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: buildDocker + uses: gradle/gradle-build-action@v2 + env: + architecture: arm64 + with: + arguments: distDocker + - name: download image test binary + run: | + mkdir -p docker/reports + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-arm -o ./docker/tests/goss-linux-arm64 + - name: testDocker + uses: gradle/gradle-build-action@v2 + env: + architecture: arm64 + with: + arguments: testDocker + - name: Login to DockerHub + uses: docker/login-action@v2 + if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} + with: + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} + - name: publish + if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} + uses: gradle/gradle-build-action@v2 + env: + architecture: arm64 + with: + arguments: "-Pbranch=${{ github.head_ref || github.ref_name }} dockerUpload" + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml index 6f44fe43e54..0007837da19 100644 --- a/.github/workflows/gradle-wrapper-validation.yml +++ b/.github/workflows/gradle-wrapper-validation.yml @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 name: "Validate Gradle Wrapper" -on: [push, pull_request] +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: jobs: validation: diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml new file mode 100644 index 00000000000..991af8952da --- /dev/null +++ b/.github/workflows/javadoc.yml @@ -0,0 +1,31 @@ +name: javadoc +on: + workflow_dispatch: + workflow_run: + workflows: [compile] + types: + - completed + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + javadoc_17: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java 17 + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/download-artifact@v3 + with: + name: classes + - name: javadoc (JDK 17) + run: ./gradlew --no-daemon javadoc diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..361161c9001 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,39 @@ +name: nightly +on: + schedule: + - cron: '0 5 * * *' # run at 5 AM UTC + +env: + GRADLE_OPTS: "-Xmx4g -Dorg.gradle.daemon=false" + +jobs: + run-acceptance-tests: + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 90 + name: "Run acceptance test suite" + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Cache classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ github.sha }} + - name: ethereum acceptance tests + uses: gradle/gradle-build-action@v2 + with: + arguments: acceptanceTest --scan + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-acceptance-test-results + path: '**/build/test-results/acceptanceTest/TEST-*.xml' + retention-days: 1 diff --git a/.github/workflows/pr-checklist-on-draft.yml b/.github/workflows/pr-checklist-on-draft.yml new file mode 100644 index 00000000000..69421e19e06 --- /dev/null +++ b/.github/workflows/pr-checklist-on-draft.yml @@ -0,0 +1,21 @@ +name: "comment on pr with checklist" +on: + pull_request_target: + types: [ converted_to_draft ] + branches: [ main ] +jobs: + checklist: + name: "add checklist as a comment on newly opened PRs" + runs-on: [besu,Linux,self-hosted,nodocker] + steps: + - uses: actions/github-script@v5 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '- [ ] I thought about running CI.\n- + [ ] If I didn't run CI, I ran as much locally as possible before pushing.\n- + }) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml new file mode 100644 index 00000000000..5a55f6aefe9 --- /dev/null +++ b/.github/workflows/reference-tests.yml @@ -0,0 +1,148 @@ +name: reference-tests +on: + workflow_call: + + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + total-runners: 6 + +jobs: + runner-indexes: + runs-on: ubuntu-22.04 + name: Generate runner indexes + outputs: + json: ${{ steps.generate-index-list.outputs.json }} + steps: + - id: generate-index-list + run: | + MAX_INDEX=$((${{ env.total-runners }}-1)) + INDEX_LIST=$(seq 0 ${MAX_INDEX}) + INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) + echo "::set-output name=json::${INDEX_JSON}" + prepareReferenceTestEthereum: + runs-on: [ self-hosted, Linux, X64 ] + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + with: + submodules: recursive + set-safe-directory: true + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: generate ethereum reference tests + uses: gradle/gradle-build-action@v2 + with: + arguments: ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + - name: Cache reference tests + uses: actions/cache@v3 + with: + path: "ethereum/referencetests/build/generated/sources/reference-test/**" + key: ${{ runner.os }}-reftests-${{ github.sha }} + referenceTestEthereum: + runs-on: [ self-hosted, X64, Linux ] + needs: + - prepareReferenceTestEthereum + - runner-indexes + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + strategy: + matrix: + runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: Restore reference tests + uses: actions/cache@v3 + with: + path: "ethereum/referencetests/build/generated/sources/reference-test/**" + key: ${{ runner.os }}-reftests-${{ github.sha }} + - name: Split tests + id: split-tests + uses: chaosaffe/split-tests@v1-alpha.1 + with: + glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' + split-total: ${{ env.total-runners }} + split-index: ${{ matrix.runner-index }} + - name: run reference tests + run: ./gradlew ethereum:referenceTests:referenceTests $(echo ${{ steps.split-tests.outputs.test-suite }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /') + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-reference-${{ matrix.runner-index }}-test-results + path: '**/build/test-results/referenceTests/TEST-*.xml' + retention-days: 1 + referenceTestQbft: + runs-on: [ self-hosted, Linux, X64 ] + timeout-minutes: 30 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: export runner UID + run: echo "runner_uid=$UID" >> $GITHUB_ENV + - name: Checkout Repo + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Restore classes + uses: actions/cache@v3 + with: + path: "**/*.class" + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + - name: qbft reference tests + uses: gradle/gradle-build-action@v2 + with: + arguments: consensus:qbft:referencetests + - name: Correct Ownership in GITHUB_WORKSPACE directory + uses: peter-murray/reset-workspace-ownership-action@v1 + with: + user_id: ${{ env.runner_uid }} + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-qbft-reference-test-results + path: '**/build/test-results/referenceTests/TEST-*.xml' + retention-days: 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16fd67f94d0..21a01ba4a9f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,7 @@ on: types: released jobs: dockerPromoteX64: - runs-on: [besu-research-ubuntu-16] + runs-on: [besu-research-ubuntu-8] steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml index 983164e1061..f7a8e820192 100644 --- a/.github/workflows/repolinter.yml +++ b/.github/workflows/repolinter.yml @@ -3,7 +3,6 @@ name: Repolinter on: - workflow_dispatch: push: branches: - master diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml new file mode 100644 index 00000000000..83ad916112d --- /dev/null +++ b/.github/workflows/report.yml @@ -0,0 +1,97 @@ +name: report +on: + workflow_call: + + +permissions: + checks: write + +jobs: + runner-indexes: + runs-on: ubuntu-22.04 + name: Generate runner indexes + outputs: + json: ${{ steps.generate-index-list.outputs.json }} + steps: + - id: generate-index-list + run: | + MAX_INDEX=$((${{ env.total-runners }}-1)) + INDEX_LIST=$(seq 0 ${MAX_INDEX}) + INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) + echo "::set-output name=json::${INDEX_JSON}" + unit-tests: + runs-on: ubuntu-22.04 + strategy: + matrix: + runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + steps: + - name: Download Test Report + uses: dawidd6/action-download-artifact@v2 + with: + name: junit-test-${{ matrix.runner-index }}-results + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + with: + commit: ${{github.event.workflow_run.head_sha}} + report_paths: '**/build/test-results/test/TEST-*.xml' + integration-tests: + runs-on: ubuntu-22.04 + steps: + - name: Download Test Report + uses: dawidd6/action-download-artifact@v2 + with: + name: junit-integration-test-results + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + with: + commit: ${{github.event.workflow_run.head_sha}} + report_paths: '**/build/test-results/test/TEST-*.xml' + acceptance-tests: + runs-on: ubuntu-22.04 + steps: + - name: Download Test Report + uses: dawidd6/action-download-artifact@v2 + with: + name: junit-acceptance-test-results + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + with: + commit: ${{github.event.workflow_run.head_sha}} + report_paths: '**/build/test-results/test/TEST-*.xml' + qbft-reference-tests: + runs-on: ubuntu-22.04 + steps: + - name: Download Test Report + uses: dawidd6/action-download-artifact@v2 + with: + name: junit-qbft-reference-test-results + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + with: + commit: ${{github.event.workflow_run.head_sha}} + report_paths: '**/build/test-results/test/TEST-*.xml' + reference-tests: + runs-on: ubuntu-22.04 + strategy: + matrix: + runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + steps: + - name: Download Test Report + uses: dawidd6/action-download-artifact@v2 + with: + name: junit-reference-${{ matrix.runner-index }}-test-results + workflow: ${{ github.event.workflow.id }} + run_id: ${{ github.event.workflow_run.id }} + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + with: + commit: ${{github.event.workflow_run.head_sha}} + report_paths: '**/build/test-results/test/TEST-*.xml' \ No newline at end of file diff --git a/.github/workflows/spotless.yml b/.github/workflows/spotless.yml new file mode 100644 index 00000000000..ab8fd3ec9fb --- /dev/null +++ b/.github/workflows/spotless.yml @@ -0,0 +1,21 @@ +name: "Spotless Code Formatting" +on: + push: + branches: [ main ] + pull_request: + +jobs: + spotless: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: spotless + run: ./gradlew --no-daemon --parallel clean spotlessCheck From ace1a6a03018f2c5959a5b9ad25e0c6a740f291e Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Thu, 23 Nov 2023 06:19:59 +1000 Subject: [PATCH 009/155] update checkout action to v4 (#6197) Signed-off-by: Sally MacFarlane Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 48 +++++++------------ .github/workflows/callee.yml | 13 +++++ .github/workflows/caller.yml | 9 ++++ .github/workflows/checks.yml | 37 ++++++++++++++ .github/workflows/compile.yml | 48 +++---------------- .../workflows/gradle-wrapper-validation.yml | 10 ++-- .github/workflows/javadoc.yml | 10 +--- .github/workflows/parallel-unit-tests.yml | 48 +++++++++++++++++++ .github/workflows/reference-tests.yml | 8 ++-- .github/workflows/repolinter.yml | 16 ++----- .github/workflows/report.yml | 20 +------- .github/workflows/spotless.yml | 7 ++- .github/workflows/unit-tests.yml | 43 +++++++++++++++++ 13 files changed, 192 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/callee.yml create mode 100644 .github/workflows/caller.yml create mode 100644 .github/workflows/parallel-unit-tests.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index ccf371fb808..f40f3c71318 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,6 +1,7 @@ name: acceptance-tests on: - workflow_call: + pull_request_review: + types: [ submitted ] env: @@ -9,7 +10,8 @@ env: jobs: runner-indexes: - runs-on: ubuntu-22.04 + runs-on: + group: "Besu Large Runners" name: Generate runner indexes outputs: json: ${{ steps.generate-index-list.outputs.json }} @@ -21,23 +23,17 @@ jobs: INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) echo "::set-output name=json::${INDEX_JSON}" acceptanceTestEthereum: - runs-on: [ self-hosted, X64, Linux ] + runs-on: + group: "Besu Large Runners" name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" needs: - runner-indexes - if: ${{ github.actor != 'dependabot[bot]' }} + if: ${{ github.actor != 'dependabot[bot]'}} strategy: fail-fast: false matrix: runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - - name: install native packages - run: | - sudo apt-get update - sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools - sudo service haveged restart - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -46,11 +42,12 @@ jobs: distribution: adopt java-version: 17 cache: gradle - - name: Restore classes - uses: actions/cache@v3 + - name: Compile + uses: gradle/gradle-build-action@v2 with: - path: "**/*.class" - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} + gradle-home-cache-cleanup: true + cache-read-only: false + arguments: compileJava compileTestJava -Dorg.gradle.parallel=true - name: Split tests id: split-tests uses: chaosaffe/split-tests@v1-alpha.1 @@ -71,21 +68,8 @@ jobs: run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} - - name: Upload Test Report - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails - with: - name: junit-acceptance-${{ matrix.runner-index }}-test-results - path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - retention-days: 1 - - name: Acceptance Test Report html - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails with: - name: html-acceptance-${{ matrix.runner-index }}-test-results - path: 'acceptance-tests/tests/build/reports/tests/acceptanceTest/**' - retention-days: 1 + report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' diff --git a/.github/workflows/callee.yml b/.github/workflows/callee.yml new file mode 100644 index 00000000000..1b7a6cc3ace --- /dev/null +++ b/.github/workflows/callee.yml @@ -0,0 +1,13 @@ +name: callee + +on: + workflow_run: + workflows: [ "caller" ] + types: [ "requested" ] + workflow_dispatch: + +jobs: + echo: + runs-on: ubuntu-22.04 + steps: + - run: echo called diff --git a/.github/workflows/caller.yml b/.github/workflows/caller.yml new file mode 100644 index 00000000000..7955282a23e --- /dev/null +++ b/.github/workflows/caller.yml @@ -0,0 +1,9 @@ +name: caller + +on: pull_request + +jobs: + echo: + runs-on: ubuntu-22.04 + steps: + - run: echo calling diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e69de29bb2d..bed65b5e2ce 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -0,0 +1,37 @@ +name: "checks" + +on: + pull_request: + +jobs: + lint: + name: "Repository Linting" + runs-on: ubuntu-22.04 + container: ghcr.io/todogroup/repolinter:v0.11.2 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Lint Repo + run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown + gradle-wrapper: + name: "Gradle Wrapper Validation" + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: gradle/wrapper-validation-action@v1.1.0 + spotless: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: spotless + uses: gradle/gradle-build-action@v2.10.0 + with: + arguments: build spotlessCheck -Dorg.gradle.parallel=true + cache: gradle diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index a53212cdd34..d5544ba6b3e 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -1,18 +1,19 @@ name: compile on: - push: - branches: [ main ] - pull_request: + workflow_run: + workflows: + - checks + types: + - completed env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 6 jobs: compile: runs-on: ubuntu-22.04 timeout-minutes: 30 - if: ${{ github.actor != 'dependabot[bot]' }} + if: ${{ github.actor != 'dependabot[bot]' && github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -25,39 +26,4 @@ jobs: - name: Compile uses: gradle/gradle-build-action@v2 with: - gradle-home-cache-cleanup: true - cache-read-only: false - arguments: compileJava compileTestJava -Dorg.gradle.parallel=true - compileJmh: - runs-on: ubuntu-22.04 - needs: compile - timeout-minutes: 30 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: compileJmh - uses: gradle/gradle-build-action@v2 - with: - arguments: compileJmh - - javadoc_17: - runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java 17 - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: javadoc (JDK 17) - run: ./gradlew --no-daemon javadoc + arguments: build -x test -x spotlessCheck -Dorg.gradle.parallel=true diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml index 0007837da19..bcad0841949 100644 --- a/.github/workflows/gradle-wrapper-validation.yml +++ b/.github/workflows/gradle-wrapper-validation.yml @@ -1,15 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 -name: "Validate Gradle Wrapper" +name: gradle-wrapper-validation on: - push: - branches: [ main ] - pull_request: - workflow_dispatch: + workflow_call: + jobs: validation: name: "Gradle Wrapper Validation" - runs-on: [besu-research-ubuntu-8] + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml index 991af8952da..b6307d97cee 100644 --- a/.github/workflows/javadoc.yml +++ b/.github/workflows/javadoc.yml @@ -1,10 +1,6 @@ name: javadoc on: - workflow_dispatch: - workflow_run: - workflows: [compile] - types: - - completed + workflow_call: env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" @@ -23,9 +19,5 @@ jobs: distribution: adopt java-version: 17 cache: gradle - - name: Restore classes - uses: actions/download-artifact@v3 - with: - name: classes - name: javadoc (JDK 17) run: ./gradlew --no-daemon javadoc diff --git a/.github/workflows/parallel-unit-tests.yml b/.github/workflows/parallel-unit-tests.yml new file mode 100644 index 00000000000..33bd2710ef3 --- /dev/null +++ b/.github/workflows/parallel-unit-tests.yml @@ -0,0 +1,48 @@ +name: parallel-unit-tests +on: + workflow_call: + +env: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + total-runners: 4 +jobs: + junit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + runner_index: + - 0 + - 1 + - 2 + - 3 + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Split tests + id: split-tests + uses: chaosaffe/split-tests@v1-alpha.1 + with: + glob: '**/src/test/java/**/*.java' + split-total: ${{ env.total-runners }} + split-index: ${{ matrix.runner_index }} + line-count: true + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: write out test list + run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt + - name: debug testfile paths + run: cat testList.txt + - name: format gradle args + # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension, + # then swap path delimiter with package delimiter + run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt + - name: debug test class list + run: cat gradleArgs.txt + - name: run unit tests + run: ./gradlew test `cat gradleArgs.txt` + diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 5a55f6aefe9..3d5149c0bbc 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -9,7 +9,7 @@ env: jobs: runner-indexes: - runs-on: ubuntu-22.04 + runs-on: besu-research-ubuntu-8 name: Generate runner indexes outputs: json: ${{ steps.generate-index-list.outputs.json }} @@ -21,7 +21,7 @@ jobs: INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) echo "::set-output name=json::${INDEX_JSON}" prepareReferenceTestEthereum: - runs-on: [ self-hosted, Linux, X64 ] + runs-on: besu-research-ubuntu-8 timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} steps: @@ -57,7 +57,7 @@ jobs: path: "ethereum/referencetests/build/generated/sources/reference-test/**" key: ${{ runner.os }}-reftests-${{ github.sha }} referenceTestEthereum: - runs-on: [ self-hosted, X64, Linux ] + runs-on: besu-research-ubuntu-8 needs: - prepareReferenceTestEthereum - runner-indexes @@ -110,7 +110,7 @@ jobs: path: '**/build/test-results/referenceTests/TEST-*.xml' retention-days: 1 referenceTestQbft: - runs-on: [ self-hosted, Linux, X64 ] + runs-on: besu-research-ubuntu-8 timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} steps: diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml index f7a8e820192..bfc97b11da6 100644 --- a/.github/workflows/repolinter.yml +++ b/.github/workflows/repolinter.yml @@ -1,21 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # Hyperledger Repolinter Action -name: Repolinter +name: repolinter on: - push: - branches: - - master - - main - pull_request: - branches: - - master - - main + workflow_call: + jobs: build: - runs-on: [besu-research-ubuntu-16] - container: ghcr.io/todogroup/repolinter:v0.10.1 + runs-on: ubuntu-22.04 + container: ghcr.io/todogroup/repolinter:v0.11.2 steps: - name: Checkout Code uses: actions/checkout@v4 diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 83ad916112d..9849e271835 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -2,37 +2,21 @@ name: report on: workflow_call: - permissions: checks: write jobs: - runner-indexes: - runs-on: ubuntu-22.04 - name: Generate runner indexes - outputs: - json: ${{ steps.generate-index-list.outputs.json }} - steps: - - id: generate-index-list - run: | - MAX_INDEX=$((${{ env.total-runners }}-1)) - INDEX_LIST=$(seq 0 ${MAX_INDEX}) - INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) - echo "::set-output name=json::${INDEX_JSON}" unit-tests: runs-on: ubuntu-22.04 - strategy: - matrix: - runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} steps: - name: Download Test Report uses: dawidd6/action-download-artifact@v2 with: - name: junit-test-${{ matrix.runner-index }}-results + name: junit-test-results workflow: ${{ github.event.workflow.id }} run_id: ${{ github.event.workflow_run.id }} - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 + uses: mikepenz/action-junit-report@v4 with: commit: ${{github.event.workflow_run.head_sha}} report_paths: '**/build/test-results/test/TEST-*.xml' diff --git a/.github/workflows/spotless.yml b/.github/workflows/spotless.yml index ab8fd3ec9fb..f1814fdccfa 100644 --- a/.github/workflows/spotless.yml +++ b/.github/workflows/spotless.yml @@ -1,8 +1,7 @@ -name: "Spotless Code Formatting" +name: spotless on: - push: - branches: [ main ] - pull_request: + workflow_call: + jobs: spotless: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000000..a65caaed36a --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,43 @@ +name: unit-tests +on: + workflow_run: + workflows: [ "compile" ] + types: [ "completed" ] + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + +jobs: + unitTests: + runs-on: besu-research-ubuntu-8 + if: ${{github.event.workflow_run.conclusion == 'success' }} + permissions: + checks: write + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + cache: gradle + - name: Build + uses: gradle/gradle-build-action@v2 + with: + gradle-home-cache-cleanup: true + cache-read-only: false + arguments: build -Dorg.gradle.parallel=true + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-xml-reports + path: '**/build/test-results/test/TEST-*.xml' + retention-days: 1 + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/test-results/test/TEST-*.xml' + annotate_only: true From 2e9593ebc60e420d0da42bf68562c52b923e1547 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 14:42:03 -0500 Subject: [PATCH 010/155] pls just run the workflow_run already Signed-off-by: Justin Florentine --- .github/workflows/checks.yml | 2 +- .github/workflows/compile.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bed65b5e2ce..a0272368460 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -33,5 +33,5 @@ jobs: - name: spotless uses: gradle/gradle-build-action@v2.10.0 with: - arguments: build spotlessCheck -Dorg.gradle.parallel=true + arguments: spotlessCheck -Dorg.gradle.parallel=true cache: gradle diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index d5544ba6b3e..e9626338a6f 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -24,6 +24,7 @@ jobs: java-version: 17 cache: gradle - name: Compile - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.10.0 with: arguments: build -x test -x spotlessCheck -Dorg.gradle.parallel=true + cache: gradle From 1ca3ae1896bde82cd663c523e1cc295de1e40a3d Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:14:41 -0500 Subject: [PATCH 011/155] updating gradle-build-action per docs Signed-off-by: Justin Florentine --- .github/workflows/checks.yml | 10 ++++++---- .github/workflows/compile.yml | 12 +++++++----- .github/workflows/pr-checklist-on-draft.yml | 2 +- .github/workflows/pr-checklist-on-open.yml | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a0272368460..43cde05e71b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,6 +3,9 @@ name: "checks" on: pull_request: +env: + GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true + jobs: lint: name: "Repository Linting" @@ -30,8 +33,7 @@ jobs: with: distribution: adopt java-version: 17 - - name: spotless + - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - with: - arguments: spotlessCheck -Dorg.gradle.parallel=true - cache: gradle + - name: run spotless + run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index e9626338a6f..3339de1414f 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -5,9 +5,12 @@ on: - checks types: - completed + workflow_dispatch: + env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + GRADLE_OPTS: "-Xmx6g" + GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true jobs: compile: @@ -23,8 +26,7 @@ jobs: distribution: adopt java-version: 17 cache: gradle - - name: Compile + - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - with: - arguments: build -x test -x spotlessCheck -Dorg.gradle.parallel=true - cache: gradle + - name: Run Build + run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/.github/workflows/pr-checklist-on-draft.yml b/.github/workflows/pr-checklist-on-draft.yml index 69421e19e06..6790b296b38 100644 --- a/.github/workflows/pr-checklist-on-draft.yml +++ b/.github/workflows/pr-checklist-on-draft.yml @@ -6,7 +6,7 @@ on: jobs: checklist: name: "add checklist as a comment on newly opened PRs" - runs-on: [besu,Linux,self-hosted,nodocker] + runs-on: ubuntu-22.04 steps: - uses: actions/github-script@v5 with: diff --git a/.github/workflows/pr-checklist-on-open.yml b/.github/workflows/pr-checklist-on-open.yml index 3a4d4342c58..6bccd0ad111 100644 --- a/.github/workflows/pr-checklist-on-open.yml +++ b/.github/workflows/pr-checklist-on-open.yml @@ -6,7 +6,7 @@ on: jobs: checklist: name: "add checklist as a comment on newly opened PRs" - runs-on: [besu-research-ubuntu-8] + runs-on: ubuntu-22.04 steps: - uses: actions/github-script@v5 with: From 873766baea6e7bc34da8327df3acd49a98b15cd2 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:25:11 -0500 Subject: [PATCH 012/155] can't meet any conditionals on dispatch Signed-off-by: Justin Florentine --- .github/workflows/compile.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 3339de1414f..fa25ce59fd3 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -16,7 +16,6 @@ jobs: compile: runs-on: ubuntu-22.04 timeout-minutes: 30 - if: ${{ github.actor != 'dependabot[bot]' && github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout Repo uses: actions/checkout@v3 From f12e5137e41b06e68968aa5ffcf64cf2baa51dab Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:32:28 -0500 Subject: [PATCH 013/155] why would dispatch to compile trigger unit, but not checks? Signed-off-by: Justin Florentine --- .github/workflows/checks.yml | 2 +- .github/workflows/compile.yml | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 43cde05e71b..410339fd0cf 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,4 @@ -name: "checks" +name: checks on: pull_request: diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index fa25ce59fd3..1109b779a36 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -1,10 +1,8 @@ name: compile on: workflow_run: - workflows: - - checks - types: - - completed + workflows: [ "checks" ] + types: [ "completed" ] workflow_dispatch: From 84c3e38ec503bd0fb5865a3720461e0fed170051 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:46:12 -0500 Subject: [PATCH 014/155] breaking this up was a fools errand Signed-off-by: Justin Florentine --- .github/workflows/checks.yml | 39 --------------- .github/workflows/pre-review.yml | 84 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 39 deletions(-) delete mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/pre-review.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index 410339fd0cf..00000000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: checks - -on: - pull_request: - -env: - GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true - -jobs: - lint: - name: "Repository Linting" - runs-on: ubuntu-22.04 - container: ghcr.io/todogroup/repolinter:v0.11.2 - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Lint Repo - run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown - gradle-wrapper: - name: "Gradle Wrapper Validation" - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1.1.0 - spotless: - runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - - name: Setup Gradle - uses: gradle/gradle-build-action@v2.10.0 - - name: run spotless - run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml new file mode 100644 index 00000000000..9d07d2e7b79 --- /dev/null +++ b/.github/workflows/pre-review.yml @@ -0,0 +1,84 @@ +name: pre-review + +on: + pull_request: + +env: + GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true + +jobs: + repolint: + name: "Repository Linting" + runs-on: ubuntu-22.04 + container: ghcr.io/todogroup/repolinter:v0.11.2 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Lint Repo + run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown + gradle-wrapper: + name: "Gradle Wrapper Validation" + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: gradle/wrapper-validation-action@v1.1.0 + spotless: + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.10.0 + - name: run spotless + run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + compile: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: [spotless, gradle-wrapper, repolint] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2.10.0 + - name: Run Build + run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + unitTests: + runs-on: besu-research-ubuntu-8 + needs: [ compile ] + permissions: + checks: write + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + run: ./gradlew build -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Upload Test Report + uses: actions/upload-artifact@v3 + if: always() # always run even if the previous step fails + with: + name: junit-xml-reports + path: '**/build/test-results/test/TEST-*.xml' + retention-days: 1 + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/test-results/test/TEST-*.xml' + annotate_only: true From 24188b8edf80c18a9f837f4886f637730cddfbf1 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:48:37 -0500 Subject: [PATCH 015/155] versions Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 9d07d2e7b79..7a2648113b2 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -67,7 +67,7 @@ jobs: distribution: adopt java-version: 17 - name: Setup Gradle - uses: gradle/gradle-build-action@v2 + uses: gradle/gradle-build-action@v2.10.0 run: ./gradlew build -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 From e43f64e1d055c1f506db059ee6672e215b7d7d9c Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:51:42 -0500 Subject: [PATCH 016/155] adds workflow_dispatch Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 7a2648113b2..a995ecc569d 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -2,6 +2,7 @@ name: pre-review on: pull_request: + workflow_dispatch: env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true From 4df5660b9edd32d50097ee2c296b4c3b41584d83 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 15:54:21 -0500 Subject: [PATCH 017/155] no uses and run Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index a995ecc569d..4b8aa5067b0 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -52,7 +52,7 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - - name: Run Build + - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true unitTests: runs-on: besu-research-ubuntu-8 @@ -69,6 +69,7 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 + - name: Gradle Unit Tests run: ./gradlew build -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 From 7a49367452a38f1c96cfe4c81d952f53c9c61188 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 16:00:22 -0500 Subject: [PATCH 018/155] any self-hosted will do for now Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 4b8aa5067b0..c96fbd6f1df 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -55,7 +55,7 @@ jobs: - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true unitTests: - runs-on: besu-research-ubuntu-8 + runs-on: [self-hosted] needs: [ compile ] permissions: checks: write From a026834eb5d3d7eae93471d3da7aa7951b246c6b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 17:14:08 -0500 Subject: [PATCH 019/155] return of parallel unit tests Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index c96fbd6f1df..73fcdcbfff9 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -55,10 +55,18 @@ jobs: - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true unitTests: - runs-on: [self-hosted] + runs-on: ubuntu-22.04 needs: [ compile ] permissions: checks: write + strategy: + fail-fast: false + matrix: + ci_node_index: + - 0 + - 1 + - 2 + - 3 steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -69,8 +77,32 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - - name: Gradle Unit Tests - run: ./gradlew build -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Download existing test timings + uses: dawidd6/action-download-artifact@v2 + with: + branch: main + name: junit-xml-reports + path: tmp/junit-xml-reports-downloaded + continue-on-error: true + - uses: r7kamura/split-tests-by-timings@v0 + id: split-tests + with: + reports: tmp/junit-xml-reports-downloaded + glob: "**/src/test/java/**/*.java" + index: ${{ matrix.ci_node_index }} + total: 4 + - name: write out test list + run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt + - name: debug testfile paths + run: cat testList.txt + - name: format gradle args + # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension, + # then swap path delimiter with package delimiter + run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt + - name: debug test class list + run: cat gradleArgs.txt + - name: run unit tests + run: ./gradlew test `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails From 4f11d3707ebfaa355fdb4d01a4705158cdbbaef4 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 17:22:47 -0500 Subject: [PATCH 020/155] everything is basically documented wrong Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 73fcdcbfff9..37df3be70b5 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -80,10 +80,9 @@ jobs: - name: Download existing test timings uses: dawidd6/action-download-artifact@v2 with: - branch: main name: junit-xml-reports path: tmp/junit-xml-reports-downloaded - continue-on-error: true + if-no-artifact-found: ignore - uses: r7kamura/split-tests-by-timings@v0 id: split-tests with: From 49efc9b7e1d0515cd8ea6a3ca3d6747b6962f759 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 6 Dec 2023 17:26:52 -0500 Subject: [PATCH 021/155] snake case Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 37df3be70b5..52b37176eca 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -82,7 +82,7 @@ jobs: with: name: junit-xml-reports path: tmp/junit-xml-reports-downloaded - if-no-artifact-found: ignore + if_no_artifact_found: ignore - uses: r7kamura/split-tests-by-timings@v0 id: split-tests with: From 15f60f420e485941e913927c0c71ec703bb0920b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 15:37:45 -0500 Subject: [PATCH 022/155] welcome to regex hell Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 19 +++++++++++++------ build.gradle | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 52b37176eca..0328168b035 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -87,21 +87,28 @@ jobs: id: split-tests with: reports: tmp/junit-xml-reports-downloaded - glob: "**/src/test/java/**/*.java" + glob: "**/src/test/java/**/*Test.java" index: ${{ matrix.ci_node_index }} total: 4 - name: write out test list - run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt + run: echo "${{ steps.split-tests.outputs.paths }}" | grep -v "acceptance-tests\/" | grep -v "ethereum\/referencetests" >> testList.txt - name: debug testfile paths run: cat testList.txt - name: format gradle args - # regex means: truncate file paths to align with package name, replacing with tests switch, then drop file extension, - # then swap path delimiter with package delimiter - run: cat testList.txt | sed -e 's/[^ ]*src\/test\/java\//--tests\ /g' -e 's/\.java//g' -e 's/\//\./g' >> gradleArgs.txt + # I'm so sorry. + # take the space separated list of paths to test files and break into newlines + # use sed on each one to: + # copy the initial path to the hold buffer, strip off the source root and start quoting, then convert / to . + # so the file path is a package path, drop the trailing .java and close the quote. swap that with the holding buffer, + # get the module prefix of the path, and convert that to a gradle module syntax referring to the test target, + # delimited with colons, and suffix with --test. Now append the holding buffer back onto patternspace. + # now pipe that to awk to collapse the newlines introduced by sed + # use tr to convert newlines to spaces, so it can all go in as one thing into gradle. + run: cat testList.txt | awk '{for(i=1; i<=NF; i++) print $i}' | sed -e 'h;s/.*src\/test\/java\//\"/;s/\//\./g;s/\.java$/\"/;x;s/\(.*\)src\/test\/java.*/\1test\ --tests/;s/\//\:/g;G' | awk 'ORS=NR%2?" ":"\n"' | tr '\n' ' ' > gradleArgs.txt - name: debug test class list run: cat gradleArgs.txt - name: run unit tests - run: ./gradlew test `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails diff --git a/build.gradle b/build.gradle index 255a878d975..978e0370052 100644 --- a/build.gradle +++ b/build.gradle @@ -174,6 +174,8 @@ allprojects { dependencies { components.all(BouncyCastleCapability) errorprone 'com.google.errorprone:error_prone_core' + testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } configurations.all { From 42b85cae0228c49611cbc63678f99c615d40e86c Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 15:46:46 -0500 Subject: [PATCH 023/155] smaller chunks maybe Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 0328168b035..edd041b16da 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -89,7 +89,7 @@ jobs: reports: tmp/junit-xml-reports-downloaded glob: "**/src/test/java/**/*Test.java" index: ${{ matrix.ci_node_index }} - total: 4 + total: 40 - name: write out test list run: echo "${{ steps.split-tests.outputs.paths }}" | grep -v "acceptance-tests\/" | grep -v "ethereum\/referencetests" >> testList.txt - name: debug testfile paths From cd2ee95643b023731e1a364c24d522468ce13227 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 15:54:22 -0500 Subject: [PATCH 024/155] grep gonna grep Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index edd041b16da..88bcd16e4f3 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -90,13 +90,13 @@ jobs: glob: "**/src/test/java/**/*Test.java" index: ${{ matrix.ci_node_index }} total: 40 + # take the space separated list of paths to test files and break into newlines to filter out other tests with grep - name: write out test list - run: echo "${{ steps.split-tests.outputs.paths }}" | grep -v "acceptance-tests\/" | grep -v "ethereum\/referencetests" >> testList.txt + run: echo "${{ steps.split-tests.outputs.paths }}" | awk '{for(i=1; i<=NF; i++) print $i}' | grep -v "acceptance-tests\/" | grep -v "ethereum\/referencetests" >> testList.txt - name: debug testfile paths run: cat testList.txt - name: format gradle args - # I'm so sorry. - # take the space separated list of paths to test files and break into newlines + # use sed on each one to: # copy the initial path to the hold buffer, strip off the source root and start quoting, then convert / to . # so the file path is a package path, drop the trailing .java and close the quote. swap that with the holding buffer, @@ -104,7 +104,7 @@ jobs: # delimited with colons, and suffix with --test. Now append the holding buffer back onto patternspace. # now pipe that to awk to collapse the newlines introduced by sed # use tr to convert newlines to spaces, so it can all go in as one thing into gradle. - run: cat testList.txt | awk '{for(i=1; i<=NF; i++) print $i}' | sed -e 'h;s/.*src\/test\/java\//\"/;s/\//\./g;s/\.java$/\"/;x;s/\(.*\)src\/test\/java.*/\1test\ --tests/;s/\//\:/g;G' | awk 'ORS=NR%2?" ":"\n"' | tr '\n' ' ' > gradleArgs.txt + run: cat testList.txt | sed -e 'h;s/.*src\/test\/java\//\"/;s/\//\./g;s/\.java$/\"/;x;s/\(.*\)src\/test\/java.*/\1test\ --tests/;s/\//\:/g;G' | awk 'ORS=NR%2?" ":"\n"' | tr '\n' ' ' > gradleArgs.txt - name: debug test class list run: cat gradleArgs.txt - name: run unit tests From 3980fd2fb13717fd1c48b607b280708dd5b719d5 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 17:12:53 -0500 Subject: [PATCH 025/155] maybe thats stupid and we can just separate by module Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 42 +++++--------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 88bcd16e4f3..de747159498 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -62,11 +62,11 @@ jobs: strategy: fail-fast: false matrix: - ci_node_index: - - 0 - - 1 - - 2 - - 3 + gradle_args: + - "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test" + - "besu:test consensus:test crypto:test" + - "ethereum:api:test" + - "ethereum:core:test" steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -77,38 +77,8 @@ jobs: java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - - name: Download existing test timings - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-xml-reports - path: tmp/junit-xml-reports-downloaded - if_no_artifact_found: ignore - - uses: r7kamura/split-tests-by-timings@v0 - id: split-tests - with: - reports: tmp/junit-xml-reports-downloaded - glob: "**/src/test/java/**/*Test.java" - index: ${{ matrix.ci_node_index }} - total: 40 - # take the space separated list of paths to test files and break into newlines to filter out other tests with grep - - name: write out test list - run: echo "${{ steps.split-tests.outputs.paths }}" | awk '{for(i=1; i<=NF; i++) print $i}' | grep -v "acceptance-tests\/" | grep -v "ethereum\/referencetests" >> testList.txt - - name: debug testfile paths - run: cat testList.txt - - name: format gradle args - - # use sed on each one to: - # copy the initial path to the hold buffer, strip off the source root and start quoting, then convert / to . - # so the file path is a package path, drop the trailing .java and close the quote. swap that with the holding buffer, - # get the module prefix of the path, and convert that to a gradle module syntax referring to the test target, - # delimited with colons, and suffix with --test. Now append the holding buffer back onto patternspace. - # now pipe that to awk to collapse the newlines introduced by sed - # use tr to convert newlines to spaces, so it can all go in as one thing into gradle. - run: cat testList.txt | sed -e 'h;s/.*src\/test\/java\//\"/;s/\//\./g;s/\.java$/\"/;x;s/\(.*\)src\/test\/java.*/\1test\ --tests/;s/\//\:/g;G' | awk 'ORS=NR%2?" ":"\n"' | tr '\n' ' ' > gradleArgs.txt - - name: debug test class list - run: cat gradleArgs.txt - name: run unit tests - run: ./gradlew `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew ${{ matrix.gradle.args }} -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails From 33526c290d0c901f96ec4c8a6c31df0ed7506b12 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 18:10:16 -0500 Subject: [PATCH 026/155] definitely don't want to run the whole build Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index de747159498..dd8755e21ba 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -78,7 +78,7 @@ jobs: - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests - run: ./gradlew ${{ matrix.gradle.args }} -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew "${{ matrix.gradle_args }}" -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails From b83ff46044689bb1d91ddba3012cc3a6c88814b1 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 18:39:18 -0500 Subject: [PATCH 027/155] need env arg for matrix val Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index dd8755e21ba..dc76097eee4 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -6,6 +6,7 @@ on: env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true + GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} jobs: repolint: @@ -78,7 +79,7 @@ jobs: - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests - run: ./gradlew "${{ matrix.gradle_args }}" -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails From cd93ad877aba17db9c7d3640887b90e15ade3095 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 7 Dec 2023 18:41:44 -0500 Subject: [PATCH 028/155] env in wrong place Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index dc76097eee4..b7500052d2d 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -6,7 +6,6 @@ on: env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true - GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} jobs: repolint: @@ -56,6 +55,8 @@ jobs: - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true unitTests: + env: + GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} runs-on: ubuntu-22.04 needs: [ compile ] permissions: From 9b0b68144fd105d98368b5c90efc1ab0472d5fa4 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 8 Dec 2023 09:30:43 -0500 Subject: [PATCH 029/155] lets try acceptance on free runners Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index f40f3c71318..c2b7f5ead40 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -10,8 +10,7 @@ env: jobs: runner-indexes: - runs-on: - group: "Besu Large Runners" + runs-on: ubuntu-22.04 name: Generate runner indexes outputs: json: ${{ steps.generate-index-list.outputs.json }} @@ -23,8 +22,7 @@ jobs: INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) echo "::set-output name=json::${INDEX_JSON}" acceptanceTestEthereum: - runs-on: - group: "Besu Large Runners" + runs-on: ubuntu-22.04 name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" needs: - runner-indexes From 892bcee1e6d0d8a3dcb2f0697e760fc619d6cf6d Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 8 Dec 2023 09:48:42 -0500 Subject: [PATCH 030/155] undoes junit deps Signed-off-by: Justin Florentine --- build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.gradle b/build.gradle index 978e0370052..255a878d975 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,6 @@ allprojects { dependencies { components.all(BouncyCastleCapability) errorprone 'com.google.errorprone:error_prone_core' - testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' - testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } configurations.all { From ec8bf4f75d02b0234cba3f43a3a3fa73774ac031 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 8 Dec 2023 14:13:28 -0500 Subject: [PATCH 031/155] saw some timeouts, increase runners Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 +- .github/workflows/callee.yml | 13 ------------- .github/workflows/caller.yml | 9 --------- 3 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 .github/workflows/callee.yml delete mode 100644 .github/workflows/caller.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index c2b7f5ead40..916e8b8c28e 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -6,7 +6,7 @@ on: env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 6 + total-runners: 10 jobs: runner-indexes: diff --git a/.github/workflows/callee.yml b/.github/workflows/callee.yml deleted file mode 100644 index 1b7a6cc3ace..00000000000 --- a/.github/workflows/callee.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: callee - -on: - workflow_run: - workflows: [ "caller" ] - types: [ "requested" ] - workflow_dispatch: - -jobs: - echo: - runs-on: ubuntu-22.04 - steps: - - run: echo called diff --git a/.github/workflows/caller.yml b/.github/workflows/caller.yml deleted file mode 100644 index 7955282a23e..00000000000 --- a/.github/workflows/caller.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: caller - -on: pull_request - -jobs: - echo: - runs-on: ubuntu-22.04 - steps: - - run: echo calling From 97357791f65424bc8d500fef12046a08273dc98e Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 8 Dec 2023 16:09:50 -0500 Subject: [PATCH 032/155] adds tests to separate out long running api tests Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 11 ++--------- .github/workflows/pre-review.yml | 4 +++- ethereum/api/build.gradle | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 916e8b8c28e..95042710080 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,7 +26,7 @@ jobs: name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" needs: - runner-indexes - if: ${{ github.actor != 'dependabot[bot]'}} + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved'}} strategy: fail-fast: false matrix: @@ -39,13 +39,8 @@ jobs: with: distribution: adopt java-version: 17 - cache: gradle - name: Compile uses: gradle/gradle-build-action@v2 - with: - gradle-home-cache-cleanup: true - cache-read-only: false - arguments: compileJava compileTestJava -Dorg.gradle.parallel=true - name: Split tests id: split-tests uses: chaosaffe/split-tests@v1-alpha.1 @@ -60,10 +55,8 @@ jobs: #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, #then drop file extension, then insert --tests option between each. run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@@g' -e 's@/@.@g' -e 's/\.java//g' -e 's/\ /\ --tests\ /g' >> gradleArgs.txt - - name: list keystores - run: find . -name "*.jks" - name: run acceptance tests - run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` + run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Publish Test Report diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index b7500052d2d..a7100db9a12 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -67,7 +67,9 @@ jobs: gradle_args: - "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test" - "besu:test consensus:test crypto:test" - - "ethereum:api:test" + - "ethereum:api:testBonsai" + - "ethereum:api:testForest" + - "ethereum:api:testRemainder" - "ethereum:core:test" steps: - name: Checkout Repo diff --git a/ethereum/api/build.gradle b/ethereum/api/build.gradle index ed395df8c5d..351a4f700c0 100644 --- a/ethereum/api/build.gradle +++ b/ethereum/api/build.gradle @@ -168,3 +168,29 @@ tasks.register('generateTestBlockchain') { } } test.dependsOn(generateTestBlockchain) +/* + Utility tasks used to separate out long running suites of tests so they can be parallelized in CI + */ +tasks.register("testBonsai", Test) { + useJUnitPlatform() + filter { + includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + } + dependsOn(generateTestBlockchain) +} + +tasks.register("testForest", Test) { + useJUnitPlatform() + filter { + includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") + } + dependsOn(generateTestBlockchain) +} + +tasks.register("testRemainder", Test) { + useJUnitPlatform() + filter { + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") + } +} From 6b25cd8433ef7279cc5eebbfa17d04e135a32750 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 8 Dec 2023 16:13:32 -0500 Subject: [PATCH 033/155] ya got me again, spotless Signed-off-by: Justin Florentine --- ethereum/api/build.gradle | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ethereum/api/build.gradle b/ethereum/api/build.gradle index 351a4f700c0..e5836c46eb6 100644 --- a/ethereum/api/build.gradle +++ b/ethereum/api/build.gradle @@ -169,13 +169,13 @@ tasks.register('generateTestBlockchain') { } test.dependsOn(generateTestBlockchain) /* - Utility tasks used to separate out long running suites of tests so they can be parallelized in CI + Utility tasks used to separate out long running suites of tests so they can be parallelized in CI */ tasks.register("testBonsai", Test) { useJUnitPlatform() - filter { - includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") - } + filter { + includeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + } dependsOn(generateTestBlockchain) } @@ -189,8 +189,8 @@ tasks.register("testForest", Test) { tasks.register("testRemainder", Test) { useJUnitPlatform() - filter { - excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") - excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") - } + filter { + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.bonsai.*") + excludeTestsMatching("org.hyperledger.besu.ethereum.api.jsonrpc.forest.*") + } } From e0ad857716cce93f3ce5d2f8770aecc72e0f5481 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 09:09:38 -0500 Subject: [PATCH 034/155] push it Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 95042710080..7a870c5c900 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -2,11 +2,13 @@ name: acceptance-tests on: pull_request_review: types: [ submitted ] + workflow_dispatch: + env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 10 + total-runners: 32 jobs: runner-indexes: From 69e7bab1825db3b2a85f09c31dd3a098ad73214f Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 10:03:32 -0500 Subject: [PATCH 035/155] reftests too Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 -- .github/workflows/reference-tests.yml | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 7a870c5c900..de0dc3f2e80 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,7 +1,5 @@ name: acceptance-tests on: - pull_request_review: - types: [ submitted ] workflow_dispatch: diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 3d5149c0bbc..ff122722942 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -1,6 +1,6 @@ name: reference-tests on: - workflow_call: + workflow_dispatch: env: @@ -9,7 +9,7 @@ env: jobs: runner-indexes: - runs-on: besu-research-ubuntu-8 + runs-on: ubuntu-22.04 name: Generate runner indexes outputs: json: ${{ steps.generate-index-list.outputs.json }} @@ -21,7 +21,7 @@ jobs: INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) echo "::set-output name=json::${INDEX_JSON}" prepareReferenceTestEthereum: - runs-on: besu-research-ubuntu-8 + runs-on: ubuntu-22.04 timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} steps: @@ -57,7 +57,7 @@ jobs: path: "ethereum/referencetests/build/generated/sources/reference-test/**" key: ${{ runner.os }}-reftests-${{ github.sha }} referenceTestEthereum: - runs-on: besu-research-ubuntu-8 + runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum - runner-indexes @@ -110,7 +110,7 @@ jobs: path: '**/build/test-results/referenceTests/TEST-*.xml' retention-days: 1 referenceTestQbft: - runs-on: besu-research-ubuntu-8 + runs-on: ubuntu-22.04 timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} steps: From 35df63a48cf283a0d60963ddea9722b9ecac6012 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 10:35:10 -0500 Subject: [PATCH 036/155] yolo Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index de0dc3f2e80..b33bf6da6f7 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,7 +26,8 @@ jobs: name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" needs: - runner-indexes - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved'}} + # TODO: return check for PR approval here! + if: ${{ github.actor != 'dependabot[bot]'}} strategy: fail-fast: false matrix: From 081486e1f61ce147bd3ba60f68504100c26a93a8 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 13:09:01 -0500 Subject: [PATCH 037/155] store acceptance test results Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index b33bf6da6f7..6919a543bcc 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -60,6 +60,11 @@ jobs: run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt + - name: Upload Acceptance Test Results + uses: actions/upload-artifact@v3.1.0 + with: + name: acceptance-test-results + path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails From dd0b46b0974e8b15fb981781124f742c6496a39a Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 14:16:40 -0500 Subject: [PATCH 038/155] try timing based splitting Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 35 +++++++++++--------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 6919a543bcc..fa7bbc2fb47 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -9,29 +9,15 @@ env: total-runners: 32 jobs: - runner-indexes: - runs-on: ubuntu-22.04 - name: Generate runner indexes - outputs: - json: ${{ steps.generate-index-list.outputs.json }} - steps: - - id: generate-index-list - run: | - MAX_INDEX=$((${{ env.total-runners }}-1)) - INDEX_LIST=$(seq 0 ${MAX_INDEX}) - INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) - echo "::set-output name=json::${INDEX_JSON}" acceptanceTestEthereum: runs-on: ubuntu-22.04 name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" - needs: - - runner-indexes # TODO: return check for PR approval here! if: ${{ github.actor != 'dependabot[bot]'}} strategy: fail-fast: false matrix: - runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -40,18 +26,25 @@ jobs: with: distribution: adopt java-version: 17 - - name: Compile + - name: get acceptance test report + uses: dawidd6/action-download-artifact@v2 + with: + branch: main + name: acceptance-test-results + path: tmp/junit-xml-reports-downloaded + continue-on-error: true + - name: setup gradle uses: gradle/gradle-build-action@v2 - name: Split tests id: split-tests - uses: chaosaffe/split-tests@v1-alpha.1 + uses: r7kamura/split-tests-by-timings@v0 with: + reports: tmp/junit-xml-reports-downloaded glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' - split-total: ${{ env.total-runners }} - split-index: ${{ matrix.runner-index }} - + total: 16 + index: ${{ matrix.runner-index }} - name: write out test list - run: echo "${{ steps.split-tests.outputs.test-suite }}" >> testList.txt + run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt - name: format gradle args #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, #then drop file extension, then insert --tests option between each. From 19aac9a069946e7725f7001e73b7eafe6419b092 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 14:19:50 -0500 Subject: [PATCH 039/155] typo in yaml Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index fa7bbc2fb47..ed0a4accacf 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -11,7 +11,7 @@ env: jobs: acceptanceTestEthereum: runs-on: ubuntu-22.04 - name: "Acceptance Runner #${{ matrix.runner-index }}: Run acceptance tests in parallel" + name: "Acceptance Runner #${{ matrix.runner_index }}: Run acceptance tests in parallel" # TODO: return check for PR approval here! if: ${{ github.actor != 'dependabot[bot]'}} strategy: @@ -42,7 +42,7 @@ jobs: reports: tmp/junit-xml-reports-downloaded glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' total: 16 - index: ${{ matrix.runner-index }} + index: ${{ matrix.runner_index }} - name: write out test list run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt - name: format gradle args From 3004ce437c47a2bafefa910051831c2abccd2781 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 14:28:32 -0500 Subject: [PATCH 040/155] bug in regex Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index ed0a4accacf..4ac1a1a63a9 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -48,7 +48,7 @@ jobs: - name: format gradle args #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, #then drop file extension, then insert --tests option between each. - run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@@g' -e 's@/@.@g' -e 's/\.java//g' -e 's/\ /\ --tests\ /g' >> gradleArgs.txt + run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: cleanup tempfiles From 19b583baf31215014800de6ddccae7aa575aeb04 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 14:30:39 -0500 Subject: [PATCH 041/155] bug in regex Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 4ac1a1a63a9..f34b9042803 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -50,7 +50,7 @@ jobs: #then drop file extension, then insert --tests option between each. run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests - run: ./gradlew acceptanceTest --tests `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results From c6284db59c7c0e184c65a63f8a1e72d0f817b524 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 15:37:47 -0500 Subject: [PATCH 042/155] lets see what you got gradle caching Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 8 +- .github/workflows/reference-tests.yml | 116 +++++-------------------- 2 files changed, 26 insertions(+), 98 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index f34b9042803..d3d58f40414 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -2,11 +2,9 @@ name: acceptance-tests on: workflow_dispatch: - - env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 32 + total-runners: 16 jobs: acceptanceTestEthereum: @@ -32,7 +30,7 @@ jobs: branch: main name: acceptance-test-results path: tmp/junit-xml-reports-downloaded - continue-on-error: true + if_no_artifact_found: true - name: setup gradle uses: gradle/gradle-build-action@v2 - name: Split tests @@ -41,7 +39,7 @@ jobs: with: reports: tmp/junit-xml-reports-downloaded glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java' - total: 16 + total: ${{env.total-runners}} index: ${{ matrix.runner_index }} - name: write out test list run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index ff122722942..21bad3b98ec 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -2,31 +2,15 @@ name: reference-tests on: workflow_dispatch: - env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" total-runners: 6 jobs: - runner-indexes: - runs-on: ubuntu-22.04 - name: Generate runner indexes - outputs: - json: ${{ steps.generate-index-list.outputs.json }} - steps: - - id: generate-index-list - run: | - MAX_INDEX=$((${{ env.total-runners }}-1)) - INDEX_LIST=$(seq 0 ${MAX_INDEX}) - INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) - echo "::set-output name=json::${INDEX_JSON}" prepareReferenceTestEthereum: runs-on: ubuntu-22.04 - timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - name: Checkout Repo uses: actions/checkout@v3 with: @@ -37,38 +21,20 @@ jobs: with: distribution: adopt java-version: 17 - cache: gradle - - name: Restore classes - uses: actions/cache@v3 - with: - path: "**/*.class" - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} - - name: generate ethereum reference tests + - name: setup gradle uses: gradle/gradle-build-action@v2 - with: - arguments: ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} - - name: Cache reference tests - uses: actions/cache@v3 - with: - path: "ethereum/referencetests/build/generated/sources/reference-test/**" - key: ${{ runner.os }}-reftests-${{ github.sha }} + - name: execute generate reference tests + run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true referenceTestEthereum: runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum - - runner-indexes - timeout-minutes: 30 if: ${{ github.actor != 'dependabot[bot]' }} strategy: + fail-fast: false matrix: - runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} + runner-index: [0,1,2,3,4,5] steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - name: Checkout Repo uses: actions/checkout@v3 with: @@ -78,63 +44,27 @@ jobs: with: distribution: adopt java-version: 17 - cache: gradle - - name: Restore classes - uses: actions/cache@v3 - with: - path: "**/*.class" - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} - - name: Restore reference tests - uses: actions/cache@v3 - with: - path: "ethereum/referencetests/build/generated/sources/reference-test/**" - key: ${{ runner.os }}-reftests-${{ github.sha }} + - name: get acceptance test report + uses: dawidd6/action-download-artifact@v2 + with: + branch: main + name: acceptance-test-results + path: tmp/ref-xml-reports-downloaded + if_no_artifact_found: true + - name: setup gradle + uses: gradle/gradle-build-action@v2 - name: Split tests id: split-tests - uses: chaosaffe/split-tests@v1-alpha.1 + uses: r7kamura/split-tests-by-timings@v0 with: + reports: tmp/ref-xml-reports-downloaded glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' - split-total: ${{ env.total-runners }} - split-index: ${{ matrix.runner-index }} + total: ${{env.total-runners}} + index: ${{ matrix.runner-index }} + - name: compose gradle args + run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt - name: run reference tests - run: ./gradlew ethereum:referenceTests:referenceTests $(echo ${{ steps.split-tests.outputs.test-suite }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /') - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} - - name: Upload Test Report - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails - with: - name: junit-reference-${{ matrix.runner-index }}-test-results - path: '**/build/test-results/referenceTests/TEST-*.xml' - retention-days: 1 - referenceTestQbft: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - - name: Checkout Repo - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: Restore classes - uses: actions/cache@v3 - with: - path: "**/*.class" - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.sha }} - - name: qbft reference tests - uses: gradle/gradle-build-action@v2 - with: - arguments: consensus:qbft:referencetests + run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Correct Ownership in GITHUB_WORKSPACE directory uses: peter-murray/reset-workspace-ownership-action@v1 with: @@ -143,6 +73,6 @@ jobs: uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails with: - name: junit-qbft-reference-test-results + name: acceptance-test-results path: '**/build/test-results/referenceTests/TEST-*.xml' - retention-days: 1 + From 216907e2b8317d55592c02adc92fe25a57b199bb Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 16:03:59 -0500 Subject: [PATCH 043/155] stash generated tests Signed-off-by: Justin Florentine --- .github/workflows/reference-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 21bad3b98ec..5e9005c49df 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -25,6 +25,12 @@ jobs: uses: gradle/gradle-build-action@v2 - name: execute generate reference tests run: ./gradlew ethereum:referencetests:blockchainReferenceTests ethereum:referencetests:generalstateReferenceTests ethereum:referencetests:generalstateRegressionReferenceTests -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: store generated tests + uses: actions/upload-artifact@v3 + with: + name: reference-tests + path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' + referenceTestEthereum: runs-on: ubuntu-22.04 needs: @@ -53,6 +59,10 @@ jobs: if_no_artifact_found: true - name: setup gradle uses: gradle/gradle-build-action@v2 + - name: retrieve generated tests + uses: dawidd6/action-download-artifact@v2 + with: + name: reference-tests - name: Split tests id: split-tests uses: r7kamura/split-tests-by-timings@v0 From 662ce1c732fd3b1a5575b2ada366e4cfaf8ac04f Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 17:24:06 -0500 Subject: [PATCH 044/155] cleanup copypasta and rearrange Signed-off-by: Justin Florentine --- .github/workflows/reference-tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 5e9005c49df..319cdc685d9 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -50,19 +50,19 @@ jobs: with: distribution: adopt java-version: 17 - - name: get acceptance test report + - name: retrieve generated tests + uses: dawidd6/action-download-artifact@v2 + with: + name: reference-tests + - name: get reference test report uses: dawidd6/action-download-artifact@v2 with: branch: main - name: acceptance-test-results + name: reference-test-results path: tmp/ref-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle uses: gradle/gradle-build-action@v2 - - name: retrieve generated tests - uses: dawidd6/action-download-artifact@v2 - with: - name: reference-tests - name: Split tests id: split-tests uses: r7kamura/split-tests-by-timings@v0 @@ -83,6 +83,6 @@ jobs: uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails with: - name: acceptance-test-results + name: reference-test-results path: '**/build/test-results/referenceTests/TEST-*.xml' From 3eaf4bd22fea3a81e2398e768fa699d3ca62c1b0 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 11 Dec 2023 17:37:45 -0500 Subject: [PATCH 045/155] idk why not finding artifact Signed-off-by: Justin Florentine --- .github/workflows/reference-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 319cdc685d9..a4c1beca09e 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -28,13 +28,14 @@ jobs: - name: store generated tests uses: actions/upload-artifact@v3 with: - name: reference-tests + name: 'reference-tests' path: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' referenceTestEthereum: runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum + #TODO: add condition to check pr is approved if: ${{ github.actor != 'dependabot[bot]' }} strategy: fail-fast: false @@ -53,7 +54,8 @@ jobs: - name: retrieve generated tests uses: dawidd6/action-download-artifact@v2 with: - name: reference-tests + name: 'reference-tests' + path: 'ethereum/referencetests/build/generated/sources/reference-test/' - name: get reference test report uses: dawidd6/action-download-artifact@v2 with: From 46006fd3e53a88e961ec2d074cdeddfde8b215f4 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 12 Dec 2023 08:51:48 -0500 Subject: [PATCH 046/155] need to use the download action that defaults to current branch Signed-off-by: jflo --- .github/workflows/reference-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index a4c1beca09e..3689bb18448 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -52,7 +52,7 @@ jobs: distribution: adopt java-version: 17 - name: retrieve generated tests - uses: dawidd6/action-download-artifact@v2 + uses: actions/download-artifact@v3.0.2 with: name: 'reference-tests' path: 'ethereum/referencetests/build/generated/sources/reference-test/' From bd1854022e6a2ff0a4d0dd3e671f1e3be852ce77 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 12 Dec 2023 08:58:22 -0500 Subject: [PATCH 047/155] no need to correct perms anymore Signed-off-by: jflo --- .github/workflows/reference-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 3689bb18448..38cd39cb657 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -77,10 +77,6 @@ jobs: run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt - name: run reference tests run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails From b10e35e495945ffb0ff961f59a66f17d5b616150 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 12 Dec 2023 16:57:32 -0500 Subject: [PATCH 048/155] needed to store all test results in different artifacts, then recombine them Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 5 +++-- .github/workflows/pre-review.yml | 7 ------- .github/workflows/reference-tests.yml | 5 +++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d3d58f40414..21c2ec2a971 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -28,7 +28,8 @@ jobs: uses: dawidd6/action-download-artifact@v2 with: branch: main - name: acceptance-test-results + name_is_regexp: true + name: 'acceptance-node-\d*\d-test-results' path: tmp/junit-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle @@ -54,7 +55,7 @@ jobs: - name: Upload Acceptance Test Results uses: actions/upload-artifact@v3.1.0 with: - name: acceptance-test-results + name: acceptance-node-${{matrix.runner_index}}-test-results path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index a7100db9a12..b581bfd0bd0 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -83,13 +83,6 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: Upload Test Report - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails - with: - name: junit-xml-reports - path: '**/build/test-results/test/TEST-*.xml' - retention-days: 1 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 38cd39cb657..5ad83b4601a 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -60,7 +60,8 @@ jobs: uses: dawidd6/action-download-artifact@v2 with: branch: main - name: reference-test-results + name_is_regexp: true + name: 'reference-test-node-\d*\d-results' path: tmp/ref-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle @@ -81,6 +82,6 @@ jobs: uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails with: - name: reference-test-results + name: reference-test-node-${{matrix.runner-index}}-results path: '**/build/test-results/referenceTests/TEST-*.xml' From 8f5713eccffbe4d9794e3c534434ab7f108b965f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 13 Dec 2023 11:55:31 -0500 Subject: [PATCH 049/155] adds integrationtest and publishes reftest results to pr Signed-off-by: jflo --- .github/workflows/integration-tests.yml | 31 +++++++++++++++++++++++++ .github/workflows/reference-tests.yml | 6 +++++ 2 files changed, 37 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000000..eb374badf9f --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,31 @@ +name: integration-tests +on: + workflow_dispatch: + +env: + GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" + +jobs: + referenceTestEthereum: + runs-on: ubuntu-22.04 + #TODO: add condition to check pr is approved + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: setup gradle + uses: gradle/gradle-build-action@v2 + - name: run integration tests + run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/build/test-results/integrationTest/TEST-*.xml' + + diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 5ad83b4601a..0cf97c7ed69 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -84,4 +84,10 @@ jobs: with: name: reference-test-node-${{matrix.runner-index}}-results path: '**/build/test-results/referenceTests/TEST-*.xml' + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() # always run even if the build step fails + with: + report_paths: '**/build/test-results/referenceTest/TEST-*.xml' + From 6fd4d2b5f185f56bdff04ce95cc931743db29ee2 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 13 Dec 2023 13:40:41 -0500 Subject: [PATCH 050/155] can we override sonar props from env? Signed-off-by: jflo --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/sonarcloud.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index eb374badf9f..9858dc76f6d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -6,7 +6,7 @@ env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" jobs: - referenceTestEthereum: + integrationTetst: runs-on: ubuntu-22.04 #TODO: add condition to check pr is approved if: ${{ github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 92d9a7713e7..437a24360be 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -34,4 +34,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }} + SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }} run: ./gradlew build sonarqube --continue --info From 3c099f03c6d29dc9b1589780cbdd9fa256442a31 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 13 Dec 2023 16:36:37 -0500 Subject: [PATCH 051/155] whoops wrong repo filter Signed-off-by: jflo --- .github/workflows/sonarcloud.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 437a24360be..293d95156e0 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -14,7 +14,6 @@ permissions: jobs: Analysis: runs-on: ubuntu-latest - if: github.repository == 'hyperledger/besu' steps: - name: checkout uses: actions/checkout@v4 From 77be8e33087f7c22dd587bbd48a8830ca66691f5 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 13 Dec 2023 17:27:52 -0500 Subject: [PATCH 052/155] gradle caching Signed-off-by: jflo --- .github/workflows/sonarcloud.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 293d95156e0..2873945499d 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -22,13 +22,14 @@ jobs: with: distribution: 'temurin' java-version: '17' - cache: gradle - name: Cache SonarCloud packages uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar + - name: setup gradle + uses: gradle/gradle-build-action@v2 - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any From e183092bb543b8fe5a3abfebda16724d6f4b86a3 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 14 Dec 2023 16:16:18 -0500 Subject: [PATCH 053/155] sonar tweaks Signed-off-by: jflo --- .github/workflows/artifacts.yml | 33 ++++++++++++++++++++++++++++++++ .github/workflows/docker.yml | 4 ++-- .github/workflows/sonarcloud.yml | 2 +- build.gradle | 11 +++++++++-- 4 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/artifacts.yml diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml new file mode 100644 index 00000000000..d1df7c6d033 --- /dev/null +++ b/.github/workflows/artifacts.yml @@ -0,0 +1,33 @@ + +name: artifacts + +on: + workflow_dispatch: + +jobs: + artifacts: + runs-on: ubuntu-22.04 + steps: + # TODO: ensure all checks have passed already + - name: checkout + uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: '17' + - name: setup gradle + uses: gradle/gradle-build-action@v2 + - name: assemble distributions + run: ./gradlew assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: upload tarball + uses: actions/upload-artifact@v3 + with: + name: 'tarball' + path: 'build/distributions/besu*.tar.gz' + - name: upload zipfile + uses: actions/upload-artifact@v3 + with: + name: 'zipfile' + path: 'build/distributions/besu*.zip' + diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2f4747f90b8..3fa216c8502 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,11 +1,11 @@ name: "Container Images" on: - workflow_call: + workflow_dispatch: jobs: buildX64Docker: - runs-on: [self-hosted, X64, Linux] + runs-on: [X64, Linux] if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: export runner UID diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 2873945499d..556ef8c8759 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -36,4 +36,4 @@ jobs: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} SONAR_ORGANIZATION: ${{ env.SONAR_ORGANIZATION }} SONAR_PROJECT_KEY: $ {{ env.SONAR_PROJECT_KEY }} - run: ./gradlew build sonarqube --continue --info + run: ./gradlew build sonarqube --continue --info -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/build.gradle b/build.gradle index 255a878d975..2d2e610d58d 100644 --- a/build.gradle +++ b/build.gradle @@ -36,8 +36,9 @@ plugins { sonarqube { properties { - property "sonar.projectKey", "hyperledger_besu" - property "sonar.organization", "hyperledger" + property "sonar.projectKey", "$System.env.SONAR_PROJECT_KEY" + property "sonar.organization", "$System.env.SONAR_ORGANIZATION" + property "sonar.gradle.skipCompile", "true" property "sonar.host.url", "https://sonarcloud.io" property "sonar.coverage.jacoco.xmlReportPaths", "${buildDir}/reports/jacoco/jacocoRootReport/jacocoRootReport.xml" property "sonar.coverage.exclusions", "acceptance-tests/**/*" @@ -983,6 +984,12 @@ task checkSpdxHeader(type: CheckSpdxHeader) { ].join("|") } +jacocoTestReport { + reports { + xml.enabled true + } +} + task jacocoRootReport(type: org.gradle.testing.jacoco.tasks.JacocoReport) { additionalSourceDirs.from files(subprojects.sourceSets.main.allSource.srcDirs) sourceDirectories.from files(subprojects.sourceSets.main.allSource.srcDirs) From 68bdb4bc963321a7740b716bb7832d4e0e5ac31f Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 15 Dec 2023 18:28:16 -0500 Subject: [PATCH 054/155] docker task cleanup Signed-off-by: jflo --- .github/workflows/docker.yml | 39 +++++++++--------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3fa216c8502..05775aee1f4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,11 +5,9 @@ on: jobs: buildX64Docker: - runs-on: [X64, Linux] + runs-on: ubuntu-22.04 if: ${{ github.actor != 'dependabot[bot]' }} steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -17,15 +15,10 @@ jobs: with: distribution: adopt java-version: 17 - cache: gradle - - name: hadoLint_openjdk_11 - run: docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile - - name: hadoLint_openjdk_11_debug - run: docker run --rm -i hadolint/hadolint < docker/openjdk-11-debug/Dockerfile - - name: hadoLint_openjdk_latest - run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - - name: hadoLint_graalvm - run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + - name: setup gradle + uses: gradle/gradle-build-action@v2 + - name: hadoLint all Dockerfiles + run: find docker -name Dockerfile -exec docker run --rm -i hadolint/hadolint < {} \; - name: distDocker uses: gradle/gradle-build-action@v2 with: @@ -34,31 +27,24 @@ jobs: run: | mkdir -p docker/reports curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-amd64 -o ./docker/tests/goss-linux-amd64 - - name: distDocker + - name: test docker uses: gradle/gradle-build-action@v2 env: architecture: amd64 with: arguments: testDocker - name: Login to DockerHub - if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USER_RW }} + username: robocopsgonemad password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} run: ./gradlew --no-daemon "-Pbranch=${{ github.head_ref || github.ref_name }}" dockerUpload - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} buildArm64Docker: - runs-on: [self-hosted, ARM64, Linux] + runs-on: ubuntu-22.04 if: ${{ github.actor != 'dependabot[bot]' }} steps: - - name: export runner UID - run: echo "runner_uid=$UID" >> $GITHUB_ENV - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -67,7 +53,6 @@ jobs: distribution: zulu architecture: arm64 java-version: 17 - cache: gradle - name: Docker login run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin @@ -89,18 +74,12 @@ jobs: arguments: testDocker - name: Login to DockerHub uses: docker/login-action@v2 - if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} with: - username: ${{ secrets.DOCKER_USER_RW }} + username: robocopsgonemad password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish - if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} uses: gradle/gradle-build-action@v2 env: architecture: arm64 with: arguments: "-Pbranch=${{ github.head_ref || github.ref_name }} dockerUpload" - - name: Correct Ownership in GITHUB_WORKSPACE directory - uses: peter-murray/reset-workspace-ownership-action@v1 - with: - user_id: ${{ env.runner_uid }} From 928a1eb78be3c30f9090fc0fdd177946514f9c1a Mon Sep 17 00:00:00 2001 From: jflo Date: Sat, 16 Dec 2023 08:36:09 -0500 Subject: [PATCH 055/155] update dockerfiles manually Signed-off-by: jflo --- .github/workflows/docker.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 05775aee1f4..66d802b39b0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,8 +17,17 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 - - name: hadoLint all Dockerfiles - run: find docker -name Dockerfile -exec docker run --rm -i hadolint/hadolint < {} \; + - name: hadoLint_openj9-jdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openj9-jdk-17/Dockerfile + - name: hadoLint_openjdk_17 + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile + - name: hadoLint_openjdk_17_debug + run: docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile + - name: hadoLint_openjdk_latest + run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile + - name: hadoLint_graalvm + run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile + - name: distDocker uses: gradle/gradle-build-action@v2 with: From 3dcaa601e5aed9ad0b0ec4f3eb317445c8d4c279 Mon Sep 17 00:00:00 2001 From: jflo Date: Sat, 16 Dec 2023 09:16:08 -0500 Subject: [PATCH 056/155] use self hosted arm for now Signed-off-by: jflo --- .github/workflows/docker.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 66d802b39b0..9056e6f9650 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,7 +6,6 @@ on: jobs: buildX64Docker: runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -27,7 +26,6 @@ jobs: run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - name: hadoLint_graalvm run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - name: distDocker uses: gradle/gradle-build-action@v2 with: @@ -48,11 +46,9 @@ jobs: username: robocopsgonemad password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish - if: ${{ github.actor != 'dependabot[bot]' && (contains('refs/heads/release-', github.ref) || github.ref == 'refs/heads/main') }} run: ./gradlew --no-daemon "-Pbranch=${{ github.head_ref || github.ref_name }}" dockerUpload buildArm64Docker: - runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} + runs-on: [self-hosted, ARM64] steps: - name: Checkout Repo uses: actions/checkout@v3 From 7739b7067473c2c1470be02eb0459819b565f1e1 Mon Sep 17 00:00:00 2001 From: jflo Date: Sat, 16 Dec 2023 09:50:59 -0500 Subject: [PATCH 057/155] orgs from env Signed-off-by: jflo --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9056e6f9650..fe9293bb4de 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -46,7 +46,7 @@ jobs: username: robocopsgonemad password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish - run: ./gradlew --no-daemon "-Pbranch=${{ github.head_ref || github.ref_name }}" dockerUpload + run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} buildArm64Docker: runs-on: [self-hosted, ARM64] steps: @@ -66,7 +66,7 @@ jobs: env: architecture: arm64 with: - arguments: distDocker + arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: download image test binary run: | mkdir -p docker/reports @@ -76,7 +76,7 @@ jobs: env: architecture: arm64 with: - arguments: testDocker + arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -87,4 +87,4 @@ jobs: env: architecture: arm64 with: - arguments: "-Pbranch=${{ github.head_ref || github.ref_name }} dockerUpload" + arguments: -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload From 9fa0bef8c3f169c4140fbd628ba3bac535c7734d Mon Sep 17 00:00:00 2001 From: jflo Date: Sat, 16 Dec 2023 11:13:57 -0500 Subject: [PATCH 058/155] don't run the whole shebang, i can't believe that actually worked on a free runner Signed-off-by: jflo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fe9293bb4de..3e1499d06cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -46,7 +46,7 @@ jobs: username: robocopsgonemad password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish - run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} + run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload buildArm64Docker: runs-on: [self-hosted, ARM64] steps: From b7a4f50656272cae0fc3996f7b8a311cef2af844 Mon Sep 17 00:00:00 2001 From: jflo Date: Mon, 18 Dec 2023 12:57:41 -0500 Subject: [PATCH 059/155] return to process Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 6 ++++-- .github/workflows/docker.yml | 1 - .github/workflows/integration-tests.yml | 7 +++++-- .github/workflows/reference-tests.yml | 10 +++++++--- .github/workflows/release.yml | 5 +++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 21c2ec2a971..7d274aed68a 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,6 +1,8 @@ name: acceptance-tests on: workflow_dispatch: + pull_request_review: + types: [submitted] env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" @@ -10,8 +12,8 @@ jobs: acceptanceTestEthereum: runs-on: ubuntu-22.04 name: "Acceptance Runner #${{ matrix.runner_index }}: Run acceptance tests in parallel" - # TODO: return check for PR approval here! - if: ${{ github.actor != 'dependabot[bot]'}} + # TODO: don't re-run on subsequent comments on an approved workflow + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3e1499d06cc..bbf3768b59f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,6 @@ name: "Container Images" on: workflow_dispatch: - jobs: buildX64Docker: runs-on: ubuntu-22.04 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9858dc76f6d..f9d7d2b1a02 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,6 +1,9 @@ name: integration-tests on: workflow_dispatch: + pull_request_review: + types: + - submitted env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" @@ -8,8 +11,8 @@ env: jobs: integrationTetst: runs-on: ubuntu-22.04 - #TODO: add condition to check pr is approved - if: ${{ github.actor != 'dependabot[bot]' }} + # TODO: don't re-run on subsequent comments on an approved workflow + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} steps: - name: Checkout Repo uses: actions/checkout@v3 diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 0cf97c7ed69..74ef586e497 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -1,6 +1,9 @@ name: reference-tests on: workflow_dispatch: + pull_request_review: + types: + - submitted env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" @@ -9,7 +12,8 @@ env: jobs: prepareReferenceTestEthereum: runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} + # TODO: don't re-run on subsequent comments on an approved workflow + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -35,8 +39,8 @@ jobs: runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum - #TODO: add condition to check pr is approved - if: ${{ github.actor != 'dependabot[bot]' }} + # TODO: don't re-run on subsequent comments on an approved workflow + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21a01ba4a9f..17442e8333e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,16 @@ name: release besu on: + workflow_dispatch: release: types: released jobs: dockerPromoteX64: - runs-on: [besu-research-ubuntu-8] + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - distribution: 'temurin' # See 'Supported distributions' for available options + distribution: 'adopt' # See 'Supported distributions' for available options java-version: '17' cache: gradle - name: Login to DockerHub From c355249239214040dd183c9a9449c5627df5bb37 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 10:08:09 -0500 Subject: [PATCH 060/155] touch Signed-off-by: jflo --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e09db4de540..3fa36ae6e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ ## 24.1.0 +- Github Actions based build. + ### Breaking Changes ### Deprecations From fc6926e2edc6a899f905fba8f8e783a61d741efd Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 12:45:46 -0500 Subject: [PATCH 061/155] can we make sure expected checks have passed? Signed-off-by: jflo --- .github/workflows/artifacts.yml | 19 +++++++++++++++++++ .github/workflows/docker.yml | 3 +++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index d1df7c6d033..41a36956e45 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -3,12 +3,31 @@ name: artifacts on: workflow_dispatch: + release: + types: + - created jobs: artifacts: runs-on: ubuntu-22.04 steps: # TODO: ensure all checks have passed already + - name: Check if other workflow has run + id: 'check pre-review has run' + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const { data: { statuses } } = await github.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + }); + const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); + return otherWorkflow && otherWorkflow.state === 'success'; + - name: Run if other workflow was successful + if: steps.check.outputs.result == 'true' + run: echo "Other workflow was successful, running this job" - name: checkout uses: actions/checkout@v4 - name: Set up JDK 17 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index bbf3768b59f..6387a8de873 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,9 @@ name: "Container Images" on: workflow_dispatch: + release: + types: + - created jobs: buildX64Docker: From 654f8711b5583a9d16b1328fb7af47e1a1b6775d Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 13:04:02 -0500 Subject: [PATCH 062/155] rename Signed-off-by: jflo --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 41a36956e45..77b96313e7b 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -13,7 +13,7 @@ jobs: steps: # TODO: ensure all checks have passed already - name: Check if other workflow has run - id: 'check pre-review has run' + id: 'confirmPreReview' uses: actions/github-script@v3 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -27,7 +27,7 @@ jobs: return otherWorkflow && otherWorkflow.state === 'success'; - name: Run if other workflow was successful if: steps.check.outputs.result == 'true' - run: echo "Other workflow was successful, running this job" + run: echo "pre-review was successful, continuing" - name: checkout uses: actions/checkout@v4 - name: Set up JDK 17 From 21ddd556927953709bf02fec85fdc0b4a8a43a85 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 14:12:15 -0500 Subject: [PATCH 063/155] caveman debugging Signed-off-by: jflo --- .github/workflows/artifacts.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 77b96313e7b..d92b3463b80 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -14,7 +14,7 @@ jobs: # TODO: ensure all checks have passed already - name: Check if other workflow has run id: 'confirmPreReview' - uses: actions/github-script@v3 + uses: actions/github-script@v7.0.1 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | @@ -23,7 +23,11 @@ jobs: repo: context.repo.repo, ref: context.sha, }); + console.log(data); + console.log(statuses); + console.log(context); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); + console.log(otherWorkflow); return otherWorkflow && otherWorkflow.state === 'success'; - name: Run if other workflow was successful if: steps.check.outputs.result == 'true' From 023206407fedef7626d150472d20b6bc1108ebfc Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 14:17:17 -0500 Subject: [PATCH 064/155] yay javascript Signed-off-by: jflo --- .github/workflows/artifacts.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index d92b3463b80..0f9c2ee9dcb 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -23,7 +23,6 @@ jobs: repo: context.repo.repo, ref: context.sha, }); - console.log(data); console.log(statuses); console.log(context); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); From 39b10ac7eb28cedd8cdaa1e7998a10cf17482d86 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 14:20:44 -0500 Subject: [PATCH 065/155] new version, new location Signed-off-by: jflo --- .github/workflows/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 0f9c2ee9dcb..251f2aa20e3 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -18,7 +18,7 @@ jobs: with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const { data: { statuses } } = await github.repos.getCombinedStatusForRef({ + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: context.sha, From adc3e3f16fd01f6c3ea0ef380a87de49514bda8c Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 14:26:01 -0500 Subject: [PATCH 066/155] most of those fields were wrong Signed-off-by: jflo --- .github/workflows/artifacts.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 251f2aa20e3..2168ad672e8 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -18,13 +18,13 @@ jobs: with: github-token: ${{secrets.GITHUB_TOKEN}} script: | + console.log(context); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, + owner: context.repository.owner, + repo: context.repository, ref: context.sha, }); console.log(statuses); - console.log(context); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(otherWorkflow); return otherWorkflow && otherWorkflow.state === 'success'; From 472774c17c70f884acfbe9cec711d60a6f77effe Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 15:13:19 -0500 Subject: [PATCH 067/155] use the class Signed-off-by: jflo --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 2168ad672e8..90a041c159b 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -20,8 +20,8 @@ jobs: script: | console.log(context); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repository.owner, - repo: context.repository, + owner: context.repo.owner, + repo: context.repo.repo, ref: context.sha, }); console.log(statuses); From 9e60f18c8cfe47efd894079d006b39d9af1bf2ef Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 15:26:08 -0500 Subject: [PATCH 068/155] still not finding checks Signed-off-by: jflo --- .github/workflows/artifacts.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 90a041c159b..00af1f66269 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -19,11 +19,15 @@ jobs: github-token: ${{secrets.GITHUB_TOKEN}} script: | console.log(context); + console.log(context.repo.owner); + console.log(context.repo.repo); + console.log(context.sha); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: context.sha, }); + console.log(data); console.log(statuses); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(otherWorkflow); From 61535cdc55eb12a1fda84c89fd873d0f0dec53f7 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 15:35:01 -0500 Subject: [PATCH 069/155] still not finding checks Signed-off-by: jflo --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 00af1f66269..918995d2413 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -22,12 +22,12 @@ jobs: console.log(context.repo.owner); console.log(context.repo.repo); console.log(context.sha); - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + const refState = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: context.sha, }); - console.log(data); + console.log(refState); console.log(statuses); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(otherWorkflow); From d23fb3cf45af6b3a9909b73d56eec6c4e8b3e279 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:02:40 -0500 Subject: [PATCH 070/155] turns out commit statuses are not free Signed-off-by: jflo --- .github/workflows/artifacts.yml | 17 +++++++++++------ .github/workflows/pre-review.yml | 11 +++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 918995d2413..1671851909d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -11,9 +11,8 @@ jobs: artifacts: runs-on: ubuntu-22.04 steps: - # TODO: ensure all checks have passed already - - name: Check if other workflow has run - id: 'confirmPreReview' + - name: Check if pre-review workflow has run + id: confirmPreReview uses: actions/github-script@v7.0.1 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -33,7 +32,8 @@ jobs: console.log(otherWorkflow); return otherWorkflow && otherWorkflow.state === 'success'; - name: Run if other workflow was successful - if: steps.check.outputs.result == 'true' + # TODO: combine unit, acceptance, integration, reference tests before creating artifacts. + if: steps.confirmPreReview.outputs.result == 'true' run: echo "pre-review was successful, continuing" - name: checkout uses: actions/checkout@v4 @@ -46,14 +46,19 @@ jobs: uses: gradle/gradle-build-action@v2 - name: assemble distributions run: ./gradlew assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: new version number + id: verNum + run: | + VER_NUM = $(grep "version=" gradle.properties | sed 's/^version=//g') + echo "::set-output name=verNum::$VER_NUM" - name: upload tarball uses: actions/upload-artifact@v3 with: - name: 'tarball' + name: besu-${{ steps.verNum.outputs.verNum }}.tar.gz path: 'build/distributions/besu*.tar.gz' - name: upload zipfile uses: actions/upload-artifact@v3 with: - name: 'zipfile' + name: besu-${{ steps.verNum.outputs.verNum }}.zip path: 'build/distributions/besu*.zip' diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index b581bfd0bd0..196d1509bc4 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -8,6 +8,11 @@ env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true jobs: + pend: + - name: Set commit status as pending + uses: myrotvorets/set-commit-status-action@master + with: + status: pending repolint: name: "Repository Linting" runs-on: ubuntu-22.04 @@ -82,7 +87,13 @@ jobs: - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests + id: unitTest run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: Set commit status as pending + uses: myrotvorets/set-commit-status-action@master + if: always() + with: + status: ${{ job.status }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails From aa6852f78fcf7a9a4d91ea94df9a96f4d97775cf Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:03:32 -0500 Subject: [PATCH 071/155] lintaaaaaar Signed-off-by: jflo --- .github/workflows/pre-review.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 196d1509bc4..68cf088b4ca 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -9,10 +9,10 @@ env: jobs: pend: - - name: Set commit status as pending - uses: myrotvorets/set-commit-status-action@master - with: - status: pending + name: Set commit status as pending + uses: myrotvorets/set-commit-status-action@master + with: + status: pending repolint: name: "Repository Linting" runs-on: ubuntu-22.04 From cfadbd7b1494331cc871114b54c9e537396e2455 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:05:40 -0500 Subject: [PATCH 072/155] wrong version Signed-off-by: jflo --- .github/workflows/pre-review.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 68cf088b4ca..cd946985153 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -10,7 +10,7 @@ env: jobs: pend: name: Set commit status as pending - uses: myrotvorets/set-commit-status-action@master + uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending repolint: @@ -90,7 +90,7 @@ jobs: id: unitTest run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Set commit status as pending - uses: myrotvorets/set-commit-status-action@master + uses: myrotvorets/set-commit-status-action@v2.0.0 if: always() with: status: ${{ job.status }} From 499c40ac4d450940fa212de5af45b48eab9a2693 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:07:34 -0500 Subject: [PATCH 073/155] it's a step not a jorb Signed-off-by: jflo --- .github/workflows/pre-review.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index cd946985153..572de56c433 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -10,9 +10,12 @@ env: jobs: pend: name: Set commit status as pending - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: pending + runs-on: ubuntu-22.04 + steps: + - name: pending + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + status: pending repolint: name: "Repository Linting" runs-on: ubuntu-22.04 From 36dd329c6e4d8708f8869223fb9931b622bd37ac Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:30:58 -0500 Subject: [PATCH 074/155] adds permissions and granular failure status Signed-off-by: jflo --- .github/workflows/pre-review.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 572de56c433..25964abcef8 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -4,12 +4,15 @@ on: pull_request: workflow_dispatch: +permissions: + statuses: write + env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true jobs: pend: - name: Set commit status as pending + name: unit testing pending runs-on: ubuntu-22.04 steps: - name: pending @@ -92,11 +95,19 @@ jobs: - name: run unit tests id: unitTest run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: Set commit status as pending + - name: flag failed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 - if: always() + if: failure() with: status: ${{ job.status }} + context: unit-tests-${{matrix.gradle_args}} + description: ${{matrix.gradle_args}} + - name: flag passed unit tests + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: success() + with: + status: ${{ job.status }} + - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails From 3e6765f69bb3c20f91789c899437ae1e701491b6 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:39:49 -0500 Subject: [PATCH 075/155] moar permissions Signed-off-by: jflo --- .github/workflows/pre-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 25964abcef8..7c140a5b386 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -72,6 +72,7 @@ jobs: needs: [ compile ] permissions: checks: write + statuses: write strategy: fail-fast: false matrix: From 14657106787b3de9431143bdd2d2f27d3df4ec3e Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 16:56:45 -0500 Subject: [PATCH 076/155] now we have statuses Signed-off-by: jflo --- .github/workflows/artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 1671851909d..72754860a23 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -27,7 +27,7 @@ jobs: ref: context.sha, }); console.log(refState); - console.log(statuses); + console.log(refState.statuses[0]); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(otherWorkflow); return otherWorkflow && otherWorkflow.state === 'success'; From 3a3e14470a3c581cf9ad56d7fda053fd4ef4395a Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 17:23:33 -0500 Subject: [PATCH 077/155] restore return value filter Signed-off-by: jflo --- .github/workflows/artifacts.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 72754860a23..9ccaaa234e7 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -16,23 +16,23 @@ jobs: uses: actions/github-script@v7.0.1 with: github-token: ${{secrets.GITHUB_TOKEN}} + # TODO: combine unit, acceptance, integration, reference tests before creating artifacts. also ensure absence of any failed statii script: | console.log(context); console.log(context.repo.owner); console.log(context.repo.repo); console.log(context.sha); - const refState = await github.rest.repos.getCombinedStatusForRef({ + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: context.sha, }); - console.log(refState); - console.log(refState.statuses[0]); + console.log(statuses[0]); const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(otherWorkflow); return otherWorkflow && otherWorkflow.state === 'success'; - name: Run if other workflow was successful - # TODO: combine unit, acceptance, integration, reference tests before creating artifacts. + if: steps.confirmPreReview.outputs.result == 'true' run: echo "pre-review was successful, continuing" - name: checkout From ad346fb743c45b2abd5fbe73df9b54a5721f2b70 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 17:42:43 -0500 Subject: [PATCH 078/155] check for failures too Signed-off-by: jflo --- .github/workflows/artifacts.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 9ccaaa234e7..1d5ab6fcd23 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -18,19 +18,17 @@ jobs: github-token: ${{secrets.GITHUB_TOKEN}} # TODO: combine unit, acceptance, integration, reference tests before creating artifacts. also ensure absence of any failed statii script: | - console.log(context); - console.log(context.repo.owner); - console.log(context.repo.repo); - console.log(context.sha); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: context.sha, }); - console.log(statuses[0]); - const otherWorkflow = statuses.find(({ context }) => context === 'pre-review'); - console.log(otherWorkflow); - return otherWorkflow && otherWorkflow.state === 'success'; + console.log(statuses); + const anyFailure = statuses.find(( { state }) => state === 'failed'); + console.log(anyFailure); + const prePRWorkflow = statuses.find(({ context }) => context === 'pre-review'); + console.log(prePRWorkflow); + return prePRWorkflow && prePRWorkflow.state === 'success' && !anyFailure; - name: Run if other workflow was successful if: steps.confirmPreReview.outputs.result == 'true' @@ -49,7 +47,7 @@ jobs: - name: new version number id: verNum run: | - VER_NUM = $(grep "version=" gradle.properties | sed 's/^version=//g') + export VER_NUM = $(grep "version=" gradle.properties | sed 's/^version=//g') echo "::set-output name=verNum::$VER_NUM" - name: upload tarball uses: actions/upload-artifact@v3 From f341f928e43fa1d302f06529753d1cfe94701bc1 Mon Sep 17 00:00:00 2001 From: jflo Date: Tue, 19 Dec 2023 18:00:03 -0500 Subject: [PATCH 079/155] abort on any failed or missing expected success Signed-off-by: jflo --- .github/workflows/artifacts.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 1d5ab6fcd23..430f2afee71 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -24,15 +24,16 @@ jobs: ref: context.sha, }); console.log(statuses); - const anyFailure = statuses.find(( { state }) => state === 'failed'); + const anyFailure = statuses.find(( { state }) => state === 'failure'); console.log(anyFailure); const prePRWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(prePRWorkflow); return prePRWorkflow && prePRWorkflow.state === 'success' && !anyFailure; - - name: Run if other workflow was successful - - if: steps.confirmPreReview.outputs.result == 'true' - run: echo "pre-review was successful, continuing" + - name: Abort on missing status + if: steps.confirmPreReview.outputs.result == 'false' + run: | + echo "::error::required status missing" + exit 1 - name: checkout uses: actions/checkout@v4 - name: Set up JDK 17 From ddb510268f435cd5619a8b7cb424c993dfc1aa36 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 08:23:26 -0500 Subject: [PATCH 080/155] manages acceptance test status, no artifacts created without it Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 52 +++++++++++++++++++++++++- .github/workflows/artifacts.yml | 12 ++++-- .github/workflows/reference-tests.yml | 2 +- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 7d274aed68a..9dea0225f85 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -9,24 +9,53 @@ env: total-runners: 16 jobs: + pend: + name: acceptance testing pending + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + permissions: + statuses: write + steps: + - name: pending + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + status: pending acceptanceTestEthereum: runs-on: ubuntu-22.04 + needs: pend name: "Acceptance Runner #${{ matrix.runner_index }}: Run acceptance tests in parallel" - # TODO: don't re-run on subsequent comments on an approved workflow + permissions: + statuses: write if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} strategy: fail-fast: false matrix: runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] steps: + - name: already tested + id: needsRun + uses: actions/github-script@v7.0.1 + with: + script: | + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + }); + const acceptanceTested = statuses.find(({ context }) => context === 'acceptance-tests'); + const needsRun = acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); + return needsRun - name: Checkout Repo + if: steps.needsRun.outputs.result == 'true' uses: actions/checkout@v3 - name: Set up Java + if: steps.needsRun.outputs.result == 'true' uses: actions/setup-java@v3 with: distribution: adopt java-version: 17 - name: get acceptance test report + if: steps.needsRun.outputs.result == 'true' uses: dawidd6/action-download-artifact@v2 with: branch: main @@ -35,8 +64,10 @@ jobs: path: tmp/junit-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle + if: steps.needsRun.outputs.result == 'true' uses: gradle/gradle-build-action@v2 - name: Split tests + if: steps.needsRun.outputs.result == 'true' id: split-tests uses: r7kamura/split-tests-by-timings@v0 with: @@ -45,22 +76,39 @@ jobs: total: ${{env.total-runners}} index: ${{ matrix.runner_index }} - name: write out test list + if: steps.needsRun.outputs.result == 'true' run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt - name: format gradle args + if: steps.needsRun.outputs.result == 'true' #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, #then drop file extension, then insert --tests option between each. run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests + if: steps.needsRun.outputs.result == 'true' run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: flag failed acceptance tests + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() && steps.needsRun.outputs.result == 'true' + with: + status: ${{ job.status }} + context: acceptance-tests-${{matrix.runner_index}} + description: fail acceptance test batch ${{matrix.runner_index}} + - name: flag passed unit tests + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: success() steps.needsRun.outputs.result == 'true' + with: + status: ${{ job.status }} - name: cleanup tempfiles + if: steps.needsRun.outputs.result == 'true' run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results + if: steps.needsRun.outputs.result == 'true' uses: actions/upload-artifact@v3.1.0 with: name: acceptance-node-${{matrix.runner_index}}-test-results path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the build step fails + if: steps.needsRun.outputs.result == 'true' && (success() || failure()) # always run even if the build step fails with: report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 430f2afee71..8a3603859b0 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -11,8 +11,8 @@ jobs: artifacts: runs-on: ubuntu-22.04 steps: - - name: Check if pre-review workflow has run - id: confirmPreReview + - name: confirm tested + id: confirmTested uses: actions/github-script@v7.0.1 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -26,11 +26,15 @@ jobs: console.log(statuses); const anyFailure = statuses.find(( { state }) => state === 'failure'); console.log(anyFailure); + const anyPending = statuses.find(( { state }) => state === 'pending'); + console.log(anyPending); const prePRWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(prePRWorkflow); - return prePRWorkflow && prePRWorkflow.state === 'success' && !anyFailure; + const acceptTests = statuses.find(({context}) => context === 'acceptance-tests'); + console.log(acceptTests) + return prePRWorkflow && prePRWorkflow.state === 'success' && acceptTests && acceptTests.state === 'success' && !anyFailure && !anyPending; - name: Abort on missing status - if: steps.confirmPreReview.outputs.result == 'false' + if: steps.confirmTested.outputs.result == 'false' run: | echo "::error::required status missing" exit 1 diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 74ef586e497..b41adb39da9 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -12,7 +12,7 @@ env: jobs: prepareReferenceTestEthereum: runs-on: ubuntu-22.04 - # TODO: don't re-run on subsequent comments on an approved workflow + # TODO: don't re-run on subsequent comments on an approved workflow, check status if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} steps: - name: Checkout Repo From 64f95f9383a7162e2fd66b87ab47c87c1b74dae3 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 08:40:57 -0500 Subject: [PATCH 081/155] LINTAAARRRR Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 9dea0225f85..ca2add620c0 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -95,7 +95,7 @@ jobs: description: fail acceptance test batch ${{matrix.runner_index}} - name: flag passed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 - if: success() steps.needsRun.outputs.result == 'true' + if: success() && steps.needsRun.outputs.result == 'true' with: status: ${{ job.status }} - name: cleanup tempfiles From ff6a33c3b4bcd8ccbd72c6c5107fc8d29f3c393f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 13:46:58 -0500 Subject: [PATCH 082/155] trigger tests on code changed after approved Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 6 +++++- .github/workflows/integration-tests.yml | 15 ++++++++++++++- .github/workflows/pre-review.yml | 9 +++++++++ .github/workflows/reference-tests.yml | 1 + 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index ca2add620c0..474060c68e1 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,6 +1,7 @@ name: acceptance-tests on: workflow_dispatch: + workflow_call: pull_request_review: types: [submitted] @@ -20,10 +21,11 @@ jobs: uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending + sha: ${{github.pull_request.head.sha}} acceptanceTestEthereum: runs-on: ubuntu-22.04 needs: pend - name: "Acceptance Runner #${{ matrix.runner_index }}: Run acceptance tests in parallel" + name: "Acceptance Runner" permissions: statuses: write if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} @@ -92,12 +94,14 @@ jobs: with: status: ${{ job.status }} context: acceptance-tests-${{matrix.runner_index}} + sha: ${{github.pull_request.head.sha}} description: fail acceptance test batch ${{matrix.runner_index}} - name: flag passed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 if: success() && steps.needsRun.outputs.result == 'true' with: status: ${{ job.status }} + sha: ${{github.pull_request.head.sha}} - name: cleanup tempfiles if: steps.needsRun.outputs.result == 'true' run: rm testList.txt gradleArgs.txt diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f9d7d2b1a02..c36c0c25bd2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,6 +1,7 @@ name: integration-tests on: workflow_dispatch: + workflow_call: pull_request_review: types: - submitted @@ -9,7 +10,19 @@ env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" jobs: - integrationTetst: + pend: + name: integration testing pending + runs-on: ubuntu-22.04 + if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + permissions: + statuses: write + steps: + - name: pending + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + status: pending + sha: ${{github.pull_request.head.sha}} + integrationTest: runs-on: ubuntu-22.04 # TODO: don't re-run on subsequent comments on an approved workflow if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 7c140a5b386..9945595073f 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -115,3 +115,12 @@ jobs: with: report_paths: '**/test-results/test/TEST-*.xml' annotate_only: true + acceptanceTest: + needs: unitTests + uses: ./.github/workflows/acceptance-tests.yml + integrationTest: + needs: unitTests + uses: ./github/workflows/integration-tests.yml + referenceTest: + needs: unitTests + uses: ./github/workflows/reference-tests.yml diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index b41adb39da9..e7eace8ee44 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -1,6 +1,7 @@ name: reference-tests on: workflow_dispatch: + workflow_call: pull_request_review: types: - submitted From e3444f48cfa7d3b88fa43b0c5cacc8d002fc8eb3 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 14:05:21 -0500 Subject: [PATCH 083/155] pin called workflows to this branch for now Signed-off-by: jflo --- .github/workflows/pre-review.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 9945595073f..3a6f7a3dfb1 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -117,10 +117,10 @@ jobs: annotate_only: true acceptanceTest: needs: unitTests - uses: ./.github/workflows/acceptance-tests.yml + uses: ./.github/workflows/acceptance-tests.yml@gha_acceptTest integrationTest: needs: unitTests - uses: ./github/workflows/integration-tests.yml + uses: ./github/workflows/integration-tests.yml@gha_acceptTest referenceTest: needs: unitTests - uses: ./github/workflows/reference-tests.yml + uses: ./github/workflows/reference-tests.yml@gha_acceptTest From d1341607a0125f3ea4db4490f63b3510fa175b8f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 14:13:08 -0500 Subject: [PATCH 084/155] fix path typo Signed-off-by: jflo --- .github/workflows/pre-review.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 3a6f7a3dfb1..494d6dd5381 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -117,10 +117,10 @@ jobs: annotate_only: true acceptanceTest: needs: unitTests - uses: ./.github/workflows/acceptance-tests.yml@gha_acceptTest + uses: ./.github/workflows/acceptance-tests.yml integrationTest: needs: unitTests - uses: ./github/workflows/integration-tests.yml@gha_acceptTest + uses: ./.github/workflows/integration-tests.yml referenceTest: needs: unitTests - uses: ./github/workflows/reference-tests.yml@gha_acceptTest + uses: ./.github/workflows/reference-tests.yml From 88b3d35096ceaea03e0bbe68c0b3536b1acbab42 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 14:59:43 -0500 Subject: [PATCH 085/155] according to docs, event should be forwarded. lies? Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 8 ++++++++ .github/workflows/integration-tests.yml | 22 ++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 474060c68e1..a02da1afcdb 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -10,6 +10,12 @@ env: total-runners: 16 jobs: + debug: + name: debugging + runs-on: ubuntu-22.04 + steps: + - name: echo event + run: echo ${{github.event}} pend: name: acceptance testing pending runs-on: ubuntu-22.04 @@ -17,6 +23,8 @@ jobs: permissions: statuses: write steps: + - name: debug + run: echo ${{github.event}} - name: pending uses: myrotvorets/set-commit-status-action@v2.0.0 with: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c36c0c25bd2..3d0060cc1e3 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -24,23 +24,41 @@ jobs: sha: ${{github.pull_request.head.sha}} integrationTest: runs-on: ubuntu-22.04 - # TODO: don't re-run on subsequent comments on an approved workflow + permissions: + statuses: write if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} steps: + - name: already tested + id: needsRun + uses: actions/github-script@v7.0.1 + with: + script: | + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + }); + const integrationTested = statuses.find(({ context }) => context === 'integration-tests'); + const needsRun = integrationTested.find(({ state }) => state != 'success' && state != 'pending'); + return needsRun - name: Checkout Repo + if: steps.needsRun.outputs.result == 'true' uses: actions/checkout@v3 - name: Set up Java + if: steps.needsRun.outputs.result == 'true' uses: actions/setup-java@v3 with: distribution: adopt java-version: 17 - name: setup gradle + if: steps.needsRun.outputs.result == 'true' uses: gradle/gradle-build-action@v2 - name: run integration tests + if: steps.needsRun.outputs.result == 'true' run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the build step fails + if: (success() || failure()) && steps.needsRun.outputs.result == 'true' with: report_paths: '**/build/test-results/integrationTest/TEST-*.xml' From b0d6b9a40bc53f4c4bfc64205255a6b0ded86c5f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 15:17:42 -0500 Subject: [PATCH 086/155] parse json Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 5 ++++- .github/workflows/artifacts.yml | 26 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index a02da1afcdb..a04c5b3c552 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -15,7 +15,10 @@ jobs: runs-on: ubuntu-22.04 steps: - name: echo event - run: echo ${{github.event}} + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + echo "$GITHUB_CONTEXT" pend: name: acceptance testing pending runs-on: ubuntu-22.04 diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 8a3603859b0..667da29154d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -31,8 +31,16 @@ jobs: const prePRWorkflow = statuses.find(({ context }) => context === 'pre-review'); console.log(prePRWorkflow); const acceptTests = statuses.find(({context}) => context === 'acceptance-tests'); - console.log(acceptTests) - return prePRWorkflow && prePRWorkflow.state === 'success' && acceptTests && acceptTests.state === 'success' && !anyFailure && !anyPending; + console.log(acceptTests); + const intTests = statuses.find(({context}) => context === 'integration-tests'); + console.log(intTests); + const refTests = statuses.find(({context}) => context === 'reference-tests'); + console.log(refTests); + return prePRWorkflow && prePRWorkflow.state === 'success' && + acceptTests && acceptTests.state === 'success' && + intTests && intTests.state === 'success' && + refTests && refTests.state === 'success'' && + !anyFailure && !anyPending; - name: Abort on missing status if: steps.confirmTested.outputs.result == 'false' run: | @@ -43,25 +51,23 @@ jobs: - name: Set up JDK 17 uses: actions/setup-java@v3 with: - distribution: 'adopt' + distribution: 'temurin' java-version: '17' - name: setup gradle uses: gradle/gradle-build-action@v2 - name: assemble distributions run: ./gradlew assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: new version number - id: verNum - run: | - export VER_NUM = $(grep "version=" gradle.properties | sed 's/^version=//g') - echo "::set-output name=verNum::$VER_NUM" + - name: short sha + id: shortSha + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: upload tarball uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.verNum.outputs.verNum }}.tar.gz + name: besu-${{ steps.shortSha.outputs.sha }}.tar.gz path: 'build/distributions/besu*.tar.gz' - name: upload zipfile uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.verNum.outputs.verNum }}.zip + name: besu-${{ steps.shortSha.outputs.sha }}.zip path: 'build/distributions/besu*.zip' From 0c26a99c611b5498b4289b51bfb9a0a88178f807 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 18:35:45 -0500 Subject: [PATCH 087/155] put all the checks in one place Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 43 +++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index a04c5b3c552..6935044f470 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -10,19 +10,46 @@ env: total-runners: 16 jobs: - debug: - name: debugging + shouldRun: + name: checks to ensure we should run runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} steps: - name: echo event - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: | - echo "$GITHUB_CONTEXT" + id: shouldRun + uses: actions/github-script@v7.0.1 + with: + script: | + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + }); + const unitTested = statuses.find(({ context }) => context === 'unit-tests'); + console.log(unitTested) + const failedUnits = unitTested.find(({state}) => state === 'failure' || state === 'pending'); + console.log(failedUnits) + const acceptanceTested = statuses.find(({ context }) => context === 'acceptance-tests'); + console.log(accpetanceTested); + const needsRun = acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); + console.log(needsRun); + console.log(github.actor); + + const { data: reviews } = await github.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews.filter(review => review.state === 'APPROVED'); + console.log(approvingReviews); + + return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0;) pend: name: acceptance testing pending runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + needs: shouldRun + if: ${{ needs.shouldRun.outputs.shouldrun }} permissions: statuses: write steps: @@ -39,7 +66,7 @@ jobs: name: "Acceptance Runner" permissions: statuses: write - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + if: ${{ needs.shouldRun.outputs.shouldrun }} strategy: fail-fast: false matrix: From 090a775f743d36e91a5fdb9fb988d0d3186b20a7 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 18:52:14 -0500 Subject: [PATCH 088/155] javascript embedded in yaml? no linter can save you Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 8 ++++---- .github/workflows/artifacts.yml | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 6935044f470..37b37f7ccaf 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -16,7 +16,7 @@ jobs: outputs: shouldRun: ${{steps.shouldRun.outputs.result}} steps: - - name: echo event + - name: required check id: shouldRun uses: actions/github-script@v7.0.1 with: @@ -27,9 +27,9 @@ jobs: ref: context.sha, }); const unitTested = statuses.find(({ context }) => context === 'unit-tests'); - console.log(unitTested) + console.log(unitTested); const failedUnits = unitTested.find(({state}) => state === 'failure' || state === 'pending'); - console.log(failedUnits) + console.log(failedUnits); const acceptanceTested = statuses.find(({ context }) => context === 'acceptance-tests'); console.log(accpetanceTested); const needsRun = acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); @@ -44,7 +44,7 @@ jobs: const approvingReviews = reviews.filter(review => review.state === 'APPROVED'); console.log(approvingReviews); - return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0;) + return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); pend: name: acceptance testing pending runs-on: ubuntu-22.04 diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 667da29154d..e75c8fa2824 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -36,11 +36,19 @@ jobs: console.log(intTests); const refTests = statuses.find(({context}) => context === 'reference-tests'); console.log(refTests); + + const { data: reviews } = await github.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews.filter(review => review.state === 'APPROVED'); + return prePRWorkflow && prePRWorkflow.state === 'success' && acceptTests && acceptTests.state === 'success' && intTests && intTests.state === 'success' && - refTests && refTests.state === 'success'' && - !anyFailure && !anyPending; + refTests && refTests.state === 'success' && + !anyFailure && !anyPending && (approvingReviews.length > 0); - name: Abort on missing status if: steps.confirmTested.outputs.result == 'false' run: | From e3448cf115fd998b3890044c035c5b1ce038106f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 19:12:29 -0500 Subject: [PATCH 089/155] TRUTHY Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 37b37f7ccaf..6ef73947552 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,13 +26,13 @@ jobs: repo: context.repo.repo, ref: context.sha, }); - const unitTested = statuses.find(({ context }) => context === 'unit-tests'); + const unitTested = statuses && statuses.find(({ context }) => context === 'pre-review'); console.log(unitTested); - const failedUnits = unitTested.find(({state}) => state === 'failure' || state === 'pending'); + const failedUnits = unitTested && unitTested.find(({state}) => state === 'failure' || state === 'pending'); console.log(failedUnits); - const acceptanceTested = statuses.find(({ context }) => context === 'acceptance-tests'); + const acceptanceTested = statuses && statuses.find(({ context }) => context === 'acceptance-tests'); console.log(accpetanceTested); - const needsRun = acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); + const needsRun = acceptanceTested && acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); console.log(needsRun); console.log(github.actor); @@ -41,7 +41,7 @@ jobs: repo: context.repo.repo, pull_number: context.issue.number, }); - const approvingReviews = reviews.filter(review => review.state === 'APPROVED'); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); console.log(approvingReviews); return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); From 005cba903c938bc5731df25afcb15031ffe34f1f Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 19:31:38 -0500 Subject: [PATCH 090/155] typos Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 6ef73947552..48efce30063 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,12 +26,13 @@ jobs: repo: context.repo.repo, ref: context.sha, }); + console.log(statuses); const unitTested = statuses && statuses.find(({ context }) => context === 'pre-review'); console.log(unitTested); const failedUnits = unitTested && unitTested.find(({state}) => state === 'failure' || state === 'pending'); console.log(failedUnits); const acceptanceTested = statuses && statuses.find(({ context }) => context === 'acceptance-tests'); - console.log(accpetanceTested); + console.log(acceptanceTested); const needsRun = acceptanceTested && acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); console.log(needsRun); console.log(github.actor); From 53c07d892ec994068545742fa1f1abb543d1d482 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 19:48:51 -0500 Subject: [PATCH 091/155] what even is in the context Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 48efce30063..fd6dd92731c 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,6 +26,7 @@ jobs: repo: context.repo.repo, ref: context.sha, }); + console.log(context); console.log(statuses); const unitTested = statuses && statuses.find(({ context }) => context === 'pre-review'); console.log(unitTested); @@ -37,7 +38,7 @@ jobs: console.log(needsRun); console.log(github.actor); - const { data: reviews } = await github.pulls.listReviews({ + const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, From 31ebfe6842130a57a4b751f998dbb51b1c50ab41 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 20:10:08 -0500 Subject: [PATCH 092/155] found approvals, but no statuses? Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index fd6dd92731c..8878080e7ed 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -21,6 +21,9 @@ jobs: uses: actions/github-script@v7.0.1 with: script: | + console.log(context.repo.owner); + console.log(context.repo.repo); + console.log(context.sha); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, From a81800e772fb91fa90a4e95879691a4e17f0cbb2 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 20:50:11 -0500 Subject: [PATCH 093/155] getting kinda weird, ngl Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 8878080e7ed..1423840bf82 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -19,15 +19,18 @@ jobs: - name: required check id: shouldRun uses: actions/github-script@v7.0.1 + env: + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: script: | + const { RELEVANT_SHA } = process.env; console.log(context.repo.owner); console.log(context.repo.repo); - console.log(context.sha); + console.log(${RELEVANT_SHA}); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, - ref: context.sha, + ref: ${RELEVANT_SHA}, }); console.log(context); console.log(statuses); From bbb00c67f1871806e48d67006f049da315bc9341 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 21:06:21 -0500 Subject: [PATCH 094/155] idk about this syntax Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 1423840bf82..5f859e52c95 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -26,11 +26,12 @@ jobs: const { RELEVANT_SHA } = process.env; console.log(context.repo.owner); console.log(context.repo.repo); - console.log(${RELEVANT_SHA}); + console.log(`working on ${RELEVANT_SHA}`); + console.log(RELEVANT_SHA); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, - ref: ${RELEVANT_SHA}, + ref: RELEVANT_SHA, }); console.log(context); console.log(statuses); From a94fa34112f35410621aa8a92c6c2085d3be6930 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 21:29:40 -0500 Subject: [PATCH 095/155] filter gets 'em all Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 5f859e52c95..1f4274c8a61 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -35,13 +35,13 @@ jobs: }); console.log(context); console.log(statuses); - const unitTested = statuses && statuses.find(({ context }) => context === 'pre-review'); + const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); console.log(unitTested); - const failedUnits = unitTested && unitTested.find(({state}) => state === 'failure' || state === 'pending'); + const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); console.log(failedUnits); - const acceptanceTested = statuses && statuses.find(({ context }) => context === 'acceptance-tests'); + const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); console.log(acceptanceTested); - const needsRun = acceptanceTested && acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); + const needsRun = acceptanceTested && acceptanceTested.filter(({ state }) => state != 'success' && state != 'pending'); console.log(needsRun); console.log(github.actor); From 8093e6ac87863ec1b1f9dc078f9e3dc2dcf52bb9 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 20 Dec 2023 21:49:41 -0500 Subject: [PATCH 096/155] no more per-step checks Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 29 +++----------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 1f4274c8a61..691a2111282 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -81,30 +81,14 @@ jobs: matrix: runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] steps: - - name: already tested - id: needsRun - uses: actions/github-script@v7.0.1 - with: - script: | - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: context.sha, - }); - const acceptanceTested = statuses.find(({ context }) => context === 'acceptance-tests'); - const needsRun = acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); - return needsRun - name: Checkout Repo - if: steps.needsRun.outputs.result == 'true' uses: actions/checkout@v3 - name: Set up Java - if: steps.needsRun.outputs.result == 'true' uses: actions/setup-java@v3 with: distribution: adopt java-version: 17 - name: get acceptance test report - if: steps.needsRun.outputs.result == 'true' uses: dawidd6/action-download-artifact@v2 with: branch: main @@ -113,10 +97,8 @@ jobs: path: tmp/junit-xml-reports-downloaded if_no_artifact_found: true - name: setup gradle - if: steps.needsRun.outputs.result == 'true' uses: gradle/gradle-build-action@v2 - name: Split tests - if: steps.needsRun.outputs.result == 'true' id: split-tests uses: r7kamura/split-tests-by-timings@v0 with: @@ -125,19 +107,16 @@ jobs: total: ${{env.total-runners}} index: ${{ matrix.runner_index }} - name: write out test list - if: steps.needsRun.outputs.result == 'true' run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt - name: format gradle args - if: steps.needsRun.outputs.result == 'true' #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter, #then drop file extension, then insert --tests option between each. run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests - if: steps.needsRun.outputs.result == 'true' run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: flag failed acceptance tests uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() && steps.needsRun.outputs.result == 'true' + if: failure() with: status: ${{ job.status }} context: acceptance-tests-${{matrix.runner_index}} @@ -145,21 +124,19 @@ jobs: description: fail acceptance test batch ${{matrix.runner_index}} - name: flag passed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 - if: success() && steps.needsRun.outputs.result == 'true' + if: success() with: status: ${{ job.status }} sha: ${{github.pull_request.head.sha}} - name: cleanup tempfiles - if: steps.needsRun.outputs.result == 'true' run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results - if: steps.needsRun.outputs.result == 'true' uses: actions/upload-artifact@v3.1.0 with: name: acceptance-node-${{matrix.runner_index}}-test-results path: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: steps.needsRun.outputs.result == 'true' && (success() || failure()) # always run even if the build step fails + if: (success() || failure()) # always run even if the build step fails with: report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' From 1f58ba873a2bafa43d9a53408844adcc65fb7e58 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 08:17:17 -0500 Subject: [PATCH 097/155] enabling ref/int tests Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 21 ++------- .github/workflows/integration-tests.yml | 62 ++++++++++++++++--------- .github/workflows/pre-review.yml | 6 +-- .github/workflows/reference-tests.yml | 56 +++++++++++++++++++--- 4 files changed, 98 insertions(+), 47 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 691a2111282..0b6e8660406 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -20,30 +20,20 @@ jobs: id: shouldRun uses: actions/github-script@v7.0.1 env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: script: | const { RELEVANT_SHA } = process.env; - console.log(context.repo.owner); - console.log(context.repo.repo); - console.log(`working on ${RELEVANT_SHA}`); - console.log(RELEVANT_SHA); const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref: RELEVANT_SHA, }); - console.log(context); - console.log(statuses); const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - console.log(unitTested); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - console.log(failedUnits); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); - console.log(acceptanceTested); const needsRun = acceptanceTested && acceptanceTested.filter(({ state }) => state != 'success' && state != 'pending'); - console.log(needsRun); - console.log(github.actor); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -51,14 +41,13 @@ jobs: pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - console.log(approvingReviews); return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); pend: name: acceptance testing pending runs-on: ubuntu-22.04 needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldrun }} + if: ${{ needs.shouldRun.outputs.shouldRun }} permissions: statuses: write steps: @@ -68,14 +57,14 @@ jobs: uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending - sha: ${{github.pull_request.head.sha}} + sha: ${{ github.event.pull_request.head.sha || github.sha }} acceptanceTestEthereum: runs-on: ubuntu-22.04 needs: pend name: "Acceptance Runner" permissions: statuses: write - if: ${{ needs.shouldRun.outputs.shouldrun }} + if: ${{ needs.shouldRun.outputs.shouldRun }} strategy: fail-fast: false matrix: @@ -86,7 +75,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: get acceptance test report uses: dawidd6/action-download-artifact@v2 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3d0060cc1e3..a1ea229ae33 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,10 +10,44 @@ env: GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" jobs: + shouldRun: + name: checks to ensure we should run + runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} + steps: + - name: required check + id: shouldRun + uses: actions/github-script@v7.0.1 + env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | #TODO: could be parameterized and put in a local js file + const { RELEVANT_SHA } = process.env; + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: RELEVANT_SHA, + }); + const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); + const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); + const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); + const needsRun = intTested && intTested.filter(({ state }) => state != 'success' && state != 'pending'); + + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + + return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); pend: name: integration testing pending + needs: shouldRun + if: ${{ needs.shouldRun.outputs.shouldRun }} runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} permissions: statuses: write steps: @@ -21,44 +55,28 @@ jobs: uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending - sha: ${{github.pull_request.head.sha}} + sha: ${{ github.event.pull_request.head.sha || github.sha }} integrationTest: runs-on: ubuntu-22.04 + needs: pend + if: ${{ needs.shouldRun.outputs.shouldRun }} permissions: statuses: write - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} steps: - - name: already tested - id: needsRun - uses: actions/github-script@v7.0.1 - with: - script: | - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: context.sha, - }); - const integrationTested = statuses.find(({ context }) => context === 'integration-tests'); - const needsRun = integrationTested.find(({ state }) => state != 'success' && state != 'pending'); - return needsRun - name: Checkout Repo - if: steps.needsRun.outputs.result == 'true' uses: actions/checkout@v3 - name: Set up Java - if: steps.needsRun.outputs.result == 'true' uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: setup gradle - if: steps.needsRun.outputs.result == 'true' uses: gradle/gradle-build-action@v2 - name: run integration tests - if: steps.needsRun.outputs.result == 'true' run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: (success() || failure()) && steps.needsRun.outputs.result == 'true' + if: (success() || failure()) with: report_paths: '**/build/test-results/integrationTest/TEST-*.xml' diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 494d6dd5381..ebf11e8dc13 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -43,7 +43,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 @@ -59,7 +59,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 @@ -89,7 +89,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index e7eace8ee44..57e5c759d05 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -11,10 +11,55 @@ env: total-runners: 6 jobs: + shouldRun: + name: checks to ensure we should run + runs-on: ubuntu-22.04 + outputs: + shouldRun: ${{steps.shouldRun.outputs.result}} + steps: + - name: required check + id: shouldRun + uses: actions/github-script@v7.0.1 + env: + # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main + RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | #TODO: could be parameterized and put in a local js file + const { RELEVANT_SHA } = process.env; + const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: RELEVANT_SHA, + }); + const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); + const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); + const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); + const needsRun = refTested && refTested.filter(({ state }) => state != 'success' && state != 'pending'); + + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }); + const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + + return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); + pend: + name: reference testing pending + needs: shouldRun + if: ${{ needs.shouldRun.outputs.shouldRun }} + runs-on: ubuntu-22.04 + permissions: + statuses: write + steps: + - name: pending + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + status: pending + sha: ${{ github.event.pull_request.head.sha || github.sha }} prepareReferenceTestEthereum: runs-on: ubuntu-22.04 - # TODO: don't re-run on subsequent comments on an approved workflow, check status - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + if: ${{ needs.shouldRun.outputs.shouldRun }} steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -24,7 +69,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 @@ -40,8 +85,7 @@ jobs: runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum - # TODO: don't re-run on subsequent comments on an approved workflow - if: ${{ github.actor != 'dependabot[bot]' && github.event.review.state == 'approved' }} + if: ${{ needs.shouldRun.outputs.shouldRun }} strategy: fail-fast: false matrix: @@ -54,7 +98,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt + distribution: adopt-openj9 java-version: 17 - name: retrieve generated tests uses: actions/download-artifact@v3.0.2 From 6622e7602ff7c0a84ba19bedd6878d1f7993e743 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 08:43:42 -0500 Subject: [PATCH 098/155] switch to temurin Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 3 +- .github/workflows/integration-tests.yml | 2 +- .github/workflows/nightly.yml | 39 ------------------------- .github/workflows/pre-review.yml | 6 ++-- .github/workflows/reference-tests.yml | 2 +- 5 files changed, 7 insertions(+), 45 deletions(-) delete mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 0b6e8660406..eba60257084 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -23,6 +23,7 @@ jobs: # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: + #TODO: add console logging for skip reason if any script: | const { RELEVANT_SHA } = process.env; const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ @@ -75,7 +76,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: get acceptance test report uses: dawidd6/action-download-artifact@v2 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a1ea229ae33..77a2190f31a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -68,7 +68,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 361161c9001..00000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: nightly -on: - schedule: - - cron: '0 5 * * *' # run at 5 AM UTC - -env: - GRADLE_OPTS: "-Xmx4g -Dorg.gradle.daemon=false" - -jobs: - run-acceptance-tests: - runs-on: [self-hosted, Linux, X64] - timeout-minutes: 90 - name: "Run acceptance test suite" - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: Cache classes - uses: actions/cache@v3 - with: - path: "**/*.class" - key: ${{ runner.os }}-build-${{ github.sha }} - - name: ethereum acceptance tests - uses: gradle/gradle-build-action@v2 - with: - arguments: acceptanceTest --scan - - name: Upload Test Report - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails - with: - name: junit-acceptance-test-results - path: '**/build/test-results/acceptanceTest/TEST-*.xml' - retention-days: 1 diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index ebf11e8dc13..52dd5e4f70b 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -43,7 +43,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 @@ -59,7 +59,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 @@ -89,7 +89,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v2.10.0 diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 57e5c759d05..f688de42dbd 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -69,7 +69,7 @@ jobs: - name: Set up Java uses: actions/setup-java@v3 with: - distribution: adopt-openj9 + distribution: temurin java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 From 2f9684af8bd688a3ebb3094b924aaa8b760c4378 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 09:32:16 -0500 Subject: [PATCH 099/155] non-approved pr shouldn't have run accept tests Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index eba60257084..d6c9dbd81d2 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -25,6 +25,7 @@ jobs: with: #TODO: add console logging for skip reason if any script: | + console.log(context); const { RELEVANT_SHA } = process.env; const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, @@ -41,6 +42,7 @@ jobs: repo: context.repo.repo, pull_number: context.issue.number, }); + console.log(reviews) const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); From 74b5efbd0278cc2c54fc6e883f3334e71710e912 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 10:01:33 -0500 Subject: [PATCH 100/155] effing javascript Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d6c9dbd81d2..423140eff7b 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -33,18 +33,21 @@ jobs: ref: RELEVANT_SHA, }); const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); + console.log(`unitTested=${unitTested}`); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); + console.log(`failedUnit=${failedUnit}`); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); + console.log(`acceptanceTested=${acceptanceTested}`); const needsRun = acceptanceTested && acceptanceTested.filter(({ state }) => state != 'success' && state != 'pending'); - + console.log(`needsRun=${needsRun}`); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, }); - console.log(reviews) + console.log(reviews); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - + console.log(`approvingReviews=${approvingReviews}`); return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); pend: name: acceptance testing pending @@ -55,7 +58,7 @@ jobs: statuses: write steps: - name: debug - run: echo ${{github.event}} + run: echo ${{ needs.shouldRun.outputs.shouldRun }} - name: pending uses: myrotvorets/set-commit-status-action@v2.0.0 with: From 28c7cc9456b5a6306f7d7a4fbceccaade7c37ca5 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 10:21:56 -0500 Subject: [PATCH 101/155] do you even toString bro Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 423140eff7b..d398d541b84 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -33,13 +33,13 @@ jobs: ref: RELEVANT_SHA, }); const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - console.log(`unitTested=${unitTested}`); + console.log(`unitTested=`);console.log(unitTested); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - console.log(`failedUnit=${failedUnit}`); + console.log(`failedUnits=`);console.log(failedUnits); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); - console.log(`acceptanceTested=${acceptanceTested}`); + console.log(`acceptanceTested=`);console.log(acceptanceTested); const needsRun = acceptanceTested && acceptanceTested.filter(({ state }) => state != 'success' && state != 'pending'); - console.log(`needsRun=${needsRun}`); + console.log(`needsRun=`);console.log(needsRun); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -47,7 +47,7 @@ jobs: }); console.log(reviews); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - console.log(`approvingReviews=${approvingReviews}`); + console.log(`approvingReviews=`);console.log(approvingReviews); return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); pend: name: acceptance testing pending From b323b78f1860f0556bb388774862122f4810e42c Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 11:29:45 -0500 Subject: [PATCH 102/155] still hate javascript Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 6 +++--- .github/workflows/pre-review.yml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d398d541b84..d56b7c75b90 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -35,10 +35,10 @@ jobs: const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); console.log(`unitTested=`);console.log(unitTested); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - console.log(`failedUnits=`);console.log(failedUnits); + console.log(`failedUnits=`);console.log(failedUnits.length); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); console.log(`acceptanceTested=`);console.log(acceptanceTested); - const needsRun = acceptanceTested && acceptanceTested.filter(({ state }) => state != 'success' && state != 'pending'); + const needsRun = acceptanceTested && acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); console.log(`needsRun=`);console.log(needsRun); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -48,7 +48,7 @@ jobs: console.log(reviews); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); console.log(`approvingReviews=`);console.log(approvingReviews); - return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); + return needsRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); pend: name: acceptance testing pending runs-on: ubuntu-22.04 diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 52dd5e4f70b..aadc87fef90 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -95,7 +95,8 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests id: unitTest - run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + #TODO: return to ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: ./gradlew --version - name: flag failed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 if: failure() From 6039307948a2ce8a430646fd46aae842e0bd16df Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 14:28:17 -0500 Subject: [PATCH 103/155] logic sucks Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d56b7c75b90..d87680a4f47 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -38,8 +38,8 @@ jobs: console.log(`failedUnits=`);console.log(failedUnits.length); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); console.log(`acceptanceTested=`);console.log(acceptanceTested); - const needsRun = acceptanceTested && acceptanceTested.find(({ state }) => state != 'success' && state != 'pending'); - console.log(`needsRun=`);console.log(needsRun); + const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success' || state === 'pending') > 0; + console.log(`alreadyRun=`);console.log(alreadyRun); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -48,7 +48,11 @@ jobs: console.log(reviews); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); console.log(`approvingReviews=`);console.log(approvingReviews); - return needsRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + if(!shouldRun) { + + } + return shouldRun; pend: name: acceptance testing pending runs-on: ubuntu-22.04 From e2fb71d9370edd05f929f1240dca0ae1ed6ca14c Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 14:44:19 -0500 Subject: [PATCH 104/155] logic sucks Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d87680a4f47..d746f5dab32 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -49,9 +49,12 @@ jobs: const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); console.log(`approvingReviews=`);console.log(approvingReviews); const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); - if(!shouldRun) { - - } + + console.log("acceptance test should be skipped"); + console.log("alreadyRun = %j", alreadyRun); + console.log("failedUnits = %j", failedUnits); + console.log("approvingReviews = %j", approvingReviews); + return shouldRun; pend: name: acceptance testing pending From a4378f4c66f95ea4c24ec67c43928704cb33eff7 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 14:52:11 -0500 Subject: [PATCH 105/155] why does it insist there is an approving review? Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index d746f5dab32..1d700e3f1c7 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -52,8 +52,8 @@ jobs: console.log("acceptance test should be skipped"); console.log("alreadyRun = %j", alreadyRun); - console.log("failedUnits = %j", failedUnits); - console.log("approvingReviews = %j", approvingReviews); + console.log("failedUnits = %j", failedUnits.length); + console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; pend: From 105d936f30b97bd6d9a122a0f06d6412d6920144 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 15:10:26 -0500 Subject: [PATCH 106/155] it's in teh conditional Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 1d700e3f1c7..b7d87cf69c5 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -50,7 +50,7 @@ jobs: console.log(`approvingReviews=`);console.log(approvingReviews); const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); - console.log("acceptance test should be skipped"); + console.log("acceptance test should be = %j", shouldRun); console.log("alreadyRun = %j", alreadyRun); console.log("failedUnits = %j", failedUnits.length); console.log("approvingReviews = %j", approvingReviews.length); @@ -60,7 +60,7 @@ jobs: name: acceptance testing pending runs-on: ubuntu-22.04 needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} permissions: statuses: write steps: @@ -77,7 +77,7 @@ jobs: name: "Acceptance Runner" permissions: statuses: write - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} strategy: fail-fast: false matrix: From 25f024767e549085681aa90364a86f5aa42ec83d Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 15:36:51 -0500 Subject: [PATCH 107/155] prechecks on all tests Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 10 +--------- .github/workflows/integration-tests.yml | 17 +++++++++++------ .github/workflows/pre-review.yml | 3 +-- .github/workflows/reference-tests.yml | 19 ++++++++++++------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index b7d87cf69c5..f7748086501 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -23,9 +23,7 @@ jobs: # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: - #TODO: add console logging for skip reason if any script: | - console.log(context); const { RELEVANT_SHA } = process.env; const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, @@ -33,24 +31,18 @@ jobs: ref: RELEVANT_SHA, }); const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - console.log(`unitTested=`);console.log(unitTested); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - console.log(`failedUnits=`);console.log(failedUnits.length); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); - console.log(`acceptanceTested=`);console.log(acceptanceTested); const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success' || state === 'pending') > 0; - console.log(`alreadyRun=`);console.log(alreadyRun); const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, }); - console.log(reviews); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - console.log(`approvingReviews=`);console.log(approvingReviews); const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); - console.log("acceptance test should be = %j", shouldRun); + console.log("tests should be run = %j", shouldRun); console.log("alreadyRun = %j", alreadyRun); console.log("failedUnits = %j", failedUnits.length); console.log("approvingReviews = %j", approvingReviews.length); diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 77a2190f31a..e36792094a8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -23,7 +23,7 @@ jobs: # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: - script: | #TODO: could be parameterized and put in a local js file + script: | const { RELEVANT_SHA } = process.env; const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, @@ -33,20 +33,25 @@ jobs: const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); - const needsRun = intTested && intTested.filter(({ state }) => state != 'success' && state != 'pending'); - + const alreadyRun = intTested && intTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + + console.log("tests should be run = %j", shouldRun); + console.log("alreadyRun = %j", alreadyRun); + console.log("failedUnits = %j", failedUnits.length); + console.log("approvingReviews = %j", approvingReviews.length); - return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); + return shouldRun; pend: name: integration testing pending needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} runs-on: ubuntu-22.04 permissions: statuses: write @@ -59,7 +64,7 @@ jobs: integrationTest: runs-on: ubuntu-22.04 needs: pend - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write steps: diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index aadc87fef90..52dd5e4f70b 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -95,8 +95,7 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: run unit tests id: unitTest - #TODO: return to ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - run: ./gradlew --version + run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: flag failed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 if: failure() diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index f688de42dbd..94d32b1401b 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -24,7 +24,7 @@ jobs: # fun fact, this changes based on incoming event, it will be different when we run this on pushes to main RELEVANT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} with: - script: | #TODO: could be parameterized and put in a local js file + script: | const { RELEVANT_SHA } = process.env; const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, @@ -34,20 +34,25 @@ jobs: const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); - const needsRun = refTested && refTested.filter(({ state }) => state != 'success' && state != 'pending'); - + const alreadyRun = refTested && refTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + + console.log("tests should be run = %j", shouldRun); + console.log("alreadyRun = %j", alreadyRun); + console.log("failedUnits = %j", failedUnits.length); + console.log("approvingReviews = %j", approvingReviews.length); - return needsRun && github.actor != 'dependabot[bot]' && !failedUnits && (approvingReviews.length > 0); + return shouldRun; pend: name: reference testing pending needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} runs-on: ubuntu-22.04 permissions: statuses: write @@ -59,7 +64,7 @@ jobs: sha: ${{ github.event.pull_request.head.sha || github.sha }} prepareReferenceTestEthereum: runs-on: ubuntu-22.04 - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -85,7 +90,7 @@ jobs: runs-on: ubuntu-22.04 needs: - prepareReferenceTestEthereum - if: ${{ needs.shouldRun.outputs.shouldRun }} + if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} strategy: fail-fast: false matrix: From b8b73008967754bce407370a0b67c42ed370543a Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 18:14:40 -0500 Subject: [PATCH 108/155] missed some deps Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 4 ++-- .github/workflows/integration-tests.yml | 7 +++++++ .github/workflows/reference-tests.yml | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index f7748086501..4b88555e11f 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -114,14 +114,14 @@ jobs: with: status: ${{ job.status }} context: acceptance-tests-${{matrix.runner_index}} - sha: ${{github.pull_request.head.sha}} + sha: ${{ github.event.pull_request.head.sha || github.sha }} description: fail acceptance test batch ${{matrix.runner_index}} - name: flag passed unit tests uses: myrotvorets/set-commit-status-action@v2.0.0 if: success() with: status: ${{ job.status }} - sha: ${{github.pull_request.head.sha}} + sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e36792094a8..3e1720a0c6f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -67,6 +67,7 @@ jobs: if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write + checks: write steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -79,6 +80,12 @@ jobs: uses: gradle/gradle-build-action@v2 - name: run integration tests run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: flag passed unit tests + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: success() + with: + status: ${{ job.status }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: (success() || failure()) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 94d32b1401b..822b55c12fe 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -64,6 +64,7 @@ jobs: sha: ${{ github.event.pull_request.head.sha || github.sha }} prepareReferenceTestEthereum: runs-on: ubuntu-22.04 + needs: shouldRun if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} steps: - name: Checkout Repo @@ -88,6 +89,9 @@ jobs: referenceTestEthereum: runs-on: ubuntu-22.04 + permissions: + statuses: write + checks: write needs: - prepareReferenceTestEthereum if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} From ff7e9bb537193526dcd0c3005c9acc20dc982f30 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 21 Dec 2023 19:21:45 -0500 Subject: [PATCH 109/155] child workflows will need check:write perms Signed-off-by: jflo --- .github/workflows/pre-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 52dd5e4f70b..104f78e59d8 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -6,6 +6,7 @@ on: permissions: statuses: write + checks: write env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true From ed0f26049d4b8e67b8e5390b437101bbdef975f4 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 3 Jan 2024 15:27:14 -0500 Subject: [PATCH 110/155] disable docker unless manual Signed-off-by: jflo --- .github/workflows/docker.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6387a8de873..3cd827e65c9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,9 +1,6 @@ -name: "Container Images" +name: docker on: workflow_dispatch: - release: - types: - - created jobs: buildX64Docker: From 0c43d96420f4a9ea1ff16248344475fde97b0b6e Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 3 Jan 2024 16:17:32 -0500 Subject: [PATCH 111/155] bad object deref Signed-off-by: jflo --- .github/workflows/acceptance-tests.yml | 1 + .github/workflows/artifacts.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 4b88555e11f..ecab3f3ae9c 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -69,6 +69,7 @@ jobs: name: "Acceptance Runner" permissions: statuses: write + checks: write if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} strategy: fail-fast: false diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index e75c8fa2824..3984e53d37b 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -37,7 +37,7 @@ jobs: const refTests = statuses.find(({context}) => context === 'reference-tests'); console.log(refTests); - const { data: reviews } = await github.pulls.listReviews({ + const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, From ba068e5f195b05bb31694c5a9eb020004da4c1c1 Mon Sep 17 00:00:00 2001 From: jflo Date: Wed, 3 Jan 2024 16:55:02 -0500 Subject: [PATCH 112/155] remove checks, and assume to be run only from main Signed-off-by: jflo --- .github/workflows/artifacts.yml | 43 --------------------------------- 1 file changed, 43 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 3984e53d37b..c0049133741 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -11,49 +11,6 @@ jobs: artifacts: runs-on: ubuntu-22.04 steps: - - name: confirm tested - id: confirmTested - uses: actions/github-script@v7.0.1 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - # TODO: combine unit, acceptance, integration, reference tests before creating artifacts. also ensure absence of any failed statii - script: | - const { data: { statuses } } = await github.rest.repos.getCombinedStatusForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: context.sha, - }); - console.log(statuses); - const anyFailure = statuses.find(( { state }) => state === 'failure'); - console.log(anyFailure); - const anyPending = statuses.find(( { state }) => state === 'pending'); - console.log(anyPending); - const prePRWorkflow = statuses.find(({ context }) => context === 'pre-review'); - console.log(prePRWorkflow); - const acceptTests = statuses.find(({context}) => context === 'acceptance-tests'); - console.log(acceptTests); - const intTests = statuses.find(({context}) => context === 'integration-tests'); - console.log(intTests); - const refTests = statuses.find(({context}) => context === 'reference-tests'); - console.log(refTests); - - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - }); - const approvingReviews = reviews.filter(review => review.state === 'APPROVED'); - - return prePRWorkflow && prePRWorkflow.state === 'success' && - acceptTests && acceptTests.state === 'success' && - intTests && intTests.state === 'success' && - refTests && refTests.state === 'success' && - !anyFailure && !anyPending && (approvingReviews.length > 0); - - name: Abort on missing status - if: steps.confirmTested.outputs.result == 'false' - run: | - echo "::error::required status missing" - exit 1 - name: checkout uses: actions/checkout@v4 - name: Set up JDK 17 From 4eaadb2818b882bcfb76978c5ae4ffd1252f47db Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 4 Jan 2024 14:39:43 -0500 Subject: [PATCH 113/155] lets build docker images Signed-off-by: jflo --- .github/workflows/docker.yml | 75 +++++++++++++++--------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3cd827e65c9..63428ed8a9e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,9 +1,12 @@ name: docker on: workflow_dispatch: + release: + types: + - created jobs: - buildX64Docker: + hadolint: runs-on: ubuntu-22.04 steps: - name: Checkout Repo @@ -25,65 +28,47 @@ jobs: run: docker run --rm -i hadolint/hadolint < docker/openjdk-latest/Dockerfile - name: hadoLint_graalvm run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile - - name: distDocker - uses: gradle/gradle-build-action@v2 - with: - arguments: distDocker - - name: test image - run: | - mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-amd64 -o ./docker/tests/goss-linux-amd64 - - name: test docker - uses: gradle/gradle-build-action@v2 - env: - architecture: amd64 - with: - arguments: testDocker - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: robocopsgonemad - password: ${{ secrets.DOCKER_PASSWORD_RW }} - - name: publish - run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload - buildArm64Docker: - runs-on: [self-hosted, ARM64] + buildDocker: + runs-on: ubuntu-22.04 + needs: hadolint + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/arm/v6 + - linux/arm/v7 + - linux/arm64 steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java uses: actions/setup-java@v3 with: - distribution: zulu - architecture: arm64 + distribution: adopt java-version: 17 - - name: Docker login - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: buildDocker + - name: setup gradle + uses: gradle/gradle-build-action@v2 + - name: distDocker uses: gradle/gradle-build-action@v2 - env: - architecture: arm64 with: - arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - - name: download image test binary + arguments: distDocker + - name: test image run: | mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-linux-arm -o ./docker/tests/goss-linux-arm64 - - name: testDocker + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-$PLATFORM-PAIR -o ./docker/tests/goss-$PLATFORM-PAIR + - name: test docker uses: gradle/gradle-build-action@v2 - env: - architecture: arm64 with: - arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} + arguments: testDocker - name: Login to DockerHub uses: docker/login-action@v2 with: - username: robocopsgonemad + username: ${{ secrets.DOCKER_USER_RW }} password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish - uses: gradle/gradle-build-action@v2 - env: - architecture: arm64 - with: - arguments: -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload + run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload From 43c4f73865553108c250d856c176071e12da5376 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 4 Jan 2024 15:57:57 -0500 Subject: [PATCH 114/155] needed org on image build, fixes platform pair Signed-off-by: jflo --- .github/workflows/docker.yml | 7 ++++--- .github/workflows/release.yml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 63428ed8a9e..6e1d309e6fa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,9 +41,10 @@ jobs: - linux/arm64 steps: - name: Prepare + id: prep run: | platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_OUTPUT - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -56,11 +57,11 @@ jobs: - name: distDocker uses: gradle/gradle-build-action@v2 with: - arguments: distDocker + arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: test image run: | mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-$PLATFORM-PAIR -o ./docker/tests/goss-$PLATFORM-PAIR + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} - name: test docker uses: gradle/gradle-build-action@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17442e8333e..d8e2627db1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: release besu on: workflow_dispatch: release: - types: released + types: [released] jobs: dockerPromoteX64: runs-on: ubuntu-22.04 From 7ab16872c2c2b8e47c5b0dbcb6e313ddc94683a1 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 4 Jan 2024 16:15:46 -0500 Subject: [PATCH 115/155] docker test needed an org too Signed-off-by: jflo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6e1d309e6fa..7d660edb057 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,7 +65,7 @@ jobs: - name: test docker uses: gradle/gradle-build-action@v2 with: - arguments: testDocker + arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: Login to DockerHub uses: docker/login-action@v2 with: From 351082b4dba2979c877cd0ebf6b634d7dcce3ca3 Mon Sep 17 00:00:00 2001 From: jflo Date: Thu, 4 Jan 2024 20:29:09 -0500 Subject: [PATCH 116/155] testDocker needs an env var Signed-off-by: jflo --- .github/workflows/docker.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7d660edb057..78e93207b5c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,8 +36,6 @@ jobs: matrix: platform: - linux/amd64 - - linux/arm/v6 - - linux/arm/v7 - linux/arm64 steps: - name: Prepare @@ -45,6 +43,7 @@ jobs: run: | platform=${{ matrix.platform }} echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_OUTPUT + echo "ARCH=${platform//linux\//}" >> $GITHUB_OUTPUT - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -54,16 +53,18 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 - - name: distDocker + - name: build image uses: gradle/gradle-build-action@v2 with: arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - - name: test image + - name: install goss run: | mkdir -p docker/reports - curl -L https://github.com/aelsabbahy/goss/releases/download/v0.3.9/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} + curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} - name: test docker uses: gradle/gradle-build-action@v2 + env: + architecture: ${{ steps.prep.outputs.ARCH }} with: arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: Login to DockerHub From 542d2ff2495a0f8b6ab1256db3cc7eca075aaaf6 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 10:05:51 -0500 Subject: [PATCH 117/155] need to run on self hosted arm Signed-off-by: jflo --- .github/workflows/docker.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 78e93207b5c..97001c6c462 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,21 +29,26 @@ jobs: - name: hadoLint_graalvm run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile buildDocker: - runs-on: ubuntu-22.04 + runs-on: ${ matrix.platform } needs: hadolint strategy: fail-fast: false matrix: platform: - - linux/amd64 - - linux/arm64 + - ubuntu-22.04 + - [self-hosted, ARM64, macOS] steps: - name: Prepare id: prep run: | platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_OUTPUT - echo "ARCH=${platform//linux\//}" >> $GITHUB_OUTPUT + if [ "$platform" = 'ubuntu-22.04' ]; then + echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT + echo "ARCH=amd64" >> $GITHUB_OUTPUT + else + echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT + echo "ARCH=arm64" >> $GITHUB_OUTPUT + fi - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java From f003559baea37d3892df10cffcae7522f4006421 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 10:13:22 -0500 Subject: [PATCH 118/155] order matters Signed-off-by: jflo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 97001c6c462..0a56b06ca20 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,6 @@ jobs: - name: hadoLint_graalvm run: docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile buildDocker: - runs-on: ${ matrix.platform } needs: hadolint strategy: fail-fast: false @@ -37,6 +36,7 @@ jobs: platform: - ubuntu-22.04 - [self-hosted, ARM64, macOS] + runs-on: ${ matrix.platform } steps: - name: Prepare id: prep From bbd914eec1b3d095b9543ccae09b06e77e38d330 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 10:16:02 -0500 Subject: [PATCH 119/155] syntax matters Signed-off-by: jflo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0a56b06ca20..6132df5b332 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,7 +36,7 @@ jobs: platform: - ubuntu-22.04 - [self-hosted, ARM64, macOS] - runs-on: ${ matrix.platform } + runs-on: ${{ matrix.platform }} steps: - name: Prepare id: prep From c188e2c8b587ae9a4eb1b878e30fb1c877f903bf Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 10:43:23 -0500 Subject: [PATCH 120/155] don't rerun image build so often, and fix tag collisions Signed-off-by: jflo --- .github/workflows/docker.yml | 2 ++ build.gradle | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6132df5b332..cc13117c621 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,6 +74,8 @@ jobs: arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: Login to DockerHub uses: docker/login-action@v2 + env: + architecture: ${{ steps.prep.outputs.ARCH }} with: username: ${{ secrets.DOCKER_USER_RW }} password: ${{ secrets.DOCKER_PASSWORD_RW }} diff --git a/build.gradle b/build.gradle index 2d2e610d58d..17496dc9cba 100644 --- a/build.gradle +++ b/build.gradle @@ -772,7 +772,6 @@ task distDocker { } task testDocker { - dependsOn distDocker def dockerReportsDir = "docker/reports/" doFirst { @@ -792,7 +791,6 @@ task testDocker { } task dockerUpload { - dependsOn distDocker def architecture = System.getenv('architecture') def image = "${dockerImageName}:${dockerBuildVersion}" def additionalTags = [] From 8082ad57951b0b4f903aa88a21e7e3b0c83467b4 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 10:53:16 -0500 Subject: [PATCH 121/155] whoops wrong step Signed-off-by: jflo --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cc13117c621..0504ff9e9ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,10 +74,10 @@ jobs: arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} - name: Login to DockerHub uses: docker/login-action@v2 - env: - architecture: ${{ steps.prep.outputs.ARCH }} with: username: ${{ secrets.DOCKER_USER_RW }} password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: publish + env: + architecture: ${{ steps.prep.outputs.ARCH }} run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload From 17167defdb240c6e73495b54feb12d0607bc151b Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 13:43:47 -0500 Subject: [PATCH 122/155] include git sha in docker naming Signed-off-by: jflo --- .github/workflows/docker.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0504ff9e9ad..af5f2cefc6c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,6 +49,9 @@ jobs: echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT echo "ARCH=arm64" >> $GITHUB_OUTPUT fi + - name: short sha + id: shortSha + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Checkout Repo uses: actions/checkout@v3 - name: Set up Java @@ -61,7 +64,7 @@ jobs: - name: build image uses: gradle/gradle-build-action@v2 with: - arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} + arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT - name: install goss run: | mkdir -p docker/reports @@ -71,7 +74,7 @@ jobs: env: architecture: ${{ steps.prep.outputs.ARCH }} with: - arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} + arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -80,4 +83,4 @@ jobs: - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload + run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT From 3dbc3fca740ff369ea99d96d1346729906c9e521 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 14:01:18 -0500 Subject: [PATCH 123/155] no more manual sha256ing Signed-off-by: jflo --- .github/workflows/artifacts.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index c0049133741..ba203039b51 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -22,17 +22,30 @@ jobs: uses: gradle/gradle-build-action@v2 - name: assemble distributions run: ./gradlew assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: short sha - id: shortSha - run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: hashes + id: hashes + run: | + echo "gitSha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "zipSha=$(md5sum -a 256 build/distributions/besu*.zip)" >> $GITHUB_OUTPUT + echo "tarSha=$(md5sum -a 256 build/distributions/besu*.tar.gz)" >> $GITHUB_OUTPUT - name: upload tarball uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.shortSha.outputs.sha }}.tar.gz + name: besu-${{ steps.hashes.outputs.gitSha }}.tar.gz path: 'build/distributions/besu*.tar.gz' - name: upload zipfile uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.shortSha.outputs.sha }}.zip + name: besu-${{ steps.hashes.outputs.gitSha }}.zip path: 'build/distributions/besu*.zip' + - name: Upload Release assets + uses: softprops/action-gh-release@v1 + with: + files: | + build/distributions/besu*.tar.gz + build/distributions/besu*.zip + body: | + tarball: ${{steps.hashes.outputs.tarSha}} + zipfile: ${{steps.hashes.outputs.zipSha}} + From 103a6c4503679afd6fd63a32b0120d650854ea2d Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 14:07:33 -0500 Subject: [PATCH 124/155] new job for multi-arch manifest Signed-off-by: jflo --- .github/workflows/docker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index af5f2cefc6c..eb1d3ff0c51 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -84,3 +84,9 @@ jobs: env: architecture: ${{ steps.prep.outputs.ARCH }} run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT + multiArch: + needs: buildDocker + runs-on: ubuntu-22.04 + steps: + - name: multi-arch docker + run: ./gradlew -PdockerOrgName=${{secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT manifestDocker From 1f3908a035fb34912e98c98ddedb7650a91c6e67 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 14:16:01 -0500 Subject: [PATCH 125/155] md5? really boomer? Signed-off-by: jflo --- .github/workflows/artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index ba203039b51..a900356713d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -26,8 +26,8 @@ jobs: id: hashes run: | echo "gitSha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - echo "zipSha=$(md5sum -a 256 build/distributions/besu*.zip)" >> $GITHUB_OUTPUT - echo "tarSha=$(md5sum -a 256 build/distributions/besu*.tar.gz)" >> $GITHUB_OUTPUT + echo "zipSha=$(shasum -a 256 build/distributions/besu*.zip)" >> $GITHUB_OUTPUT + echo "tarSha=$(shasum -a 256 build/distributions/besu*.tar.gz)" >> $GITHUB_OUTPUT - name: upload tarball uses: actions/upload-artifact@v3 with: From 8b9e132832ffff65c2965d6e65382fe83748abd7 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 14:52:18 -0500 Subject: [PATCH 126/155] whoops can't get sha till checkout Signed-off-by: jflo --- .github/workflows/artifacts.yml | 2 +- .github/workflows/docker.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index a900356713d..5065a8344fa 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: release: types: - - created + - prereleased jobs: artifacts: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eb1d3ff0c51..4905a8d71d5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: release: types: - - created + - prereleased jobs: hadolint: @@ -49,11 +49,11 @@ jobs: echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT echo "ARCH=arm64" >> $GITHUB_OUTPUT fi + - name: Checkout Repo + uses: actions/checkout@v3 - name: short sha id: shortSha run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - name: Checkout Repo - uses: actions/checkout@v3 - name: Set up Java uses: actions/setup-java@v3 with: From 17f22b589a864060da07dcb88222213df67ce8e5 Mon Sep 17 00:00:00 2001 From: jflo Date: Fri, 5 Jan 2024 16:10:35 -0500 Subject: [PATCH 127/155] only need artifacts and docker images on release prep Signed-off-by: jflo --- .github/workflows/artifacts.yml | 7 ++----- .github/workflows/docker.yml | 9 ++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 5065a8344fa..02ae116845a 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -2,7 +2,6 @@ name: artifacts on: - workflow_dispatch: release: types: - prereleased @@ -21,22 +20,20 @@ jobs: - name: setup gradle uses: gradle/gradle-build-action@v2 - name: assemble distributions - run: ./gradlew assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + run: + ./gradlew -Prelease.releaseVersion=${{github.ref_name}} assemble -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - name: hashes id: hashes run: | - echo "gitSha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT echo "zipSha=$(shasum -a 256 build/distributions/besu*.zip)" >> $GITHUB_OUTPUT echo "tarSha=$(shasum -a 256 build/distributions/besu*.tar.gz)" >> $GITHUB_OUTPUT - name: upload tarball uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.hashes.outputs.gitSha }}.tar.gz path: 'build/distributions/besu*.tar.gz' - name: upload zipfile uses: actions/upload-artifact@v3 with: - name: besu-${{ steps.hashes.outputs.gitSha }}.zip path: 'build/distributions/besu*.zip' - name: Upload Release assets uses: softprops/action-gh-release@v1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4905a8d71d5..2947a9ada91 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,5 @@ name: docker on: - workflow_dispatch: release: types: - prereleased @@ -64,7 +63,7 @@ jobs: - name: build image uses: gradle/gradle-build-action@v2 with: - arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT + arguments: distDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} - name: install goss run: | mkdir -p docker/reports @@ -74,7 +73,7 @@ jobs: env: architecture: ${{ steps.prep.outputs.ARCH }} with: - arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT + arguments: testDocker -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -83,10 +82,10 @@ jobs: - name: publish env: architecture: ${{ steps.prep.outputs.ARCH }} - run: ./gradlew --no-daemon -PdockerOrgName=${{ secrets.DOCKER_ORG }} dockerUpload -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT + run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} multiArch: needs: buildDocker runs-on: ubuntu-22.04 steps: - name: multi-arch docker - run: ./gradlew -PdockerOrgName=${{secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ steps.shortSha.outputs.sha }}-SNAPSHOT manifestDocker + run: ./gradlew manifestDocker -PdockerOrgName=${{secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} From aee1fdeddd2a47c8d6d5ec99e7d6aa5b5bb89d58 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Mon, 8 Jan 2024 14:18:34 -0500 Subject: [PATCH 128/155] filename already included Signed-off-by: Justin Florentine --- .github/workflows/artifacts.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 02ae116845a..19594e1f14e 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -38,11 +38,12 @@ jobs: - name: Upload Release assets uses: softprops/action-gh-release@v1 with: + append_body: true files: | build/distributions/besu*.tar.gz build/distributions/besu*.zip body: | - tarball: ${{steps.hashes.outputs.tarSha}} - zipfile: ${{steps.hashes.outputs.zipSha}} + ${{steps.hashes.outputs.tarSha}} + ${{steps.hashes.outputs.zipSha}} From ab0ed41ee7c0e868813cdc3ba47ce5d27edf7f76 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 14:02:16 -0500 Subject: [PATCH 129/155] consolidate status to get rid of shard-specific ones Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 +- .github/workflows/pre-review.yml | 63 +++++++++++++++++++------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index ecab3f3ae9c..c7f906341cc 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -117,7 +117,7 @@ jobs: context: acceptance-tests-${{matrix.runner_index}} sha: ${{ github.event.pull_request.head.sha || github.sha }} description: fail acceptance test batch ${{matrix.runner_index}} - - name: flag passed unit tests + - name: flag passed acceptance tests uses: myrotvorets/set-commit-status-action@v2.0.0 if: success() with: diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 104f78e59d8..c69f8cecc3b 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -12,14 +12,6 @@ env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true jobs: - pend: - name: unit testing pending - runs-on: ubuntu-22.04 - steps: - - name: pending - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: pending repolint: name: "Repository Linting" runs-on: ubuntu-22.04 @@ -29,12 +21,26 @@ jobs: uses: actions/checkout@v4 - name: Lint Repo run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown + - name: flag repolint failure + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() + with: + status: ${{ job.status }} + context: pre-review + description: repo linting failed gradle-wrapper: name: "Gradle Wrapper Validation" runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1.1.0 + - name: flag gradle wrapper failure + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() + with: + status: ${{ job.status }} + context: pre-review + description: gradlew validation failed spotless: runs-on: ubuntu-22.04 if: ${{ github.actor != 'dependabot[bot]' }} @@ -50,6 +56,13 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: run spotless run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: flag spotless failure + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() + with: + status: ${{ job.status }} + context: pre-review + description: spotless failed compile: runs-on: ubuntu-22.04 timeout-minutes: 30 @@ -66,6 +79,13 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: flag compile failure + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() + with: + status: ${{ job.status }} + context: pre-review + description: compilation failed unitTests: env: GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} @@ -74,6 +94,8 @@ jobs: permissions: checks: write statuses: write + outputs: + unitShardFailed: ${{ steps.unitTest.outputs. strategy: fail-fast: false matrix: @@ -102,20 +124,29 @@ jobs: if: failure() with: status: ${{ job.status }} - context: unit-tests-${{matrix.gradle_args}} - description: ${{matrix.gradle_args}} - - name: flag passed unit tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: success() - with: - status: ${{ job.status }} - + context: pre-review + description: unit test shard failed - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails with: report_paths: '**/test-results/test/TEST-*.xml' annotate_only: true + consolidateStatus: + runs-on: ubuntu-22.04 + needs: unitTests + permissions: + statuses: write + checks: write + steps: + - name: status success + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: ${{jobs.unitTests.status == 'success' }} + with: + status: success + context: pre-review + description: unit test shard failed + acceptanceTest: needs: unitTests uses: ./.github/workflows/acceptance-tests.yml From f95bdaabd318143857d4dd58b0f3456a9d876a19 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 14:10:30 -0500 Subject: [PATCH 130/155] distracted mid-expression Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index c69f8cecc3b..f1be469ad4d 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -94,8 +94,6 @@ jobs: permissions: checks: write statuses: write - outputs: - unitShardFailed: ${{ steps.unitTest.outputs. strategy: fail-fast: false matrix: From a01da79d2decf4dd369c54967c92d964207c186e Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 14:11:34 -0500 Subject: [PATCH 131/155] typo in expression Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index f1be469ad4d..f86619f2c4f 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -139,7 +139,7 @@ jobs: steps: - name: status success uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{jobs.unitTests.status == 'success' }} + if: ${{job.unitTests.status == 'success' }} with: status: success context: pre-review From e8fc83a277c6de1a4a92dcd870da5b3d9611a50b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 14:31:37 -0500 Subject: [PATCH 132/155] allow pre-review to conclude prior to approval Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 3 +-- .github/workflows/integration-tests.yml | 3 +-- .github/workflows/pre-review.yml | 9 --------- .github/workflows/reference-tests.yml | 3 +-- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index c7f906341cc..a423e25d34e 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -1,7 +1,6 @@ name: acceptance-tests on: - workflow_dispatch: - workflow_call: + pull_request: pull_request_review: types: [submitted] diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3e1720a0c6f..7548bc638aa 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,7 +1,6 @@ name: integration-tests on: - workflow_dispatch: - workflow_call: + pull_request: pull_request_review: types: - submitted diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index f86619f2c4f..1f5e0e7ebf3 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -145,12 +145,3 @@ jobs: context: pre-review description: unit test shard failed - acceptanceTest: - needs: unitTests - uses: ./.github/workflows/acceptance-tests.yml - integrationTest: - needs: unitTests - uses: ./.github/workflows/integration-tests.yml - referenceTest: - needs: unitTests - uses: ./.github/workflows/reference-tests.yml diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 822b55c12fe..1185f807894 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -1,7 +1,6 @@ name: reference-tests on: - workflow_dispatch: - workflow_call: + pull_request: pull_request_review: types: - submitted From fdf7d386ceb64215651e280f9558303adaf7cb70 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 15:08:55 -0500 Subject: [PATCH 133/155] conditional should use needs result? Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 1f5e0e7ebf3..23c35dc75a0 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -139,7 +139,7 @@ jobs: steps: - name: status success uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{job.unitTests.status == 'success' }} + if: ${{needs.unitTests.result == "success" }} with: status: success context: pre-review From 823de9eb8b577bd319a46f4495bcd76cae7ef215 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 15:18:23 -0500 Subject: [PATCH 134/155] single quotes? Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 23c35dc75a0..f4cade5e276 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -139,7 +139,7 @@ jobs: steps: - name: status success uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{needs.unitTests.result == "success" }} + if: ${{needs.unitTests.steps.unitTest.result == 'success' }} with: status: success context: pre-review From 0001955f2182b219e804592ef9e6c5715546ca54 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 15:59:39 -0500 Subject: [PATCH 135/155] i wonder what happens with both success/fail statuses? Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index f4cade5e276..4b6b22b25b9 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -122,7 +122,7 @@ jobs: if: failure() with: status: ${{ job.status }} - context: pre-review + context: unit-tested description: unit test shard failed - name: Publish Test Report uses: mikepenz/action-junit-report@v4 @@ -139,9 +139,8 @@ jobs: steps: - name: status success uses: myrotvorets/set-commit-status-action@v2.0.0 - if: ${{needs.unitTests.steps.unitTest.result == 'success' }} with: status: success - context: pre-review - description: unit test shard failed + context: unit-tested + description: unit tests passed From d2398ea07b3af3d2d8f24da63a1dce37ce6e5a2a Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 16:27:10 -0500 Subject: [PATCH 136/155] intentional test failure Signed-off-by: Justin Florentine --- .../src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 26296556302..e19305824f9 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3898,7 +3898,7 @@ public void futureEipsValuesAreUsed() { verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); verify(mockControllerBuilder).build(); - assertThat(networkArg.getValue()).isEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); + assertThat(networkArg.getValue()).isNotEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From 14b95a222fd5000474a2db5ce9e1099e83f88286 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Tue, 9 Jan 2024 17:33:07 -0500 Subject: [PATCH 137/155] fix intentional failed test Signed-off-by: Justin Florentine --- .../src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index e19305824f9..26296556302 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3898,7 +3898,7 @@ public void futureEipsValuesAreUsed() { verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); verify(mockControllerBuilder).build(); - assertThat(networkArg.getValue()).isNotEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); + assertThat(networkArg.getValue()).isEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From f0db89831a036cab7e03dbde2b62d308e36919a9 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 11:15:16 -0500 Subject: [PATCH 138/155] enables status required before merging Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index a423e25d34e..6b8a6e4cf6a 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -11,6 +11,7 @@ env: jobs: shouldRun: name: checks to ensure we should run + # necessary because there is no single PR approved event, need to check all comments/approvals/denials runs-on: ubuntu-22.04 outputs: shouldRun: ${{steps.shouldRun.outputs.result}} @@ -29,9 +30,7 @@ jobs: repo: context.repo.repo, ref: RELEVANT_SHA, }); - const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); + const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tested'); const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -39,11 +38,10 @@ jobs: pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); console.log("tests should be run = %j", shouldRun); console.log("alreadyRun = %j", alreadyRun); - console.log("failedUnits = %j", failedUnits.length); console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; @@ -113,15 +111,9 @@ jobs: if: failure() with: status: ${{ job.status }} - context: acceptance-tests-${{matrix.runner_index}} + context: acceptance-tested sha: ${{ github.event.pull_request.head.sha || github.sha }} description: fail acceptance test batch ${{matrix.runner_index}} - - name: flag passed acceptance tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: success() - with: - status: ${{ job.status }} - sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results @@ -134,3 +126,16 @@ jobs: if: (success() || failure()) # always run even if the build step fails with: report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' + consolidateStatus: + runs-on: ubuntu-22.04 + needs: acceptanceTestEthereum + permissions: + statuses: write + checks: write + steps: + - name: status success + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + status: success + context: acceptance-tested + description: acceptance tests passed \ No newline at end of file From 4e30e3b181b282deb31aae3e3e6cd68419ef558c Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 12:58:41 -0500 Subject: [PATCH 139/155] default sha not always provided Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 3 ++ .github/workflows/reference-tests.yml | 41 ++++++++++++++++++++------ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 6b8a6e4cf6a..f31395cb50f 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -12,6 +12,7 @@ jobs: shouldRun: name: checks to ensure we should run # necessary because there is no single PR approved event, need to check all comments/approvals/denials + # might also be a job running, and additional approvals runs-on: ubuntu-22.04 outputs: shouldRun: ${{steps.shouldRun.outputs.result}} @@ -60,6 +61,7 @@ jobs: with: status: pending sha: ${{ github.event.pull_request.head.sha || github.sha }} + context: acceptance-tested acceptanceTestEthereum: runs-on: ubuntu-22.04 needs: pend @@ -136,6 +138,7 @@ jobs: - name: status success uses: myrotvorets/set-commit-status-action@v2.0.0 with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} status: success context: acceptance-tested description: acceptance tests passed \ No newline at end of file diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 1185f807894..9e5dd51c8b0 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -12,6 +12,8 @@ env: jobs: shouldRun: name: checks to ensure we should run + # necessary because there is no single PR approved event, need to check all comments/approvals/denials + # might also be a job running, and additional approvals runs-on: ubuntu-22.04 outputs: shouldRun: ${{steps.shouldRun.outputs.result}} @@ -30,9 +32,9 @@ jobs: repo: context.repo.repo, ref: RELEVANT_SHA, }); - const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); + + + const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tested'); const alreadyRun = refTested && refTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -40,11 +42,10 @@ jobs: pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); console.log("tests should be run = %j", shouldRun); console.log("alreadyRun = %j", alreadyRun); - console.log("failedUnits = %j", failedUnits.length); console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; @@ -61,6 +62,7 @@ jobs: with: status: pending sha: ${{ github.event.pull_request.head.sha || github.sha }} + context: reference-tested prepareReferenceTestEthereum: runs-on: ubuntu-22.04 needs: shouldRun @@ -97,7 +99,7 @@ jobs: strategy: fail-fast: false matrix: - runner-index: [0,1,2,3,4,5] + runner_index: [0,1,2,3,4,5] steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -130,21 +132,42 @@ jobs: reports: tmp/ref-xml-reports-downloaded glob: 'ethereum/referencetests/build/generated/sources/reference-test/**/*.java' total: ${{env.total-runners}} - index: ${{ matrix.runner-index }} + index: ${{ matrix.runner_index }} - name: compose gradle args run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt - name: run reference tests run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true + - name: flag failed reference tests + uses: myrotvorets/set-commit-status-action@v2.0.0 + if: failure() + with: + status: ${{ job.status }} + context: reference-tested + sha: ${{ github.event.pull_request.head.sha || github.sha }} + description: fail reference test batch ${{matrix.runner_index}} - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails with: - name: reference-test-node-${{matrix.runner-index}}-results + name: reference-test-node-${{matrix.runner_index}}-results path: '**/build/test-results/referenceTests/TEST-*.xml' - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails with: report_paths: '**/build/test-results/referenceTest/TEST-*.xml' - + consolidateStatus: + runs-on: ubuntu-22.04 + needs: referenceTestEthereum + permissions: + statuses: write + checks: write + steps: + - name: status success + uses: myrotvorets/set-commit-status-action@v2.0.0 + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + status: success + context: reference-tested + description: reference tests passed From 73498ce854df7eed12ca7af457e7804c08881c67 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 14:03:02 -0500 Subject: [PATCH 140/155] integration testing required Signed-off-by: Justin Florentine --- .github/workflows/integration-tests.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7548bc638aa..3cd7926d83b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -29,9 +29,8 @@ jobs: repo: context.repo.repo, ref: RELEVANT_SHA, }); - const unitTested = statuses && statuses.filter(({ context }) => context === 'pre-review'); - const failedUnits = unitTested && unitTested.filter(({state}) => state === 'failure' || state === 'pending'); - const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); + + const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tested'); const alreadyRun = intTested && intTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -39,11 +38,10 @@ jobs: pull_number: context.issue.number, }); const approvingReviews = reviews && reviews.filter(review => review.state === 'APPROVED'); - const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (failedUnits.length === 0) && (approvingReviews.length > 0); + const shouldRun = !alreadyRun && github.actor != 'dependabot[bot]' && (approvingReviews.length > 0); console.log("tests should be run = %j", shouldRun); console.log("alreadyRun = %j", alreadyRun); - console.log("failedUnits = %j", failedUnits.length); console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; @@ -59,6 +57,7 @@ jobs: uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending + context: integration-tested sha: ${{ github.event.pull_request.head.sha || github.sha }} integrationTest: runs-on: ubuntu-22.04 @@ -79,11 +78,11 @@ jobs: uses: gradle/gradle-build-action@v2 - name: run integration tests run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag passed unit tests + - name: flag passed integration tests uses: myrotvorets/set-commit-status-action@v2.0.0 - if: success() with: status: ${{ job.status }} + context: integration-tested sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 From 015521d16d0bd8969aa8557e76dc4236d4674f6b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 14:18:56 -0500 Subject: [PATCH 141/155] testing default status check behavior with a broken test Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 49 ------------------- .../hyperledger/besu/cli/BesuCommandTest.java | 2 +- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 4b6b22b25b9..d93ab7d3853 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -21,26 +21,12 @@ jobs: uses: actions/checkout@v4 - name: Lint Repo run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown - - name: flag repolint failure - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: pre-review - description: repo linting failed gradle-wrapper: name: "Gradle Wrapper Validation" runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1.1.0 - - name: flag gradle wrapper failure - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: pre-review - description: gradlew validation failed spotless: runs-on: ubuntu-22.04 if: ${{ github.actor != 'dependabot[bot]' }} @@ -56,13 +42,6 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: run spotless run: ./gradlew spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag spotless failure - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: pre-review - description: spotless failed compile: runs-on: ubuntu-22.04 timeout-minutes: 30 @@ -79,13 +58,6 @@ jobs: uses: gradle/gradle-build-action@v2.10.0 - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag compile failure - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: pre-review - description: compilation failed unitTests: env: GRADLEW_UNIT_TEST_ARGS: ${{matrix.gradle_args}} @@ -117,30 +89,9 @@ jobs: - name: run unit tests id: unitTest run: ./gradlew $GRADLEW_UNIT_TEST_ARGS -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag failed unit tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: unit-tested - description: unit test shard failed - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the build step fails with: report_paths: '**/test-results/test/TEST-*.xml' annotate_only: true - consolidateStatus: - runs-on: ubuntu-22.04 - needs: unitTests - permissions: - statuses: write - checks: write - steps: - - name: status success - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: success - context: unit-tested - description: unit tests passed - diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 26296556302..e19305824f9 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3898,7 +3898,7 @@ public void futureEipsValuesAreUsed() { verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); verify(mockControllerBuilder).build(); - assertThat(networkArg.getValue()).isEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); + assertThat(networkArg.getValue()).isNotEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From 9949869fe791069eb6edcccd44c22c44405493f9 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 15:06:54 -0500 Subject: [PATCH 142/155] testing default status check behavior with corrected test Signed-off-by: Justin Florentine --- .../src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index e19305824f9..26296556302 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3898,7 +3898,7 @@ public void futureEipsValuesAreUsed() { verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any(), any()); verify(mockControllerBuilder).build(); - assertThat(networkArg.getValue()).isNotEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); + assertThat(networkArg.getValue()).isEqualTo(EthNetworkConfig.getNetworkConfig(FUTURE_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From b0cf8dd2c51d48e695b7514f8a2464705654d21b Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 15:49:04 -0500 Subject: [PATCH 143/155] might only need to manage pending status Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 25 +------------------------ .github/workflows/integration-tests.yml | 9 +-------- .github/workflows/reference-tests.yml | 25 +------------------------ 3 files changed, 3 insertions(+), 56 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index f31395cb50f..ae547782d73 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -31,7 +31,7 @@ jobs: repo: context.repo.repo, ref: RELEVANT_SHA, }); - const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tested'); + const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -61,7 +61,6 @@ jobs: with: status: pending sha: ${{ github.event.pull_request.head.sha || github.sha }} - context: acceptance-tested acceptanceTestEthereum: runs-on: ubuntu-22.04 needs: pend @@ -108,14 +107,6 @@ jobs: run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g' > gradleArgs.txt - name: run acceptance tests run: ./gradlew acceptanceTest `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag failed acceptance tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: acceptance-tested - sha: ${{ github.event.pull_request.head.sha || github.sha }} - description: fail acceptance test batch ${{matrix.runner_index}} - name: cleanup tempfiles run: rm testList.txt gradleArgs.txt - name: Upload Acceptance Test Results @@ -128,17 +119,3 @@ jobs: if: (success() || failure()) # always run even if the build step fails with: report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' - consolidateStatus: - runs-on: ubuntu-22.04 - needs: acceptanceTestEthereum - permissions: - statuses: write - checks: write - steps: - - name: status success - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - status: success - context: acceptance-tested - description: acceptance tests passed \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3cd7926d83b..b2d94e3d5b8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ jobs: ref: RELEVANT_SHA, }); - const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tested'); + const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); const alreadyRun = intTested && intTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -57,7 +57,6 @@ jobs: uses: myrotvorets/set-commit-status-action@v2.0.0 with: status: pending - context: integration-tested sha: ${{ github.event.pull_request.head.sha || github.sha }} integrationTest: runs-on: ubuntu-22.04 @@ -78,12 +77,6 @@ jobs: uses: gradle/gradle-build-action@v2 - name: run integration tests run: ./gradlew integrationTest -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag passed integration tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: ${{ job.status }} - context: integration-tested - sha: ${{ github.event.pull_request.head.sha || github.sha }} - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: (success() || failure()) diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 9e5dd51c8b0..a1be0837362 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -34,7 +34,7 @@ jobs: }); - const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tested'); + const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); const alreadyRun = refTested && refTested.find(({ state }) => state === 'success' || state === 'pending') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -62,7 +62,6 @@ jobs: with: status: pending sha: ${{ github.event.pull_request.head.sha || github.sha }} - context: reference-tested prepareReferenceTestEthereum: runs-on: ubuntu-22.04 needs: shouldRun @@ -137,14 +136,6 @@ jobs: run: echo ${{ steps.split-tests.outputs.paths }} | sed -e 's/^.*java\///' -e 's@/@.@g' -e 's/\.java//' -e 's/^/--tests /' > refTestArgs.txt - name: run reference tests run: ./gradlew ethereum:referenceTests:referenceTests `cat refTestArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true - - name: flag failed reference tests - uses: myrotvorets/set-commit-status-action@v2.0.0 - if: failure() - with: - status: ${{ job.status }} - context: reference-tested - sha: ${{ github.event.pull_request.head.sha || github.sha }} - description: fail reference test batch ${{matrix.runner_index}} - name: Upload Test Report uses: actions/upload-artifact@v3 if: always() # always run even if the previous step fails @@ -156,18 +147,4 @@ jobs: if: success() || failure() # always run even if the build step fails with: report_paths: '**/build/test-results/referenceTest/TEST-*.xml' - consolidateStatus: - runs-on: ubuntu-22.04 - needs: referenceTestEthereum - permissions: - statuses: write - checks: write - steps: - - name: status success - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - status: success - context: reference-tested - description: reference tests passed From 435391b6308a119816bfcddbb1b06c8390a49851 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 17:34:00 -0500 Subject: [PATCH 144/155] lets see if a dummy job after a matrix works Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index d93ab7d3853..c37d256e792 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -67,7 +67,7 @@ jobs: checks: write statuses: write strategy: - fail-fast: false + fail-fast: true matrix: gradle_args: - "test -x besu:test -x consensus:test -x crypto:test -x ethereum:eth:test -x ethereum:api:test -x ethereum:core:test" @@ -95,3 +95,11 @@ jobs: with: report_paths: '**/test-results/test/TEST-*.xml' annotate_only: true + pre-review: + runs-on: ubuntu-22.04 + needs: [unitTests] + permissions: + checks: write + statuses: write + + From 9b50f2f3a2fa8520fe1754c39a47b6dacc2a2d66 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 17:49:23 -0500 Subject: [PATCH 145/155] lets see if a dummy job after a matrix works Signed-off-by: Justin Florentine --- .github/workflows/pre-review.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index c37d256e792..23484171ffc 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -101,5 +101,6 @@ jobs: permissions: checks: write statuses: write - - + steps: + - name: consolidation + run: echo "consolidating statuses" From 59c227c91b8756e5939d5648f802d0c961fcf7d7 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Wed, 10 Jan 2024 18:10:14 -0500 Subject: [PATCH 146/155] intentional test failure Signed-off-by: Justin Florentine --- .../src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 26296556302..3e313722e2f 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3915,7 +3915,7 @@ public void experimentalEipsValuesAreUsed() { verify(mockControllerBuilder).build(); assertThat(networkArg.getValue()) - .isEqualTo(EthNetworkConfig.getNetworkConfig(EXPERIMENTAL_EIPS)); + .isNotEqualTo(EthNetworkConfig.getNetworkConfig(EXPERIMENTAL_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From 884366de584422afc08c957cac1031ddd9718df4 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 10:07:35 -0500 Subject: [PATCH 147/155] needing to name a job, not a workflow after a check is dumb Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 9 +++++++++ .github/workflows/integration-tests.yml | 2 +- .github/workflows/reference-tests.yml | 9 +++++++++ .../java/org/hyperledger/besu/cli/BesuCommandTest.java | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index ae547782d73..fe35d153c01 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -119,3 +119,12 @@ jobs: if: (success() || failure()) # always run even if the build step fails with: report_paths: 'acceptance-tests/tests/build/test-results/acceptanceTest/TEST-*.xml' + acceptance-tests: + runs-on: ubuntu-22.04 + needs: [ acceptanceTestEthereum ] + permissions: + checks: write + statuses: write + steps: + - name: consolidation + run: echo "consolidating statuses" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b2d94e3d5b8..fe52aac9e93 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -58,7 +58,7 @@ jobs: with: status: pending sha: ${{ github.event.pull_request.head.sha || github.sha }} - integrationTest: + integration-tests: runs-on: ubuntu-22.04 needs: pend if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index a1be0837362..24de65c8ff9 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -147,4 +147,13 @@ jobs: if: success() || failure() # always run even if the build step fails with: report_paths: '**/build/test-results/referenceTest/TEST-*.xml' + reference-tests: + runs-on: ubuntu-22.04 + needs: [ referenceTestEthereum ] + permissions: + checks: write + statuses: write + steps: + - name: consolidation + run: echo "consolidating statuses" diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 3e313722e2f..26296556302 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3915,7 +3915,7 @@ public void experimentalEipsValuesAreUsed() { verify(mockControllerBuilder).build(); assertThat(networkArg.getValue()) - .isNotEqualTo(EthNetworkConfig.getNetworkConfig(EXPERIMENTAL_EIPS)); + .isEqualTo(EthNetworkConfig.getNetworkConfig(EXPERIMENTAL_EIPS)); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); From 804fcae7760d507f1bbcb5a7da29ef97ce3b7f65 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 12:02:45 -0500 Subject: [PATCH 148/155] fail fast Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 2 +- .github/workflows/reference-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index fe35d153c01..449d44b9a95 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -70,7 +70,7 @@ jobs: checks: write if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} strategy: - fail-fast: false + fail-fast: true matrix: runner_index: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] steps: diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 24de65c8ff9..802d789ab35 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -96,7 +96,7 @@ jobs: - prepareReferenceTestEthereum if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} strategy: - fail-fast: false + fail-fast: true matrix: runner_index: [0,1,2,3,4,5] steps: From ecca1956e722ebe5eeab78c3a63824044740c8d7 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 14:16:53 -0500 Subject: [PATCH 149/155] doing your own pending means clearing it. yuck Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 17 +---------------- .github/workflows/integration-tests.yml | 15 +-------------- .github/workflows/reference-tests.yml | 16 ++-------------- 3 files changed, 4 insertions(+), 44 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 449d44b9a95..fca52af7e3e 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -32,7 +32,7 @@ jobs: ref: RELEVANT_SHA, }); const acceptanceTested = statuses && statuses.filter(({ context }) => context === 'acceptance-tests'); - const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success' || state === 'pending') > 0; + const alreadyRun = acceptanceTested && acceptanceTested.find(({ state }) => state === 'success') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -46,21 +46,6 @@ jobs: console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; - pend: - name: acceptance testing pending - runs-on: ubuntu-22.04 - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true'}} - permissions: - statuses: write - steps: - - name: debug - run: echo ${{ needs.shouldRun.outputs.shouldRun }} - - name: pending - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: pending - sha: ${{ github.event.pull_request.head.sha || github.sha }} acceptanceTestEthereum: runs-on: ubuntu-22.04 needs: pend diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fe52aac9e93..b14bfce74ae 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,7 +31,7 @@ jobs: }); const intTested = statuses && statuses.filter(({ context }) => context === 'integration-tests'); - const alreadyRun = intTested && intTested.find(({ state }) => state === 'success' || state === 'pending') > 0; + const alreadyRun = intTested && intTested.find(({ state }) => state === 'success') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -45,19 +45,6 @@ jobs: console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; - pend: - name: integration testing pending - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} - runs-on: ubuntu-22.04 - permissions: - statuses: write - steps: - - name: pending - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: pending - sha: ${{ github.event.pull_request.head.sha || github.sha }} integration-tests: runs-on: ubuntu-22.04 needs: pend diff --git a/.github/workflows/reference-tests.yml b/.github/workflows/reference-tests.yml index 802d789ab35..640c3e5de4b 100644 --- a/.github/workflows/reference-tests.yml +++ b/.github/workflows/reference-tests.yml @@ -35,7 +35,7 @@ jobs: const refTested = statuses && statuses.filter(({ context }) => context === 'reference-tests'); - const alreadyRun = refTested && refTested.find(({ state }) => state === 'success' || state === 'pending') > 0; + const alreadyRun = refTested && refTested.find(({ state }) => state === 'success') > 0; const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -49,19 +49,7 @@ jobs: console.log("approvingReviews = %j", approvingReviews.length); return shouldRun; - pend: - name: reference testing pending - needs: shouldRun - if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} - runs-on: ubuntu-22.04 - permissions: - statuses: write - steps: - - name: pending - uses: myrotvorets/set-commit-status-action@v2.0.0 - with: - status: pending - sha: ${{ github.event.pull_request.head.sha || github.sha }} + prepareReferenceTestEthereum: runs-on: ubuntu-22.04 needs: shouldRun From a88a17fd28a2b30171ccb248d8e3e30a138be576 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 14:27:09 -0500 Subject: [PATCH 150/155] can't depend on missing job Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 1 - .github/workflows/integration-tests.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index fca52af7e3e..142c83f9f7a 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -48,7 +48,6 @@ jobs: return shouldRun; acceptanceTestEthereum: runs-on: ubuntu-22.04 - needs: pend name: "Acceptance Runner" permissions: statuses: write diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b14bfce74ae..66abdf006f3 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -47,7 +47,6 @@ jobs: return shouldRun; integration-tests: runs-on: ubuntu-22.04 - needs: pend if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write From 93f4c83256003c524c102035c0ed386121045d23 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 15:16:48 -0500 Subject: [PATCH 151/155] can't switch on an output you're not waiting for Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 1 + .github/workflows/integration-tests.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 142c83f9f7a..de5e46858ba 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -49,6 +49,7 @@ jobs: acceptanceTestEthereum: runs-on: ubuntu-22.04 name: "Acceptance Runner" + needs: shouldRun permissions: statuses: write checks: write diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 66abdf006f3..7ef7e5cfb13 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -47,6 +47,7 @@ jobs: return shouldRun; integration-tests: runs-on: ubuntu-22.04 + needs: shouldRun if: ${{ needs.shouldRun.outputs.shouldRun == 'true' }} permissions: statuses: write From 5c9c8b5b2ea21d53c0b5bcec0cd240c515c4ee51 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Thu, 11 Jan 2024 17:22:13 -0500 Subject: [PATCH 152/155] manifest needs gradle Signed-off-by: Justin Florentine --- .github/workflows/acceptance-tests.yml | 1 - .github/workflows/codeql.yml | 24 +++----- .github/workflows/compile.yml | 29 --------- .github/workflows/docker.yml | 4 ++ .github/workflows/javadoc.yml | 23 -------- .github/workflows/repolinter.yml | 17 ------ .github/workflows/report.yml | 81 -------------------------- .github/workflows/spotless.yml | 20 ------- .github/workflows/unit-tests.yml | 43 -------------- 9 files changed, 12 insertions(+), 230 deletions(-) delete mode 100644 .github/workflows/compile.yml delete mode 100644 .github/workflows/javadoc.yml delete mode 100644 .github/workflows/repolinter.yml delete mode 100644 .github/workflows/report.yml delete mode 100644 .github/workflows/spotless.yml delete mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index de5e46858ba..d7172aee698 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -12,7 +12,6 @@ jobs: shouldRun: name: checks to ensure we should run # necessary because there is no single PR approved event, need to check all comments/approvals/denials - # might also be a job running, and additional approvals runs-on: ubuntu-22.04 outputs: shouldRun: ${{steps.shouldRun.outputs.result}} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d42aa6f368f..28a3fbd64b6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -12,34 +12,25 @@ name: "CodeQL" on: - workflow_call: + push: + branches: + - main jobs: analyze: name: Analyze - runs-on: [besu-research-ubuntu-16] + runs-on: ubuntu-22.04 permissions: actions: read contents: read security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Java uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 - cache: gradle - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 @@ -50,9 +41,10 @@ jobs: # Prefix the list here with "+" to use these queries and those in the config file. queries: security-and-quality,security-extended - # Autobuild failed (OOM) - # Hence, supply memory args for gradle build - - run: | + - name: setup gradle + uses: gradle/gradle-build-action@v2 + - name: compileJava noscan + run: | JAVA_OPTS="-Xmx2048M" ./gradlew --no-scan compileJava - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml deleted file mode 100644 index 1109b779a36..00000000000 --- a/.github/workflows/compile.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: compile -on: - workflow_run: - workflows: [ "checks" ] - types: [ "completed" ] - workflow_dispatch: - - -env: - GRADLE_OPTS: "-Xmx6g" - GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true - -jobs: - compile: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: Setup Gradle - uses: gradle/gradle-build-action@v2.10.0 - - name: Run Build - run: ./gradlew build -x test -x spotlessCheck -Dorg.gradle.parallel=true -Dorg.gradle.caching=true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2947a9ada91..327909a2734 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -87,5 +87,9 @@ jobs: needs: buildDocker runs-on: ubuntu-22.04 steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: setup gradle + uses: gradle/gradle-build-action@v2 - name: multi-arch docker run: ./gradlew manifestDocker -PdockerOrgName=${{secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} diff --git a/.github/workflows/javadoc.yml b/.github/workflows/javadoc.yml deleted file mode 100644 index b6307d97cee..00000000000 --- a/.github/workflows/javadoc.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: javadoc -on: - workflow_call: - -env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - total-runners: 6 - -jobs: - javadoc_17: - runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java 17 - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: javadoc (JDK 17) - run: ./gradlew --no-daemon javadoc diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml deleted file mode 100644 index bfc97b11da6..00000000000 --- a/.github/workflows/repolinter.yml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Hyperledger Repolinter Action -name: repolinter - -on: - workflow_call: - - -jobs: - build: - runs-on: ubuntu-22.04 - container: ghcr.io/todogroup/repolinter:v0.11.2 - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Lint Repo - run: bundle exec /app/bin/repolinter.js --rulesetUrl https://raw.githubusercontent.com/hyperledger-labs/hyperledger-community-management-tools/main/repo_structure/repolint.json --format markdown diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml deleted file mode 100644 index 9849e271835..00000000000 --- a/.github/workflows/report.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: report -on: - workflow_call: - -permissions: - checks: write - -jobs: - unit-tests: - runs-on: ubuntu-22.04 - steps: - - name: Download Test Report - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-test-results - workflow: ${{ github.event.workflow.id }} - run_id: ${{ github.event.workflow_run.id }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - with: - commit: ${{github.event.workflow_run.head_sha}} - report_paths: '**/build/test-results/test/TEST-*.xml' - integration-tests: - runs-on: ubuntu-22.04 - steps: - - name: Download Test Report - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-integration-test-results - workflow: ${{ github.event.workflow.id }} - run_id: ${{ github.event.workflow_run.id }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - with: - commit: ${{github.event.workflow_run.head_sha}} - report_paths: '**/build/test-results/test/TEST-*.xml' - acceptance-tests: - runs-on: ubuntu-22.04 - steps: - - name: Download Test Report - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-acceptance-test-results - workflow: ${{ github.event.workflow.id }} - run_id: ${{ github.event.workflow_run.id }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - with: - commit: ${{github.event.workflow_run.head_sha}} - report_paths: '**/build/test-results/test/TEST-*.xml' - qbft-reference-tests: - runs-on: ubuntu-22.04 - steps: - - name: Download Test Report - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-qbft-reference-test-results - workflow: ${{ github.event.workflow.id }} - run_id: ${{ github.event.workflow_run.id }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - with: - commit: ${{github.event.workflow_run.head_sha}} - report_paths: '**/build/test-results/test/TEST-*.xml' - reference-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} - steps: - - name: Download Test Report - uses: dawidd6/action-download-artifact@v2 - with: - name: junit-reference-${{ matrix.runner-index }}-test-results - workflow: ${{ github.event.workflow.id }} - run_id: ${{ github.event.workflow_run.id }} - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - with: - commit: ${{github.event.workflow_run.head_sha}} - report_paths: '**/build/test-results/test/TEST-*.xml' \ No newline at end of file diff --git a/.github/workflows/spotless.yml b/.github/workflows/spotless.yml deleted file mode 100644 index f1814fdccfa..00000000000 --- a/.github/workflows/spotless.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: spotless -on: - workflow_call: - - -jobs: - spotless: - runs-on: ubuntu-22.04 - if: ${{ github.actor != 'dependabot[bot]' }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: spotless - run: ./gradlew --no-daemon --parallel clean spotlessCheck diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index a65caaed36a..00000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: unit-tests -on: - workflow_run: - workflows: [ "compile" ] - types: [ "completed" ] - -env: - GRADLE_OPTS: "-Xmx6g -Dorg.gradle.daemon=false" - -jobs: - unitTests: - runs-on: besu-research-ubuntu-8 - if: ${{github.event.workflow_run.conclusion == 'success' }} - permissions: - checks: write - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Set up Java - uses: actions/setup-java@v3 - with: - distribution: adopt - java-version: 17 - cache: gradle - - name: Build - uses: gradle/gradle-build-action@v2 - with: - gradle-home-cache-cleanup: true - cache-read-only: false - arguments: build -Dorg.gradle.parallel=true - - name: Upload Test Report - uses: actions/upload-artifact@v3 - if: always() # always run even if the previous step fails - with: - name: junit-xml-reports - path: '**/build/test-results/test/TEST-*.xml' - retention-days: 1 - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the build step fails - with: - report_paths: '**/test-results/test/TEST-*.xml' - annotate_only: true From 0df78ad6d2ba07dec4ccfac6284cfaed6f0ad394 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 12 Jan 2024 13:59:47 -0500 Subject: [PATCH 153/155] you will also need java ser Signed-off-by: Justin Florentine --- .github/workflows/docker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 327909a2734..2ff3204574c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -89,6 +89,11 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v3 + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 - name: multi-arch docker From 4db11bfbdcc30312d398af01a8bbc9803078af33 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 12 Jan 2024 14:44:56 -0500 Subject: [PATCH 154/155] logging into dockerhub Signed-off-by: Justin Florentine --- .github/workflows/docker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2ff3204574c..8458696d100 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -96,5 +96,10 @@ jobs: java-version: 17 - name: setup gradle uses: gradle/gradle-build-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USER_RW }} + password: ${{ secrets.DOCKER_PASSWORD_RW }} - name: multi-arch docker run: ./gradlew manifestDocker -PdockerOrgName=${{secrets.DOCKER_ORG }} -Prelease.releaseVersion=${{ github.ref_name }} From ecc5e1a011bc21844374471ef9213d6379f366c4 Mon Sep 17 00:00:00 2001 From: Justin Florentine Date: Fri, 12 Jan 2024 16:06:27 -0500 Subject: [PATCH 155/155] test windows output Signed-off-by: Justin Florentine --- .github/workflows/artifacts.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 19594e1f14e..36a03371d14 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -45,5 +45,27 @@ jobs: body: | ${{steps.hashes.outputs.tarSha}} ${{steps.hashes.outputs.zipSha}} - + testWindows: + runs-on: windows-2022 + needs: assemble + timeout-minutes: 10 + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: adopt + java-version: 17 + - name: Download zip + uses: actions/download-artifact@v3 + with: + name: besu*.zip + - name: test Besu + run: | + unzip besu-*.zip -d besu-tmp + cd besu-tmp + mv besu-* ../besu + cd .. + besu\bin\besu.bat --help + besu\bin\besu.bat --version