Merge remote-tracking branch 'upstream/master' into pavel/p2p-peerkey…

…-gen

.circleci/config.yml

@@ -715,7 +715,7 @@ commands:
             command: |
               if [ "${CIRCLE_BRANCH}" = "rel/nightly" ]
               then
-                export NO_BUILD="true"
+                export NIGHTLY_BUILD="true"
               fi
               export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g')
               export GOPATH="<< parameters.build_dir >>/go"

Makefile

@@ -179,19 +179,24 @@ ifeq ($(OS_TYPE),darwin)
 	mkdir -p $(GOPATH1)/bin-darwin-arm64
 	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
 
+	# same for buildsrc-special
+	cd tools/block-generator && \
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
 	# lipo together
-	mkdir -p $(GOPATH1)/bin-darwin-universal
+	mkdir -p $(GOPATH1)/bin
 	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
 		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
-			lipo -create -output $(GOPATH1)/bin-darwin-universal/$$binary \
+			lipo -create -output $(GOPATH1)/bin/$$binary \
 			$(GOPATH1)/bin-darwin-arm64/$$binary \
 			$(GOPATH1)/bin-darwin-amd64/$$binary; \
 		else \
 			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
 		fi \
 	done
 else
-	$(error OS_TYPE must be darwin for universal builds)
+	echo "OS_TYPE must be darwin for universal builds, skipping"
 endif
 
 deps:

catchup/fetcher_test.go

@@ -141,6 +141,13 @@ func (b *basicRPCNode) RegisterHTTPHandler(path string, handler http.Handler) {
 	b.rmux.Handle(path, handler)
 }
 
+func (b *basicRPCNode) RegisterHTTPHandlerFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	if b.rmux == nil {
+		b.rmux = mux.NewRouter()
+	}
+	b.rmux.HandleFunc(path, handler)
+}
+
 func (b *basicRPCNode) RegisterHandlers(dispatch []network.TaggedMessageHandler) {
 }
 

catchup/pref_test.go

@@ -50,7 +50,7 @@ func BenchmarkServiceFetchBlocks(b *testing.B) {
 	net := &httpTestPeerSource{}
 	ls := rpcs.MakeBlockService(logging.TestingLog(b), config.GetDefaultLocal(), remote, net, "test genesisID")
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()

catchup/service_test.go

@@ -151,7 +151,7 @@ func TestServiceFetchBlocksSameRange(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -223,7 +223,7 @@ func TestSyncRound(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -313,7 +313,7 @@ func TestPeriodicSync(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -379,7 +379,7 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -443,7 +443,7 @@ func TestAbruptWrites(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -501,7 +501,7 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -555,7 +555,7 @@ func TestServiceFetchBlocksMalformed(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -709,7 +709,7 @@ func helperTestOnSwitchToUnSupportedProtocol(
 	ls := rpcs.MakeBlockService(logging.Base(), config, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -932,7 +932,7 @@ func TestCatchupUnmatchedCertificate(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -1064,7 +1064,7 @@ func TestServiceLedgerUnavailable(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()
@@ -1110,7 +1110,7 @@ func TestServiceNoBlockForRound(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()

catchup/universalFetcher_test.go

@@ -101,7 +101,7 @@ func TestUGetBlockHTTP(t *testing.T) {
 	ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, ledger, net, "test genesisID")
 
 	nodeA := basicRPCNode{}
-	nodeA.RegisterHTTPHandler(rpcs.BlockServiceBlockPath, ls)
+	ls.RegisterHandlers(&nodeA)
 	nodeA.start()
 	defer nodeA.stop()
 	rootURL := nodeA.rootURL()

cmd/updater/update.sh

@@ -185,10 +185,6 @@ function get_updater_url() {
         UNAME=$(uname -m)
         if [[ "${UNAME}" = "x86_64" ]]; then
             ARCH="amd64"
-        elif [[ "${UNAME}" = "armv6l" ]]; then
-            ARCH="arm"
-        elif [[ "${UNAME}" = "armv7l" ]]; then
-            ARCH="arm"
         elif [[ "${UNAME}" = "aarch64" ]]; then
             ARCH="arm64"
         else

components/mocks/mockNetwork.go

@@ -103,6 +103,10 @@ func (network *MockNetwork) ClearProcessors() {
 func (network *MockNetwork) RegisterHTTPHandler(path string, handler http.Handler) {
 }
 
+// RegisterHTTPHandlerFunc - empty implementation
+func (network *MockNetwork) RegisterHTTPHandlerFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+}
+
 // OnNetworkAdvance - empty implementation
 func (network *MockNetwork) OnNetworkAdvance() {}
 

data/appRateLimiter.go

@@ -81,8 +81,8 @@ func makeAppRateLimiter(maxCacheSize int, maxAppPeerRate uint64, serviceRateWind
 	serviceRatePerWindow := maxAppPeerRate * uint64(serviceRateWindow/time.Second)
 	maxBucketSize := maxCacheSize / numBuckets
 	if maxBucketSize == 0 {
-		// got the max size less then buckets, use maps of 1
-		maxBucketSize = 1
+		// got the max size less then buckets, use maps of 2 to avoid eviction on each insert
+		maxBucketSize = 2
 	}
 	r := &appRateLimiter{
 		maxBucketSize:        maxBucketSize,

data/appRateLimiter_test.go

@@ -40,7 +40,7 @@ func TestAppRateLimiter_Make(t *testing.T) {
 	window := 1 * time.Second
 	rm := makeAppRateLimiter(10, rate, window)
 
-	require.Equal(t, 1, rm.maxBucketSize)
+	require.Equal(t, 2, rm.maxBucketSize)
 	require.NotEmpty(t, rm.seed)
 	require.NotEmpty(t, rm.salt)
 	for i := 0; i < len(rm.buckets); i++ {

data/txHandler_test.go

@@ -2515,8 +2515,14 @@ func TestTxHandlerAppRateLimiterERLEnabled(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	t.Parallel()
 
+	// technically we don't need any users for this test
+	// but we need to create the genesis accounts to prevent this warning:
+	// "cannot start evaluator: overflowed subtracting rewards for block 1"
+	_, _, genesis := makeTestGenesisAccounts(t, 0)
+	genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
 	ledgerName := fmt.Sprintf("%s-mem", t.Name())
 	const inMem = true
+
 	log := logging.TestingLog(t)
 	log.SetLevel(logging.Panic)
 
@@ -2525,11 +2531,9 @@ func TestTxHandlerAppRateLimiterERLEnabled(t *testing.T) {
 	cfg.TxBacklogServiceRateWindowSeconds = 1
 	cfg.TxBacklogAppTxPerSecondRate = 3
 	cfg.TxBacklogSize = 3
-	ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, bookkeeping.GenesisBalances{}, genesisID, genesisHash, cfg)
+	l, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, cfg)
 	require.NoError(t, err)
-	defer ledger.Close()
-
-	l := ledger
+	defer l.Close()
 
 	func() {
 		cfg.EnableTxBacklogRateLimiting = false
@@ -2618,9 +2622,10 @@ func TestTxHandlerAppRateLimiterERLEnabled(t *testing.T) {
 	require.Equal(t, 1, handler.appLimiter.len())
 }
 
+// TestTxHandlerAppRateLimiter submits few app txns to make the app rate limit to filter one the last txn
+// to ensure it is propely integrated with the txHandler
 func TestTxHandlerAppRateLimiter(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	t.Parallel()
 
 	const numUsers = 10
 	log := logging.TestingLog(t)
@@ -2637,16 +2642,16 @@ func TestTxHandlerAppRateLimiter(t *testing.T) {
 	cfg.TxBacklogAppTxRateLimiterMaxSize = 100
 	cfg.TxBacklogServiceRateWindowSeconds = 1
 	cfg.TxBacklogAppTxPerSecondRate = 3
-	ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, cfg)
+	l, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, cfg)
 	require.NoError(t, err)
-	defer ledger.Close()
+	defer l.Close()
 
-	l := ledger
 	handler, err := makeTestTxHandler(l, cfg)
 	require.NoError(t, err)
 	defer handler.txVerificationPool.Shutdown()
 	defer close(handler.streamVerifierDropped)
 
+	handler.appLimiterBacklogThreshold = -1 // force the rate limiter to start checking transactions
 	tx := transactions.Transaction{
 		Type: protocol.ApplicationCallTx,
 		Header: transactions.Header{
@@ -2667,21 +2672,21 @@ func TestTxHandlerAppRateLimiter(t *testing.T) {
 	require.Equal(t, network.OutgoingMessage{Action: network.Ignore}, action)
 	require.Equal(t, 1, len(handler.backlogQueue))
 
+	counterBefore := transactionMessagesAppLimiterDrop.GetUint64Value()
 	// trigger the rate limiter and ensure the txn is ignored
-	tx2 := tx
-	for i := 0; i < cfg.TxBacklogAppTxPerSecondRate*cfg.TxBacklogServiceRateWindowSeconds; i++ {
-		tx2.ForeignApps = append(tx2.ForeignApps, 1)
+	numTxnToTriggerARL := cfg.TxBacklogAppTxPerSecondRate * cfg.TxBacklogServiceRateWindowSeconds
+	for i := 0; i < numTxnToTriggerARL; i++ {
+		tx2 := tx
+		tx2.Header.Sender = addresses[i+1]
+		signedTx2 := tx2.Sign(secrets[i+1])
+		blob2 := protocol.Encode(&signedTx2)
+
+		action = handler.processIncomingTxn(network.IncomingMessage{Data: blob2, Sender: mockSender{}})
+		require.Equal(t, network.OutgoingMessage{Action: network.Ignore}, action)
 	}
-	signedTx2 := tx.Sign(secrets[1])
-	blob2 := protocol.Encode(&signedTx2)
-
-	action = handler.processIncomingTxn(network.IncomingMessage{Data: blob2, Sender: mockSender{}})
-	require.Equal(t, network.OutgoingMessage{Action: network.Ignore}, action)
-	require.Equal(t, 1, len(handler.backlogQueue))
-
-	// backlogQueue has the first txn, but the second one is dropped
-	msg := <-handler.backlogQueue
-	require.Equal(t, msg.rawmsg.Data, blob, blob)
+	// last txn should be dropped
+	require.Equal(t, 1+numTxnToTriggerARL-1, len(handler.backlogQueue))
+	require.Equal(t, counterBefore+1, transactionMessagesAppLimiterDrop.GetUint64Value())
 }
 
 // TestTxHandlerCapGuard checks there is no cap guard leak in case of invalid input.

docker/build/cicd.centos8.Dockerfile → docker/build/cicd.centos9.Dockerfile

@@ -1,18 +1,13 @@
 ARG ARCH="amd64"
 
-FROM quay.io/centos/centos:stream8
+FROM quay.io/centos/centos:stream9
 ARG GOLANG_VERSION
 ARG ARCH="amd64"
-RUN dnf update rpm -y && \
-    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+RUN dnf install -y epel-release epel-next-release && dnf config-manager --set-enabled crb && \
     dnf update -y && \
     dnf install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel expect jq \
     libtool gcc-c++ libstdc++-devel rpmdevtools createrepo rpm-sign bzip2 which \
-    libffi-devel openssl-devel
-RUN dnf install -y epel-release && \
-    dnf update && \
-    dnf -y --enablerepo=powertools install libstdc++-static && \
-    dnf -y install make
+    libffi-devel openssl-devel libstdc++-static
 RUN echo "${BOLD}Downloading and installing binaries...${RESET}" && \
     curl -Of https://shellcheck.storage.googleapis.com/shellcheck-v0.7.0.linux.x86_64.tar.xz && \
     tar -C /usr/local/bin/ -xf shellcheck-v0.7.0.linux.x86_64.tar.xz --no-anchored 'shellcheck' --strip=1

network/gossipNode.go

@@ -57,8 +57,9 @@ type GossipNode interface {
 	Disconnect(badnode DisconnectablePeer)
 	DisconnectPeers() // only used by testing
 
-	// RegisterHTTPHandler path accepts gorilla/mux path annotations
+	// RegisterHTTPHandler and RegisterHTTPHandlerFunc: path accepts gorilla/mux path annotations
 	RegisterHTTPHandler(path string, handler http.Handler)
+	RegisterHTTPHandlerFunc(path string, handler func(http.ResponseWriter, *http.Request))
 
 	// RequestConnectOutgoing asks the system to actually connect to peers.
 	// `replace` optionally drops existing connections before making new ones.

network/hybridNetwork.go

@@ -146,6 +146,12 @@ func (n *HybridP2PNetwork) RegisterHTTPHandler(path string, handler http.Handler
 	n.wsNetwork.RegisterHTTPHandler(path, handler)
 }
 
+// RegisterHTTPHandlerFunc implements GossipNode
+func (n *HybridP2PNetwork) RegisterHTTPHandlerFunc(path string, handlerFunc func(http.ResponseWriter, *http.Request)) {
+	n.p2pNetwork.RegisterHTTPHandlerFunc(path, handlerFunc)
+	n.wsNetwork.RegisterHTTPHandlerFunc(path, handlerFunc)
+}
+
 // RequestConnectOutgoing implements GossipNode
 func (n *HybridP2PNetwork) RequestConnectOutgoing(replace bool, quit <-chan struct{}) {}
 

network/p2p/http.go

@@ -57,6 +57,14 @@ func (s *HTTPServer) RegisterHTTPHandler(path string, handler http.Handler) {
 	})
 }
 
+// RegisterHTTPHandlerFunc registers a http handler with a given path.
+func (s *HTTPServer) RegisterHTTPHandlerFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	s.p2phttpMux.HandleFunc(path, handler)
+	s.p2phttpMuxRegistrarOnce.Do(func() {
+		s.Host.SetHTTPHandlerAtPath(algorandP2pHTTPProtocol, "/", s.p2phttpMux)
+	})
+}
+
 // MakeHTTPClient creates a http.Client that uses libp2p transport for a given protocol and peer address.
 func MakeHTTPClient(addrInfo *peer.AddrInfo) (*http.Client, error) {
 	clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs)