Merge remote-tracking branch 'upstream/master' into parallelizable-te…

…st-txn

.aptly.conf

@@ -1,5 +1,4 @@
 {
-  "rootDir": "/root/aptly",
   "downloadConcurrency": 4,
   "downloadSpeedLimit": 0,
   "architectures": [],
@@ -27,4 +26,3 @@
   },
   "SwiftPublishEndpoints": {}
 }
-

.circleci/config.yml

@@ -45,30 +45,18 @@ executors:
     machine:
       image: << pipeline.parameters.ubuntu_image >>
     resource_class: arm.large
-  mac_amd64_medium:
+  mac_arm64_medium:
     macos:
       xcode: 14.2.0
-    resource_class: macos.x86.medium.gen2
+    resource_class: macos.m1.medium.gen1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
-  mac_amd64_large:
+  mac_arm64_large:
     macos:
       xcode: 14.2.0
-    # Since they removed the large class for amd64, we will use medium here too.
-    resource_class: macos.x86.medium.gen2
+    resource_class: macos.m1.large.gen1
     environment:
       HOMEBREW_NO_AUTO_UPDATE: "true"
-  mac_arm64: &executor-mac-arm64
-    machine: true
-    resource_class: algorand/macstadium-m1
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: "true"
-  # these are required b/c jobs explicitly assign sizes to the executors
-  # for `mac_arm64` there is only one size
-  mac_arm64_medium:
-    <<: *executor-mac-arm64
-  mac_arm64_large:
-    <<: *executor-mac-arm64
 
 slack-fail-stop-step: &slack-fail-post-step
   post-steps:
@@ -86,7 +74,7 @@ workflows:
           name: << matrix.platform >>_build_nightly
           matrix: &matrix-nightly
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
+              platform: ["amd64", "arm64", "mac_arm64"]
           filters: &filters-nightly
             branches:
               only:
@@ -137,7 +125,7 @@ workflows:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
           matrix:
             parameters:
-              platform: ["amd64", "arm64", "mac_amd64", "mac_arm64"]
+              platform: ["amd64", "arm64", "mac_arm64"]
               job_type: ["test_nightly", "integration_nightly", "e2e_expect_nightly"]
           requires:
             - << matrix.platform >>_<< matrix.job_type >>
@@ -303,7 +291,7 @@ jobs:
     parameters:
       platform:
         type: string
-    executor: << parameters.platform >>_medium
+    executor: << parameters.platform >>_large
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 10
     environment:
@@ -319,7 +307,7 @@ jobs:
     parameters:
       platform:
         type: string
-    executor: << parameters.platform >>_medium
+    executor: << parameters.platform >>_large
     working_directory: << pipeline.parameters.build_dir >>/project
     parallelism: 2
     environment:
@@ -727,12 +715,13 @@ commands:
             command: |
               if [ "${CIRCLE_BRANCH}" = "rel/nightly" ]
               then
-                export NO_BUILD="true"
+                export NIGHTLY_BUILD="true"
               fi
               export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g')
               export GOPATH="<< parameters.build_dir >>/go"
               export TRAVIS_BRANCH=${CIRCLE_BRANCH}
               scripts/travis/deploy_packages.sh
+            no_output_timeout: 20m
         - when:
             condition:
               equal: [ "amd64", << parameters.platform >> ]

.github/workflows/benchmarks.yml

@@ -17,7 +17,7 @@ jobs:
     name: Performance regression check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
       - uses: actions/[email protected]
         with:
           go-version-file: 'go.mod'

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
           update: true
           path-type: inherit
       - name: Check out code
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Determine Go version

.github/workflows/codegen_verification.yml

@@ -7,15 +7,22 @@ on:
 jobs:
   codegen_verification:
     runs-on: ubuntu-20.04
+    services:
+      converter:
+        image: swaggerapi/swagger-converter@sha256:dcfd1c2537f5f271cb4ec942d08aa59ca41b9a24078040061a772afca7e548ae # v1.0.4
+        ports:
+          - 8080:8080
     steps:
       - name: Check out code
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           path: go-algorand
       - name: Uninstall existing go installation
         run: sudo apt-get -y -q purge golang-go
       - name: Run codegen_verification.sh
+        env:
+          SWAGGER_CONVERTER_API: "http://localhost:8080"
         run: |
           export GOPATH="${GITHUB_WORKSPACE}/go"
           cd go-algorand

.github/workflows/container.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
 
       - name: Generate Container Metadata
         id: meta
@@ -59,7 +59,7 @@ jobs:
   #   if: github.ref == format('refs/heads/{0}', 'master')
   #   steps:
   #     - name: Checkout Code
-  #       uses: actions/checkout@v3.5.3
+  #       uses: actions/checkout@v4
 
   #     - name: Update DockerHub Repository Description
   #       uses: peter-evans/dockerhub-description@v3

.github/workflows/reviewdog.yml

@@ -10,17 +10,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
         # move go out of the way temporarily to avoid "go list ./..." from installing modules
       - name: Make libsodium.a
         run: sudo mv /usr/bin/go /usr/bin/go.bak && make crypto/libs/linux/amd64/lib/libsodium.a && sudo mv /usr/bin/go.bak /usr/bin/go
       - name: reviewdog-golangci-lint
-        uses: reviewdog/action-golangci-lint@v2.3.1
+        uses: reviewdog/action-golangci-lint@v2.6.1
         with:
           go_version_file: go.mod
-          golangci_lint_version: "v1.53.2"
+          golangci_lint_version: "v1.58.0"
           golangci_lint_flags: "-c .golangci.yml --allow-parallel-runners"
           reporter: "github-pr-check"
           tool_name: "Lint Errors"
@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
       # move go out of the way temporarily to avoid "go list ./..." from installing modules
@@ -63,13 +63,13 @@ jobs:
         run: |
           cd cicdtmp/golangci-lint
           git clone https://github.com/golangci/golangci-lint.git .
-          git checkout tags/v1.53.2
+          git checkout tags/v1.58.0
           CGO_ENABLED=true go build -trimpath -o golangci-lint-cgo ./cmd/golangci-lint
           ./golangci-lint-cgo --version
           cd ../../
       - name: Install reviewdog
         run: |
-          curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/v0.14.1/install.sh | sh -s
+          curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/v0.18.1/install.sh | sh -s -- v0.18.1
           reviewdog --version
       - name: Build custom linters
         run: |
@@ -102,3 +102,16 @@ jobs:
         run: |
           curl -X POST --data-urlencode "payload={\"text\": \"Reviewdog failed. ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \"}" $SLACK_WEBHOOK
         if: ${{ failure() && (contains(github.ref_name, 'rel/nightly') || contains(github.ref_name, 'rel/beta') || contains(github.ref_name, 'rel/stable') || contains(github.ref_name, 'master')) }}
+  reviewdog-shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: shellcheck
+        uses: reviewdog/action-shellcheck@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          reporter: "github-pr-check"
+          shellcheck_flags: "-e SC2034,SC2046,SC2053,SC2207,SC2145 -S warning"
+          fail_on_error: true
+          path: |
+            test/scripts/e2e_subs

.github/workflows/tools.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
       # move go out of the way temporarily to avoid "go list ./..." from installing modules
       - name: Make libsodium.a
         run: sudo mv /usr/bin/go /usr/bin/go.bak && make crypto/libs/linux/amd64/lib/libsodium.a && sudo mv /usr/bin/go.bak /usr/bin/go

.golangci.yml

@@ -36,7 +36,9 @@ linters-settings:
     - (*github.com/spf13/pflag.FlagSet).MarkDeprecated
     - (*github.com/spf13/pflag.FlagSet).MarkShorthandDeprecated
   govet:
-    check-shadowing: true
+    # Enables these linters in addition to the default ones.
+    enable:
+        - shadow
     settings:
       shadow:
         # explanation of strict vs non-strict:
@@ -131,6 +133,10 @@ issues:
       linters:
         - staticcheck
       text: "SA4006: this value" # of X is never used
+    - path: _test\.go
+      linters:
+        - revive
+      text: "dot-imports: should not use dot imports"
     - linters:
         - staticcheck
       text: "SA1019: rand*"

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:20.04 as builder
 
-ARG GO_VERSION="1.20.7"
+ARG GO_VERSION="1.21.10"
 
 ARG CHANNEL
 ARG URL
@@ -41,12 +41,12 @@ RUN /dist/files/build/install.sh \
     -b "${BRANCH}" \
     -s "${SHA}"
 
-FROM debian:bookworm-20230703-slim as final
+FROM debian:bookworm-20240311-slim as final
 
 ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" KMD_PORT="7833" ALGORAND_DATA="/algod/data"
 
 # curl is needed to lookup the fast catchup url
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl gosu && \
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \
     update-ca-certificates && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \

Makefile

@@ -11,6 +11,13 @@ endif
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
+# overrides for cross-compiling platform-specific binaries
+ifdef CROSS_COMPILE_ARCH
+  ARCH := $(CROSS_COMPILE_ARCH)
+  GO_INSTALL := CGO_ENABLED=1 GOOS=$(OS_TYPE) GOARCH=$(ARCH) go build -o $(GOPATH1)/bin-$(OS_TYPE)-$(ARCH)
+else
+  GO_INSTALL := go install
+endif
 S3_RELEASE_BUCKET = $$S3_RELEASE_BUCKET
 
 GOLANG_VERSIONS				:= $(shell ./scripts/get_golang_version.sh all)
@@ -42,8 +49,13 @@ else
 export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
 endif
 
-# M1 Mac--homebrew install location in /opt/homebrew
 ifeq ($(OS_TYPE), darwin)
+# For Xcode >= 15, set -no_warn_duplicate_libraries linker option
+CLANG_MAJOR_VERSION := $(shell clang --version | grep '^Apple clang version ' | awk '{print $$4}' | cut -d. -f1)
+ifeq ($(shell [ $(CLANG_MAJOR_VERSION) -ge 15 ] && echo true), true)
+EXTLDFLAGS := -Wl,-no_warn_duplicate_libraries
+endif
+# M1 Mac--homebrew install location in /opt/homebrew
 ifeq ($(ARCH), arm64)
 export CPATH=/opt/homebrew/include
 export LIBRARY_PATH=/opt/homebrew/lib
@@ -102,6 +114,9 @@ fix: build
 lint: deps
 	$(GOPATH1)/bin/golangci-lint run -c .golangci.yml
 
+expectlint:
+	cd test/e2e-go/cli/goal/expect && python3 expect_linter.py *.exp
+
 check_go_version:
 	@if [ $(CURRENT_GO_VERSION_MAJOR) != $(GOLANG_VERSION_BUILD_MAJOR) ]; then \
 		echo "Wrong major version of Go installed ($(CURRENT_GO_VERSION_MAJOR)). Please use $(GOLANG_VERSION_BUILD_MAJOR)"; \
@@ -153,10 +168,40 @@ crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a:
 	cp -R crypto/libsodium-fork/. crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork
 	cd crypto/copies/$(OS_TYPE)/$(ARCH)/libsodium-fork && \
 		./autogen.sh --prefix $(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH) && \
-		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" && \
+		./configure --disable-shared --prefix="$(SRCPATH)/crypto/libs/$(OS_TYPE)/$(ARCH)" $(EXTRA_CONFIGURE_FLAGS) && \
 		$(MAKE) && \
 		$(MAKE) install
 
+universal:
+ifeq ($(OS_TYPE),darwin)
+	# build amd64 Mac binaries
+	mkdir -p $(GOPATH1)/bin-darwin-amd64
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+
+	# build arm64 Mac binaries
+	mkdir -p $(GOPATH1)/bin-darwin-arm64
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
+	# same for buildsrc-special
+	cd tools/block-generator && \
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
+	# lipo together
+	mkdir -p $(GOPATH1)/bin
+	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
+		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
+			lipo -create -output $(GOPATH1)/bin/$$binary \
+			$(GOPATH1)/bin-darwin-arm64/$$binary \
+			$(GOPATH1)/bin-darwin-amd64/$$binary; \
+		else \
+			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
+		fi \
+	done
+else
+	echo "OS_TYPE must be darwin for universal builds, skipping"
+endif
+
 deps:
 	./scripts/check_deps.sh
 
@@ -212,11 +257,11 @@ ${GOCACHE}/file.txt:
 	touch "${GOCACHE}"/file.txt
 
 buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN ${GOCACHE}/file.txt
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	$(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 buildsrc-special:
 	cd tools/block-generator && \
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	$(GO_INSTALL) $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 check-go-version:
 	./scripts/check_golang_version.sh build
@@ -331,7 +376,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger
+.PHONY: default fmt lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_kmd_swagger universal
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

README.md

@@ -96,7 +96,7 @@ Please refer to our [CONTRIBUTING](CONTRIBUTING.md) document.
 
 ## Project Layout
 
-`go-algorand` is split into various subsystems containing varius packages.
+`go-algorand` is split into various subsystems containing various packages.
 
 ### Core
 
@@ -199,6 +199,6 @@ Provides utilities for the various components:
 
 Please see the [COPYING_FAQ](COPYING_FAQ) for details about how to apply our license.
 
-Copyright (C) 2019-2023, Algorand Inc.
+Copyright (C) 2019-2024, Algorand Inc.
 
 [developer site url]: https://developer.algorand.org/