Merge branch 'main' into metrics-base

.github/workflows/golangci-build-test.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - name: Build
         run: go build -v ./cmd/...

.github/workflows/golangci-lint.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.github/workflows/publish.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
 
       - name: Create package
         run: |

Dockerfile

@@ -21,4 +21,4 @@ COPY --from=builder /es-node/run.sh /es-node/
 RUN chmod +x /es-node/run.sh
 WORKDIR /es-node
 
-EXPOSE 9545 9222 30305/udp
+EXPOSE 9545 9222 9222/udp

GUIDE.md

@@ -10,7 +10,7 @@ For a detailed explanation for es-node please consult the [README](/README.md).
 
 - Layer 1: [dencun-devnet-12](https://dencun-devnet-12.ethpandaops.io/)
 - storage-contracts-v1: v0.1.0
-- es-node: v0.1.3
+- es-node: v0.1.4
 
 ## Minimum Hardware Requirements 
 
@@ -19,7 +19,7 @@ Please refer to [this section](/README.md/#minimum-hardware-requirements) for ha
 ## System Environment
  - Ubuntu 20.04+ (tested and verified)
  - (Optional) Docker 24.0.5+ (would simplify the process)
- - (Optional) Go 1.20.* (not compatible with Go 1.21 yet) and Node.js 16+ (can be installed following the [steps](#install-dependencies))
+ - (Optional) Go 1.20+ and Node.js 16+ (can be installed following the [steps](#install-dependencies))
 
 You can choose [the method of running es-node](#options-to-run-es-node) based on your current environment.
 
@@ -51,19 +51,19 @@ Download the pre-built package suitable for your platform:
 
 Linux x86-64 or AMD64:
 ```sh
-curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.3/es-node.v0.1.3.linux-amd64.tar.gz | tar -xz
+curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.4/es-node.v0.1.4.linux-amd64.tar.gz | tar -xz
 ```
 MacOS x86-64 or AMD64:
 ```sh
-curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.3/es-node.v0.1.3.darwin-amd64.tar.gz | tar -xz
+curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.4/es-node.v0.1.4.darwin-amd64.tar.gz | tar -xz
 ```
 MacOS ARM64:
 ```sh
-curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.3/es-node.v0.1.3.darwin-arm64.tar.gz | tar -xz
+curl -L https://github.com/ethstorage/es-node/releases/download/v0.1.4/es-node.v0.1.4.darwin-arm64.tar.gz | tar -xz
 ```
 Run es-node
 ```
-cd es-node.v0.1.3
+cd es-node.v0.1.4
 env ES_NODE_STORAGE_MINER=<miner> ES_NODE_SIGNER_PRIVATE_KEY=<private_key> ./run.sh
 ```
 
@@ -77,9 +77,9 @@ docker run --name es  -d  \
           -e ES_NODE_SIGNER_PRIVATE_KEY=<private_key> \
           -p 9545:9545 \
           -p 9222:9222 \
-          -p 30305:30305/udp \
+          -p 9222:9222/udp \
           --entrypoint /es-node/run.sh \
-          ghcr.io/ethstorage/es-node
+          ghcr.io/ethstorage/es-node:v0.1.4
 ```
 
 You can check docker logs using the following command:
@@ -94,7 +94,7 @@ Download source code and switch to the latest release branch:
 ```sh
 git clone https://github.com/ethstorage/es-node.git
 cd es-node
-git checkout v0.1.3
+git checkout v0.1.4
 ```
 Build es-node:
 ```sh
@@ -123,13 +123,13 @@ _Please note that not all steps in this section are required; they depend on you
 
 ### Install Go
 
-Download a stable Go release, e.g., go1.20.10 (can't be built on go1.21.* yet). 
+Download a stable Go release, e.g., go1.21.4. 
 ```sh
-curl -OL https://golang.org/dl/go1.20.10.linux-amd64.tar.gz
+curl -OL https://golang.org/dl/go1.21.4.linux-amd64.tar.gz
 ```
 Extract and install
 ```sh
-tar -C /usr/local -xf go1.20.10.linux-amd64.tar.gz
+tar -C /usr/local -xf go1.21.4.linux-amd64.tar.gz
 ```
 Update `$PATH`
 ```

README.md

@@ -97,6 +97,9 @@ The full list of options that you can use to configure an es-node are as follows
 |`--p2p.listen.tcp`|TCP port to bind LibP2P to. Any available system port if set to 0.|`9222`||
 |`--p2p.listen.udp`|UDP port to bind Discv5 to. Same as TCP port if left 0.|`0`||
 |`--p2p.nat`|Enable NAT traversal with PMP/UPNP devices to learn external IP.|||
+|`--p2p.max.request.size`|max request size is the maximum number of bytes to request from a remote peer. It is value should not larger than 8 * 1024 * 1024. if you have good network condition, you can increase the max request size to improve the sync performance.|1048576||
+|`--p2p.sync.concurrency`|sync concurrency is the number of chunks to split a shard into to allow concurrent retrievals.|16||
+|`--p2p.fill-empty.concurrency`|fill empty concurrency is the number of threads to concurrently fill encoded empty blobs.|NumCPU - 2||
 |`--p2p.no-discovery`|Disable Discv5 (node discovery)|||
 |`--p2p.peers.grace`|Grace period to keep a newly connected peer around, if it is not misbehaving.|`30s`||
 |`--p2p.peers.hi`|High-tide peer count. The node starts pruning peer connections slowly after reaching this number.|`30`||

cmd/es-node/main.go

@@ -29,7 +29,7 @@ import (
 var (
 	GitCommit     = ""
 	GitDate       = ""
-	Version       = "v0.1.3"
+	Version       = "v0.1.4"
 	Meta          = "dev"
 	BuildTime     = ""
 	systemVersion = fmt.Sprintf("%s/%s", runtime.GOARCH, runtime.GOOS)

cmd/es-utils/utils/utils.go

@@ -320,10 +320,10 @@ func UploadBlobs(
 		needEncoding,
 		-1,
 		value,
-		1000000,
+		5000000,
 		"",
-		"400000000",
-		"4000000000",
+		"",
+		"300000000",
 		chainID,
 		calldata,
 	)

cmd/es-utils/utils/utils_test.go

@@ -1,14 +1,11 @@
 // Copyright 2022-2023, EthStorage.
 // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE
 
-//go:build !ci
-
 package utils
 
 import (
 	"bytes"
-	"io"
-	"net/http"
+	"os"
 	"testing"
 )
 
@@ -40,12 +37,14 @@ func generateSequentialBytes(t *testing.T, n int) []byte {
 }
 
 func readTxt(t *testing.T, n int) []byte {
-	resp, err := http.Get("https://www.gutenberg.org/cache/epub/11/pg11.txt")
+	f, err := os.Open("../blob_8k.dat")
 	if err != nil {
-		t.Fatal("read txt error")
+		t.Fatal("read file error")
 	}
-	defer resp.Body.Close()
-	data, err := io.ReadAll(resp.Body)
+	defer f.Close()
+
+	data := make([]byte, 1024)
+	_, err = f.Read(data)
 	if err != nil {
 		t.Fatal("read txt error")
 	}

docker-compose.yaml

@@ -5,7 +5,7 @@ services:
     ports:
       - '9545:9545'
       - '9222:9222'
-      - '30305:30305/udp'
+      - '9222:9222/udp'
     volumes:
       - ./es-data:/es-node/es-data
     environment:

ethstorage/flags/p2p_flags.go

@@ -161,14 +161,22 @@ var (
 		Value:    1 * 1024 * 1024,
 		EnvVar:   p2pEnv("MAX_REQUEST_SIZE"),
 	}
-	MaxConcurrency = cli.Uint64Flag{
-		Name: "p2p.max.concurrency",
-		Usage: "max concurrency is the number of chunks to split a shard into to allow concurrent retrievals. " +
+	SyncConcurrency = cli.Uint64Flag{
+		Name: "p2p.sync.concurrency",
+		Usage: "sync concurrency is the number of chunks to split a shard into to allow concurrent retrievals. " +
 			"The default value is 16, the min value is 1.",
 		Required: false,
 		Value:    16,
 		EnvVar:   p2pEnv("MAX_CONCURRENCY"),
 	}
+	FillEmptyConcurrency = cli.IntFlag{
+		Name: "p2p.fill-empty.concurrency",
+		Usage: "fill empty concurrency is the number of threads to concurrently fill encoded empty blobs. " +
+			"The default value is NumCPU - 2 and the min value is 1.",
+		Required: false,
+		Value:    0,
+		EnvVar:   p2pEnv("Fill_Empty_Concurrency"),
+	}
 	MetaDownloadBatchSize = cli.Uint64Flag{
 		Name:     "p2p.meta.download.batch",
 		Usage:    "Batch size for requesting the blob metadatas stored in the storage contract in one RPC call.",
@@ -338,7 +346,8 @@ var p2pFlags = []cli.Flag{
 	HostMux,
 	HostSecurity,
 	MaxRequestSize,
-	MaxConcurrency,
+	SyncConcurrency,
+	FillEmptyConcurrency,
 	MetaDownloadBatchSize,
 	PeersLo,
 	PeersHi,

ethstorage/metrics/metrics.go

@@ -10,7 +10,7 @@ import (
 	"strconv"
 	"time"
 
-	ophttp "github.com/ethereum-optimism/optimism/op-node/http"
+	ophttp "github.com/ethereum-optimism/optimism/op-service/httputil"
 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 	pb "github.com/libp2p/go-libp2p-pubsub/pb"
 	libp2pmetrics "github.com/libp2p/go-libp2p/core/metrics"

ethstorage/miner/l1_mining_api.go

@@ -222,7 +222,7 @@ func (m *l1MiningAPI) estimateReward(ctx context.Context, cfg Config, contract c
 	}
 
 	minedTs := curBlock.Time - (new(big.Int).Sub(curBlock.Number, block).Uint64())*12
-	var reward *big.Int
+	reward := big.NewInt(0)
 	if shard < lastShard {
 		basePayment := new(big.Int).Mul(cfg.StorageCost, new(big.Int).SetUint64(cfg.ShardEntry))
 		reward = paymentIn(basePayment, cfg.DcfFactor, lastMineTime, minedTs, cfg.StartTime)
@@ -236,7 +236,7 @@ func (m *l1MiningAPI) estimateReward(ctx context.Context, cfg Config, contract c
 		}
 	}
 	minerReward := new(big.Int).Div(
-		new(big.Int).Mul(new(big.Int).SetUint64((rewardDenominator-cfg.TreasuryShare)), reward),
+		new(big.Int).Mul(new(big.Int).SetUint64(rewardDenominator-cfg.TreasuryShare), reward),
 		new(big.Int).SetUint64(rewardDenominator),
 	)
 	return minerReward, nil