indexer cleanup

update version Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer update version Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer Addressing issue coinbase#41 - Allow max concurrency limits configuration on the indexer
smeyerhot · Dec 24, 2020 · b73d1ad · b73d1ad
1 parent d415efb
commit b73d1ad
Show file tree

Hide file tree

Showing 13 changed files with 238 additions and 55 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
 rosetta-bitcoin
 bitcoin-data
 cli-data
+rosetta-sdk-go
+container.log
+.DS_Store
diff --git a/Dockerfile b/Dockerfile
@@ -18,7 +18,7 @@ FROM ubuntu:18.04 as bitcoind-builder
 RUN mkdir -p /app \
   && chown -R nobody:nogroup /app
 WORKDIR /app
-
+COPY ./rosetta-sdk-go/* /app/
 # Source: https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md#ubuntu--debian
 RUN apt-get update && apt-get install -y make gcc g++ autoconf autotools-dev bsdmainutils build-essential git libboost-all-dev \
   libcurl4-openssl-dev libdb++-dev libevent-dev libssl-dev libtool pkg-config python python-pip libzmq3-dev wget
@@ -48,6 +48,7 @@ ENV GOLANG_VERSION 1.15.5
 ENV GOLANG_DOWNLOAD_SHA256 9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d
 ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
 
+
 RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
   && echo "$GOLANG_DOWNLOAD_SHA256  golang.tar.gz" | sha256sum -c - \
   && tar -C /usr/local -xzf golang.tar.gz \
@@ -85,7 +86,6 @@ COPY --from=bitcoind-builder /app/bitcoind /app/bitcoind
 
 # Copy binary from rosetta-builder
 COPY --from=rosetta-builder /app/* /app/
-
 # Set permissions for everything added to /app
 RUN chmod -R 755 /app/*
 

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -0,0 +1,93 @@
+# Copyright 2020 Coinbase, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build bitcoind
+FROM ubuntu:18.04 as bitcoind-builder
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app
+WORKDIR /app
+# Copy the local sdk to your container
+COPY ./rosetta-sdk-go/* /app/
+# Source: https://github.com/bitcoin/bitcoin/blob/master/doc/build-unix.md#ubuntu--debian
+RUN apt-get update && apt-get install -y make gcc g++ autoconf autotools-dev bsdmainutils build-essential git libboost-all-dev \
+  libcurl4-openssl-dev libdb++-dev libevent-dev libssl-dev libtool pkg-config python python-pip libzmq3-dev wget
+
+# VERSION: Bitcoin Core 0.20.1
+RUN git clone https://github.com/bitcoin/bitcoin \
+  && cd bitcoin \
+  && git checkout 7ff64311bee570874c4f0dfa18f518552188df08
+
+RUN cd bitcoin \
+  && ./autogen.sh \
+  && ./configure --disable-tests --without-miniupnpc --without-gui --with-incompatible-bdb --disable-hardening --disable-zmq --disable-bench --disable-wallet \
+  && make
+
+RUN mv bitcoin/src/bitcoind /app/bitcoind \
+  && rm -rf bitcoin
+
+# Build Rosetta Server Components
+FROM ubuntu:18.04 as rosetta-builder
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y curl make gcc g++
+ENV GOLANG_VERSION 1.15.5
+ENV GOLANG_DOWNLOAD_SHA256 9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d
+ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
+
+
+RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \
+  && echo "$GOLANG_DOWNLOAD_SHA256  golang.tar.gz" | sha256sum -c - \
+  && tar -C /usr/local -xzf golang.tar.gz \
+  && rm golang.tar.gz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Use native remote build context to build in any directory
+COPY . src
+RUN cd src \
+  && go build \
+  && cd .. \
+  && mv src/rosetta-bitcoin /app/rosetta-bitcoin \
+  && mv src/assets/* /app \
+  && rm -rf src
+
+## Build Final Image
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+  apt-get install --no-install-recommends -y libevent-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev && \
+  apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN mkdir -p /app \
+  && chown -R nobody:nogroup /app \
+  && mkdir -p /data \
+  && chown -R nobody:nogroup /data
+
+WORKDIR /app
+
+# Copy binary from bitcoind-builder
+COPY --from=bitcoind-builder /app/bitcoind /app/bitcoind
+
+# Copy binary from rosetta-builder
+COPY --from=rosetta-builder /app/* /app/
+# Set permissions for everything added to /app
+RUN chmod -R 755 /app/*
+
+CMD ["/app/rosetta-bitcoin"]
diff --git a/Makefile b/Makefile
@@ -23,24 +23,27 @@ build:
 	docker build -t rosetta-bitcoin:latest https://github.com/coinbase/rosetta-bitcoin.git
 
 build-local:
-	docker build -t rosetta-bitcoin:latest .
+	docker build -t rosetta-bitcoin:latest .build-local:
+
+build-local-dev:
+	docker build -f Dockerfile.dev -t rosetta-bitcoin:latest .
 
 build-release:
 	# make sure to always set version with vX.X.X
 	docker build -t rosetta-bitcoin:$(version) .;
 	docker save rosetta-bitcoin:$(version) | gzip > rosetta-bitcoin-$(version).tar.gz;
 
 run-mainnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 
 run-mainnet-offline:
-	docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 run-testnet-online:
-	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+	docker run -d --rm --ulimit "nofile=${NOFILE}:${NOFILE}" -v "${PWD}/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 
 run-testnet-offline:
-	docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+	docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 
 train:
 	./zstd-train.sh $(network) transaction $(data-directory)

diff --git a/README.md b/README.md
@@ -52,7 +52,16 @@ After cloning this repository, run:
 ```text
 make build-local
 ```
+#### From Source and Developing Locally
+After cloning this repository, run:
+```text
+make build-local-dev
+```
+Note: The purpose of this command is to build rosetta-bitcoin using a local sdk - this means you must have a local copy of rosetta-sdk-go and you must add the following to the bottom of your go.mod
 
+```text
+replace github.com/coinbase/rosetta-sdk-go v0.6.5 => ./rosetta-sdk-go
+```
 ### Run
 Running the following commands will start a Docker container in
 [detached mode](https://docs.docker.com/engine/reference/run/#detached--d) with
@@ -61,25 +70,25 @@ at port `8080`.
 
 #### Mainnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=MAINNET" -e "PORT=8080" -p 8080:8080 -p 8333:8333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-online`._
 
 #### Mainnet:Offline
 ```text
-docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=MAINNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-mainnet-offline`._
 
 #### Testnet:Online
 ```text
-docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
+docker run -d --rm --ulimit "nofile=100000:100000" -v "$(pwd)/bitcoin-data:/data" -e "MAXSYNC"="256" -e "MODE=ONLINE" -e "NETWORK=TESTNET" -e "PORT=8080" -p 8080:8080 -p 18333:18333 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-online`._
 
 #### Testnet:Offline
 ```text
-docker run -d --rm -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
+docker run -d --rm -e "MAXSYNC"="256" -e "MODE=OFFLINE" -e "NETWORK=TESTNET" -e "PORT=8081" -p 8081:8081 rosetta-bitcoin:latest
 ```
 _If you cloned the repository, you can run `make run-testnet-offline`._
 

diff --git a/configuration/configuration.go b/configuration/configuration.go
@@ -22,6 +22,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/coinbase/rosetta-sdk-go/syncer"
+
 	"github.com/coinbase/rosetta-bitcoin/bitcoin"
 
 	"github.com/btcsuite/btcd/chaincfg"
@@ -90,6 +92,10 @@ const (
 	// to determine mode.
 	ModeEnv = "MODE"
 
+	// MaxSyncConcurrency is an environment variable
+	// used to cap the syncer concurrency
+	MaxSyncConcurrency = "MAXSYNC"
+
 	// NetworkEnv is the environment variable
 	// read to determine network.
 	NetworkEnv = "NETWORK"
@@ -122,18 +128,28 @@ type Configuration struct {
 	IndexerPath            string
 	BitcoindPath           string
 	Compressors            []*encoder.CompressorEntry
+	MaxSyncConcurrency     int64
 }
 
 // LoadConfiguration attempts to create a new Configuration
 // using the ENVs in the environment.
 func LoadConfiguration(baseDirectory string) (*Configuration, error) {
+
 	config := &Configuration{}
 	config.Pruning = &PruningConfiguration{
 		Frequency: pruneFrequency,
 		Depth:     pruneDepth,
 		MinHeight: minPruneHeight,
 	}
-
+	maxSyncValue := os.Getenv(MaxSyncConcurrency)
+	switch maxSyncValue {
+	case "":
+		config.MaxSyncConcurrency = syncer.DefaultMaxConcurrency
+	case "0":
+		return nil, errors.New("syncer concurrency must be greater than zero")
+	default:
+		config.MaxSyncConcurrency, _ = strconv.ParseInt(maxSyncValue, 10, 64)
+	}
 	modeValue := Mode(os.Getenv(ModeEnv))
 	switch modeValue {
 	case Online:
@@ -180,6 +196,7 @@ func LoadConfiguration(baseDirectory string) (*Configuration, error) {
 		}
 		config.GenesisBlockIdentifier = bitcoin.TestnetGenesisBlockIdentifier
 		config.Params = bitcoin.TestnetParams
+
 		config.Currency = bitcoin.TestnetCurrency
 		config.ConfigPath = testnetConfigPath
 		config.RPCPort = testnetRPCPort

diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go
@@ -30,9 +30,10 @@ import (
 
 func TestLoadConfiguration(t *testing.T) {
 	tests := map[string]struct {
-		Mode    string
-		Network string
-		Port    string
+		Mode               string
+		Network            string
+		Port               string
+		MaxSyncConcurrency string
 
 		cfg *Configuration
 		err error
@@ -44,15 +45,17 @@ func TestLoadConfiguration(t *testing.T) {
 			Mode: string(Online),
 			err:  errors.New("NETWORK must be populated"),
 		},
-		"only mode and network set": {
-			Mode:    string(Online),
-			Network: Mainnet,
-			err:     errors.New("PORT must be populated"),
+		"port not set": {
+			Mode:               string(Online),
+			Network:            Mainnet,
+			MaxSyncConcurrency: "256",
+			err:                errors.New("PORT must be populated"),
 		},
 		"all set (mainnet)": {
-			Mode:    string(Online),
-			Network: Mainnet,
-			Port:    "1000",
+			Mode:               string(Online),
+			Network:            Mainnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "256",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -63,6 +66,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.MainnetCurrency,
 				GenesisBlockIdentifier: bitcoin.MainnetGenesisBlockIdentifier,
 				Port:                   1000,
+				MaxSyncConcurrency:     256,
 				RPCPort:                mainnetRPCPort,
 				ConfigPath:             mainnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -79,9 +83,10 @@ func TestLoadConfiguration(t *testing.T) {
 			},
 		},
 		"all set (testnet)": {
-			Mode:    string(Online),
-			Network: Testnet,
-			Port:    "1000",
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "256",
 			cfg: &Configuration{
 				Mode: Online,
 				Network: &types.NetworkIdentifier{
@@ -92,6 +97,7 @@ func TestLoadConfiguration(t *testing.T) {
 				Currency:               bitcoin.TestnetCurrency,
 				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
 				Port:                   1000,
+				MaxSyncConcurrency:     256,
 				RPCPort:                testnetRPCPort,
 				ConfigPath:             testnetConfigPath,
 				Pruning: &PruningConfiguration{
@@ -107,6 +113,44 @@ func TestLoadConfiguration(t *testing.T) {
 				},
 			},
 		},
+		"max sync set": {
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "",
+			cfg: &Configuration{
+				Mode: Online,
+				Network: &types.NetworkIdentifier{
+					Network:    bitcoin.TestnetNetwork,
+					Blockchain: bitcoin.Blockchain,
+				},
+				Params:                 bitcoin.TestnetParams,
+				Currency:               bitcoin.TestnetCurrency,
+				GenesisBlockIdentifier: bitcoin.TestnetGenesisBlockIdentifier,
+				Port:                   1000,
+				MaxSyncConcurrency:     256,
+				RPCPort:                testnetRPCPort,
+				ConfigPath:             testnetConfigPath,
+				Pruning: &PruningConfiguration{
+					Frequency: pruneFrequency,
+					Depth:     pruneDepth,
+					MinHeight: minPruneHeight,
+				},
+				Compressors: []*encoder.CompressorEntry{
+					{
+						Namespace:      transactionNamespace,
+						DictionaryPath: testnetTransactionDictionary,
+					},
+				},
+			},
+		},
+		"invalid sync concurrency ": {
+			Mode:               string(Online),
+			Network:            Testnet,
+			Port:               "1000",
+			MaxSyncConcurrency: "0",
+			err:                errors.New("syncer concurrency must be greater than zero"),
+		},
 		"invalid mode": {
 			Mode:    "bad mode",
 			Network: Testnet,
@@ -136,6 +180,7 @@ func TestLoadConfiguration(t *testing.T) {
 			os.Setenv(ModeEnv, test.Mode)
 			os.Setenv(NetworkEnv, test.Network)
 			os.Setenv(PortEnv, test.Port)
+			os.Setenv(MaxSyncConcurrency, test.MaxSyncConcurrency)
 
 			cfg, err := LoadConfiguration(newDir)
 			if test.err != nil {

diff --git a/go.mod b/go.mod
@@ -14,3 +14,7 @@ require (
 	golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect
 	honnef.co/go/tools v0.0.1-2020.1.5 // indirect
 )
+
+
+
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,3 +14,7 @@ require (
		golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect
		honnef.co/go/tools v0.0.1-2020.1.5 // indirect
		)