'24 sync

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*.md]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = false
+max_line_length = 80
+insert_final_newline = true
+charset = utf-8
+end_of_line = lf

.gitlab-ci.yml

@@ -157,15 +157,4 @@ backend-deploy-and-test-zcash_testnet:
       - configs/coins/zcash_testnet.json
   tags:
     - blockbook
-  script: ./contrib/scripts/backend-deploy-and-test.sh zcash_testnet zcash-testnet zcash=test testnet3/debug.log
-
-backend-deploy-and-test-goerli-archive:
-  stage: backend-deploy-and-test
-  only:
-    refs:
-      - master
-    changes:
-      - configs/coins/ethereum_testnet_goerli_archive.json
-  tags:
-    - blockbook
-  script: ./contrib/scripts/backend-deploy-and-test.sh ethereum_testnet_goerli_archive ethereum-testnet-goerli-archive ethereum=test ethereum_testnet_goerli_archive.log
+  script: ./contrib/scripts/backend-deploy-and-test.sh zcash_testnet zcash-testnet zcash=test testnet3/debug.log

Makefile

@@ -1,9 +1,10 @@
 BIN_IMAGE = blockbook-build
 DEB_IMAGE = blockbook-build-deb
 PACKAGER = $(shell id -u):$(shell id -g)
+DOCKER_VERSION = $(shell docker version --format '{{.Client.Version}}')
 BASE_IMAGE = $$(awk -F= '$$1=="ID" { print $$2 ;}' /etc/os-release):$$(awk -F= '$$1=="VERSION_ID" { print $$2 ;}' /etc/os-release | tr -d '"')
 NO_CACHE = false
-TCMALLOC = 
+TCMALLOC =
 PORTABLE = 0
 ARGS ?=
 
@@ -27,7 +28,7 @@ test-all: .bin-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" --network="host" $(BIN_IMAGE) make test-all ARGS="$(ARGS)"
 
 deb-backend-%: .deb-image
-	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
+	docker run -t --rm -e PACKAGER=$(PACKAGER) -v /var/run/docker.sock:/var/run/docker.sock -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh backend $* $(ARGS)
 
 deb-blockbook-%: .deb-image
 	docker run -t --rm -e PACKAGER=$(PACKAGER) -v "$(CURDIR):/src" -v "$(CURDIR)/build:/out" $(DEB_IMAGE) /build/build-deb.sh blockbook $* $(ARGS)
@@ -55,7 +56,7 @@ build-images: clean-images
 .deb-image: .bin-image
 	@if [ $$(build/tools/image_status.sh $(DEB_IMAGE):latest build/docker) != "ok" ]; then \
 		echo "Building image $(DEB_IMAGE)..."; \
-		docker build --no-cache=$(NO_CACHE) -t $(DEB_IMAGE) build/docker/deb; \
+		docker build --no-cache=$(NO_CACHE) --build-arg DOCKER_VERSION=$(DOCKER_VERSION) -t $(DEB_IMAGE) build/docker/deb; \
 	else \
 		echo "Image $(DEB_IMAGE) is up to date"; \
 	fi
@@ -79,3 +80,6 @@ clean-bin-image:
 
 clean-deb-image:
 	- docker rmi $(DEB_IMAGE)
+
+style:
+	find . -name "*.go" -exec gofmt -w {} \;

README.md

@@ -2,14 +2,14 @@
 
 # Blockbook
 
-**Blockbook** is back-end service for Trezor wallet. Main features of **Blockbook** are:
+**Blockbook** is a back-end service for Trezor Suite. The main features of **Blockbook** are:
 
-- index of addresses and address balances of the connected block chain
-- fast index search
-- simple blockchain explorer
-- websocket, API and legacy Bitcore Insight compatible socket.io interfaces
-- support of multiple coins (Bitcoin and Ethereum type) with easy extensibility to other coins
-- scripts for easy creation of debian packages for backend and blockbook
+-   index of addresses and address balances of the connected block chain
+-   fast index search
+-   simple blockchain explorer
+-   websocket, API and legacy Bitcore Insight compatible socket.io interfaces
+-   support of multiple coins (Bitcoin and Ethereum type) with easy extensibility to other coins
+-   scripts for easy creation of debian packages for backend and blockbook
 
 ## Build and installation instructions
 
@@ -19,42 +19,43 @@ Memory and disk requirements for initial synchronization of **Bitcoin mainnet**
 Other coins should have lower requirements, depending on the size of their block chain. Note that fast SSD disks are highly
 recommended.
 
-User installation guide is [here](https://wiki.trezor.io/User_manual:Running_a_local_instance_of_Trezor_Wallet_backend_(Blockbook)).
+User installation guide is [here](<https://wiki.trezor.io/User_manual:Running_a_local_instance_of_Trezor_Wallet_backend_(Blockbook)>).
 
 Developer build guide is [here](/docs/build.md).
 
 Contribution guide is [here](CONTRIBUTING.md).
 
 ## Implemented coins
 
-Blockbook currently supports over 30 coins. The Trezor team implemented 
+Blockbook currently supports over 30 coins. The Trezor team implemented
 
-- Bitcoin, Bitcoin Cash, Zcash, Dash, Litecoin, Bitcoin Gold, Ethereum, Ethereum Classic, Dogecoin, Namecoin, Vertcoin, DigiByte, Liquid
+-   Bitcoin, Bitcoin Cash, Zcash, Dash, Litecoin, Bitcoin Gold, Ethereum, Ethereum Classic, Dogecoin, Namecoin, Vertcoin, DigiByte, Liquid
 
 the rest of coins were implemented by the community.
 
 Testnets for some coins are also supported, for example:
-- Bitcoin Testnet, Bitcoin Cash Testnet, ZCash Testnet, Ethereum Testnet Ropsten
+
+-   Bitcoin Testnet, Bitcoin Cash Testnet, ZCash Testnet, Ethereum Testnets (Sepolia, Holesky)
 
 List of all implemented coins is in [the registry of ports](/docs/ports.md).
 
 ## Common issues when running Blockbook or implementing additional coins
 
 #### Out of memory when doing initial synchronization
 
-How to reduce memory footprint of the initial sync: 
+How to reduce memory footprint of the initial sync:
 
-- disable rocksdb cache by parameter `-dbcache=0`, the default size is 500MB
-- run blockbook with parameter `-workers=1`. This disables bulk import mode, which caches a lot of data in memory (not in rocksdb cache). It will run about twice as slowly but especially for smaller blockchains it is no problem at all.
+-   disable rocksdb cache by parameter `-dbcache=0`, the default size is 500MB
+-   run blockbook with parameter `-workers=1`. This disables bulk import mode, which caches a lot of data in memory (not in rocksdb cache). It will run about twice as slowly but especially for smaller blockchains it is no problem at all.
 
 Please add your experience to this [issue](https://github.com/trezor/blockbook/issues/43).
 
 #### Error `internalState: database is in inconsistent state and cannot be used`
 
-Blockbook was killed during the initial import, most commonly by OOM killer. 
-By default, Blockbook performs the initial import in bulk import mode, which for performance reasons does not store all data immediately to the database. If Blockbook is killed during this phase, the database is left in an inconsistent state. 
+Blockbook was killed during the initial import, most commonly by OOM killer.
+By default, Blockbook performs the initial import in bulk import mode, which for performance reasons does not store all data immediately to the database. If Blockbook is killed during this phase, the database is left in an inconsistent state.
 
-See above how to reduce the memory footprint, delete the database files and run the import again. 
+See above how to reduce the memory footprint, delete the database files and run the import again.
 
 Check [this](https://github.com/trezor/blockbook/issues/89) or [this](https://github.com/trezor/blockbook/issues/147) issue for more info.
 
@@ -73,3 +74,7 @@ Blockbook stores data the key-value store RocksDB. Database format is described
 ## API
 
 Blockbook API is described [here](/docs/api.md).
+
+## Environment variables
+
+List of environment variables that affect Blockbook's behavior is [here](/docs/env.md).

api/ethereumtype.go

@@ -0,0 +1,92 @@
+package api
+
+import (
+	"sync"
+
+	"github.com/golang/glog"
+	"github.com/linxGnu/grocksdb"
+	"github.com/trezor/blockbook/bchain"
+	"github.com/trezor/blockbook/db"
+)
+
+// refetch internal data
+var refetchingInternalData = false
+var refetchInternalDataMux sync.Mutex
+
+func (w *Worker) IsRefetchingInternalData() bool {
+	refetchInternalDataMux.Lock()
+	defer refetchInternalDataMux.Unlock()
+	return refetchingInternalData
+}
+
+func (w *Worker) RefetchInternalData() error {
+	refetchInternalDataMux.Lock()
+	defer refetchInternalDataMux.Unlock()
+	if !refetchingInternalData {
+		refetchingInternalData = true
+		go w.RefetchInternalDataRoutine()
+	}
+	return nil
+}
+
+const maxNumberOfRetires = 25
+
+func (w *Worker) incrementRefetchInternalDataRetryCount(ie *db.BlockInternalDataError) {
+	wb := grocksdb.NewWriteBatch()
+	defer wb.Destroy()
+	err := w.db.StoreBlockInternalDataErrorEthereumType(wb, &bchain.Block{
+		BlockHeader: bchain.BlockHeader{
+			Hash:   ie.Hash,
+			Height: ie.Height,
+		},
+	}, ie.ErrorMessage, ie.Retries+1)
+	if err != nil {
+		glog.Errorf("StoreBlockInternalDataErrorEthereumType %d %s, error %v", ie.Height, ie.Hash, err)
+	} else {
+		w.db.WriteBatch(wb)
+	}
+}
+
+func (w *Worker) RefetchInternalDataRoutine() {
+	internalErrors, err := w.db.GetBlockInternalDataErrorsEthereumType()
+	if err == nil {
+		for i := range internalErrors {
+			ie := &internalErrors[i]
+			if ie.Retries >= maxNumberOfRetires {
+				glog.Infof("Refetching internal data for %d %s, retries exceeded", ie.Height, ie.Hash)
+				continue
+			}
+			glog.Infof("Refetching internal data for %d %s, retries %d", ie.Height, ie.Hash, ie.Retries)
+			block, err := w.chain.GetBlock(ie.Hash, ie.Height)
+			var blockSpecificData *bchain.EthereumBlockSpecificData
+			if block != nil {
+				blockSpecificData, _ = block.CoinSpecificData.(*bchain.EthereumBlockSpecificData)
+			}
+			if err != nil || block == nil || blockSpecificData == nil || blockSpecificData.InternalDataError != "" {
+				glog.Errorf("Refetching internal data for %d %s, error %v, retrying", ie.Height, ie.Hash, err)
+				// try for second time to fetch the data - the 2nd attempt after the first unsuccessful has many times higher probability of success
+				// probably something to do with data preloaded to cache on the backend
+				block, err = w.chain.GetBlock(ie.Hash, ie.Height)
+				if err != nil || block == nil {
+					glog.Errorf("Refetching internal data for %d %s, error %v", ie.Height, ie.Hash, err)
+					continue
+				}
+			}
+			blockSpecificData, _ = block.CoinSpecificData.(*bchain.EthereumBlockSpecificData)
+			if blockSpecificData != nil && blockSpecificData.InternalDataError != "" {
+				glog.Errorf("Refetching internal data for %d %s, internal data error %v", ie.Height, ie.Hash, blockSpecificData.InternalDataError)
+				w.incrementRefetchInternalDataRetryCount(ie)
+			} else {
+				err = w.db.ReconnectInternalDataToBlockEthereumType(block)
+				if err != nil {
+					glog.Errorf("ReconnectInternalDataToBlockEthereumType %d %s, error %v", ie.Height, ie.Hash, err)
+				} else {
+					glog.Infof("Refetching internal data for %d %s, success", ie.Height, ie.Hash)
+				}
+			}
+		}
+	}
+	refetchInternalDataMux.Lock()
+	refetchingInternalData = false
+	refetchInternalDataMux.Unlock()
+}

api/types.go

@@ -208,9 +208,9 @@ type TokenTransfer struct {
 	From             string               `json:"from"`
 	To               string               `json:"to"`
 	Contract         string               `json:"contract"`
-	Name             string               `json:"name"`
-	Symbol           string               `json:"symbol"`
-	Decimals         int                  `json:"decimals"`
+	Name             string               `json:"name,omitempty"`
+	Symbol           string               `json:"symbol,omitempty"`
+	Decimals         int                  `json:"decimals,omitempty"`
 	Value            *Amount              `json:"value,omitempty"`
 	MultiTokenValues []MultiTokenValue    `json:"multiTokenValues,omitempty"`
 }
@@ -231,7 +231,11 @@ type EthereumSpecific struct {
 	Nonce             uint64                                 `json:"nonce"`
 	GasLimit          *big.Int                               `json:"gasLimit"`
 	GasUsed           *big.Int                               `json:"gasUsed,omitempty"`
-	GasPrice          *Amount                                `json:"gasPrice"`
+	GasPrice          *Amount                                `json:"gasPrice,omitempty"`
+	L1Fee             *big.Int                               `json:"l1Fee,omitempty"`
+	L1FeeScalar       string                                 `json:"l1FeeScalar,omitempty"`
+	L1GasPrice        *Amount                                `json:"l1GasPrice,omitempty"`
+	L1GasUsed         *big.Int                               `json:"l1GasUsed,omitempty"`
 	Data              string                                 `json:"data,omitempty"`
 	ParsedData        *bchain.EthereumParsedInputData        `json:"parsedData,omitempty"`
 	InternalTransfers []EthereumInternalTransfer             `json:"internalTransfers,omitempty"`
@@ -316,6 +320,19 @@ type AddressFilter struct {
 	OnlyConfirmed bool
 }
 
+// StakingPool holds data about address participation in a staking pool contract
+type StakingPool struct {
+	Contract                string  `json:"contract"`
+	Name                    string  `json:"name"`
+	PendingBalance          *Amount `json:"pendingBalance"`
+	PendingDepositedBalance *Amount `json:"pendingDepositedBalance"`
+	DepositedBalance        *Amount `json:"depositedBalance"`
+	WithdrawTotalAmount     *Amount `json:"withdrawTotalAmount"`
+	ClaimableAmount         *Amount `json:"claimableAmount"`
+	RestakedReward          *Amount `json:"restakedReward"`
+	AutocompoundBalance     *Amount `json:"autocompoundBalance"`
+}
+
 // Address holds information about address and its transactions
 type Address struct {
 	Paging
@@ -326,6 +343,7 @@ type Address struct {
 	UnconfirmedBalanceSat *Amount              `json:"unconfirmedBalance"`
 	UnconfirmedTxs        int                  `json:"unconfirmedTxs"`
 	Txs                   int                  `json:"txs"`
+	AddrTxCount           int                  `json:"addrTxCount,omitempty"`
 	NonTokenTxs           int                  `json:"nonTokenTxs,omitempty"`
 	InternalTxs           int                  `json:"internalTxs,omitempty"`
 	Transactions          []*Tx                `json:"transactions,omitempty"`
@@ -341,6 +359,7 @@ type Address struct {
 	ContractInfo          *bchain.ContractInfo `json:"contractInfo,omitempty"`
 	Erc20Contract         *bchain.ContractInfo `json:"erc20Contract,omitempty"` // deprecated
 	AddressAliases        AddressAliasesMap    `json:"addressAliases,omitempty"`
+	StakingPools          []StakingPool        `json:"stakingPools,omitempty"`
 	// helpers for explorer
 	Filter        string              `json:"-"`
 	XPubAddresses map[string]struct{} `json:"-"`
@@ -484,6 +503,7 @@ type BlockRaw struct {
 // BlockbookInfo contains information about the running blockbook instance
 type BlockbookInfo struct {
 	Coin                         string                       `json:"coin"`
+	Network                      string                       `json:"network"`
 	Host                         string                       `json:"host"`
 	Version                      string                       `json:"version"`
 	GitCommit                    string                       `json:"gitCommit"`
@@ -503,6 +523,7 @@ type BlockbookInfo struct {
 	CurrentFiatRatesTime         *time.Time                   `json:"currentFiatRatesTime,omitempty"`
 	HistoricalFiatRatesTime      *time.Time                   `json:"historicalFiatRatesTime,omitempty"`
 	HistoricalTokenFiatRatesTime *time.Time                   `json:"historicalTokenFiatRatesTime,omitempty"`
+	SupportedStakingPools        []string                     `json:"supportedStakingPools,omitempty"`
 	DbSizeFromColumns            int64                        `json:"dbSizeFromColumns,omitempty"`
 	DbColumns                    []common.InternalStateColumn `json:"dbColumns,omitempty"`
 	About                        string                       `json:"about"`