bumped firehose-core to

CHANGELOG.md

@@ -4,6 +4,52 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md)
 for instructions to keep up to date.
 
+
+## Unreleased
+
+* bumped firehose-core to `v0.2.1`
+
+### Operators
+
+> [!IMPORTANT]
+> We have had reports of older versions of this software creating corrupted merged-blocks-files (with duplicate or extra out-of-bound blocks)
+> This release adds additional validation of merged-blocks to prevent serving duplicate blocks from the firehose or substreams service. 
+> This may cause service outage if you have produced those blocks or downloaded them from another party who was affected by this bug.
+
+1. Find the affected files by running the following command (can be run multiple times in parallel, over smaller ranges) 
+
+```
+tools check merged-blocks-batch <merged-blocks-store> <start> <stop>
+```
+
+2. If you see any affected range, produce fixed merged-blocks files with the following command, on each range:
+
+```
+tools fix-bloated-merged-blocks <merged-blocks-store> <output-store> <start>:<stop>
+```
+
+3. Copy the merged-blocks files created in output-store over to the your merged-blocks-store, replacing the corrupted files.
+
+### Added
+
+* Added `tools check merged-blocks-batch` to simplify checking blocks continuity in batched mode, optionally writing results to a store
+* Added the command `tools fix-bloated-merged-blocks` to try to fix merged-blocks that contain duplicates and blocks outside of their range.
+* Command `tools print one-block and merged-blocks` now supports a new `--output-format` `jsonl` format. Bytes data can now printed as hex or base58 string instead of base64 string.
+* Added retry loop for merger when walking one block files. Some use-cases where the bundle reader was sending files too fast and the merger was not waiting to accumulate enough files to start bundling merged files
+
+### Fixed
+
+* Bumped `bstream`: the `filesource` will now refuse to read blocks from a merged-files if they are not ordered or if there are any duplicate.
+* The command `tools download-from-firehose` will now fail if it is being served blocks "out of order", to prevent any corrupted merged-blocks from being created.
+* The command `tools print merged-blocks` did not print the whole merged-blocks file, the arguments were confusing: now it will parse <start_block> as a uint64.
+* The command `tools unmerge-blocks` did not cover the whole given range, now fixed
+
+### Removed
+
+* **Breaking** The `reader-node-log-to-zap` flag has been removed. This was a source of confusion for operators reporting Firehose on <Chain> bugs because the node's logs where merged within normal Firehose on <Chain> logs and it was not super obvious.
+
+  Now, logs from the node will be printed to `stdout` unformatted exactly like presented by the chain. Filtering of such logs must now be delegated to the node's implementation and how it deals depends on the node's binary. Refer to it to determine how you can tweak the logging verbosity emitted by the node.
+
 ## [1.1.10](https://github.com/streamingfast/firehose-near/releases/tag/v1.1.10)
 
 * bump firehose-core to `v0.1.12`

cmd/firenear/main.go

@@ -64,11 +64,11 @@ func main() {
 				return nil
 			},
 
-			TransformFlags: map[string]*firecore.TransformFlag{
-				"receipt-account-filters": {
-					Description: "Comma-separated accounts to use as filter/index. If it contains a colon (:), it will be interpreted as <prefix>:<suffix> (each of which can be empty, ex: 'hello:' or ':world')",
-					Parser:      parseReceiptAccountFilters,
+			TransformFlags: &firecore.TransformFlags{
+				Register: func(flags *pflag.FlagSet) {
+					flags.String("receipt-account-filters", "", "Comma-separated accounts to use as filter/index. If it contains a colon (:), it will be interpreted as <prefix>:<suffix> (each of which can be empty, ex: 'hello:' or ':world')")
 				},
+				Parse: parseReceiptAccountFilters,
 			},
 		},
 	})

cmd/firenear/reader_node_bootstraper.go

@@ -12,10 +12,14 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/streamingfast/dstore"
+	firecore "github.com/streamingfast/firehose-core"
 	"github.com/streamingfast/node-manager/operator"
+	"go.uber.org/zap"
 )
 
-func newReaderNodeBootstrapper(cmd *cobra.Command, nodeDataDir string) (operator.Bootstrapper, error) {
+func newReaderNodeBootstrapper(ctx context.Context, logger *zap.Logger, cmd *cobra.Command, resolvedNodeArguments []string, resolver firecore.ReaderNodeArgumentResolver) (operator.Bootstrapper, error) {
+	nodeDataDir := resolver("{node-data-dir}")
+
 	hostname, _ := os.Hostname()
 
 	fmt.Println("Hostname", hostname)

cmd/firenear/tools.go

@@ -5,9 +5,12 @@ import (
 	"io"
 	"strings"
 
+	"github.com/spf13/cobra"
 	"github.com/streamingfast/bstream"
+	"github.com/streamingfast/cli/sflags"
 	pbtransform "github.com/streamingfast/firehose-near/pb/sf/near/transform/v1"
 	pbnear "github.com/streamingfast/firehose-near/pb/sf/near/type/v1"
+	"go.uber.org/zap"
 	"google.golang.org/protobuf/types/known/anypb"
 )
 
@@ -55,7 +58,8 @@ func printBlock(blk *bstream.Block, alsoPrintTransactions bool, out io.Writer) e
 	return nil
 }
 
-func parseReceiptAccountFilters(in string) (*anypb.Any, error) {
+func parseReceiptAccountFilters(cmd *cobra.Command, logger *zap.Logger) ([]*anypb.Any, error) {
+	in := sflags.MustGetString(cmd, "receipt-account-filters")
 	if in == "" {
 		return nil, nil
 	}
@@ -79,5 +83,9 @@ func parseReceiptAccountFilters(in string) (*anypb.Any, error) {
 		PrefixAndSuffixPairs: pairs,
 	}
 
-	return anypb.New(filters)
+	any, err := anypb.New(filters)
+	if err != nil {
+		return nil, err
+	}
+	return []*anypb.Any{any}, nil
 }

go.mod

@@ -8,11 +8,11 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.15.0
-	github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a
+	github.com/streamingfast/bstream v0.0.2-0.20231108161055-1092e9150ee8
 	github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80
 	github.com/streamingfast/dbin v0.9.1-0.20220513054835-1abebbb944ad
 	github.com/streamingfast/dstore v0.1.1-0.20230620124109-3924b3b36c77
-	github.com/streamingfast/firehose-core v0.1.12
+	github.com/streamingfast/firehose-core v0.2.1
 	github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091
 	github.com/streamingfast/near-go v0.0.0-20220302163233-b638f5b48a2d
 	github.com/streamingfast/node-manager v0.0.2-0.20230406142433-692298a8b8d2
@@ -73,6 +73,7 @@ require (
 	github.com/envoyproxy/go-control-plane v0.11.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
@@ -162,7 +163,7 @@ require (
 	github.com/streamingfast/firehose v0.1.1-0.20230731171526-3428fc0ff8d9 // indirect
 	github.com/streamingfast/index-builder v0.0.0-20221031203737-fa2e70f09dc2 // indirect
 	github.com/streamingfast/jsonpb v0.0.0-20210811021341-3670f0aa02d0 // indirect
-	github.com/streamingfast/merger v0.0.3-0.20230413191947-fe3ce68cb9f8 // indirect
+	github.com/streamingfast/merger v0.0.3-0.20231027161314-209c2ddd8d96 // indirect
 	github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308 // indirect
 	github.com/streamingfast/relayer v0.0.2-0.20220909122435-e67fbc964fd9 // indirect
 	github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 // indirect

go.sum

@@ -287,6 +287,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d h1:zqfo2jECgX5eYQseB/X+uV4Y5ocGOG/vG/LTztUCyPA=
+github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -750,8 +752,8 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
 github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680 h1:fGJnUx0shX9Y312QOlz+/+yLquihXRhNqctJ26jtZZM=
 github.com/streamingfast/atm v0.0.0-20220131151839-18c87005e680/go.mod h1:iISPGAstbUsPgyC3auLLi7PYUTi9lHv5z0COam0OPOY=
 github.com/streamingfast/bstream v0.0.2-0.20221017131819-2a7e38be1047/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q=
-github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a h1:NeCO5JLz38HRK1uaV1Emo9u5gUSRtmtZZGNK8BKyLIE=
-github.com/streamingfast/bstream v0.0.2-0.20230829131224-b9272048dc6a/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q=
+github.com/streamingfast/bstream v0.0.2-0.20231108161055-1092e9150ee8 h1:97vTsS3HZ5Q/ADapuQGbOWlOl4VGlUmG6TDtk8vmL/E=
+github.com/streamingfast/bstream v0.0.2-0.20231108161055-1092e9150ee8/go.mod h1:Njkx972HcZiz0djWBylxqO/eq686eDGr+egQ1lePj3Q=
 github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 h1:UxJUTcEVkdZy8N77E3exz0iNlgQuxl4m220GPvzdZ2s=
 github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80/go.mod h1:QxjVH73Lkqk+mP8bndvhMuQDUINfkgsYhdCH/5TJFKI=
 github.com/streamingfast/dauth v0.0.0-20210812020920-1c83ba29add1/go.mod h1:FIYpVqt+ICVuNBoOH3ZIicIctpVoCq3393+RpfXsPEM=
@@ -784,8 +786,8 @@ github.com/streamingfast/dtracing v0.0.0-20220305214756-b5c0e8699839/go.mod h1:h
 github.com/streamingfast/firehose v0.1.1-0.20221017171248-8fd3adbe7b4d/go.mod h1:weGz9xDNJMBNmn03XiJZ/b5Ngw8UAUoLirarqG7OwQY=
 github.com/streamingfast/firehose v0.1.1-0.20230731171526-3428fc0ff8d9 h1:SybLEPFkJk8EINJGDlijSWxEZGNXDul4zqlDpttQUCI=
 github.com/streamingfast/firehose v0.1.1-0.20230731171526-3428fc0ff8d9/go.mod h1:lGC1T6mpAAApjBQNF5COSXb3SbrYRI3dBR1f6/PZE54=
-github.com/streamingfast/firehose-core v0.1.12 h1:eNmAfuyKkJXd1rAqpUxNPPHBa8gdU3CJ6nORA1nab+Q=
-github.com/streamingfast/firehose-core v0.1.12/go.mod h1:RUwLigWHqoBipsf9SwQczRAnuGFM/MDHcbrSWMZ0f2k=
+github.com/streamingfast/firehose-core v0.2.1 h1:yErirlPp1gkgj9GiO+TfBIsr3UDCM6PnnwbJbgBqr4A=
+github.com/streamingfast/firehose-core v0.2.1/go.mod h1:5Bf1vpTAhxI93isYYc7zcGrW1QzAZxioIll6siC6FTw=
 github.com/streamingfast/index-builder v0.0.0-20221031203737-fa2e70f09dc2 h1:dgYLhP3STiPi30fISAijFPEB11D4r1fQFc8D3cpgV5s=
 github.com/streamingfast/index-builder v0.0.0-20221031203737-fa2e70f09dc2/go.mod h1:OYv1UX/kRsV9aP4SEwa9zpt34qGzdtJzOvdGn+n56as=
 github.com/streamingfast/jsonpb v0.0.0-20210811021341-3670f0aa02d0 h1:g8eEYbFSykyzIyuxNMmHEUGGUvJE0ivmqZagLDK42gw=
@@ -797,8 +799,8 @@ github.com/streamingfast/logging v0.0.0-20220304183711-ddba33d79e27/go.mod h1:4G
 github.com/streamingfast/logging v0.0.0-20220304214715-bc750a74b424/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
 github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo=
 github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU=
-github.com/streamingfast/merger v0.0.3-0.20230413191947-fe3ce68cb9f8 h1:riOEu4aTwjy7Gjq8safA8kYUs+jVKi/gZ0+k7S/w/u8=
-github.com/streamingfast/merger v0.0.3-0.20230413191947-fe3ce68cb9f8/go.mod h1:WGMs+zwpPQNfzRnOqnyNdQfyGSG4lXYWQacicAGiP4s=
+github.com/streamingfast/merger v0.0.3-0.20231027161314-209c2ddd8d96 h1:aq5hUjo+Y+3OUH2z1egyJ9fSepRvOzxgR+TYICdSEgE=
+github.com/streamingfast/merger v0.0.3-0.20231027161314-209c2ddd8d96/go.mod h1:WGMs+zwpPQNfzRnOqnyNdQfyGSG4lXYWQacicAGiP4s=
 github.com/streamingfast/near-go v0.0.0-20220302163233-b638f5b48a2d h1:qaIxiajgUitO2bk9QaMwH+K6y9zopWKgw4D8HplTbXI=
 github.com/streamingfast/near-go v0.0.0-20220302163233-b638f5b48a2d/go.mod h1:24+rnDf/tjrOC2ZlC3CjC5x6DWehS0LQGdDxeTXYapM=
 github.com/streamingfast/node-manager v0.0.2-0.20230406142433-692298a8b8d2 h1:6Jdu6LBwaW38n2jjInFk1fM460cq+5paEAHGPPRWWN0=

pb/sf/near/type/v1/type.go

@@ -68,6 +68,14 @@ func (b *Block) GetFirehoseBlockNumber() uint64 {
 	return b.Header.Height
 }
 
+func (b *Block) GetFirehoseBlockParentNumber() uint64 {
+	return b.Header.PrevHeight
+}
+
+func (b *Block) GetFirehoseBlockVersion() int32 {
+	return int32(b.Header.LatestProtocolVersion)
+}
+
 func (b *Block) GetFirehoseBlockParentID() string {
 	return b.PreviousID()
 }