polygon/sync: ignore safe peer not found err on tip event (#11893)

fixes another crash on chain tip, happened due to a rare and unfortunate
situation where the peer that gave us the new block hash event
disconnected before we have processed that event and didn't give us a
chance to fetch the block from it:
```
INFO[09-05|17:58:53.469] [2/6 PolygonSync] forward                progress=11604698
DBUG[09-05|17:58:55.192] [p2p.peerTracker] peer disconnected      peerId=de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f
DBUG[09-05|17:59:13.075] Error while executing stage              err="[2/6 PolygonSync] stopped: peer not found: de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f"
EROR[09-05|17:59:13.075] [2/6 PolygonSync] stopping node          err="peer not found: de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f"
```

Full logs
```
INFO[09-05|17:58:53.426] [2/6 PolygonSync] update fork choice     block=11604698 age=0 hash=0x5765c68670a5a0d9435d7ace7c07db48f2b725dbb93c59d5141afd600f8e3b2d
DBUG[09-05|17:58:53.426] [2/6 PolygonSync] connecting tip         blockNum=11604698 blockHash=0x5765c68670a5a0d9435d7ace7c07db48f2b725dbb93c59d5141afd600f8e3b2d
DBUG[09-05|17:58:53.426] [2/6 PolygonSync] DONE                   in=1.955560708s
DBUG[09-05|17:58:53.427] [3/6 Senders] DONE                       in=544.584µs
INFO[09-05|17:58:53.446] [4/6 Execution] Done Commit every block  blk=11604698 blks=1 blk/s=55.3 txs=3 tx/s=165 gas/s=7.10M buf=85.0KB/512.0MB stepsInDB=0.00 step=24.4 alloc=968.1MB sys=6.3GB
DBUG[09-05|17:58:53.446] [4/6 Execution] DONE                     in=18.908167ms
DBUG[09-05|17:58:53.446] [5/6 TxLookup] DONE                      in=282.583µs
DBUG[09-05|17:58:53.446] [6/6 Finish] DONE                        in=13.541µs
DBUG[09-05|17:58:53.446] RPC Daemon notified of new headers       from=11604697 to=11604698 amount=1 hash=0x5765c68670a5a0d9435d7ace7c07db48f2b725dbb93c59d5141afd600f8e3b2d header sending=11.583µs log sending=291ns
INFO[09-05|17:58:53.446] Timings (slower than 50ms)               PolygonSync=1.955s alloc=968.1MB sys=6.3GB
DBUG[09-05|17:58:53.447] [6/6 Finish] Prune done                  in=5.416µs
DBUG[09-05|17:58:53.447] [5/6 TxLookup] Prune done                in=349.917µs
DBUG[09-05|17:58:53.448] [4/6 Execution] Prune done               in=672.291µs
DBUG[09-05|17:58:53.448] [3/6 Senders] Prune done                 in=2.5µs
DBUG[09-05|17:58:53.448] [2/6 PolygonSync] Prune done             in=1.542µs
DBUG[09-05|17:58:53.448] [snapshots] Prune Blocks                 to=11602976 limit=10
DBUG[09-05|17:58:53.449] [snapshots] Prune Bor Blocks             to=11602976 limit=10
DBUG[09-05|17:58:53.449] [1/6 OtterSync] Prune done               in=1.254625ms
DBUG[09-05|17:58:53.469] [1/6 OtterSync] DONE                     in=37.291µs
INFO[09-05|17:58:53.469] [2/6 PolygonSync] forward                progress=11604698
DBUG[09-05|17:58:55.192] [p2p.peerTracker] peer disconnected      peerId=de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f
DBUG[09-05|17:59:13.075] Error while executing stage              err="[2/6 PolygonSync] stopped: peer not found: de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f"
EROR[09-05|17:59:13.075] [2/6 PolygonSync] stopping node          err="peer not found: de55d16b6e1fca28cdd3d11eb0dd89e3b77b96d4722172bd5e04ac255922324076a87748e97bc021af2307dccbb5ef8062389cfcba1845f77219eee7935dea9f"
DBUG[09-05|17:59:13.076] rpcdaemon: the subscription to pending blocks channel was closed 
INFO[09-05|17:59:13.075] Exiting... 
```

polygon/p2p/message_sender.go

@@ -19,8 +19,10 @@ package p2p
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	sentry "github.com/erigontech/erigon-lib/gointerfaces/sentryproto"
+	libsentry "github.com/erigontech/erigon-lib/p2p/sentry"
 	"github.com/erigontech/erigon/eth/protocols/eth"
 	"github.com/erigontech/erigon/rlp"
 )
@@ -64,10 +66,13 @@ func (ms *messageSender) sendMessage(ctx context.Context, messageId sentry.Messa
 		},
 	})
 	if err != nil {
+		if libsentry.IsPeerNotFoundErr(err) {
+			return fmt.Errorf("%w: %s", ErrPeerNotFound, peerId.String())
+		}
 		return err
 	}
 	if len(sent.Peers) == 0 {
-		return ErrPeerNotFound
+		return fmt.Errorf("%w: %s", ErrPeerNotFound, peerId.String())
 	}
 
 	return nil

polygon/sync/sync.go

@@ -181,7 +181,7 @@ func (s *Sync) applyNewBlockOnTip(
 	} else {
 		blocks, err := s.p2pService.FetchBlocks(ctx, rootNum, newBlockHeaderNum+1, event.PeerId)
 		if err != nil {
-			if (p2p.ErrIncompleteHeaders{}).Is(err) || (p2p.ErrMissingBodies{}).Is(err) {
+			if s.ignoreFetchBlocksErrOnTipEvent(err) {
 				s.logger.Debug(
 					syncLogPrefix("applyNewBlockOnTip: failed to fetch complete blocks, ignoring event"),
 					"err", err,
@@ -256,7 +256,7 @@ func (s *Sync) applyNewBlockHashesOnTip(
 
 		newBlocks, err := s.p2pService.FetchBlocks(ctx, headerHashNum.Number, headerHashNum.Number+1, event.PeerId)
 		if err != nil {
-			if (p2p.ErrIncompleteHeaders{}).Is(err) || (p2p.ErrMissingBodies{}).Is(err) {
+			if s.ignoreFetchBlocksErrOnTipEvent(err) {
 				s.logger.Debug(
 					syncLogPrefix("applyNewBlockHashesOnTip: failed to fetch complete blocks, ignoring event"),
 					"err", err,
@@ -383,3 +383,9 @@ func (s *Sync) sync(ctx context.Context, tip *types.Header, tipDownloader tipDow
 
 	return tip, nil
 }
+
+func (s *Sync) ignoreFetchBlocksErrOnTipEvent(err error) bool {
+	return errors.Is(err, &p2p.ErrIncompleteHeaders{}) ||
+		errors.Is(err, &p2p.ErrMissingBodies{}) ||
+		errors.Is(err, p2p.ErrPeerNotFound)
+}