Caplin: Remotion of bad peers giving us bad blocks (#7411)

cl/rpc/rpc.go

@@ -49,7 +49,7 @@ func NewBeaconRpcP2P(ctx context.Context, sentinel sentinel.SentinelClient, beac
 	}
 }
 
-func (b *BeaconRpcP2P) sendBlocksRequest(topic string, reqData []byte, count uint64) ([]*cltypes.SignedBeaconBlock, error) {
+func (b *BeaconRpcP2P) sendBlocksRequest(topic string, reqData []byte, count uint64) ([]*cltypes.SignedBeaconBlock, string, error) {
 	// Prepare output slice.
 	responsePacket := []*cltypes.SignedBeaconBlock{}
 
@@ -58,11 +58,11 @@ func (b *BeaconRpcP2P) sendBlocksRequest(topic string, reqData []byte, count uin
 		Topic: topic,
 	})
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	if message.Error {
 		log.Debug("received range req error", "err", string(message.Data))
-		return nil, nil
+		return nil, message.Peer.Pid, nil
 	}
 
 	r := bytes.NewReader(message.Data)
@@ -72,17 +72,17 @@ func (b *BeaconRpcP2P) sendBlocksRequest(topic string, reqData []byte, count uin
 			if err == io.EOF {
 				break
 			}
-			return nil, err
+			return nil, message.Peer.Pid, err
 		}
 
 		// Read varint for length of message.
 		encodedLn, _, err := ssz_snappy.ReadUvarint(r)
 		if err != nil {
-			return nil, fmt.Errorf("unable to read varint from message prefix: %v", err)
+			return nil, message.Peer.Pid, fmt.Errorf("unable to read varint from message prefix: %v", err)
 		}
 		// Sanity check for message size.
 		if encodedLn > uint64(maxMessageLength) {
-			return nil, fmt.Errorf("received message too big")
+			return nil, message.Peer.Pid, fmt.Errorf("received message too big")
 		}
 
 		// Read bytes using snappy into a new raw buffer of side encodedLn.
@@ -92,55 +92,55 @@ func (b *BeaconRpcP2P) sendBlocksRequest(topic string, reqData []byte, count uin
 		for bytesRead < int(encodedLn) {
 			n, err := sr.Read(raw[bytesRead:])
 			if err != nil {
-				return nil, fmt.Errorf("read error: %w", err)
+				return nil, message.Peer.Pid, fmt.Errorf("read error: %w", err)
 			}
 			bytesRead += n
 		}
 		// Fork digests
 		respForkDigest := binary.BigEndian.Uint32(forkDigest)
 		if respForkDigest == 0 {
-			return nil, fmt.Errorf("null fork digest")
+			return nil, message.Peer.Pid, fmt.Errorf("null fork digest")
 		}
 
 		version, err := fork.ForkDigestVersion(utils.Uint32ToBytes4(respForkDigest), b.beaconConfig, b.genesisConfig.GenesisValidatorRoot)
 		if err != nil {
-			return nil, err
+			return nil, message.Peer.Pid, err
 		}
 		responseChunk := &cltypes.SignedBeaconBlock{}
 
 		if err = responseChunk.DecodeSSZWithVersion(raw, int(version)); err != nil {
-			return nil, err
+			return nil, message.Peer.Pid, err
 		}
 		responsePacket = append(responsePacket, responseChunk)
 		// TODO(issues/5884): figure out why there is this extra byte.
 		r.ReadByte()
 	}
 
-	return responsePacket, nil
+	return responsePacket, message.Peer.Pid, nil
 }
 
 // SendBeaconBlocksByRangeReq retrieves blocks range from beacon chain.
-func (b *BeaconRpcP2P) SendBeaconBlocksByRangeReq(start, count uint64) ([]*cltypes.SignedBeaconBlock, error) {
+func (b *BeaconRpcP2P) SendBeaconBlocksByRangeReq(start, count uint64) ([]*cltypes.SignedBeaconBlock, string, error) {
 	req := &cltypes.BeaconBlocksByRangeRequest{
 		StartSlot: start,
 		Count:     count,
 		Step:      1, // deprecated, and must be set to 1.
 	}
 	var buffer buffer.Buffer
 	if err := ssz_snappy.EncodeAndWrite(&buffer, req); err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	data := common.CopyBytes(buffer.Bytes())
 	return b.sendBlocksRequest(communication.BeaconBlocksByRangeProtocolV2, data, count)
 }
 
 // SendBeaconBlocksByRootReq retrieves blocks by root from beacon chain.
-func (b *BeaconRpcP2P) SendBeaconBlocksByRootReq(roots [][32]byte) ([]*cltypes.SignedBeaconBlock, error) {
+func (b *BeaconRpcP2P) SendBeaconBlocksByRootReq(roots [][32]byte) ([]*cltypes.SignedBeaconBlock, string, error) {
 	var req cltypes.BeaconBlocksByRootRequest = roots
 	var buffer buffer.Buffer
 	if err := ssz_snappy.EncodeAndWrite(&buffer, &req); err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	data := common.CopyBytes(buffer.Bytes())
 	return b.sendBlocksRequest(communication.BeaconBlocksByRootProtocolV2, data, uint64(len(roots)))
@@ -181,3 +181,7 @@ func (b *BeaconRpcP2P) PropagateBlock(block *cltypes.SignedBeaconBlock) error {
 	})
 	return err
 }
+
+func (b *BeaconRpcP2P) BanPeer(pid string) {
+	b.sentinel.BanPeer(b.ctx, &sentinel.Peer{Pid: pid})
+}

cmd/erigon-cl/forkchoice/fork_graph/fork_graph.go

@@ -21,7 +21,7 @@ const (
 	PreValidated   ChainSegmentInsertionResult = 5
 )
 
-const snapshotStateEverySlot = 128
+const snapshotStateEverySlot = 64
 
 /*
 * The state store process is related to graph theory in the sense that the Ethereum blockchain can be thought of as a directed graph,

cmd/erigon-cl/network/backward_beacon_downloader.go

@@ -85,7 +85,7 @@ func (b *BackwardBeaconDownloader) RequestMore() {
 	if start > b.slotToDownload {
 		start = 0
 	}
-	responses, err := b.rpc.SendBeaconBlocksByRangeReq(start, count)
+	responses, _, err := b.rpc.SendBeaconBlocksByRangeReq(start, count)
 	if err != nil {
 		return
 	}

cmd/erigon-cl/network/beacon_downloader.go

@@ -78,8 +78,9 @@ func (f *ForwardBeaconDownloader) addSegment(block *cltypes.SignedBeaconBlock) {
 
 func (f *ForwardBeaconDownloader) RequestMore() {
 	count := uint64(4) // dont need many
-	responses, err := f.rpc.SendBeaconBlocksByRangeReq(f.highestSlotProcessed+1, count)
+	responses, pid, err := f.rpc.SendBeaconBlocksByRangeReq(f.highestSlotProcessed+1, count)
 	if err != nil {
+		f.rpc.BanPeer(pid)
 		// Wait a bit in this case (we do not need to be super performant here).
 		time.Sleep(time.Second)
 		return

cmd/erigon-cl/network/gossip_manager.go

@@ -83,6 +83,10 @@ func (g *GossipManager) Start() {
 			log.Debug("Received block via gossip", "slot", block.Block.Slot)
 
 			if err := g.forkChoice.OnBlock(block, true, true); err != nil {
+				// if we are within a quarter of an epoch within chain tip we ban it
+				if currentSlotByTime < g.forkChoice.HighestSeen()+(g.beaconConfig.SlotsPerEpoch/4) {
+					g.sentinel.BanPeer(g.ctx, data.Peer)
+				}
 				log.Debug("[Beacon Gossip] Failure in processing block", "err", err)
 				continue
 			}

cmd/erigon-cl/stages/stage_fork_choice.go

@@ -128,8 +128,8 @@ func startDownloadService(s *stagedsync.StageState, cfg StageForkChoiceCfg) {
 		// Checks done, update all internals accordingly
 		return highestSlotProcessed, libcommon.Hash{}, nil
 	})
-	maxBlockBehindBeforeDownload := int64(5)
-	overtimeMargin := uint64(2) // how much time has passed before trying download the next block in seconds
+	maxBlockBehindBeforeDownload := int64(32)
+	overtimeMargin := uint64(6) // how much time has passed before trying download the next block in seconds
 MainLoop:
 	for {
 		targetSlot := utils.GetCurrentSlot(cfg.genesisCfg.GenesisTime, cfg.beaconCfg.SecondsPerSlot)

cmd/sentinel/sentinel/service/service.go

@@ -109,7 +109,7 @@ func (s *SentinelServer) SendRequest(_ context.Context, req *sentinelrpc.Request
 	defer s.mu.RUnlock()
 	retryReqInterval := time.NewTicker(200 * time.Millisecond)
 	defer retryReqInterval.Stop()
-	timeout := time.NewTimer(1 * time.Second)
+	timeout := time.NewTimer(2 * time.Second)
 	defer retryReqInterval.Stop()
 	doneCh := make(chan *sentinelrpc.ResponseData)
 	// Try finding the data to our peers
@@ -124,6 +124,10 @@ func (s *SentinelServer) SendRequest(_ context.Context, req *sentinelrpc.Request
 				// Wait a bit to not exhaust CPU and skip.
 				continue
 			}
+			pidText, err := pid.MarshalText()
+			if err != nil {
+				return nil, err
+			}
 			//log.Trace("[sentinel] Sent request", "pid", pid)
 			s.sentinel.Peers().PeerDoRequest(pid)
 			go func() {
@@ -139,6 +143,9 @@ func (s *SentinelServer) SendRequest(_ context.Context, req *sentinelrpc.Request
 				case doneCh <- &sentinelrpc.ResponseData{
 					Data:  data,
 					Error: isError,
+					Peer: &sentinelrpc.Peer{
+						Pid: string(pidText),
+					},
 				}:
 				default:
 				}
@@ -149,6 +156,9 @@ func (s *SentinelServer) SendRequest(_ context.Context, req *sentinelrpc.Request
 			return &sentinelrpc.ResponseData{
 				Data:  []byte("sentinel timeout"),
 				Error: true,
+				Peer: &sentinelrpc.Peer{
+					Pid: "",
+				},
 			}, nil
 		}
 	}

go.mod

@@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
 go 1.19
 
 require (
-	github.com/ledgerwatch/erigon-lib v0.0.0-20230427033621-8ed42874e3e2
+	github.com/ledgerwatch/erigon-lib v0.0.0-20230429180756-1b57bef163a0
 	github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336
 	github.com/ledgerwatch/log/v3 v3.7.0
 	github.com/ledgerwatch/secp256k1 v1.0.0

go.sum

@@ -438,8 +438,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230427033621-8ed42874e3e2 h1:upWpgnnGdc/CZLok0F0uaVJmFhaHPDrjKsXpE6KeDsY=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230427033621-8ed42874e3e2/go.mod h1:D05f9OXc/2cnYxCyBexlu5HeIeQW9GKXynyWYzJ1F5I=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230429180756-1b57bef163a0 h1:OScZjxP4sf0UU2PRcFa6pnGPFmTYiIsEKlw4qfuGtqU=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230429180756-1b57bef163a0/go.mod h1:NMvXxA0hP92i39cdY4f79JYLfi7nJjWppX9Ati2KPbs=
 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM=
 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=